From d3fc38a073ee454e21d8d1ca094182c8602ce24d Mon Sep 17 00:00:00 2001 From: Tao Bao Date: Thu, 8 Mar 2018 16:09:01 -0800 Subject: [PATCH] releasetools: Create PropertyFiles class. And move StreamingPropertyFiles as its subclass. We will need similar PropertyFiles instance for non-A/B OTA as well (to expose the offset/size for the METADATA entry). Bug: 74210298 Test: python -m unittest test_ota_from_target_files Test: Generate an A/B OTA. Check the generated property-files string. Test: pylint --rcfile=pylintrc \ ota_from_target_files.py \ test_ota_from_target_files.py Change-Id: If90d97f0b330749fd8a6cde2ed9d0d6cd6ea60a8 (cherry picked from commit 432f374a1701909ca324f8b047666614684568c0) --- tools/releasetools/ota_from_target_files.py | 73 +++++--- .../test_ota_from_target_files.py | 162 +++++++++++++++--- 2 files changed, 184 insertions(+), 51 deletions(-) diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index 34db7063f..816b8f5a2 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -955,8 +955,15 @@ def GetPackageMetadata(target_info, source_info=None): return metadata -class StreamingPropertyFiles(object): - """Computes the ota-streaming-property-files string for streaming A/B OTA. +class PropertyFiles(object): + """A class that computes the property-files string for an OTA package. + + A property-files string is a comma-separated string that contains the + offset/size info for an OTA package. The entries, which must be ZIP_STORED, + can be fetched directly with the package URL along with the offset/size info. + These strings can be used for streaming A/B OTAs, or allowing an updater to + download package metadata entry directly, without paying the cost of + downloading entire package. Computing the final property-files string requires two passes. Because doing the whole package signing (with signapk.jar) will possibly reorder the ZIP @@ -966,7 +973,7 @@ class StreamingPropertyFiles(object): This class provides functions to be called for each pass. The general flow is as follows. - property_files = StreamingPropertyFiles() + property_files = PropertyFiles() # The first pass, which writes placeholders before doing initial signing. property_files.Compute() SignOutput() @@ -981,17 +988,9 @@ class StreamingPropertyFiles(object): """ def __init__(self): - self.required = ( - # payload.bin and payload_properties.txt must exist. - 'payload.bin', - 'payload_properties.txt', - ) - self.optional = ( - # care_map.txt is available only if dm-verity is enabled. - 'care_map.txt', - # compatibility.zip is available only if target supports Treble. - 'compatibility.zip', - ) + self.name = None + self.required = () + self.optional = () def Compute(self, input_zip): """Computes and returns a property-files string with placeholders. @@ -1083,7 +1082,26 @@ class StreamingPropertyFiles(object): return ','.join(tokens) -def FinalizeMetadata(metadata, input_file, output_file): +class StreamingPropertyFiles(PropertyFiles): + """A subclass for computing the property-files for streaming A/B OTAs.""" + + def __init__(self): + super(StreamingPropertyFiles, self).__init__() + self.name = 'ota-streaming-property-files' + self.required = ( + # payload.bin and payload_properties.txt must exist. + 'payload.bin', + 'payload_properties.txt', + ) + self.optional = ( + # care_map.txt is available only if dm-verity is enabled. + 'care_map.txt', + # compatibility.zip is available only if target supports Treble. + 'compatibility.zip', + ) + + +def FinalizeMetadata(metadata, input_file, output_file, needed_property_files): """Finalizes the metadata and signs an A/B OTA package. In order to stream an A/B OTA package, we need 'ota-streaming-property-files' @@ -1101,14 +1119,14 @@ def FinalizeMetadata(metadata, input_file, output_file): input_file: The input ZIP filename that doesn't contain the package METADATA entry yet. output_file: The final output ZIP filename. + needed_property_files: The list of PropertyFiles' to be generated. """ output_zip = zipfile.ZipFile( input_file, 'a', compression=zipfile.ZIP_DEFLATED) - property_files = StreamingPropertyFiles() - # Write the current metadata entry with placeholders. - metadata['ota-streaming-property-files'] = property_files.Compute(output_zip) + for property_files in needed_property_files: + metadata[property_files.name] = property_files.Compute(output_zip) WriteMetadata(metadata, output_zip) common.ZipClose(output_zip) @@ -1122,14 +1140,14 @@ def FinalizeMetadata(metadata, input_file, output_file): # Open the signed zip. Compute the final metadata that's needed for streaming. with zipfile.ZipFile(prelim_signing, 'r') as prelim_signing_zip: - expected_length = len(metadata['ota-streaming-property-files']) - metadata['ota-streaming-property-files'] = property_files.Finalize( - prelim_signing_zip, expected_length) + for property_files in needed_property_files: + metadata[property_files.name] = property_files.Finalize( + prelim_signing_zip, len(metadata[property_files.name])) # Replace the METADATA entry. common.ZipDelete(prelim_signing, METADATA_NAME) - output_zip = zipfile.ZipFile(prelim_signing, 'a', - compression=zipfile.ZIP_DEFLATED) + output_zip = zipfile.ZipFile( + prelim_signing, 'a', compression=zipfile.ZIP_DEFLATED) WriteMetadata(metadata, output_zip) common.ZipClose(output_zip) @@ -1138,8 +1156,8 @@ def FinalizeMetadata(metadata, input_file, output_file): # Reopen the final signed zip to double check the streaming metadata. with zipfile.ZipFile(output_file, 'r') as output_zip: - property_files.Verify( - output_zip, metadata['ota-streaming-property-files'].strip()) + for property_files in needed_property_files: + property_files.Verify(output_zip, metadata[property_files.name].strip()) def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): @@ -1564,7 +1582,10 @@ def WriteABOTAPackageWithBrilloScript(target_file, output_file, # FinalizeMetadata(). common.ZipClose(output_zip) - FinalizeMetadata(metadata, staging_file, output_file) + needed_property_files = ( + StreamingPropertyFiles(), + ) + FinalizeMetadata(metadata, staging_file, output_file, needed_property_files) def main(argv): diff --git a/tools/releasetools/test_ota_from_target_files.py b/tools/releasetools/test_ota_from_target_files.py index c8e87bff4..e2e85fd4b 100644 --- a/tools/releasetools/test_ota_from_target_files.py +++ b/tools/releasetools/test_ota_from_target_files.py @@ -26,8 +26,8 @@ from ota_from_target_files import ( _LoadOemDicts, BuildInfo, GetPackageMetadata, GetTargetFilesZipForSecondaryImages, GetTargetFilesZipWithoutPostinstallConfig, - Payload, PayloadSigner, POSTINSTALL_CONFIG, StreamingPropertyFiles, - WriteFingerprintAssertion) + Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles, + StreamingPropertyFiles, WriteFingerprintAssertion) def construct_target_files(secondary=False): @@ -590,7 +590,23 @@ class OtaFromTargetFilesTest(unittest.TestCase): self.assertNotIn(POSTINSTALL_CONFIG, verify_zip.namelist()) -class StreamingPropertyFilesTest(unittest.TestCase): +class TestPropertyFiles(PropertyFiles): + """A class that extends PropertyFiles for testing purpose.""" + + def __init__(self): + super(TestPropertyFiles, self).__init__() + self.name = 'ota-test-property-files' + self.required = ( + 'required-entry1', + 'required-entry2', + ) + self.optional = ( + 'optional-entry1', + 'optional-entry2', + ) + + +class PropertyFilesTest(unittest.TestCase): def tearDown(self): common.Cleanup() @@ -607,7 +623,7 @@ class StreamingPropertyFilesTest(unittest.TestCase): return zip_file @staticmethod - def _parse_streaming_metadata_string(data): + def _parse_property_files_string(data): result = {} for token in data.split(','): name, info = token.split(':', 1) @@ -627,47 +643,57 @@ class StreamingPropertyFilesTest(unittest.TestCase): def test_Compute(self): entries = ( - 'payload.bin', - 'payload_properties.txt', + 'required-entry1', + 'required-entry2', ) zip_file = self._construct_zip_package(entries) - property_files = StreamingPropertyFiles() + property_files = TestPropertyFiles() with zipfile.ZipFile(zip_file, 'r') as zip_fp: - streaming_metadata = property_files.Compute(zip_fp) + property_files_string = property_files.Compute(zip_fp) - tokens = self._parse_streaming_metadata_string(streaming_metadata) + tokens = self._parse_property_files_string(property_files_string) self.assertEqual(3, len(tokens)) self._verify_entries(zip_file, tokens, entries) - def test_Compute_withCareMapTxtAndCompatibilityZip(self): + def test_Compute_withOptionalEntries(self): entries = ( - 'payload.bin', - 'payload_properties.txt', - 'care_map.txt', - 'compatibility.zip', + 'required-entry1', + 'required-entry2', + 'optional-entry1', + 'optional-entry2', ) zip_file = self._construct_zip_package(entries) - property_files = StreamingPropertyFiles() + property_files = TestPropertyFiles() with zipfile.ZipFile(zip_file, 'r') as zip_fp: - streaming_metadata = property_files.Compute(zip_fp) + property_files_string = property_files.Compute(zip_fp) - tokens = self._parse_streaming_metadata_string(streaming_metadata) + tokens = self._parse_property_files_string(property_files_string) self.assertEqual(5, len(tokens)) self._verify_entries(zip_file, tokens, entries) + def test_Compute_missingRequiredEntry(self): + entries = ( + 'required-entry2', + ) + zip_file = self._construct_zip_package(entries) + property_files = TestPropertyFiles() + with zipfile.ZipFile(zip_file, 'r') as zip_fp: + self.assertRaises(KeyError, property_files.Compute, zip_fp) + def test_Finalize(self): entries = [ - 'payload.bin', - 'payload_properties.txt', + 'required-entry1', + 'required-entry2', 'META-INF/com/android/metadata', ] zip_file = self._construct_zip_package(entries) - property_files = StreamingPropertyFiles() + property_files = TestPropertyFiles() with zipfile.ZipFile(zip_file, 'r') as zip_fp: + # pylint: disable=protected-access raw_metadata = property_files._GetPropertyFilesString( zip_fp, reserve_space=False) streaming_metadata = property_files.Finalize(zip_fp, len(raw_metadata)) - tokens = self._parse_streaming_metadata_string(streaming_metadata) + tokens = self._parse_property_files_string(streaming_metadata) self.assertEqual(3, len(tokens)) # 'META-INF/com/android/metadata' will be key'd as 'metadata' in the @@ -677,15 +703,17 @@ class StreamingPropertyFilesTest(unittest.TestCase): def test_Finalize_assertReservedLength(self): entries = ( - 'payload.bin', - 'payload_properties.txt', - 'care_map.txt', + 'required-entry1', + 'required-entry2', + 'optional-entry1', + 'optional-entry2', 'META-INF/com/android/metadata', ) zip_file = self._construct_zip_package(entries) - property_files = StreamingPropertyFiles() + property_files = TestPropertyFiles() with zipfile.ZipFile(zip_file, 'r') as zip_fp: # First get the raw metadata string (i.e. without padding space). + # pylint: disable=protected-access raw_metadata = property_files._GetPropertyFilesString( zip_fp, reserve_space=False) raw_length = len(raw_metadata) @@ -709,16 +737,100 @@ class StreamingPropertyFilesTest(unittest.TestCase): self.assertEqual(' ' * 20, streaming_metadata[raw_length:]) def test_Verify(self): + entries = ( + 'required-entry1', + 'required-entry2', + 'optional-entry1', + 'optional-entry2', + 'META-INF/com/android/metadata', + ) + zip_file = self._construct_zip_package(entries) + property_files = TestPropertyFiles() + with zipfile.ZipFile(zip_file, 'r') as zip_fp: + # First get the raw metadata string (i.e. without padding space). + # pylint: disable=protected-access + raw_metadata = property_files._GetPropertyFilesString( + zip_fp, reserve_space=False) + + # Should pass the test if verification passes. + property_files.Verify(zip_fp, raw_metadata) + + # Or raise on verification failure. + self.assertRaises( + AssertionError, property_files.Verify, zip_fp, raw_metadata + 'x') + + +class StreamingPropertyFilesTest(PropertyFilesTest): + """Additional sanity checks specialized for StreamingPropertyFiles.""" + + def test_init(self): + property_files = StreamingPropertyFiles() + self.assertEqual('ota-streaming-property-files', property_files.name) + self.assertEqual( + ( + 'payload.bin', + 'payload_properties.txt', + ), + property_files.required) + self.assertEqual( + ( + 'care_map.txt', + 'compatibility.zip', + ), + property_files.optional) + + def test_Compute(self): entries = ( 'payload.bin', 'payload_properties.txt', 'care_map.txt', + 'compatibility.zip', + ) + zip_file = self._construct_zip_package(entries) + property_files = StreamingPropertyFiles() + with zipfile.ZipFile(zip_file, 'r') as zip_fp: + property_files_string = property_files.Compute(zip_fp) + + tokens = self._parse_property_files_string(property_files_string) + self.assertEqual(5, len(tokens)) + self._verify_entries(zip_file, tokens, entries) + + def test_Finalize(self): + entries = [ + 'payload.bin', + 'payload_properties.txt', + 'care_map.txt', + 'compatibility.zip', + 'META-INF/com/android/metadata', + ] + zip_file = self._construct_zip_package(entries) + property_files = StreamingPropertyFiles() + with zipfile.ZipFile(zip_file, 'r') as zip_fp: + # pylint: disable=protected-access + raw_metadata = property_files._GetPropertyFilesString( + zip_fp, reserve_space=False) + streaming_metadata = property_files.Finalize(zip_fp, len(raw_metadata)) + tokens = self._parse_property_files_string(streaming_metadata) + + self.assertEqual(5, len(tokens)) + # 'META-INF/com/android/metadata' will be key'd as 'metadata' in the + # streaming metadata. + entries[4] = 'metadata' + self._verify_entries(zip_file, tokens, entries) + + def test_Verify(self): + entries = ( + 'payload.bin', + 'payload_properties.txt', + 'care_map.txt', + 'compatibility.zip', 'META-INF/com/android/metadata', ) zip_file = self._construct_zip_package(entries) property_files = StreamingPropertyFiles() with zipfile.ZipFile(zip_file, 'r') as zip_fp: # First get the raw metadata string (i.e. without padding space). + # pylint: disable=protected-access raw_metadata = property_files._GetPropertyFilesString( zip_fp, reserve_space=False)