Merge "Reland "Calculate the runtime fingerprint prefixes from build prop""

This commit is contained in:
Tianjie Xu 2020-05-12 18:51:25 +00:00 committed by Gerrit Code Review
commit e9ab85956f
4 changed files with 429 additions and 23 deletions

View File

@ -738,18 +738,22 @@ class PartitionBuildProps(object):
partition: name of the partition.
props_allow_override: a list of build properties to search for the
alternative values during runtime.
build_props: a dictionary of build properties for the given partition.
prop_overrides: a dict of list. And each list holds the overridden values
for props_allow_override.
build_props: a dict of build properties for the given partition.
prop_overrides: a set of props that are overridden by import.
placeholder_values: A dict of runtime variables' values to replace the
placeholders in the build.prop file. We expect exactly one value for
each of the variables.
"""
def __init__(self, input_file, name):
def __init__(self, input_file, name, placeholder_values=None):
self.input_file = input_file
self.partition = name
self.props_allow_override = [props.format(name) for props in [
'ro.product.{}.name', 'ro.product.{}.device']]
'ro.product.{}.brand', 'ro.product.{}.name', 'ro.product.{}.device']]
self.build_props = {}
self.prop_overrides = {}
self.prop_overrides = set()
self.placeholder_values = {}
if placeholder_values:
self.placeholder_values = copy.deepcopy(placeholder_values)
@staticmethod
def FromDictionary(name, build_props):
@ -760,9 +764,8 @@ class PartitionBuildProps(object):
return props
@staticmethod
def FromInputFile(input_file, name):
def FromInputFile(input_file, name, placeholder_values=None):
"""Loads the build.prop file and builds the attributes."""
data = ''
for prop_file in ['{}/etc/build.prop'.format(name.upper()),
'{}/build.prop'.format(name.upper())]:
@ -772,10 +775,62 @@ class PartitionBuildProps(object):
except KeyError:
logger.warning('Failed to read %s', prop_file)
props = PartitionBuildProps(input_file, name)
props.build_props = LoadDictionaryFromLines(data.split('\n'))
props = PartitionBuildProps(input_file, name, placeholder_values)
props._LoadBuildProp(data)
return props
def _LoadBuildProp(self, data):
for line in data.split('\n'):
line = line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("import"):
overrides = self._ImportParser(line)
duplicates = self.prop_overrides.intersection(overrides.keys())
if duplicates:
raise ValueError('prop {} is overridden multiple times'.format(
','.join(duplicates)))
self.prop_overrides = self.prop_overrides.union(overrides.keys())
self.build_props.update(overrides)
elif "=" in line:
name, value = line.split("=", 1)
if name in self.prop_overrides:
raise ValueError('prop {} is set again after overridden by import '
'statement'.format(name))
self.build_props[name] = value
def _ImportParser(self, line):
"""Parses the build prop in a given import statement."""
tokens = line.split()
if len(tokens) != 2 or tokens[0] != 'import':
raise ValueError('Unrecognized import statement {}'.format(line))
import_path = tokens[1]
if not re.match(r'^/{}/.*\.prop$'.format(self.partition), import_path):
raise ValueError('Unrecognized import path {}'.format(line))
# We only recognize a subset of import statement that the init process
# supports. And we can loose the restriction based on how the dynamic
# fingerprint is used in practice. The placeholder format should be
# ${placeholder}, and its value should be provided by the caller through
# the placeholder_values.
for prop, value in self.placeholder_values.items():
prop_place_holder = '${{{}}}'.format(prop)
if prop_place_holder in import_path:
import_path = import_path.replace(prop_place_holder, value)
if '$' in import_path:
logger.info('Unresolved place holder in import path %s', import_path)
return {}
import_path = import_path.replace('/{}'.format(self.partition),
self.partition.upper())
logger.info('Parsing build props override from %s', import_path)
lines = ReadFromInputFile(self.input_file, import_path).split('\n')
d = LoadDictionaryFromLines(lines)
return {key: val for key, val in d.items()
if key in self.props_allow_override}
def GetProp(self, prop):
return self.build_props.get(prop)

View File

@ -193,6 +193,8 @@ A/B OTA specific options
from __future__ import print_function
import collections
import copy
import itertools
import logging
import multiprocessing
import os.path
@ -229,6 +231,7 @@ OPTIONS.include_secondary = False
OPTIONS.no_signing = False
OPTIONS.block_based = True
OPTIONS.updater_binary = None
OPTIONS.oem_dicts = None
OPTIONS.oem_source = None
OPTIONS.oem_no_mount = False
OPTIONS.full_radio = False
@ -247,6 +250,7 @@ OPTIONS.retrofit_dynamic_partitions = False
OPTIONS.skip_compatibility_check = False
OPTIONS.output_metadata_path = None
OPTIONS.disable_fec_computation = False
OPTIONS.boot_variable_values = None
METADATA_NAME = 'META-INF/com/android/metadata'
@ -1959,6 +1963,36 @@ def GenerateNonAbOtaPackage(target_file, output_file, source_file=None):
output_file)
def CalculateRuntimeFingerprints():
"""Returns a set of runtime fingerprints based on the boot variables."""
build_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
fingerprints = {build_info.fingerprint}
if not OPTIONS.boot_variable_values:
return fingerprints
# Calculate all possible combinations of the values for the boot variables.
keys = OPTIONS.boot_variable_values.keys()
value_list = OPTIONS.boot_variable_values.values()
combinations = [dict(zip(keys, values))
for values in itertools.product(*value_list)]
for placeholder_values in combinations:
# Reload the info_dict as some build properties may change their values
# based on the value of ro.boot* properties.
info_dict = copy.deepcopy(OPTIONS.info_dict)
for partition in common.PARTITIONS_WITH_CARE_MAP:
partition_prop_key = "{}.build.prop".format(partition)
old_props = info_dict[partition_prop_key]
info_dict[partition_prop_key] = common.PartitionBuildProps.FromInputFile(
old_props.input_file, partition, placeholder_values)
info_dict["build.prop"] = info_dict["system.build.prop"]
build_info = common.BuildInfo(info_dict, OPTIONS.oem_dicts)
fingerprints.add(build_info.fingerprint)
return fingerprints
def main(argv):
def option_handler(o, a):

View File

@ -1899,7 +1899,7 @@ super_group_foo_group_size={group_foo_size}
class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
def setUp(self):
self.build_prop = [
self.odm_build_prop = [
'ro.odm.build.date.utc=1578430045',
'ro.odm.build.fingerprint='
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
@ -1918,13 +1918,81 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
def test_parseBuildProps_noImportStatement(self):
build_prop = [
'ro.odm.build.date.utc=1578430045',
'ro.odm.build.fingerprint='
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device=coral',
'ro.odm.build.date.utc=1578430045',
'ro.odm.build.fingerprint='
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device=coral',
]
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(build_prop),
'ODM/etc/build.prop': '\n'.join(build_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': ['std', 'pro']
}
partition_props = common.PartitionBuildProps.FromInputFile(
input_zip, 'odm', placeholder_values)
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coral',
}, partition_props.build_props)
self.assertEqual(set(), partition_props.prop_overrides)
def test_parseBuildProps_singleImportStatement(self):
build_std_prop = [
'ro.product.odm.device=coral',
'ro.product.odm.name=product1',
]
build_pro_prop = [
'ro.product.odm.device=coralpro',
'ro.product.odm.name=product2',
]
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(self.odm_build_prop),
'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'std'
}
partition_props = common.PartitionBuildProps.FromInputFile(
input_zip, 'odm', placeholder_values)
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coral',
'ro.product.odm.name': 'product1',
}, partition_props.build_props)
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'pro'
}
partition_props = common.PartitionBuildProps.FromInputFile(
input_zip, 'odm', placeholder_values)
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coralpro',
'ro.product.odm.name': 'product2',
}, partition_props.build_props)
def test_parseBuildProps_noPlaceHolders(self):
build_prop = copy.copy(self.odm_build_prop)
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(build_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
@ -1932,10 +2000,136 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
input_zip, 'odm')
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coral',
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coral',
}, partition_props.build_props)
self.assertEqual({}, partition_props.prop_overrides)
self.assertEqual(set(), partition_props.prop_overrides)
def test_parseBuildProps_multipleImportStatements(self):
build_prop = copy.deepcopy(self.odm_build_prop)
build_prop.append(
'import /odm/etc/build_${ro.boot.product.product_name}.prop')
build_std_prop = [
'ro.product.odm.device=coral',
]
build_pro_prop = [
'ro.product.odm.device=coralpro',
]
product1_prop = [
'ro.product.odm.name=product1',
'ro.product.not_care=not_care',
]
product2_prop = [
'ro.product.odm.name=product2',
'ro.product.not_care=not_care',
]
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(build_prop),
'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'std',
'ro.boot.product.product_name': 'product1',
'ro.boot.product.not_care': 'not_care',
}
partition_props = common.PartitionBuildProps.FromInputFile(
input_zip, 'odm', placeholder_values)
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coral',
'ro.product.odm.name': 'product1'
}, partition_props.build_props)
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'pro',
'ro.boot.product.product_name': 'product2',
'ro.boot.product.not_care': 'not_care',
}
partition_props = common.PartitionBuildProps.FromInputFile(
input_zip, 'odm', placeholder_values)
self.assertEqual({
'ro.odm.build.date.utc': '1578430045',
'ro.odm.build.fingerprint':
'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
'ro.product.odm.device': 'coralpro',
'ro.product.odm.name': 'product2'
}, partition_props.build_props)
def test_parseBuildProps_defineAfterOverride(self):
build_prop = copy.deepcopy(self.odm_build_prop)
build_prop.append('ro.product.odm.device=coral')
build_std_prop = [
'ro.product.odm.device=coral',
]
build_pro_prop = [
'ro.product.odm.device=coralpro',
]
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(build_prop),
'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'std',
}
self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
input_zip, 'odm', placeholder_values)
def test_parseBuildProps_duplicateOverride(self):
build_prop = copy.deepcopy(self.odm_build_prop)
build_prop.append(
'import /odm/etc/build_${ro.boot.product.product_name}.prop')
build_std_prop = [
'ro.product.odm.device=coral',
'ro.product.odm.name=product1',
]
build_pro_prop = [
'ro.product.odm.device=coralpro',
]
product1_prop = [
'ro.product.odm.name=product1',
]
product2_prop = [
'ro.product.odm.name=product2',
]
input_file = self._BuildZipFile({
'ODM/etc/build.prop': '\n'.join(build_prop),
'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
})
with zipfile.ZipFile(input_file, 'r') as input_zip:
placeholder_values = {
'ro.boot.product.device_name': 'std',
'ro.boot.product.product_name': 'product1',
}
self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
input_zip, 'odm', placeholder_values)

View File

@ -26,7 +26,8 @@ from ota_from_target_files import (
GetPackageMetadata, GetTargetFilesZipForSecondaryImages,
GetTargetFilesZipWithoutPostinstallConfig, NonAbOtaPropertyFiles,
Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles,
StreamingPropertyFiles, WriteFingerprintAssertion)
StreamingPropertyFiles, WriteFingerprintAssertion,
CalculateRuntimeFingerprints)
def construct_target_files(secondary=False):
@ -1318,3 +1319,125 @@ class PayloadTest(test_utils.ReleaseToolsTestCase):
Payload.SECONDARY_PAYLOAD_PROPERTIES_TXT):
continue
self.assertEqual(zipfile.ZIP_STORED, entry_info.compress_type)
class RuntimeFingerprintTest(test_utils.ReleaseToolsTestCase):
MISC_INFO = [
'recovery_api_version=3',
'fstab_version=2',
'recovery_as_boot=true',
]
BUILD_PROP = [
'ro.build.version.release=version-release',
'ro.build.id=build-id',
'ro.build.version.incremental=version-incremental',
'ro.build.type=build-type',
'ro.build.tags=build-tags',
]
VENDOR_BUILD_PROP = [
'ro.product.vendor.brand=vendor-product-brand',
'ro.product.vendor.name=vendor-product-name',
'ro.product.vendor.device=vendor-product-device'
]
def setUp(self):
common.OPTIONS.oem_dicts = None
self.test_dir = common.MakeTempDir()
self.writeFiles({'META/misc_info.txt': '\n'.join(self.MISC_INFO)})
def writeFiles(self, contents_dict):
for path, content in contents_dict.items():
abs_path = os.path.join(self.test_dir, path)
dir_name = os.path.dirname(abs_path)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
with open(abs_path, 'w') as f:
f.write(content)
@staticmethod
def constructFingerprint(prefix):
return '{}:version-release/build-id/version-incremental:' \
'build-type/build-tags'.format(prefix)
def test_CalculatePossibleFingerprints_no_dynamic_fingerprint(self):
build_prop = copy.deepcopy(self.BUILD_PROP)
build_prop.extend([
'ro.product.brand=product-brand',
'ro.product.name=product-name',
'ro.product.device=product-device',
])
self.writeFiles({
'SYSTEM/build.prop': '\n'.join(build_prop),
'VENDOR/build.prop': '\n'.join(self.VENDOR_BUILD_PROP),
})
common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
self.assertEqual({
self.constructFingerprint('product-brand/product-name/product-device')
}, CalculateRuntimeFingerprints())
def test_CalculatePossibleFingerprints_single_override(self):
vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
vendor_build_prop.extend([
'import /vendor/etc/build_${ro.boot.sku_name}.prop',
])
self.writeFiles({
'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
'VENDOR/build.prop': '\n'.join(vendor_build_prop),
'VENDOR/etc/build_std.prop':
'ro.product.vendor.name=vendor-product-std',
'VENDOR/etc/build_pro.prop':
'ro.product.vendor.name=vendor-product-pro',
})
common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
common.OPTIONS.boot_variable_values = {
'ro.boot.sku_name': ['std', 'pro']
}
self.assertEqual({
self.constructFingerprint(
'vendor-product-brand/vendor-product-name/vendor-product-device'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-std/vendor-product-device'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-pro/vendor-product-device'),
}, CalculateRuntimeFingerprints())
def test_CalculatePossibleFingerprints_multiple_overrides(self):
vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
vendor_build_prop.extend([
'import /vendor/etc/build_${ro.boot.sku_name}.prop',
'import /vendor/etc/build_${ro.boot.device_name}.prop',
])
self.writeFiles({
'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
'VENDOR/build.prop': '\n'.join(vendor_build_prop),
'VENDOR/etc/build_std.prop':
'ro.product.vendor.name=vendor-product-std',
'VENDOR/etc/build_product1.prop':
'ro.product.vendor.device=vendor-device-product1',
'VENDOR/etc/build_pro.prop':
'ro.product.vendor.name=vendor-product-pro',
'VENDOR/etc/build_product2.prop':
'ro.product.vendor.device=vendor-device-product2',
})
common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
common.OPTIONS.boot_variable_values = {
'ro.boot.sku_name': ['std', 'pro'],
'ro.boot.device_name': ['product1', 'product2'],
}
self.assertEqual({
self.constructFingerprint(
'vendor-product-brand/vendor-product-name/vendor-product-device'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-std/vendor-device-product1'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-pro/vendor-device-product1'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-std/vendor-device-product2'),
self.constructFingerprint(
'vendor-product-brand/vendor-product-pro/vendor-device-product2'),
}, CalculateRuntimeFingerprints())