Merge "Runs python formatter and linter over merge_target_files."

This commit is contained in:
Treehugger Robot 2019-04-18 02:31:19 +00:00 committed by Gerrit Code Review
commit 8551d21cef
1 changed files with 149 additions and 203 deletions

View File

@ -13,11 +13,11 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""This script merges two partial target files packages.
"""
This script merges two partial target files packages (one of which contains
system files, and the other contains non-system files) together, producing a
complete target files package that can be used to generate an OTA package.
One package contains system files, and the other contains non-system files.
It produces a complete target files package that can be used to generate an
OTA package.
Usage: merge_target_files.py [args]
@ -126,8 +126,8 @@ system_extract_special_item_list = [
'META/*',
]
# default_system_misc_info_keys is a list of keys to obtain from the system instance of
# META/misc_info.txt. The remaining keys from the other instance.
# default_system_misc_info_keys is a list of keys to obtain from the system
# instance of META/misc_info.txt. The remaining keys from the other instance.
default_system_misc_info_keys = [
'avb_system_hashtree_enable',
@ -185,10 +185,8 @@ def extract_items(target_files, target_files_temp_dir, extract_item_list):
Args:
target_files: The target files zip archive from which to extract items.
target_files_temp_dir: The temporary directory where the extracted items
will land.
will land.
extract_item_list: A list of items to extract.
"""
@ -198,9 +196,7 @@ def extract_items(target_files, target_files_temp_dir, extract_item_list):
# zip file. Otherwise, the extraction step will fail.
with zipfile.ZipFile(
target_files,
'r',
allowZip64=True) as target_files_zipfile:
target_files, 'r', allowZip64=True) as target_files_zipfile:
target_files_namelist = target_files_zipfile.namelist()
filtered_extract_item_list = []
@ -214,18 +210,17 @@ def extract_items(target_files, target_files_temp_dir, extract_item_list):
# Extract from target_files into target_files_temp_dir the
# filtered_extract_item_list.
common.UnzipToDir(
target_files,
target_files_temp_dir,
filtered_extract_item_list)
common.UnzipToDir(target_files, target_files_temp_dir,
filtered_extract_item_list)
def copy_items(from_dir, to_dir, patterns):
"""Similar to extract_items() except uses an input dir instead of zip."""
file_paths = []
for dirpath, _, filenames in os.walk(from_dir):
file_paths.extend(os.path.relpath(path=os.path.join(dirpath, filename),
start=from_dir) for filename in filenames)
file_paths.extend(
os.path.relpath(path=os.path.join(dirpath, filename), start=from_dir)
for filename in filenames)
filtered_file_paths = set()
for pattern in patterns:
@ -250,26 +245,25 @@ def read_config_list(config_file_path):
Args:
config_file_path: The path to the config file to open and read.
Returns:
The list of strings in the config file.
"""
with open(config_file_path) as config_file:
return config_file.read().splitlines()
def validate_config_lists(
system_item_list,
system_misc_info_keys,
other_item_list):
def validate_config_lists(system_item_list, system_misc_info_keys,
other_item_list):
"""Performs validations on the merge config lists.
Args:
system_item_list: The list of items to extract from the partial
system target files package as is.
system_misc_info_keys: A list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
other_item_list: The list of items to extract from the partial
other target files package as is.
system_item_list: The list of items to extract from the partial system
target files package as is.
system_misc_info_keys: A list of keys to obtain from the system instance of
META/misc_info.txt. The remaining keys from the other instance.
other_item_list: The list of items to extract from the partial other target
files package as is.
Returns:
False if a validation fails, otherwise true.
@ -284,7 +278,7 @@ def validate_config_lists(
# by the default config lists.
difference = default_combined_item_set.difference(combined_item_set)
if difference:
logger.error('Missing merge config items: %s' % list(difference))
logger.error('Missing merge config items: %s', list(difference))
logger.error('Please ensure missing items are in either the '
'system-item-list or other-item-list files provided to '
'this script.')
@ -299,11 +293,10 @@ def validate_config_lists(
return True
def process_ab_partitions_txt(
system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir):
"""Perform special processing for META/ab_partitions.txt
def process_ab_partitions_txt(system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir):
"""Perform special processing for META/ab_partitions.txt.
This function merges the contents of the META/ab_partitions.txt files from
the system directory and the other directory, placing the merged result in
@ -314,22 +307,20 @@ def process_ab_partitions_txt(
names.
Args:
system_target_files_temp_dir: The name of a directory containing the
special items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the
special items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used
to create the output target files package after all the special cases
are processed.
system_target_files_temp_dir: The name of a directory containing the special
items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the special
items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used to
create the output target files package after all the special cases are
processed.
"""
system_ab_partitions_txt = os.path.join(
system_target_files_temp_dir, 'META', 'ab_partitions.txt')
system_ab_partitions_txt = os.path.join(system_target_files_temp_dir, 'META',
'ab_partitions.txt')
other_ab_partitions_txt = os.path.join(
other_target_files_temp_dir, 'META', 'ab_partitions.txt')
other_ab_partitions_txt = os.path.join(other_target_files_temp_dir, 'META',
'ab_partitions.txt')
with open(system_ab_partitions_txt) as f:
system_ab_partitions = f.read().splitlines()
@ -339,8 +330,8 @@ def process_ab_partitions_txt(
output_ab_partitions = set(system_ab_partitions + other_ab_partitions)
output_ab_partitions_txt = os.path.join(
output_target_files_temp_dir, 'META', 'ab_partitions.txt')
output_ab_partitions_txt = os.path.join(output_target_files_temp_dir, 'META',
'ab_partitions.txt')
with open(output_ab_partitions_txt, 'w') as output:
for partition in sorted(output_ab_partitions):
@ -348,42 +339,35 @@ def process_ab_partitions_txt(
def append_recovery_to_filesystem_config(output_target_files_temp_dir):
"""Perform special processing for META/filesystem_config.txt
"""Perform special processing for META/filesystem_config.txt.
This function appends recovery information to META/filesystem_config.txt
so that recovery patch regeneration will succeed.
Args:
output_target_files_temp_dir: The name of a directory that will be used
to create the output target files package after all the special cases
are processed. We find filesystem_config.txt here.
output_target_files_temp_dir: The name of a directory that will be used to
create the output target files package after all the special cases are
processed. We find filesystem_config.txt here.
"""
filesystem_config_txt = os.path.join(
output_target_files_temp_dir,
'META',
'filesystem_config.txt')
filesystem_config_txt = os.path.join(output_target_files_temp_dir, 'META',
'filesystem_config.txt')
with open(filesystem_config_txt, 'a') as f:
# TODO(bpeckham) this data is hard coded. It should be generated
# programmatically.
f.write(
'system/bin/install-recovery.sh 0 0 750 '
'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
f.write(
'system/recovery-from-boot.p 0 0 644 '
'selabel=u:object_r:system_file:s0 capabilities=0x0\n')
f.write(
'system/etc/recovery.img 0 0 440 '
'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
f.write('system/bin/install-recovery.sh 0 0 750 '
'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
f.write('system/recovery-from-boot.p 0 0 644 '
'selabel=u:object_r:system_file:s0 capabilities=0x0\n')
f.write('system/etc/recovery.img 0 0 440 '
'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
def process_misc_info_txt(
system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir,
system_misc_info_keys):
"""Perform special processing for META/misc_info.txt
def process_misc_info_txt(system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir, system_misc_info_keys):
"""Perform special processing for META/misc_info.txt.
This function merges the contents of the META/misc_info.txt files from the
system directory and the other directory, placing the merged result in the
@ -392,18 +376,15 @@ def process_misc_info_txt(
content.
Args:
system_target_files_temp_dir: The name of a directory containing the
special items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the
special items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used
to create the output target files package after all the special cases
are processed.
system_misc_info_keys: A list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
system_target_files_temp_dir: The name of a directory containing the special
items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the special
items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used to
create the output target files package after all the special cases are
processed.
system_misc_info_keys: A list of keys to obtain from the system instance of
META/misc_info.txt. The remaining keys from the other instance.
"""
def read_helper(d):
@ -434,19 +415,18 @@ def process_misc_info_txt(
# Partition groups and group sizes are defined by the other (non-system)
# misc info file because these values may vary for each board that uses
# a shared system image.
for partition_group in merged_info_dict['super_partition_groups'].split(' '):
for partition_group in merged_info_dict['super_partition_groups'].split(
' '):
if ('super_%s_group_size' % partition_group) not in merged_info_dict:
raise ValueError(
'Other META/misc_info.txt does not contain required key '
'super_%s_group_size.' % partition_group)
key = 'super_%s_partition_list' % partition_group
merged_info_dict[key] = '%s %s' % (
system_info_dict.get(key, ''),
merged_info_dict.get(key, ''))
merged_info_dict[key] = '%s %s' % (system_info_dict.get(
key, ''), merged_info_dict.get(key, ''))
output_misc_info_txt = os.path.join(
output_target_files_temp_dir,
'META', 'misc_info.txt')
output_misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',
'misc_info.txt')
sorted_keys = sorted(merged_info_dict.keys())
@ -464,12 +444,11 @@ def process_file_contexts_bin(temp_dir, output_target_files_temp_dir):
Args:
temp_dir: The name of a scratch directory that this function can use for
intermediate files generated during processing.
intermediate files generated during processing.
output_target_files_temp_dir: The name of the working directory that must
already contain plat_file_contexts and vendor_file_contexts (in the
appropriate sub directories), and to which META/file_contexts.bin will be
written.
already contain plat_file_contexts and vendor_file_contexts (in the
appropriate sub directories), and to which META/file_contexts.bin will be
written.
"""
# To create a merged file_contexts.bin file, we use the system and vendor
@ -490,9 +469,8 @@ def process_file_contexts_bin(temp_dir, output_target_files_temp_dir):
for partition in ['SYSTEM', 'VENDOR', 'PRODUCT', 'ODM']:
prefix = 'plat' if partition == 'SYSTEM' else partition.lower()
file_contexts = os.path.join(
output_target_files_temp_dir,
partition, 'etc', 'selinux', prefix + '_file_contexts')
file_contexts = os.path.join(output_target_files_temp_dir, partition, 'etc',
'selinux', prefix + '_file_contexts')
mandatory = partition in ['SYSTEM', 'VENDOR']
@ -518,27 +496,23 @@ def process_file_contexts_bin(temp_dir, output_target_files_temp_dir):
# Finally, the compile step creates the final META/file_contexts.bin.
file_contexts_bin = os.path.join(
output_target_files_temp_dir,
'META', 'file_contexts.bin')
file_contexts_bin = os.path.join(output_target_files_temp_dir, 'META',
'file_contexts.bin')
command = [
'sefcontext_compile',
'-o', file_contexts_bin,
'-o',
file_contexts_bin,
sorted_file_contexts_txt,
]
common.RunAndWait(command, verbose=True)
def process_special_cases(
temp_dir,
system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir,
system_misc_info_keys,
rebuild_recovery
):
def process_special_cases(temp_dir, system_target_files_temp_dir,
other_target_files_temp_dir,
output_target_files_temp_dir, system_misc_info_keys,
rebuild_recovery):
"""Perform special-case processing for certain target files items.
Certain files in the output target files package require special-case
@ -546,23 +520,18 @@ def process_special_cases(
Args:
temp_dir: The name of a scratch directory that this function can use for
intermediate files generated during processing.
system_target_files_temp_dir: The name of a directory containing the
special items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the
special items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used
to create the output target files package after all the special cases
are processed.
system_misc_info_keys: A list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
intermediate files generated during processing.
system_target_files_temp_dir: The name of a directory containing the special
items extracted from the system target files package.
other_target_files_temp_dir: The name of a directory containing the special
items extracted from the other target files package.
output_target_files_temp_dir: The name of a directory that will be used to
create the output target files package after all the special cases are
processed.
system_misc_info_keys: A list of keys to obtain from the system instance of
META/misc_info.txt. The remaining keys from the other instance.
rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
devices and write it to the system image.
devices and write it to the system image.
"""
if 'ab_update' in system_misc_info_keys:
@ -586,19 +555,11 @@ def process_special_cases(
output_target_files_temp_dir=output_target_files_temp_dir)
def merge_target_files(
temp_dir,
system_target_files,
system_item_list,
system_misc_info_keys,
other_target_files,
other_item_list,
output_target_files,
output_dir,
output_item_list,
output_ota,
output_super_empty,
rebuild_recovery):
def merge_target_files(temp_dir, system_target_files, system_item_list,
system_misc_info_keys, other_target_files,
other_item_list, output_target_files, output_dir,
output_item_list, output_ota, output_super_empty,
rebuild_recovery):
"""Merge two target files packages together.
This function takes system and other target files packages as input, performs
@ -607,45 +568,35 @@ def merge_target_files(
Args:
temp_dir: The name of a directory we use when we extract items from the
input target files packages, and also a scratch directory that we use for
temporary files.
input target files packages, and also a scratch directory that we use for
temporary files.
system_target_files: The name of the zip archive containing the system
partial target files package.
partial target files package.
system_item_list: The list of items to extract from the partial system
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial system
target files package.
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial system
target files package.
system_misc_info_keys: The list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
other_target_files: The name of the zip archive containing the other
partial target files package.
other_item_list: The list of items to extract from the partial other
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial other
target files package.
output_target_files: The name of the output zip archive target files
package created by merging system and other.
of META/misc_info.txt. The remaining keys from the other instance.
other_target_files: The name of the zip archive containing the other partial
target files package.
other_item_list: The list of items to extract from the partial other target
files package as is, meaning these items will land in the output target
files package exactly as they appear in the input partial other target
files package.
output_target_files: The name of the output zip archive target files package
created by merging system and other.
output_dir: The destination directory for saving merged files.
output_item_list: The list of items to copy into the output_dir.
output_ota: The name of the output zip archive ota package.
output_super_empty: If provided, creates a super_empty.img file from the
merged target files package and saves it at this path.
merged target files package and saves it at this path.
rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
devices and write it to the system image.
devices and write it to the system image.
"""
logger.info(
'starting: merge system %s and other %s into output %s',
system_target_files,
other_target_files,
output_target_files)
logger.info('starting: merge system %s and other %s into output %s',
system_target_files, other_target_files, output_target_files)
# Create directory names that we'll use when we extract files from system,
# and other, and for zipping the final output.
@ -706,8 +657,9 @@ def merge_target_files(
# Create super_empty.img using the merged misc_info.txt.
if output_super_empty:
misc_info_txt = os.path.join(output_target_files_temp_dir,
'META', 'misc_info.txt')
misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',
'misc_info.txt')
def read_helper():
with open(misc_info_txt) as f:
return list(f.read().splitlines())
@ -718,9 +670,9 @@ def merge_target_files(
'Building super_empty.img requires use_dynamic_partitions=true.')
build_super_image_args = [
'--verbose',
misc_info_txt,
output_super_empty,
'--verbose',
misc_info_txt,
output_super_empty,
]
build_super_image.main(build_super_image_args)
@ -744,8 +696,8 @@ def merge_target_files(
output_zip = os.path.abspath(output_target_files)
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_target_files_meta_dir = os.path.join(
output_target_files_temp_dir, 'META')
output_target_files_meta_dir = os.path.join(output_target_files_temp_dir,
'META')
command = [
'find',
@ -754,13 +706,8 @@ def merge_target_files(
# TODO(bpeckham): sort this to be more like build.
meta_content = common.RunAndCheckOutput(command, verbose=False)
command = [
'find',
output_target_files_temp_dir,
'-path',
output_target_files_meta_dir,
'-prune',
'-o',
'-print'
'find', output_target_files_temp_dir, '-path',
output_target_files_meta_dir, '-prune', '-o', '-print'
]
# TODO(bpeckham): sort this to be more like build.
other_content = common.RunAndCheckOutput(command, verbose=False)
@ -772,9 +719,12 @@ def merge_target_files(
command = [
'soong_zip',
'-d',
'-o', output_zip,
'-C', output_target_files_temp_dir,
'-l', output_target_files_list,
'-o',
output_zip,
'-C',
output_target_files_temp_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_target_files)
common.RunAndWait(command, verbose=True)
@ -796,9 +746,8 @@ def call_func_with_temp_dir(func, keep_tmp):
directory. It also cleans up the temporary directory.
Args:
func: The function to call. Should accept one parameter, the path to
the temporary directory.
func: The function to call. Should accept one parameter, the path to the
temporary directory.
keep_tmp: Keep the temporary directory after processing is complete.
"""
@ -858,7 +807,8 @@ def main():
return True
args = common.ParseOptions(
sys.argv[1:], __doc__,
sys.argv[1:],
__doc__,
extra_long_opts=[
'system-target-files=',
'system-item-list=',
@ -875,13 +825,10 @@ def main():
],
extra_option_handler=option_handler)
if (len(args) != 0 or
OPTIONS.system_target_files is None or
OPTIONS.other_target_files is None or (
OPTIONS.output_target_files is None and
OPTIONS.output_dir is None) or (
OPTIONS.output_dir is not None and
OPTIONS.output_item_list is None)):
if (args or OPTIONS.system_target_files is None or
OPTIONS.other_target_files is None or
(OPTIONS.output_target_files is None and OPTIONS.output_dir is None) or
(OPTIONS.output_dir is not None and OPTIONS.output_item_list is None)):
common.Usage(__doc__)
sys.exit(1)
@ -924,8 +871,7 @@ def main():
output_item_list=output_item_list,
output_ota=OPTIONS.output_ota,
output_super_empty=OPTIONS.output_super_empty,
rebuild_recovery=OPTIONS.rebuild_recovery),
OPTIONS.keep_tmp)
rebuild_recovery=OPTIONS.rebuild_recovery), OPTIONS.keep_tmp)
if __name__ == '__main__':