diff --git a/core/Makefile b/core/Makefile index c281d195a..b7e155b5b 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1911,6 +1911,13 @@ else $(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := RECOVERY endif +# Run fs_config while creating the target files package +# $1: root directory +# $2: add prefix +define fs_config +(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) +endef + # Depending on the various images guarantees that the underlying # directories are up-to-date. $(BUILT_TARGET_FILES_PACKAGE): \ @@ -2134,28 +2141,29 @@ ifdef BOARD_PREBUILT_VENDORIMAGE $(hide) mkdir -p $(zip_root)/IMAGES $(hide) cp $(INSTALLED_VENDORIMAGE_TARGET) $(zip_root)/IMAGES/ endif + @# Run fs_config on all the system, vendor, boot ramdisk, + @# and recovery ramdisk files in the zip, and save the output + $(hide) $(call fs_config,$(zip_root)/SYSTEM,system/) > $(zip_root)/META/filesystem_config.txt +ifdef BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE + $(hide) $(call fs_config,$(zip_root)/VENDOR,vendor/) > $(zip_root)/META/vendor_filesystem_config.txt +endif +ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true) + $(hide) $(call fs_config,$(zip_root)/ROOT,) > $(zip_root)/META/root_filesystem_config.txt +endif + $(hide) $(call fs_config,$(zip_root)/BOOT/RAMDISK,) > $(zip_root)/META/boot_filesystem_config.txt +ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),) + $(hide) $(call fs_config,$(zip_root)/RECOVERY/RAMDISK,) > $(zip_root)/META/recovery_filesystem_config.txt +endif +ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET + $(hide) $(call fs_config,$(zip_root)/SYSTEM_OTHER,system/) > $(zip_root)/META/system_other_filesystem_config.txt +endif + $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \ + ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $(zip_root) @# Zip everything up, preserving symlinks and placing META/ files first to @# help early validation of the .zip file while uploading it. $(hide) find $(zip_root)/META | sort >$@.list $(hide) find $(zip_root) | grep -v "^$(zip_root)/META/" | sort >>$@.list $(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -l $@.list - @# Run fs_config on all the system, vendor, boot ramdisk, - @# and recovery ramdisk files in the zip, and save the output - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM/" } /^SYSTEM\// {print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/filesystem_config.txt - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="VENDOR/" } /^VENDOR\// {print "vendor/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/vendor_filesystem_config.txt -ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true) - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="ROOT/" } /^ROOT\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/root_filesystem_config.txt -endif - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="BOOT/RAMDISK/" } /^BOOT\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/boot_filesystem_config.txt -ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),) - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="RECOVERY/RAMDISK/" } /^RECOVERY\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/recovery_filesystem_config.txt -endif -ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET - $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM_OTHER/" } /^SYSTEM_OTHER\// { print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/system_other_filesystem_config.txt -endif - $(hide) (cd $(zip_root) && zip -qX ../$(notdir $@) META/*filesystem_config.txt) - $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \ - ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $@ .PHONY: target-files-package target-files-package: $(BUILT_TARGET_FILES_PACKAGE) diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py index 9403a77f2..0b8f3a0d3 100755 --- a/tools/releasetools/add_img_to_target_files.py +++ b/tools/releasetools/add_img_to_target_files.py @@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False OPTIONS.replace_verity_private_key = False OPTIONS.is_signing = False + +class OutputFile(object): + def __init__(self, output_zip, input_dir, prefix, name): + self._output_zip = output_zip + self.input_name = os.path.join(input_dir, prefix, name) + + if self._output_zip: + self._zip_name = os.path.join(prefix, name) + + root, suffix = os.path.splitext(name) + self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix) + else: + self.name = self.input_name + + def Write(self): + if self._output_zip: + common.ZipWrite(self._output_zip, self.name, self._zip_name) + + def GetCareMap(which, imgname): """Generate care_map of system (or vendor) partition""" @@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): """Turn the contents of SYSTEM into a system image and store it in output_zip. Returns the name of the system image file.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img") + if os.path.exists(img.input_name): print("system.img already exists in %s, no need to rebuild..." % (prefix,)) - return prebuilt_path + return img.input_name def output_sink(fn, data): ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") @@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, boot_img, info_dict=OPTIONS.info_dict) - block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") - imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, - block_list=block_list) + block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map") + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img, + block_list=block_list) - common.ZipWrite(output_zip, imgname, prefix + "system.img") - common.ZipWrite(output_zip, block_list, prefix + "system.map") - return imgname - - -def BuildSystem(input_dir, info_dict, block_list=None): - """Build the (sparse) system image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "system", block_list=block_list) + return img.name def AddSystemOther(output_zip, prefix="IMAGES/"): """Turn the contents of SYSTEM_OTHER into a system_other image and store it in output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img") + if os.path.exists(img.input_name): print("system_other.img already exists in %s, no need to rebuild..." % ( prefix,)) return - imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) - common.ZipWrite(output_zip, imgname, prefix + "system_other.img") - -def BuildSystemOther(input_dir, info_dict): - """Build the (sparse) system_other image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "system_other", block_list=None) + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img) def AddVendor(output_zip, prefix="IMAGES/"): """Turn the contents of VENDOR into a vendor image and store in it output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img") + if os.path.exists(img.input_name): print("vendor.img already exists in %s, no need to rebuild..." % (prefix,)) - return prebuilt_path + return img.input_name - block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") - imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, - block_list=block_list) - common.ZipWrite(output_zip, imgname, prefix + "vendor.img") - common.ZipWrite(output_zip, block_list, prefix + "vendor.map") - return imgname + block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map") + CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img, + block_list=block_list) + return img.name -def BuildVendor(input_dir, info_dict, block_list=None): - """Build the (sparse) vendor image and return the name of a temp - file containing it.""" - return CreateImage(input_dir, info_dict, "vendor", block_list=block_list) - - -def CreateImage(input_dir, info_dict, what, block_list=None): +def CreateImage(input_dir, info_dict, what, output_file, block_list=None): print("creating " + what + ".img...") - img = common.MakeTempFile(prefix=what + "-", suffix=".img") - # The name of the directory it is making an image out of matters to # mkyaffs2image. It wants "system" but we have a directory named # "SYSTEM", so create a symlink. + temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) try: os.symlink(os.path.join(input_dir, what.upper()), - os.path.join(input_dir, what)) + os.path.join(temp_dir, what)) except OSError as e: # bogus error on my mac version? # File "./build/tools/releasetools/img_from_target_files" @@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None): if fs_config: image_props["fs_config"] = fs_config if block_list: - image_props["block_list"] = block_list + image_props["block_list"] = block_list.name - succ = build_image.BuildImage(os.path.join(input_dir, what), - image_props, img) + succ = build_image.BuildImage(os.path.join(temp_dir, what), + image_props, output_file.name) assert succ, "build " + what + ".img image failed" + output_file.Write() + if block_list: + block_list.Write() + is_verity_partition = "verity_block_device" in image_props verity_supported = image_props.get("verity") == "true" if is_verity_partition and verity_supported: @@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None): adjusted_blocks_key = what + "_adjusted_partition_size" info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1 - return img - def AddUserdata(output_zip, prefix="IMAGES/"): """Create a userdata image and store it in output_zip. @@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"): in OPTIONS.info_dict. """ - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img") + if os.path.exists(img.input_name): print("userdata.img already exists in %s, no need to rebuild..." % ( prefix,)) return @@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"): # empty dir named "data", or a symlink to the DATA dir, # and build the image from that. temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) user_dir = os.path.join(temp_dir, "data") empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") if empty: @@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"): os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), user_dir) - img = tempfile.NamedTemporaryFile() - fstab = OPTIONS.info_dict["fstab"] if fstab: image_props["fs_type"] = fstab["/data"].fs_type @@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"): assert succ, "build userdata.img image failed" common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) - common.ZipWrite(output_zip, img.name, prefix + "userdata.img") - img.close() - shutil.rmtree(temp_dir) + img.Write() def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): """Create a VBMeta image and store it in output_zip.""" - _, img_file_name = tempfile.mkstemp() + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img") avbtool = os.getenv('AVBTOOL') or "avbtool" cmd = [avbtool, "make_vbmeta_image", - "--output", img_file_name, + "--output", img.name, "--include_descriptors_from_image", boot_img_path, "--include_descriptors_from_image", system_img_path, "--generate_dm_verity_cmdline_from_hashtree", system_img_path] @@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() assert p.returncode == 0, "avbtool make_vbmeta_image failed" - common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img") + img.Write() def AddPartitionTable(output_zip, prefix="IMAGES/"): """Create a partition table image and store it in output_zip.""" - _, img_file_name = tempfile.mkstemp() - _, bpt_file_name = tempfile.mkstemp() + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img") + bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt") # use BPTTOOL from environ, or "bpttool" if empty or not set. bpttool = os.getenv("BPTTOOL") or "bpttool" - cmd = [bpttool, "make_table", "--output_json", bpt_file_name, - "--output_gpt", img_file_name] + cmd = [bpttool, "make_table", "--output_json", bpt.name, + "--output_gpt", img.name] input_files_str = OPTIONS.info_dict["board_bpt_input_files"] input_files = input_files_str.split(" ") for i in input_files: @@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"): p.communicate() assert p.returncode == 0, "bpttool make_table failed" - common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") - common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") + img.Write() + bpt.Write() def AddCache(output_zip, prefix="IMAGES/"): """Create an empty cache image and store it in output_zip.""" - prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") - if os.path.exists(prebuilt_path): + img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img") + if os.path.exists(img.input_name): print("cache.img already exists in %s, no need to rebuild..." % (prefix,)) return @@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"): # mkyaffs2image. So we create a temp dir, and within it we create an # empty dir named "cache", and build the image from that. temp_dir = tempfile.mkdtemp() + OPTIONS.tempfiles.append(temp_dir) user_dir = os.path.join(temp_dir, "cache") os.mkdir(user_dir) - img = tempfile.NamedTemporaryFile() fstab = OPTIONS.info_dict["fstab"] if fstab: @@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"): assert succ, "build cache.img image failed" common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) - common.ZipWrite(output_zip, img.name, prefix + "cache.img") - img.close() - os.rmdir(user_dir) - os.rmdir(temp_dir) + img.Write() def AddImagesToTargetFiles(filename): - OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) + if os.path.isdir(filename): + OPTIONS.input_tmp = os.path.abspath(filename) + input_zip = None + else: + OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) if not OPTIONS.add_missing: - for n in input_zip.namelist(): - if n.startswith("IMAGES/"): - print("target_files appears to already contain images.") - sys.exit(1) + if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")): + print("target_files appears to already contain images.") + sys.exit(1) - try: - input_zip.getinfo("VENDOR/") - has_vendor = True - except KeyError: - has_vendor = False + has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR")) + has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp, + "SYSTEM_OTHER")) - has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() + if input_zip: + OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) - OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) - - common.ZipClose(input_zip) - output_zip = zipfile.ZipFile(filename, "a", - compression=zipfile.ZIP_DEFLATED, - allowZip64=True) + common.ZipClose(input_zip) + output_zip = zipfile.ZipFile(filename, "a", + compression=zipfile.ZIP_DEFLATED, + allowZip64=True) + else: + OPTIONS.info_dict = common.LoadInfoDict(filename, filename) + output_zip = None + images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES") + if not os.path.isdir(images_dir): + os.makedirs(images_dir) + images_dir = None has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") @@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename): boot_image = common.GetBootableImage( "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") if boot_image: - boot_image.AddToZip(output_zip) + if output_zip: + boot_image.AddToZip(output_zip) + else: + boot_image.WriteToDir(OPTIONS.input_tmp) recovery_image = None if has_recovery: @@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename): recovery_image = common.GetBootableImage( "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") if recovery_image: - recovery_image.AddToZip(output_zip) + if output_zip: + recovery_image.AddToZip(output_zip) + else: + recovery_image.WriteToDir(OPTIONS.input_tmp) banner("recovery (two-step image)") # The special recovery.img for two-step package use. @@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename): "IMAGES/recovery-two-step.img", "recovery-two-step.img", OPTIONS.input_tmp, "RECOVERY", two_step_image=True) if recovery_two_step_image: - recovery_two_step_image.AddToZip(output_zip) + if output_zip: + recovery_two_step_image.AddToZip(output_zip) + else: + recovery_two_step_image.WriteToDir(OPTIONS.input_tmp) banner("system") system_img_path = AddSystem( @@ -502,24 +511,39 @@ def AddImagesToTargetFiles(filename): img_vendor_dir = os.path.join( OPTIONS.input_tmp, "VENDOR_IMAGES") if os.path.exists(img_radio_path): - common.ZipWrite(output_zip, img_radio_path, - os.path.join("IMAGES", img_name)) + if output_zip: + common.ZipWrite(output_zip, img_radio_path, + os.path.join("IMAGES", img_name)) + else: + shutil.copy(img_radio_path, prebuilt_path) else: for root, _, files in os.walk(img_vendor_dir): if img_name in files: - common.ZipWrite(output_zip, os.path.join(root, img_name), - os.path.join("IMAGES", img_name)) + if output_zip: + common.ZipWrite(output_zip, os.path.join(root, img_name), + os.path.join("IMAGES", img_name)) + else: + shutil.copy(os.path.join(root, img_name), prebuilt_path) break - # Zip spec says: All slashes MUST be forward slashes. - img_path = 'IMAGES/' + img_name - assert img_path in output_zip.namelist(), "cannot find " + img_name + if output_zip: + # Zip spec says: All slashes MUST be forward slashes. + img_path = 'IMAGES/' + img_name + assert img_path in output_zip.namelist(), "cannot find " + img_name + else: + img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) + assert os.path.exists(img_path), "cannot find " + img_name if care_map_list: file_path = "META/care_map.txt" - common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) + if output_zip: + common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) + else: + with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp: + fp.write('\n'.join(care_map_list)) - common.ZipClose(output_zip) + if output_zip: + common.ZipClose(output_zip) def main(argv): def option_handler(o, a): diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 2b975181b..322433313 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -1191,6 +1191,10 @@ class File(object): t.flush() return t + def WriteToDir(self, d): + with open(os.path.join(d, self.name), "wb") as fp: + fp.write(self.data) + def AddToZip(self, z, compression=None): ZipWriteStr(z, self.name, self.data, compress_type=compression)