From 37974731fcb4e32b1de5f213d34bd832ca889869 Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Thu, 16 Sep 2010 17:44:38 -0700 Subject: [PATCH] consolidate target_files metadata into one key-value file Instead of separate files for recovery api version, tool extensions, and mkyaffs2 options, put those all in the generic key-value file. Change-Id: Ib642311632844d52e4895fd4747093fc7e86232d --- core/Makefile | 8 +- tools/releasetools/common.py | 121 ++++++++++++----------- tools/releasetools/edify_generator.py | 2 +- tools/releasetools/img_from_target_files | 29 +++--- tools/releasetools/ota_from_target_files | 65 ++++-------- 5 files changed, 102 insertions(+), 123 deletions(-) diff --git a/core/Makefile b/core/Makefile index 2f208178b..a97d8bfe9 100644 --- a/core/Makefile +++ b/core/Makefile @@ -995,9 +995,9 @@ endif $(hide) mkdir -p $(zip_root)/META $(hide) $(ACP) $(APKCERTS_FILE) $(zip_root)/META/apkcerts.txt $(hide) echo "$(PRODUCT_OTA_PUBLIC_KEYS)" > $(zip_root)/META/otakeys.txt - $(hide) echo "$(PRIVATE_RECOVERY_API_VERSION)" > $(zip_root)/META/recovery-api-version.txt + $(hide) echo "recovery_api_version=$(PRIVATE_RECOVERY_API_VERSION)" > $(zip_root)/META/misc_info.txt ifdef BOARD_FLASH_BLOCK_SIZE - $(hide) echo "blocksize=$(BOARD_FLASH_BLOCK_SIZE)" > $(zip_root)/META/misc_info.txt + $(hide) echo "blocksize=$(BOARD_FLASH_BLOCK_SIZE)" >> $(zip_root)/META/misc_info.txt endif ifdef BOARD_BOOTIMAGE_PARTITION_SIZE $(hide) echo "boot_size=$(BOARD_BOOTIMAGE_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt @@ -1020,9 +1020,9 @@ else $(hide) echo "fs_type=yaffs2" >> $(zip_root)/META/misc_info.txt $(hide) echo "partition_type=MTD" >> $(zip_root)/META/misc_info.txt endif - $(hide) echo "$(tool_extensions)" > $(zip_root)/META/tool-extensions.txt + $(hide) echo "tool_extensions=$(tool_extensions)" >> $(zip_root)/META/misc_info.txt ifdef mkyaffs2_extra_flags - $(hide) echo "$(mkyaffs2_extra_flags)" > $(zip_root)/META/mkyaffs2-extra-flags.txt + $(hide) echo "mkyaffs2_extra_flags=$(mkyaffs2_extra_flags)" >> $(zip_root)/META/misc_info.txt endif @# Zip everything up, preserving symlinks $(hide) (cd $(zip_root) && zip -qry ../$(notdir $@) .) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 44291d025..3ec4c2d3a 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -40,7 +40,6 @@ OPTIONS.verbose = False OPTIONS.tempfiles = [] OPTIONS.device_specific = None OPTIONS.extras = {} -OPTIONS.mkyaffs2_extra_flags = None OPTIONS.info_dict = None @@ -59,68 +58,78 @@ def Run(args, **kwargs): return subprocess.Popen(args, **kwargs) -def LoadInfoDict(): +def LoadInfoDict(zip): """Read and parse the META/misc_info.txt key/value pairs from the input target files and return a dict.""" d = {} try: - for line in open(os.path.join(OPTIONS.input_tmp, "META", "misc_info.txt")): + for line in zip.read("META/misc_info.txt").split("\n"): line = line.strip() if not line or line.startswith("#"): continue k, v = line.split("=", 1) d[k] = v - except IOError, e: - if e.errno == errno.ENOENT: - # ok if misc_info.txt file doesn't exist - pass - else: - raise + except KeyError: + # ok if misc_info.txt doesn't exist + pass - if "fs_type" not in d: info["fs_type"] = "yaffs2" - if "partition_type" not in d: info["partition_type"] = "MTD" + if "fs_type" not in d: d["fs_type"] = "yaffs2" + if "partition_type" not in d: d["partition_type"] = "MTD" + + # backwards compatibility: These values used to be in their own + # files. Look for them, in case we're processing an old + # target_files zip. + + if "mkyaffs2_extra_flags" not in d: + try: + d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() + except KeyError: + # ok if flags don't exist + pass + + if "recovery_api_version" not in d: + try: + d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() + except KeyError: + raise ValueError("can't find recovery API version in input target-files") + + if "tool_extensions" not in d: + try: + d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() + except KeyError: + # ok if extensions don't exist + pass + + try: + data = zip.read("META/imagesizes.txt") + for line in data.split("\n"): + if not line: continue + name, value = line.strip().split(None, 1) + if name == "blocksize": + d[name] = value + else: + d[name + "_size"] = value + except KeyError: + pass + + def makeint(key): + if key in d: + d[key] = int(d[key], 0) + + makeint("recovery_api_version") + makeint("blocksize") + makeint("system_size") + makeint("userdata_size") + makeint("recovery_size") + makeint("boot_size") return d +def DumpInfoDict(d): + for k, v in sorted(d.items()): + print "%-25s = (%s) %s" % (k, type(v).__name__, v) -def LoadMaxSizes(info): - """Load the maximum allowable images sizes from the input - target_files. Uses the imagesizes.txt file if it's available - (pre-gingerbread target_files), or the more general info dict (which - must be passed in) if not.""" - OPTIONS.max_image_size = {} - try: - for line in open(os.path.join(OPTIONS.input_tmp, "META", "imagesizes.txt")): - pieces = line.split() - if len(pieces) != 2: continue - image = pieces[0] - size = int(pieces[1], 0) - OPTIONS.max_image_size[image + ".img"] = size - except IOError, e: - if e.errno == errno.ENOENT: - def copy(x, y): - if x+y in info: OPTIONS.max_image_size[x+".img"] = int(info[x+y], 0) - copy("blocksize", "") - copy("boot", "_size") - copy("recovery", "_size") - copy("system", "_size") - copy("userdata", "_size") - else: - raise - - -def LoadMkyaffs2ExtraFlags(): - """Load mkyaffs2 extra flags.""" - try: - fn = os.path.join(OPTIONS.input_tmp, "META", "mkyaffs2-extra-flags.txt"); - if os.access(fn, os.F_OK): - OPTIONS.mkyaffs2_extra_flags = open(fn).read().rstrip("\n") - except IOError, e: - if e.errno == errno.ENOENT: - pass - - -def BuildAndAddBootableImage(sourcedir, targetname, output_zip): +def BuildAndAddBootableImage(sourcedir, targetname, output_zip, info_dict): """Take a kernel, cmdline, and ramdisk directory from the input (in 'sourcedir'), and turn them into a boot image. Put the boot image into the output zip file under the name 'targetname'. Returns @@ -133,7 +142,7 @@ def BuildAndAddBootableImage(sourcedir, targetname, output_zip): if img is None: return None - CheckSize(img, targetname) + CheckSize(img, targetname, info_dict) ZipWriteStr(output_zip, targetname, img) return targetname @@ -194,13 +203,13 @@ def BuildBootableImage(sourcedir): return data -def AddRecovery(output_zip): +def AddRecovery(output_zip, info_dict): BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), - "recovery.img", output_zip) + "recovery.img", output_zip, info_dict) -def AddBoot(output_zip): +def AddBoot(output_zip, info_dict): BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), - "boot.img", output_zip) + "boot.img", output_zip, info_dict) def UnzipTemp(filename, pattern=None): """Unzip the given archive into a temporary directory and return the name.""" @@ -294,12 +303,12 @@ def SignFile(input_name, output_name, key, password, align=None, temp.close() -def CheckSize(data, target): +def CheckSize(data, target, info_dict): """Check the data string passed against the max size limit, if any, for the given target. Raise exception if the data is too big. Print a warning if the data is nearing the maximum size.""" - fs_type = OPTIONS.info_dict.get("fs_type", None) + fs_type = info_dict.get("fs_type", None) if not fs_type: return limit = OPTIONS.max_image_size.get(target, None) diff --git a/tools/releasetools/edify_generator.py b/tools/releasetools/edify_generator.py index 3334b438f..390bd4bfc 100644 --- a/tools/releasetools/edify_generator.py +++ b/tools/releasetools/edify_generator.py @@ -31,7 +31,7 @@ class EdifyGenerator(object): """Make a temporary script object whose commands can latter be appended to the parent script with AppendScript(). Used when the caller wants to generate script commands out-of-order.""" - x = EdifyGenerator(self.version) + x = EdifyGenerator(self.version, self.info) x.mounts = self.mounts return x diff --git a/tools/releasetools/img_from_target_files b/tools/releasetools/img_from_target_files index a74ef890b..8250b0e5d 100755 --- a/tools/releasetools/img_from_target_files +++ b/tools/releasetools/img_from_target_files @@ -68,8 +68,9 @@ def AddUserdata(output_zip): build_command.append(str(OPTIONS.max_image_size["userdata.img"])) else: build_command = ["mkyaffs2image", "-f"] - if OPTIONS.mkyaffs2_extra_flags is not None: - build_command.append(OPTIONS.mkyaffs2_extra_flags); + extra = OPTIONS.info_dict.get("mkyaffs2_extra_flags", None) + if extra: + build_command.extend(extra.split()) build_command.append(user_dir) build_command.append(img.name) @@ -77,7 +78,7 @@ def AddUserdata(output_zip): p.communicate() assert p.returncode == 0, "build userdata.img image failed" - common.CheckSize(img.name, "userdata.img") + common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) output_zip.write(img.name, "userdata.img") img.close() os.rmdir(user_dir) @@ -115,8 +116,9 @@ def AddSystem(output_zip): build_command.append(str(OPTIONS.max_image_size["system.img"])) else: build_command = ["mkyaffs2image", "-f"] - if OPTIONS.mkyaffs2_extra_flags is not None: - build_command.extend(OPTIONS.mkyaffs2_extra_flags.split()); + extra = OPTIONS.info_dict.get("mkyaffs2_extra_flags", None) + if extra: + build_command.extend(extra.split()) build_command.append(os.path.join(OPTIONS.input_tmp, "system")) build_command.append(img.name) @@ -128,7 +130,7 @@ def AddSystem(output_zip): data = img.read() img.close() - common.CheckSize(data, "system.img") + common.CheckSize(data, "system.img", OPTIONS.info_dict) common.ZipWriteStr(output_zip, "system.img", data) @@ -158,20 +160,13 @@ def main(argv): OPTIONS.input_tmp = common.UnzipTemp(args[0]) - OPTIONS.info_dict = common.LoadInfoDict() - common.LoadMaxSizes(OPTIONS.info_dict) - if not OPTIONS.max_image_size: - print - print " WARNING: Failed to load max image sizes; will not enforce" - print " image size limits." - print - - common.LoadMkyaffs2ExtraFlags() + input_zip = zipfile.ZipFile(args[0], "r") + OPTIONS.info_dict = common.LoadInfoDict(input_zip) output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED) - common.AddBoot(output_zip) - common.AddRecovery(output_zip) + common.AddBoot(output_zip, OPTIONS.info_dict) + common.AddRecovery(output_zip, OPTIONS.info_dict) AddSystem(output_zip) AddUserdata(output_zip) CopyInfo(output_zip) diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files index 89caa5c90..37715bc70 100755 --- a/tools/releasetools/ota_from_target_files +++ b/tools/releasetools/ota_from_target_files @@ -347,7 +347,7 @@ def WriteFullOTAPackage(input_zip, output_zip): device_specific = common.DeviceSpecificParams( input_zip=input_zip, - input_version=GetRecoveryAPIVersion(input_zip), + input_version=OPTIONS.info_dict["recovery_api_version"], output_zip=output_zip, script=script, input_tmp=OPTIONS.input_tmp, @@ -382,7 +382,7 @@ def WriteFullOTAPackage(input_zip, output_zip): Item.GetMetadata(input_zip) Item.Get("system").SetPermissions(script) - common.CheckSize(boot_img.data, "boot.img") + common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict) common.ZipWriteStr(output_zip, "boot.img", boot_img.data) script.ShowProgress(0.2, 0) @@ -432,25 +432,9 @@ def GetBuildProp(property, z): return m.group(1).strip() -def GetRecoveryAPIVersion(zip): - """Returns the version of the recovery API. Version 0 is the older - amend code (no separate binary).""" - try: - version = zip.read("META/recovery-api-version.txt") - return int(version) - except KeyError: - try: - # version one didn't have the recovery-api-version.txt file, but - # it did include an updater binary. - zip.getinfo("OTA/bin/updater") - return 1 - except KeyError: - return 0 - - def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): - source_version = GetRecoveryAPIVersion(source_zip) - target_version = GetRecoveryAPIVersion(target_zip) + source_version = OPTIONS.source_info_dict["recovery_api_version"] + target_version = OPTIONS.target_info_dict["recovery_api_version"] if source_version == 0: print ("WARNING: generating edify script for a source that " @@ -738,33 +722,19 @@ def main(argv): print "unzipping target target-files..." OPTIONS.input_tmp = common.UnzipTemp(args[0]) - if OPTIONS.device_specific is None: - # look for the device-specific tools extension location in the input - try: - f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt")) - ds = f.read().strip() - f.close() - if ds: - ds = os.path.normpath(ds) - print "using device-specific extensions in", ds - OPTIONS.device_specific = ds - except IOError, e: - if e.errno == errno.ENOENT: - # nothing specified in the file - pass - else: - raise - - OPTIONS.info_dict = common.LoadInfoDict() - common.LoadMaxSizes(OPTIONS.info_dict) - if not OPTIONS.max_image_size: - print - print " WARNING: Failed to load max image sizes; will not enforce" - print " image size limits." - print - OPTIONS.target_tmp = OPTIONS.input_tmp input_zip = zipfile.ZipFile(args[0], "r") + OPTIONS.info_dict = common.LoadInfoDict(input_zip) + if OPTIONS.verbose: + print "--- target info ---" + common.DumpInfoDict(OPTIONS.info_dict) + + if OPTIONS.device_specific is None: + OPTIONS.device_specific = OPTIONS.info_dict.get("tool_extensions", None) + if OPTIONS.device_specific is not None: + OPTIONS.device_specific = os.path.normpath(OPTIONS.device_specific) + print "using device-specific extensions in", OPTIONS.device_specific + if OPTIONS.package_key: temp_zip_file = tempfile.NamedTemporaryFile() output_zip = zipfile.ZipFile(temp_zip_file, "w", @@ -779,6 +749,11 @@ def main(argv): print "unzipping source target-files..." OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source) source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r") + OPTIONS.target_info_dict = OPTIONS.info_dict + OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) + if OPTIONS.verbose: + print "--- source info ---" + common.DumpInfoDict(OPTIONS.source_info_dict) WriteIncrementalOTAPackage(input_zip, source_zip, output_zip) output_zip.close()