Allow add_img_to_target_files to work on a directory
In addition to the current behavior of add_img_to_target_files working on an existing zip file, allow passing in a directory where the target files have already been extracted. When in this mode, it writes the images out to that directory instead of the zip file. This allows us to call add_img_to_target_files on the temp directory used during the build to create the target files package, saving the time and space of unzipping what we just zipped. This also allows us to use the parallel soong_zip, which compresses the images much faster. Test: aosp_marlin target_files zip is the same before/after this change Test: marlin target_files zip is the same before/after this change Test: bullhead target_files zip is the same before/after this change Change-Id: I155654cdc7ad7754ba4ef76ec69c31f504a58198
This commit is contained in:
Dan Willemsen
parent
7dc54e9761
commit
2ee00d5f66
|
|
@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False
|
|||
OPTIONS.replace_verity_private_key = False
|
||||
OPTIONS.is_signing = False
|
||||
|
||||
|
||||
class OutputFile(object):
|
||||
def __init__(self, output_zip, input_dir, prefix, name):
|
||||
self._output_zip = output_zip
|
||||
self.input_name = os.path.join(input_dir, prefix, name)
|
||||
|
||||
if self._output_zip:
|
||||
self._zip_name = os.path.join(prefix, name)
|
||||
|
||||
root, suffix = os.path.splitext(name)
|
||||
self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)
|
||||
else:
|
||||
self.name = self.input_name
|
||||
|
||||
def Write(self):
|
||||
if self._output_zip:
|
||||
common.ZipWrite(self._output_zip, self.name, self._zip_name)
|
||||
|
||||
|
||||
def GetCareMap(which, imgname):
|
||||
"""Generate care_map of system (or vendor) partition"""
|
||||
|
||||
|
|
@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
|
|||
"""Turn the contents of SYSTEM into a system image and store it in
|
||||
output_zip. Returns the name of the system image file."""
|
||||
|
||||
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img")
|
||||
if os.path.exists(prebuilt_path):
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")
|
||||
if os.path.exists(img.input_name):
|
||||
print("system.img already exists in %s, no need to rebuild..." % (prefix,))
|
||||
return prebuilt_path
|
||||
return img.input_name
|
||||
|
||||
def output_sink(fn, data):
|
||||
ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
|
||||
|
|
@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
|
|||
common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
|
||||
boot_img, info_dict=OPTIONS.info_dict)
|
||||
|
||||
block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")
|
||||
imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,
|
||||
block_list=block_list)
|
||||
block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")
|
||||
CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,
|
||||
block_list=block_list)
|
||||
|
||||
common.ZipWrite(output_zip, imgname, prefix + "system.img")
|
||||
common.ZipWrite(output_zip, block_list, prefix + "system.map")
|
||||
return imgname
|
||||
|
||||
|
||||
def BuildSystem(input_dir, info_dict, block_list=None):
|
||||
"""Build the (sparse) system image and return the name of a temp
|
||||
file containing it."""
|
||||
return CreateImage(input_dir, info_dict, "system", block_list=block_list)
|
||||
return img.name
|
||||
|
||||
|
||||
def AddSystemOther(output_zip, prefix="IMAGES/"):
|
||||
"""Turn the contents of SYSTEM_OTHER into a system_other image
|
||||
and store it in output_zip."""
|
||||
|
||||
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img")
|
||||
if os.path.exists(prebuilt_path):
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")
|
||||
if os.path.exists(img.input_name):
|
||||
print("system_other.img already exists in %s, no need to rebuild..." % (
|
||||
prefix,))
|
||||
return
|
||||
|
||||
imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict)
|
||||
common.ZipWrite(output_zip, imgname, prefix + "system_other.img")
|
||||
|
||||
def BuildSystemOther(input_dir, info_dict):
|
||||
"""Build the (sparse) system_other image and return the name of a temp
|
||||
file containing it."""
|
||||
return CreateImage(input_dir, info_dict, "system_other", block_list=None)
|
||||
CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
|
||||
|
||||
|
||||
def AddVendor(output_zip, prefix="IMAGES/"):
|
||||
"""Turn the contents of VENDOR into a vendor image and store in it
|
||||
output_zip."""
|
||||
|
||||
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img")
|
||||
if os.path.exists(prebuilt_path):
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")
|
||||
if os.path.exists(img.input_name):
|
||||
print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))
|
||||
return prebuilt_path
|
||||
return img.input_name
|
||||
|
||||
block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")
|
||||
imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,
|
||||
block_list=block_list)
|
||||
common.ZipWrite(output_zip, imgname, prefix + "vendor.img")
|
||||
common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
|
||||
return imgname
|
||||
block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")
|
||||
CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,
|
||||
block_list=block_list)
|
||||
return img.name
|
||||
|
||||
|
||||
def BuildVendor(input_dir, info_dict, block_list=None):
|
||||
"""Build the (sparse) vendor image and return the name of a temp
|
||||
file containing it."""
|
||||
return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)
|
||||
|
||||
|
||||
def CreateImage(input_dir, info_dict, what, block_list=None):
|
||||
def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
|
||||
print("creating " + what + ".img...")
|
||||
|
||||
img = common.MakeTempFile(prefix=what + "-", suffix=".img")
|
||||
|
||||
# The name of the directory it is making an image out of matters to
|
||||
# mkyaffs2image. It wants "system" but we have a directory named
|
||||
# "SYSTEM", so create a symlink.
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
OPTIONS.tempfiles.append(temp_dir)
|
||||
try:
|
||||
os.symlink(os.path.join(input_dir, what.upper()),
|
||||
os.path.join(input_dir, what))
|
||||
os.path.join(temp_dir, what))
|
||||
except OSError as e:
|
||||
# bogus error on my mac version?
|
||||
# File "./build/tools/releasetools/img_from_target_files"
|
||||
|
|
@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
|
|||
if fs_config:
|
||||
image_props["fs_config"] = fs_config
|
||||
if block_list:
|
||||
image_props["block_list"] = block_list
|
||||
image_props["block_list"] = block_list.name
|
||||
|
||||
succ = build_image.BuildImage(os.path.join(input_dir, what),
|
||||
image_props, img)
|
||||
succ = build_image.BuildImage(os.path.join(temp_dir, what),
|
||||
image_props, output_file.name)
|
||||
assert succ, "build " + what + ".img image failed"
|
||||
|
||||
output_file.Write()
|
||||
if block_list:
|
||||
block_list.Write()
|
||||
|
||||
is_verity_partition = "verity_block_device" in image_props
|
||||
verity_supported = image_props.get("verity") == "true"
|
||||
if is_verity_partition and verity_supported:
|
||||
|
|
@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None):
|
|||
adjusted_blocks_key = what + "_adjusted_partition_size"
|
||||
info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1
|
||||
|
||||
return img
|
||||
|
||||
|
||||
def AddUserdata(output_zip, prefix="IMAGES/"):
|
||||
"""Create a userdata image and store it in output_zip.
|
||||
|
|
@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
|
|||
in OPTIONS.info_dict.
|
||||
"""
|
||||
|
||||
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img")
|
||||
if os.path.exists(prebuilt_path):
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")
|
||||
if os.path.exists(img.input_name):
|
||||
print("userdata.img already exists in %s, no need to rebuild..." % (
|
||||
prefix,))
|
||||
return
|
||||
|
|
@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
|
|||
# empty dir named "data", or a symlink to the DATA dir,
|
||||
# and build the image from that.
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
OPTIONS.tempfiles.append(temp_dir)
|
||||
user_dir = os.path.join(temp_dir, "data")
|
||||
empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
|
||||
if empty:
|
||||
|
|
@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
|
|||
os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
|
||||
user_dir)
|
||||
|
||||
img = tempfile.NamedTemporaryFile()
|
||||
|
||||
fstab = OPTIONS.info_dict["fstab"]
|
||||
if fstab:
|
||||
image_props["fs_type"] = fstab["/data"].fs_type
|
||||
|
|
@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"):
|
|||
assert succ, "build userdata.img image failed"
|
||||
|
||||
common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
|
||||
common.ZipWrite(output_zip, img.name, prefix + "userdata.img")
|
||||
img.close()
|
||||
shutil.rmtree(temp_dir)
|
||||
img.Write()
|
||||
|
||||
|
||||
def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
|
||||
"""Create a VBMeta image and store it in output_zip."""
|
||||
_, img_file_name = tempfile.mkstemp()
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")
|
||||
avbtool = os.getenv('AVBTOOL') or "avbtool"
|
||||
cmd = [avbtool, "make_vbmeta_image",
|
||||
"--output", img_file_name,
|
||||
"--output", img.name,
|
||||
"--include_descriptors_from_image", boot_img_path,
|
||||
"--include_descriptors_from_image", system_img_path,
|
||||
"--generate_dm_verity_cmdline_from_hashtree", system_img_path]
|
||||
|
|
@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
|
|||
p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
p.communicate()
|
||||
assert p.returncode == 0, "avbtool make_vbmeta_image failed"
|
||||
common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img")
|
||||
img.Write()
|
||||
|
||||
|
||||
def AddPartitionTable(output_zip, prefix="IMAGES/"):
|
||||
"""Create a partition table image and store it in output_zip."""
|
||||
|
||||
_, img_file_name = tempfile.mkstemp()
|
||||
_, bpt_file_name = tempfile.mkstemp()
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")
|
||||
bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")
|
||||
|
||||
# use BPTTOOL from environ, or "bpttool" if empty or not set.
|
||||
bpttool = os.getenv("BPTTOOL") or "bpttool"
|
||||
cmd = [bpttool, "make_table", "--output_json", bpt_file_name,
|
||||
"--output_gpt", img_file_name]
|
||||
cmd = [bpttool, "make_table", "--output_json", bpt.name,
|
||||
"--output_gpt", img.name]
|
||||
input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
|
||||
input_files = input_files_str.split(" ")
|
||||
for i in input_files:
|
||||
|
|
@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"):
|
|||
p.communicate()
|
||||
assert p.returncode == 0, "bpttool make_table failed"
|
||||
|
||||
common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img")
|
||||
common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt")
|
||||
img.Write()
|
||||
bpt.Write()
|
||||
|
||||
|
||||
def AddCache(output_zip, prefix="IMAGES/"):
|
||||
"""Create an empty cache image and store it in output_zip."""
|
||||
|
||||
prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img")
|
||||
if os.path.exists(prebuilt_path):
|
||||
img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")
|
||||
if os.path.exists(img.input_name):
|
||||
print("cache.img already exists in %s, no need to rebuild..." % (prefix,))
|
||||
return
|
||||
|
||||
|
|
@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"):
|
|||
# mkyaffs2image. So we create a temp dir, and within it we create an
|
||||
# empty dir named "cache", and build the image from that.
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
OPTIONS.tempfiles.append(temp_dir)
|
||||
user_dir = os.path.join(temp_dir, "cache")
|
||||
os.mkdir(user_dir)
|
||||
img = tempfile.NamedTemporaryFile()
|
||||
|
||||
fstab = OPTIONS.info_dict["fstab"]
|
||||
if fstab:
|
||||
|
|
@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"):
|
|||
assert succ, "build cache.img image failed"
|
||||
|
||||
common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
|
||||
common.ZipWrite(output_zip, img.name, prefix + "cache.img")
|
||||
img.close()
|
||||
os.rmdir(user_dir)
|
||||
os.rmdir(temp_dir)
|
||||
img.Write()
|
||||
|
||||
|
||||
def AddImagesToTargetFiles(filename):
|
||||
OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
|
||||
if os.path.isdir(filename):
|
||||
OPTIONS.input_tmp = os.path.abspath(filename)
|
||||
input_zip = None
|
||||
else:
|
||||
OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
|
||||
|
||||
if not OPTIONS.add_missing:
|
||||
for n in input_zip.namelist():
|
||||
if n.startswith("IMAGES/"):
|
||||
print("target_files appears to already contain images.")
|
||||
sys.exit(1)
|
||||
if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
|
||||
print("target_files appears to already contain images.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
input_zip.getinfo("VENDOR/")
|
||||
has_vendor = True
|
||||
except KeyError:
|
||||
has_vendor = False
|
||||
has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))
|
||||
has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,
|
||||
"SYSTEM_OTHER"))
|
||||
|
||||
has_system_other = "SYSTEM_OTHER/" in input_zip.namelist()
|
||||
if input_zip:
|
||||
OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
|
||||
|
||||
OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
|
||||
|
||||
common.ZipClose(input_zip)
|
||||
output_zip = zipfile.ZipFile(filename, "a",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
allowZip64=True)
|
||||
common.ZipClose(input_zip)
|
||||
output_zip = zipfile.ZipFile(filename, "a",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
allowZip64=True)
|
||||
else:
|
||||
OPTIONS.info_dict = common.LoadInfoDict(filename, filename)
|
||||
output_zip = None
|
||||
images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")
|
||||
if not os.path.isdir(images_dir):
|
||||
os.makedirs(images_dir)
|
||||
images_dir = None
|
||||
|
||||
has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
|
||||
system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
|
||||
|
|
@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename):
|
|||
boot_image = common.GetBootableImage(
|
||||
"IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
|
||||
if boot_image:
|
||||
boot_image.AddToZip(output_zip)
|
||||
if output_zip:
|
||||
boot_image.AddToZip(output_zip)
|
||||
else:
|
||||
boot_image.WriteToDir(OPTIONS.input_tmp)
|
||||
|
||||
recovery_image = None
|
||||
if has_recovery:
|
||||
|
|
@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename):
|
|||
recovery_image = common.GetBootableImage(
|
||||
"IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
|
||||
if recovery_image:
|
||||
recovery_image.AddToZip(output_zip)
|
||||
if output_zip:
|
||||
recovery_image.AddToZip(output_zip)
|
||||
else:
|
||||
recovery_image.WriteToDir(OPTIONS.input_tmp)
|
||||
|
||||
banner("recovery (two-step image)")
|
||||
# The special recovery.img for two-step package use.
|
||||
|
|
@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename):
|
|||
"IMAGES/recovery-two-step.img", "recovery-two-step.img",
|
||||
OPTIONS.input_tmp, "RECOVERY", two_step_image=True)
|
||||
if recovery_two_step_image:
|
||||
recovery_two_step_image.AddToZip(output_zip)
|
||||
if output_zip:
|
||||
recovery_two_step_image.AddToZip(output_zip)
|
||||
else:
|
||||
recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)
|
||||
|
||||
banner("system")
|
||||
system_img_path = AddSystem(
|
||||
|
|
@ -502,24 +511,39 @@ def AddImagesToTargetFiles(filename):
|
|||
img_vendor_dir = os.path.join(
|
||||
OPTIONS.input_tmp, "VENDOR_IMAGES")
|
||||
if os.path.exists(img_radio_path):
|
||||
common.ZipWrite(output_zip, img_radio_path,
|
||||
os.path.join("IMAGES", img_name))
|
||||
if output_zip:
|
||||
common.ZipWrite(output_zip, img_radio_path,
|
||||
os.path.join("IMAGES", img_name))
|
||||
else:
|
||||
shutil.copy(img_radio_path, prebuilt_path)
|
||||
else:
|
||||
for root, _, files in os.walk(img_vendor_dir):
|
||||
if img_name in files:
|
||||
common.ZipWrite(output_zip, os.path.join(root, img_name),
|
||||
os.path.join("IMAGES", img_name))
|
||||
if output_zip:
|
||||
common.ZipWrite(output_zip, os.path.join(root, img_name),
|
||||
os.path.join("IMAGES", img_name))
|
||||
else:
|
||||
shutil.copy(os.path.join(root, img_name), prebuilt_path)
|
||||
break
|
||||
|
||||
# Zip spec says: All slashes MUST be forward slashes.
|
||||
img_path = 'IMAGES/' + img_name
|
||||
assert img_path in output_zip.namelist(), "cannot find " + img_name
|
||||
if output_zip:
|
||||
# Zip spec says: All slashes MUST be forward slashes.
|
||||
img_path = 'IMAGES/' + img_name
|
||||
assert img_path in output_zip.namelist(), "cannot find " + img_name
|
||||
else:
|
||||
img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
|
||||
assert os.path.exists(img_path), "cannot find " + img_name
|
||||
|
||||
if care_map_list:
|
||||
file_path = "META/care_map.txt"
|
||||
common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
|
||||
if output_zip:
|
||||
common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
|
||||
else:
|
||||
with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:
|
||||
fp.write('\n'.join(care_map_list))
|
||||
|
||||
common.ZipClose(output_zip)
|
||||
if output_zip:
|
||||
common.ZipClose(output_zip)
|
||||
|
||||
def main(argv):
|
||||
def option_handler(o, a):
|
||||
|
|
|
|||
|
|
@ -1191,6 +1191,10 @@ class File(object):
|
|||
t.flush()
|
||||
return t
|
||||
|
||||
def WriteToDir(self, d):
|
||||
with open(os.path.join(d, self.name), "wb") as fp:
|
||||
fp.write(self.data)
|
||||
|
||||
def AddToZip(self, z, compression=None):
|
||||
ZipWriteStr(z, self.name, self.data, compress_type=compression)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue