forked from openkylin/platform_build
releasetools: Detect incomplete block ranges.
This CL detects incomplete block ranges (e.g. due to the holes in
mke2fs created images). Such block ranges will be tagged, so we won't
attempt to imgdiff those files. Note that the change to blockimgdiff.py,
which uses the tag info, will come in a separate CL.
An 'extra' attribute is added to RangeSet class, which defaults to an
empty dict. An 'incomplete' tag will be added into the dict by the
caller of the class. Not adding this tag as an immediate attribute,
because it is not a property regarding the ranges being represented, but
rather some storage space for the caller.
This CL also refactors GetSparseImage and RoundUpTo4K into common.py, so
the same code can be called from both of ota_from_target_files.py and
validate_target_files.py. Not able to add unittests for
GetSparseImage(), as SparseImage requires data in specific format.
Bug: 68016761
Test: Run validate_target_files.py on target-files.zip. It skips
validating files with missing holes as before.
Test: Run ota_from_target_files.py on angler target-files.zip. It gives
identical packages w/ and w/o the CL.
Test: pylint on changed files. There're warnings with common.py, but
unrelated to this change.
Change-Id: I126ccfea13c0d5ebcc8c1b4ff1a4f9200e97423a
This commit is contained in:
Tao Bao
parent
bb20e8c5f2
commit
c765cca38b
|
|
@ -25,6 +25,7 @@ import platform
|
|||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
|
@ -34,6 +35,7 @@ import zipfile
|
|||
from hashlib import sha1, sha256
|
||||
|
||||
import blockimgdiff
|
||||
import sparse_img
|
||||
|
||||
class Options(object):
|
||||
def __init__(self):
|
||||
|
|
@ -124,6 +126,11 @@ def Run(args, verbose=None, **kwargs):
|
|||
return subprocess.Popen(args, **kwargs)
|
||||
|
||||
|
||||
def RoundUpTo4K(value):
|
||||
rounded_up = value + 4095
|
||||
return rounded_up - (rounded_up % 4096)
|
||||
|
||||
|
||||
def CloseInheritedPipes():
|
||||
""" Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
|
||||
before doing other work."""
|
||||
|
|
@ -618,6 +625,56 @@ def UnzipTemp(filename, pattern=None):
|
|||
return tmp, zipfile.ZipFile(filename, "r")
|
||||
|
||||
|
||||
def GetSparseImage(which, tmpdir, input_zip):
|
||||
"""Returns a SparseImage object suitable for passing to BlockImageDiff.
|
||||
|
||||
This function loads the specified sparse image from the given path, and
|
||||
performs additional processing for OTA purpose. For example, it always adds
|
||||
block 0 to clobbered blocks list. It also detects files that cannot be
|
||||
reconstructed from the block list, for whom we should avoid applying imgdiff.
|
||||
|
||||
Args:
|
||||
which: The partition name, which must be "system" or "vendor".
|
||||
tmpdir: The directory that contains the prebuilt image and block map file.
|
||||
input_zip: The target-files ZIP archive.
|
||||
|
||||
Returns:
|
||||
A SparseImage object, with file_map info loaded.
|
||||
"""
|
||||
assert which in ("system", "vendor")
|
||||
|
||||
path = os.path.join(tmpdir, "IMAGES", which + ".img")
|
||||
mappath = os.path.join(tmpdir, "IMAGES", which + ".map")
|
||||
|
||||
# The image and map files must have been created prior to calling
|
||||
# ota_from_target_files.py (since LMP).
|
||||
assert os.path.exists(path) and os.path.exists(mappath)
|
||||
|
||||
# In ext4 filesystems, block 0 might be changed even being mounted R/O. We add
|
||||
# it to clobbered_blocks so that it will be written to the target
|
||||
# unconditionally. Note that they are still part of care_map. (Bug: 20939131)
|
||||
clobbered_blocks = "0"
|
||||
|
||||
image = sparse_img.SparseImage(path, mappath, clobbered_blocks)
|
||||
|
||||
# block.map may contain less blocks, because mke2fs may skip allocating blocks
|
||||
# if they contain all zeros. We can't reconstruct such a file from its block
|
||||
# list. Tag such entries accordingly. (Bug: 65213616)
|
||||
for entry in image.file_map:
|
||||
# "/system/framework/am.jar" => "SYSTEM/framework/am.jar".
|
||||
arcname = string.replace(entry, which, which.upper(), 1)[1:]
|
||||
# Skip artificial names, such as "__ZERO", "__NONZERO-1".
|
||||
if arcname not in input_zip.namelist():
|
||||
continue
|
||||
|
||||
info = input_zip.getinfo(arcname)
|
||||
ranges = image.file_map[entry]
|
||||
if RoundUpTo4K(info.file_size) > ranges.size() * 4096:
|
||||
ranges.extra['incomplete'] = True
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def GetKeyPasswords(keylist):
|
||||
"""Given a list of keys, prompt the user to enter passwords for
|
||||
those which require them. Return a {key: password} dict. password
|
||||
|
|
|
|||
|
|
@ -140,7 +140,6 @@ import zipfile
|
|||
|
||||
import common
|
||||
import edify_generator
|
||||
import sparse_img
|
||||
|
||||
if sys.hexversion < 0x02070000:
|
||||
print("Python 2.7 or newer is required.", file=sys.stderr)
|
||||
|
|
@ -452,31 +451,6 @@ def WriteFingerprintAssertion(script, target_info, source_info):
|
|||
source_info.GetBuildProp("ro.build.thumbprint"))
|
||||
|
||||
|
||||
def GetImage(which, tmpdir):
|
||||
"""Returns an image object suitable for passing to BlockImageDiff.
|
||||
|
||||
'which' partition must be "system" or "vendor". A prebuilt image and file
|
||||
map must already exist in tmpdir.
|
||||
"""
|
||||
|
||||
assert which in ("system", "vendor")
|
||||
|
||||
path = os.path.join(tmpdir, "IMAGES", which + ".img")
|
||||
mappath = os.path.join(tmpdir, "IMAGES", which + ".map")
|
||||
|
||||
# The image and map files must have been created prior to calling
|
||||
# ota_from_target_files.py (since LMP).
|
||||
assert os.path.exists(path) and os.path.exists(mappath)
|
||||
|
||||
# Bug: http://b/20939131
|
||||
# In ext4 filesystems, block 0 might be changed even being mounted
|
||||
# R/O. We add it to clobbered_blocks so that it will be written to the
|
||||
# target unconditionally. Note that they are still part of care_map.
|
||||
clobbered_blocks = "0"
|
||||
|
||||
return sparse_img.SparseImage(path, mappath, clobbered_blocks)
|
||||
|
||||
|
||||
def AddCompatibilityArchiveIfTrebleEnabled(target_zip, output_zip, target_info,
|
||||
source_info=None):
|
||||
"""Adds compatibility info into the output zip if it's Treble-enabled target.
|
||||
|
|
@ -662,7 +636,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then
|
|||
# has the effect of writing new data from the package to the entire
|
||||
# partition, but lets us reuse the updater code that writes incrementals to
|
||||
# do it.
|
||||
system_tgt = GetImage("system", OPTIONS.input_tmp)
|
||||
system_tgt = common.GetSparseImage("system", OPTIONS.input_tmp, input_zip)
|
||||
system_tgt.ResetFileMap()
|
||||
system_diff = common.BlockDifference("system", system_tgt, src=None)
|
||||
system_diff.WriteScript(script, output_zip)
|
||||
|
|
@ -673,7 +647,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then
|
|||
if HasVendorPartition(input_zip):
|
||||
script.ShowProgress(0.1, 0)
|
||||
|
||||
vendor_tgt = GetImage("vendor", OPTIONS.input_tmp)
|
||||
vendor_tgt = common.GetSparseImage("vendor", OPTIONS.input_tmp, input_zip)
|
||||
vendor_tgt.ResetFileMap()
|
||||
vendor_diff = common.BlockDifference("vendor", vendor_tgt)
|
||||
vendor_diff.WriteScript(script, output_zip)
|
||||
|
|
@ -846,8 +820,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):
|
|||
target_recovery = common.GetBootableImage(
|
||||
"/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")
|
||||
|
||||
system_src = GetImage("system", OPTIONS.source_tmp)
|
||||
system_tgt = GetImage("system", OPTIONS.target_tmp)
|
||||
system_src = common.GetSparseImage("system", OPTIONS.source_tmp, source_zip)
|
||||
system_tgt = common.GetSparseImage("system", OPTIONS.target_tmp, target_zip)
|
||||
|
||||
blockimgdiff_version = max(
|
||||
int(i) for i in target_info.get("blockimgdiff_versions", "1").split(","))
|
||||
|
|
@ -872,8 +846,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):
|
|||
if HasVendorPartition(target_zip):
|
||||
if not HasVendorPartition(source_zip):
|
||||
raise RuntimeError("can't generate incremental that adds /vendor")
|
||||
vendor_src = GetImage("vendor", OPTIONS.source_tmp)
|
||||
vendor_tgt = GetImage("vendor", OPTIONS.target_tmp)
|
||||
vendor_src = common.GetSparseImage("vendor", OPTIONS.source_tmp, source_zip)
|
||||
vendor_tgt = common.GetSparseImage("vendor", OPTIONS.target_tmp, target_zip)
|
||||
|
||||
# Check first block of vendor partition for remount R/W only if
|
||||
# disk type is ext4
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class RangeSet(object):
|
|||
|
||||
def __init__(self, data=None):
|
||||
self.monotonic = False
|
||||
self._extra = {}
|
||||
if isinstance(data, str):
|
||||
self._parse_internal(data)
|
||||
elif data:
|
||||
|
|
@ -56,6 +57,10 @@ class RangeSet(object):
|
|||
def __repr__(self):
|
||||
return '<RangeSet("' + self.to_string() + '")>'
|
||||
|
||||
@property
|
||||
def extra(self):
|
||||
return self._extra
|
||||
|
||||
@classmethod
|
||||
def parse(cls, text):
|
||||
"""Parse a text string consisting of a space-separated list of
|
||||
|
|
|
|||
|
|
@ -29,35 +29,17 @@ import re
|
|||
import sys
|
||||
|
||||
import common
|
||||
import sparse_img
|
||||
|
||||
|
||||
def _GetImage(which, tmpdir):
|
||||
assert which in ('system', 'vendor')
|
||||
|
||||
path = os.path.join(tmpdir, 'IMAGES', which + '.img')
|
||||
mappath = os.path.join(tmpdir, 'IMAGES', which + '.map')
|
||||
|
||||
# Map file must exist (allowed to be empty).
|
||||
assert os.path.exists(path) and os.path.exists(mappath)
|
||||
|
||||
clobbered_blocks = '0'
|
||||
return sparse_img.SparseImage(path, mappath, clobbered_blocks)
|
||||
|
||||
|
||||
def _ReadFile(file_name, unpacked_name, round_up=False):
|
||||
"""Constructs and returns a File object. Rounds up its size if needed."""
|
||||
|
||||
def RoundUpTo4K(value):
|
||||
rounded_up = value + 4095
|
||||
return rounded_up - (rounded_up % 4096)
|
||||
|
||||
assert os.path.exists(unpacked_name)
|
||||
with open(unpacked_name, 'r') as f:
|
||||
file_data = f.read()
|
||||
file_size = len(file_data)
|
||||
if round_up:
|
||||
file_size_rounded_up = RoundUpTo4K(file_size)
|
||||
file_size_rounded_up = common.RoundUpTo4K(file_size)
|
||||
file_data += '\0' * (file_size_rounded_up - file_size)
|
||||
return common.File(file_name, file_data)
|
||||
|
||||
|
|
@ -79,33 +61,28 @@ def ValidateFileConsistency(input_zip, input_tmp):
|
|||
|
||||
def CheckAllFiles(which):
|
||||
logging.info('Checking %s image.', which)
|
||||
image = _GetImage(which, input_tmp)
|
||||
image = common.GetSparseImage(which, input_tmp, input_zip)
|
||||
prefix = '/' + which
|
||||
for entry in image.file_map:
|
||||
# Skip entries like '__NONZERO-0'.
|
||||
if not entry.startswith(prefix):
|
||||
continue
|
||||
|
||||
# Read the blocks that the file resides. Note that it will contain the
|
||||
# bytes past the file length, which is expected to be padded with '\0's.
|
||||
ranges = image.file_map[entry]
|
||||
|
||||
incomplete = ranges.extra.get('incomplete', False)
|
||||
if incomplete:
|
||||
logging.warning('Skipping %s that has incomplete block list', entry)
|
||||
continue
|
||||
|
||||
blocks_sha1 = image.RangeSha1(ranges)
|
||||
|
||||
# The filename under unpacked directory, such as SYSTEM/bin/sh.
|
||||
unpacked_name = os.path.join(
|
||||
input_tmp, which.upper(), entry[(len(prefix) + 1):])
|
||||
unpacked_file = _ReadFile(entry, unpacked_name, True)
|
||||
file_size = unpacked_file.size
|
||||
|
||||
# block.map may contain less blocks, because mke2fs may skip allocating
|
||||
# blocks if they contain all zeros. We can't reconstruct such a file from
|
||||
# its block list. (Bug: 65213616)
|
||||
if file_size > ranges.size() * 4096:
|
||||
logging.warning(
|
||||
'Skipping %s that has less blocks: file size %d-byte,'
|
||||
' ranges %s (%d-byte)', entry, file_size, ranges,
|
||||
ranges.size() * 4096)
|
||||
continue
|
||||
|
||||
file_sha1 = unpacked_file.sha1
|
||||
assert blocks_sha1 == file_sha1, \
|
||||
'file: %s, range: %s, blocks_sha1: %s, file_sha1: %s' % (
|
||||
|
|
|
|||
Loading…
Reference in New Issue