From ea5d7a9de7660bef5b9c68f6372a92d4b2f2f1f6 Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Sun, 12 Sep 2010 15:26:16 -0700 Subject: [PATCH] move File and Difference classes into common script This makes them accessible from device-specific extensions (so they can be used to send radio images as binary patches, for instance). Change-Id: I2f2174b93b4265abf9400f9e5a0982caca0771e9 --- tools/releasetools/common.py | 125 ++++++++++++++++++ tools/releasetools/ota_from_target_files | 161 +++-------------------- 2 files changed, 144 insertions(+), 142 deletions(-) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 6bb37ae3e..46cef11d7 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -12,16 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import errno import getopt import getpass import imp import os import re +import sha import shutil import subprocess import sys import tempfile +import threading +import time import zipfile # missing in Python 2.4 and before @@ -547,3 +551,124 @@ class DeviceSpecificParams(object): this is used to install the image for the device's baseband processor.""" return self._DoCall("IncrementalOTA_InstallEnd") + +class File(object): + def __init__(self, name, data): + self.name = name + self.data = data + self.size = len(data) + self.sha1 = sha.sha(data).hexdigest() + + def WriteToTemp(self): + t = tempfile.NamedTemporaryFile() + t.write(self.data) + t.flush() + return t + + def AddToZip(self, z): + ZipWriteStr(z, self.name, self.data) + +DIFF_PROGRAM_BY_EXT = { + ".gz" : "imgdiff", + ".zip" : ["imgdiff", "-z"], + ".jar" : ["imgdiff", "-z"], + ".apk" : ["imgdiff", "-z"], + ".img" : "imgdiff", + } + +class Difference(object): + def __init__(self, tf, sf): + self.tf = tf + self.sf = sf + self.patch = None + + def ComputePatch(self): + """Compute the patch (as a string of data) needed to turn sf into + tf. Returns the same tuple as GetPatch().""" + + tf = self.tf + sf = self.sf + + ext = os.path.splitext(tf.name)[1] + diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") + + ttemp = tf.WriteToTemp() + stemp = sf.WriteToTemp() + + ext = os.path.splitext(tf.name)[1] + + try: + ptemp = tempfile.NamedTemporaryFile() + if isinstance(diff_program, list): + cmd = copy.copy(diff_program) + else: + cmd = [diff_program] + cmd.append(stemp.name) + cmd.append(ttemp.name) + cmd.append(ptemp.name) + p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, err = p.communicate() + if err or p.returncode != 0: + print "WARNING: failure running %s:\n%s\n" % (diff_program, err) + return None + diff = ptemp.read() + finally: + ptemp.close() + stemp.close() + ttemp.close() + + self.patch = diff + return self.tf, self.sf, self.patch + + + def GetPatch(self): + """Return a tuple (target_file, source_file, patch_data). + patch_data may be None if ComputePatch hasn't been called, or if + computing the patch failed.""" + return self.tf, self.sf, self.patch + + +def ComputeDifferences(diffs): + """Call ComputePatch on all the Difference objects in 'diffs'.""" + print len(diffs), "diffs to compute" + + # Do the largest files first, to try and reduce the long-pole effect. + by_size = [(i.tf.size, i) for i in diffs] + by_size.sort(reverse=True) + by_size = [i[1] for i in by_size] + + lock = threading.Lock() + diff_iter = iter(by_size) # accessed under lock + + def worker(): + try: + lock.acquire() + for d in diff_iter: + lock.release() + start = time.time() + d.ComputePatch() + dur = time.time() - start + lock.acquire() + + tf, sf, patch = d.GetPatch() + if sf.name == tf.name: + name = tf.name + else: + name = "%s (%s)" % (tf.name, sf.name) + if patch is None: + print "patching failed! %s" % (name,) + else: + print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( + dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) + lock.release() + except Exception, e: + print e + raise + + # start worker threads; wait for them all to finish. + threads = [threading.Thread(target=worker) + for i in range(OPTIONS.worker_threads)] + for th in threads: + th.start() + while threads: + threads.pop().join() diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files index 29911bbda..ad7e0e4bb 100755 --- a/tools/releasetools/ota_from_target_files +++ b/tools/releasetools/ota_from_target_files @@ -59,7 +59,6 @@ import re import sha import subprocess import tempfile -import threading import time import zipfile @@ -308,7 +307,7 @@ def MakeRecoveryPatch(output_zip, recovery_img, boot_img): executable. """ - d = Difference(recovery_img, boot_img) + d = common.Difference(recovery_img, boot_img) _, _, patch = d.ComputePatch() common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch) Item.Get("system/recovery-from-boot.p", dir=False) @@ -374,9 +373,9 @@ def WriteFullOTAPackage(input_zip, output_zip): symlinks = CopySystemFiles(input_zip, output_zip) script.MakeSymlinks(symlinks) - boot_img = File("boot.img", common.BuildBootableImage( + boot_img = common.File("boot.img", common.BuildBootableImage( os.path.join(OPTIONS.input_tmp, "BOOT"))) - recovery_img = File("recovery.img", common.BuildBootableImage( + recovery_img = common.File("recovery.img", common.BuildBootableImage( os.path.join(OPTIONS.input_tmp, "RECOVERY"))) MakeRecoveryPatch(output_zip, recovery_img, boot_img) @@ -407,21 +406,6 @@ def WriteMetadata(metadata, output_zip): for kv in sorted(metadata.iteritems())])) -class File(object): - def __init__(self, name, data): - self.name = name - self.data = data - self.size = len(data) - self.sha1 = sha.sha(data).hexdigest() - - def WriteToTemp(self): - t = tempfile.NamedTemporaryFile() - t.write(self.data) - t.flush() - return t - - def AddToZip(self, z): - common.ZipWriteStr(z, self.name, self.data) def LoadSystemFiles(z): @@ -432,117 +416,10 @@ def LoadSystemFiles(z): if info.filename.startswith("SYSTEM/") and not IsSymlink(info): fn = "system/" + info.filename[7:] data = z.read(info.filename) - out[fn] = File(fn, data) + out[fn] = common.File(fn, data) return out -DIFF_PROGRAM_BY_EXT = { - ".gz" : "imgdiff", - ".zip" : ["imgdiff", "-z"], - ".jar" : ["imgdiff", "-z"], - ".apk" : ["imgdiff", "-z"], - ".img" : "imgdiff", - } - - -class Difference(object): - def __init__(self, tf, sf): - self.tf = tf - self.sf = sf - self.patch = None - - def ComputePatch(self): - """Compute the patch (as a string of data) needed to turn sf into - tf. Returns the same tuple as GetPatch().""" - - tf = self.tf - sf = self.sf - - ext = os.path.splitext(tf.name)[1] - diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") - - ttemp = tf.WriteToTemp() - stemp = sf.WriteToTemp() - - ext = os.path.splitext(tf.name)[1] - - try: - ptemp = tempfile.NamedTemporaryFile() - if isinstance(diff_program, list): - cmd = copy.copy(diff_program) - else: - cmd = [diff_program] - cmd.append(stemp.name) - cmd.append(ttemp.name) - cmd.append(ptemp.name) - p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, err = p.communicate() - if err or p.returncode != 0: - print "WARNING: failure running %s:\n%s\n" % (diff_program, err) - return None - diff = ptemp.read() - finally: - ptemp.close() - stemp.close() - ttemp.close() - - self.patch = diff - return self.tf, self.sf, self.patch - - - def GetPatch(self): - """Return a tuple (target_file, source_file, patch_data). - patch_data may be None if ComputePatch hasn't been called, or if - computing the patch failed.""" - return self.tf, self.sf, self.patch - - -def ComputeDifferences(diffs): - """Call ComputePatch on all the Difference objects in 'diffs'.""" - print len(diffs), "diffs to compute" - - # Do the largest files first, to try and reduce the long-pole effect. - by_size = [(i.tf.size, i) for i in diffs] - by_size.sort(reverse=True) - by_size = [i[1] for i in by_size] - - lock = threading.Lock() - diff_iter = iter(by_size) # accessed under lock - - def worker(): - try: - lock.acquire() - for d in diff_iter: - lock.release() - start = time.time() - d.ComputePatch() - dur = time.time() - start - lock.acquire() - - tf, sf, patch = d.GetPatch() - if sf.name == tf.name: - name = tf.name - else: - name = "%s (%s)" % (tf.name, sf.name) - if patch is None: - print "patching failed! %s" % (name,) - else: - print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( - dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) - lock.release() - except Exception, e: - print e - raise - - # start worker threads; wait for them all to finish. - threads = [threading.Thread(target=worker) - for i in range(OPTIONS.worker_threads)] - for th in threads: - th.start() - while threads: - threads.pop().join() - - def GetBuildProp(property, z): """Return the fingerprint of the build of a given target-files ZipFile object.""" @@ -616,12 +493,12 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): verbatim_targets.append((fn, tf.size)) elif tf.sha1 != sf.sha1: # File is different; consider sending as a patch - diffs.append(Difference(tf, sf)) + diffs.append(common.Difference(tf, sf)) else: # Target file identical to source. pass - ComputeDifferences(diffs) + common.ComputeDifferences(diffs) for diff in diffs: tf, sf, d = diff.GetPatch() @@ -642,20 +519,20 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): script.Mount("MTD", "system", "/system") script.AssertSomeFingerprint(source_fp, target_fp) - source_boot = File("/tmp/boot.img", - common.BuildBootableImage( - os.path.join(OPTIONS.source_tmp, "BOOT"))) - target_boot = File("/tmp/boot.img", - common.BuildBootableImage( - os.path.join(OPTIONS.target_tmp, "BOOT"))) + source_boot = common.File("/tmp/boot.img", + common.BuildBootableImage( + os.path.join(OPTIONS.source_tmp, "BOOT"))) + target_boot = common.File("/tmp/boot.img", + common.BuildBootableImage( + os.path.join(OPTIONS.target_tmp, "BOOT"))) updating_boot = (source_boot.data != target_boot.data) - source_recovery = File("system/recovery.img", - common.BuildBootableImage( - os.path.join(OPTIONS.source_tmp, "RECOVERY"))) - target_recovery = File("system/recovery.img", - common.BuildBootableImage( - os.path.join(OPTIONS.target_tmp, "RECOVERY"))) + source_recovery = common.File("system/recovery.img", + common.BuildBootableImage( + os.path.join(OPTIONS.source_tmp, "RECOVERY"))) + target_recovery = common.File("system/recovery.img", + common.BuildBootableImage( + os.path.join(OPTIONS.target_tmp, "RECOVERY"))) updating_recovery = (source_recovery.data != target_recovery.data) # Here's how we divide up the progress bar: @@ -681,7 +558,7 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): script.SetProgress(so_far / total_verify_size) if updating_boot: - d = Difference(target_boot, source_boot) + d = common.Difference(target_boot, source_boot) _, _, d = d.ComputePatch() print "boot target: %d source: %d diff: %d" % ( target_boot.size, source_boot.size, len(d))