move File and Difference classes into common script

This makes them accessible from device-specific extensions (so they
can be used to send radio images as binary patches, for instance).

Change-Id: I2f2174b93b4265abf9400f9e5a0982caca0771e9

tools/releasetools/common.py

@@ -12,16 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import errno
 import getopt
 import getpass
 import imp
 import os
 import re
+import sha
 import shutil
 import subprocess
 import sys
 import tempfile
+import threading
+import time
 import zipfile
 
 # missing in Python 2.4 and before
@@ -547,3 +551,124 @@ class DeviceSpecificParams(object):
     this is used to install the image for the device's baseband
     processor."""
     return self._DoCall("IncrementalOTA_InstallEnd")
+
+class File(object):
+  def __init__(self, name, data):
+    self.name = name
+    self.data = data
+    self.size = len(data)
+    self.sha1 = sha.sha(data).hexdigest()
+
+  def WriteToTemp(self):
+    t = tempfile.NamedTemporaryFile()
+    t.write(self.data)
+    t.flush()
+    return t
+
+  def AddToZip(self, z):
+    ZipWriteStr(z, self.name, self.data)
+
+DIFF_PROGRAM_BY_EXT = {
+    ".gz" : "imgdiff",
+    ".zip" : ["imgdiff", "-z"],
+    ".jar" : ["imgdiff", "-z"],
+    ".apk" : ["imgdiff", "-z"],
+    ".img" : "imgdiff",
+    }
+
+class Difference(object):
+  def __init__(self, tf, sf):
+    self.tf = tf
+    self.sf = sf
+    self.patch = None
+
+  def ComputePatch(self):
+    """Compute the patch (as a string of data) needed to turn sf into
+    tf.  Returns the same tuple as GetPatch()."""
+
+    tf = self.tf
+    sf = self.sf
+
+    ext = os.path.splitext(tf.name)[1]
+    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
+
+    ttemp = tf.WriteToTemp()
+    stemp = sf.WriteToTemp()
+
+    ext = os.path.splitext(tf.name)[1]
+
+    try:
+      ptemp = tempfile.NamedTemporaryFile()
+      if isinstance(diff_program, list):
+        cmd = copy.copy(diff_program)
+      else:
+        cmd = [diff_program]
+      cmd.append(stemp.name)
+      cmd.append(ttemp.name)
+      cmd.append(ptemp.name)
+      p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      _, err = p.communicate()
+      if err or p.returncode != 0:
+        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
+        return None
+      diff = ptemp.read()
+    finally:
+      ptemp.close()
+      stemp.close()
+      ttemp.close()
+
+    self.patch = diff
+    return self.tf, self.sf, self.patch
+
+
+  def GetPatch(self):
+    """Return a tuple (target_file, source_file, patch_data).
+    patch_data may be None if ComputePatch hasn't been called, or if
+    computing the patch failed."""
+    return self.tf, self.sf, self.patch
+
+
+def ComputeDifferences(diffs):
+  """Call ComputePatch on all the Difference objects in 'diffs'."""
+  print len(diffs), "diffs to compute"
+
+  # Do the largest files first, to try and reduce the long-pole effect.
+  by_size = [(i.tf.size, i) for i in diffs]
+  by_size.sort(reverse=True)
+  by_size = [i[1] for i in by_size]
+
+  lock = threading.Lock()
+  diff_iter = iter(by_size)   # accessed under lock
+
+  def worker():
+    try:
+      lock.acquire()
+      for d in diff_iter:
+        lock.release()
+        start = time.time()
+        d.ComputePatch()
+        dur = time.time() - start
+        lock.acquire()
+
+        tf, sf, patch = d.GetPatch()
+        if sf.name == tf.name:
+          name = tf.name
+        else:
+          name = "%s (%s)" % (tf.name, sf.name)
+        if patch is None:
+          print "patching failed!                                  %s" % (name,)
+        else:
+          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
+              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
+      lock.release()
+    except Exception, e:
+      print e
+      raise
+
+  # start worker threads; wait for them all to finish.
+  threads = [threading.Thread(target=worker)
+             for i in range(OPTIONS.worker_threads)]
+  for th in threads:
+    th.start()
+  while threads:
+    threads.pop().join()

tools/releasetools/ota_from_target_files

@@ -59,7 +59,6 @@ import re
 import sha
 import subprocess
 import tempfile
-import threading
 import time
 import zipfile
 
@@ -308,7 +307,7 @@ def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
   executable.
   """
 
-  d = Difference(recovery_img, boot_img)
+  d = common.Difference(recovery_img, boot_img)
   _, _, patch = d.ComputePatch()
   common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
   Item.Get("system/recovery-from-boot.p", dir=False)
@@ -374,9 +373,9 @@ def WriteFullOTAPackage(input_zip, output_zip):
   symlinks = CopySystemFiles(input_zip, output_zip)
   script.MakeSymlinks(symlinks)
 
-  boot_img = File("boot.img", common.BuildBootableImage(
+  boot_img = common.File("boot.img", common.BuildBootableImage(
       os.path.join(OPTIONS.input_tmp, "BOOT")))
-  recovery_img = File("recovery.img", common.BuildBootableImage(
+  recovery_img = common.File("recovery.img", common.BuildBootableImage(
       os.path.join(OPTIONS.input_tmp, "RECOVERY")))
   MakeRecoveryPatch(output_zip, recovery_img, boot_img)
 
@@ -407,21 +406,6 @@ def WriteMetadata(metadata, output_zip):
                               for kv in sorted(metadata.iteritems())]))
 
 
-class File(object):
-  def __init__(self, name, data):
-    self.name = name
-    self.data = data
-    self.size = len(data)
-    self.sha1 = sha.sha(data).hexdigest()
-
-  def WriteToTemp(self):
-    t = tempfile.NamedTemporaryFile()
-    t.write(self.data)
-    t.flush()
-    return t
-
-  def AddToZip(self, z):
-    common.ZipWriteStr(z, self.name, self.data)
 
 
 def LoadSystemFiles(z):
@@ -432,117 +416,10 @@ def LoadSystemFiles(z):
     if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
       fn = "system/" + info.filename[7:]
       data = z.read(info.filename)
-      out[fn] = File(fn, data)
+      out[fn] = common.File(fn, data)
   return out
 
 
-DIFF_PROGRAM_BY_EXT = {
-    ".gz" : "imgdiff",
-    ".zip" : ["imgdiff", "-z"],
-    ".jar" : ["imgdiff", "-z"],
-    ".apk" : ["imgdiff", "-z"],
-    ".img" : "imgdiff",
-    }
-
-
-class Difference(object):
-  def __init__(self, tf, sf):
-    self.tf = tf
-    self.sf = sf
-    self.patch = None
-
-  def ComputePatch(self):
-    """Compute the patch (as a string of data) needed to turn sf into
-    tf.  Returns the same tuple as GetPatch()."""
-
-    tf = self.tf
-    sf = self.sf
-
-    ext = os.path.splitext(tf.name)[1]
-    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
-
-    ttemp = tf.WriteToTemp()
-    stemp = sf.WriteToTemp()
-
-    ext = os.path.splitext(tf.name)[1]
-
-    try:
-      ptemp = tempfile.NamedTemporaryFile()
-      if isinstance(diff_program, list):
-        cmd = copy.copy(diff_program)
-      else:
-        cmd = [diff_program]
-      cmd.append(stemp.name)
-      cmd.append(ttemp.name)
-      cmd.append(ptemp.name)
-      p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      _, err = p.communicate()
-      if err or p.returncode != 0:
-        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
-        return None
-      diff = ptemp.read()
-    finally:
-      ptemp.close()
-      stemp.close()
-      ttemp.close()
-
-    self.patch = diff
-    return self.tf, self.sf, self.patch
-
-
-  def GetPatch(self):
-    """Return a tuple (target_file, source_file, patch_data).
-    patch_data may be None if ComputePatch hasn't been called, or if
-    computing the patch failed."""
-    return self.tf, self.sf, self.patch
-
-
-def ComputeDifferences(diffs):
-  """Call ComputePatch on all the Difference objects in 'diffs'."""
-  print len(diffs), "diffs to compute"
-
-  # Do the largest files first, to try and reduce the long-pole effect.
-  by_size = [(i.tf.size, i) for i in diffs]
-  by_size.sort(reverse=True)
-  by_size = [i[1] for i in by_size]
-
-  lock = threading.Lock()
-  diff_iter = iter(by_size)   # accessed under lock
-
-  def worker():
-    try:
-      lock.acquire()
-      for d in diff_iter:
-        lock.release()
-        start = time.time()
-        d.ComputePatch()
-        dur = time.time() - start
-        lock.acquire()
-
-        tf, sf, patch = d.GetPatch()
-        if sf.name == tf.name:
-          name = tf.name
-        else:
-          name = "%s (%s)" % (tf.name, sf.name)
-        if patch is None:
-          print "patching failed!                                  %s" % (name,)
-        else:
-          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
-              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
-      lock.release()
-    except Exception, e:
-      print e
-      raise
-
-  # start worker threads; wait for them all to finish.
-  threads = [threading.Thread(target=worker)
-             for i in range(OPTIONS.worker_threads)]
-  for th in threads:
-    th.start()
-  while threads:
-    threads.pop().join()
-
-
 def GetBuildProp(property, z):
   """Return the fingerprint of the build of a given target-files
   ZipFile object."""
@@ -616,12 +493,12 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
       verbatim_targets.append((fn, tf.size))
     elif tf.sha1 != sf.sha1:
       # File is different; consider sending as a patch
-      diffs.append(Difference(tf, sf))
+      diffs.append(common.Difference(tf, sf))
     else:
       # Target file identical to source.
       pass
 
-  ComputeDifferences(diffs)
+  common.ComputeDifferences(diffs)
 
   for diff in diffs:
     tf, sf, d = diff.GetPatch()
@@ -642,18 +519,18 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
   script.Mount("MTD", "system", "/system")
   script.AssertSomeFingerprint(source_fp, target_fp)
 
-  source_boot = File("/tmp/boot.img",
+  source_boot = common.File("/tmp/boot.img",
                             common.BuildBootableImage(
                                 os.path.join(OPTIONS.source_tmp, "BOOT")))
-  target_boot = File("/tmp/boot.img",
+  target_boot = common.File("/tmp/boot.img",
                             common.BuildBootableImage(
                                 os.path.join(OPTIONS.target_tmp, "BOOT")))
   updating_boot = (source_boot.data != target_boot.data)
 
-  source_recovery = File("system/recovery.img",
+  source_recovery = common.File("system/recovery.img",
                                 common.BuildBootableImage(
                                     os.path.join(OPTIONS.source_tmp, "RECOVERY")))
-  target_recovery = File("system/recovery.img",
+  target_recovery = common.File("system/recovery.img",
                                 common.BuildBootableImage(
                                     os.path.join(OPTIONS.target_tmp, "RECOVERY")))
   updating_recovery = (source_recovery.data != target_recovery.data)
@@ -681,7 +558,7 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
     script.SetProgress(so_far / total_verify_size)
 
   if updating_boot:
-    d = Difference(target_boot, source_boot)
+    d = common.Difference(target_boot, source_boot)
     _, _, d = d.ComputePatch()
     print "boot      target: %d  source: %d  diff: %d" % (
         target_boot.size, source_boot.size, len(d))