make full OTAs block based

Instead of writing individual files and fixing up their metadata, make
full OTAs contain a system image and simply write it to the block
device.

This is only done for target-files that already contain the recovery
flashing information, older target-files still get a file-based full
OTA.

Bug: 12893978
Change-Id: If7586083c8f275e24fec49d260af5b5aff4a0a88

tools/releasetools/img_from_target_files

@@ -1,270 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Given a target-files zipfile, produces an image zipfile suitable for
-use with 'fastboot update'.
-
-Usage:  img_from_target_files [flags] input_target_files output_image_zip
-
-  -b  (--board_config)  <file>
-      Deprecated.
-
-  -z  (--bootable_zip)
-      Include only the bootable images (eg 'boot' and 'recovery') in
-      the output.
-
-"""
-
-import sys
-
-if sys.hexversion < 0x02040000:
-  print >> sys.stderr, "Python 2.4 or newer is required."
-  sys.exit(1)
-
-import errno
-import os
-import re
-import shutil
-import subprocess
-import tempfile
-import zipfile
-
-# missing in Python 2.4 and before
-if not hasattr(os, "SEEK_SET"):
-  os.SEEK_SET = 0
-
-import build_image
-import common
-
-OPTIONS = common.OPTIONS
-
-
-def AddSystem(output_zip):
-  """Turn the contents of SYSTEM into a system image and store it in
-  output_zip."""
-
-  print "creating system.img..."
-
-  img = tempfile.NamedTemporaryFile()
-
-  # The name of the directory it is making an image out of matters to
-  # mkyaffs2image.  It wants "system" but we have a directory named
-  # "SYSTEM", so create a symlink.
-  try:
-    os.symlink(os.path.join(OPTIONS.input_tmp, "SYSTEM"),
-               os.path.join(OPTIONS.input_tmp, "system"))
-  except OSError, e:
-      # bogus error on my mac version?
-      #   File "./build/tools/releasetools/img_from_target_files", line 86, in AddSystem
-      #     os.path.join(OPTIONS.input_tmp, "system"))
-      # OSError: [Errno 17] File exists
-    if (e.errno == errno.EEXIST):
-      pass
-
-  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
-                                                    "system")
-  fstab = OPTIONS.info_dict["fstab"]
-  if fstab:
-    image_props["fs_type" ] = fstab["/system"].fs_type
-  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "system"),
-                                image_props, img.name)
-  assert succ, "build system.img image failed"
-
-  img.seek(os.SEEK_SET, 0)
-  data = img.read()
-  img.close()
-
-  common.CheckSize(data, "system.img", OPTIONS.info_dict)
-  common.ZipWriteStr(output_zip, "system.img", data)
-
-
-def AddVendor(output_zip):
-  """Turn the contents of VENDOR into vendor.img and store it in
-  output_zip."""
-
-  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
-                                                    "vendor")
-  # The build system has to explicitly request for vendor.img.
-  if "fs_type" not in image_props:
-    return
-
-  print "creating vendor.img..."
-
-  img = tempfile.NamedTemporaryFile()
-
-  # The name of the directory it is making an image out of matters to
-  # mkyaffs2image.  It wants "vendor" but we have a directory named
-  # "VENDOR", so create a symlink or an empty directory if VENDOR does not
-  # exist.
-  if not os.path.exists(os.path.join(OPTIONS.input_tmp, "vendor")):
-    if os.path.exists(os.path.join(OPTIONS.input_tmp, "VENDOR")):
-      os.symlink(os.path.join(OPTIONS.input_tmp, "VENDOR"),
-                 os.path.join(OPTIONS.input_tmp, "vendor"))
-    else:
-      os.mkdir(os.path.join(OPTIONS.input_tmp, "vendor"))
-
-  img = tempfile.NamedTemporaryFile()
-
-  fstab = OPTIONS.info_dict["fstab"]
-  if fstab:
-    image_props["fs_type" ] = fstab["/vendor"].fs_type
-  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "vendor"),
-                                image_props, img.name)
-  assert succ, "build vendor.img image failed"
-
-  common.CheckSize(img.name, "vendor.img", OPTIONS.info_dict)
-  output_zip.write(img.name, "vendor.img")
-  img.close()
-
-
-def AddUserdata(output_zip):
-  """Create an empty userdata image and store it in output_zip."""
-
-  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
-                                                    "data")
-  # If no userdata_size is provided for extfs, skip userdata.img.
-  if (image_props.get("fs_type", "").startswith("ext") and
-      not image_props.get("partition_size")):
-    return
-
-  print "creating userdata.img..."
-
-  # The name of the directory it is making an image out of matters to
-  # mkyaffs2image.  So we create a temp dir, and within it we create an
-  # empty dir named "data", and build the image from that.
-  temp_dir = tempfile.mkdtemp()
-  user_dir = os.path.join(temp_dir, "data")
-  os.mkdir(user_dir)
-  img = tempfile.NamedTemporaryFile()
-
-  fstab = OPTIONS.info_dict["fstab"]
-  if fstab:
-    image_props["fs_type" ] = fstab["/data"].fs_type
-  succ = build_image.BuildImage(user_dir, image_props, img.name)
-  assert succ, "build userdata.img image failed"
-
-  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
-  output_zip.write(img.name, "userdata.img")
-  img.close()
-  os.rmdir(user_dir)
-  os.rmdir(temp_dir)
-
-
-def AddCache(output_zip):
-  """Create an empty cache image and store it in output_zip."""
-
-  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
-                                                    "cache")
-  # The build system has to explicitly request for cache.img.
-  if "fs_type" not in image_props:
-    return
-
-  print "creating cache.img..."
-
-  # The name of the directory it is making an image out of matters to
-  # mkyaffs2image.  So we create a temp dir, and within it we create an
-  # empty dir named "cache", and build the image from that.
-  temp_dir = tempfile.mkdtemp()
-  user_dir = os.path.join(temp_dir, "cache")
-  os.mkdir(user_dir)
-  img = tempfile.NamedTemporaryFile()
-
-  fstab = OPTIONS.info_dict["fstab"]
-  if fstab:
-    image_props["fs_type" ] = fstab["/cache"].fs_type
-  succ = build_image.BuildImage(user_dir, image_props, img.name)
-  assert succ, "build cache.img image failed"
-
-  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
-  output_zip.write(img.name, "cache.img")
-  img.close()
-  os.rmdir(user_dir)
-  os.rmdir(temp_dir)
-
-
-def CopyInfo(output_zip):
-  """Copy the android-info.txt file from the input to the output."""
-  output_zip.write(os.path.join(OPTIONS.input_tmp, "OTA", "android-info.txt"),
-                   "android-info.txt")
-
-
-def main(argv):
-  bootable_only = [False]
-
-  def option_handler(o, a):
-    if o in ("-b", "--board_config"):
-      pass       # deprecated
-    if o in ("-z", "--bootable_zip"):
-      bootable_only[0] = True
-    else:
-      return False
-    return True
-
-  args = common.ParseOptions(argv, __doc__,
-                             extra_opts="b:z",
-                             extra_long_opts=["board_config=",
-                                              "bootable_zip"],
-                             extra_option_handler=option_handler)
-
-  bootable_only = bootable_only[0]
-
-  if len(args) != 2:
-    common.Usage(__doc__)
-    sys.exit(1)
-
-  OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0])
-  OPTIONS.info_dict = common.LoadInfoDict(input_zip)
-
-  # If this image was originally labelled with SELinux contexts, make sure we
-  # also apply the labels in our new image. During building, the "file_contexts"
-  # is in the out/ directory tree, but for repacking from target-files.zip it's
-  # in the root directory of the ramdisk.
-  if "selinux_fc" in OPTIONS.info_dict:
-    OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK",
-        "file_contexts")
-
-  output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED)
-
-  common.GetBootableImage(
-      "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT").AddToZip(output_zip)
-  common.GetBootableImage(
-      "recovery.img", "recovery.img", OPTIONS.input_tmp,
-      "RECOVERY").AddToZip(output_zip)
-
-  if not bootable_only:
-    AddSystem(output_zip)
-    AddVendor(output_zip)
-    AddUserdata(output_zip)
-    AddCache(output_zip)
-    CopyInfo(output_zip)
-
-  print "cleaning up..."
-  output_zip.close()
-  shutil.rmtree(OPTIONS.input_tmp)
-
-  print "done."
-
-
-if __name__ == '__main__':
-  try:
-    common.CloseInheritedPipes()
-    main(sys.argv[1:])
-  except common.ExternalError, e:
-    print
-    print "   ERROR: %s" % (e,)
-    print
-    sys.exit(1)

tools/releasetools/img_from_target_files

@@ -0,0 +1 @@
+img_from_target_files.py

tools/releasetools/img_from_target_files.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Given a target-files zipfile, produces an image zipfile suitable for
+use with 'fastboot update'.
+
+Usage:  img_from_target_files [flags] input_target_files output_image_zip
+
+  -b  (--board_config)  <file>
+      Deprecated.
+
+  -z  (--bootable_zip)
+      Include only the bootable images (eg 'boot' and 'recovery') in
+      the output.
+
+"""
+
+import sys
+
+if sys.hexversion < 0x02040000:
+  print >> sys.stderr, "Python 2.4 or newer is required."
+  sys.exit(1)
+
+import errno
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import zipfile
+
+# missing in Python 2.4 and before
+if not hasattr(os, "SEEK_SET"):
+  os.SEEK_SET = 0
+
+import build_image
+import common
+
+OPTIONS = common.OPTIONS
+
+
+def AddSystem(output_zip, sparse=True):
+  """Turn the contents of SYSTEM into a system image and store it in
+  output_zip."""
+
+  print "creating system.img..."
+
+  img = tempfile.NamedTemporaryFile()
+
+  # The name of the directory it is making an image out of matters to
+  # mkyaffs2image.  It wants "system" but we have a directory named
+  # "SYSTEM", so create a symlink.
+  try:
+    os.symlink(os.path.join(OPTIONS.input_tmp, "SYSTEM"),
+               os.path.join(OPTIONS.input_tmp, "system"))
+  except OSError, e:
+      # bogus error on my mac version?
+      #   File "./build/tools/releasetools/img_from_target_files", line 86, in AddSystem
+      #     os.path.join(OPTIONS.input_tmp, "system"))
+      # OSError: [Errno 17] File exists
+    if (e.errno == errno.EEXIST):
+      pass
+
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "system")
+  fstab = OPTIONS.info_dict["fstab"]
+  if fstab:
+    image_props["fs_type" ] = fstab["/system"].fs_type
+  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "system"),
+                                image_props, img.name)
+  assert succ, "build system.img image failed"
+
+  if sparse:
+    img.seek(os.SEEK_SET, 0)
+    data = img.read()
+    img.close()
+  else:
+    success, name = build_image.UnsparseImage(img.name, replace=False)
+    if not success:
+      assert False, "unsparsing system.img failed"
+    try:
+      with open(name) as f:
+        data = f.read()
+    finally:
+      os.unlink(name)
+
+  common.ZipWriteStr(output_zip, "system.img", data)
+
+
+def AddVendor(output_zip):
+  """Turn the contents of VENDOR into vendor.img and store it in
+  output_zip."""
+
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "vendor")
+  # The build system has to explicitly request for vendor.img.
+  if "fs_type" not in image_props:
+    return
+
+  print "creating vendor.img..."
+
+  img = tempfile.NamedTemporaryFile()
+
+  # The name of the directory it is making an image out of matters to
+  # mkyaffs2image.  It wants "vendor" but we have a directory named
+  # "VENDOR", so create a symlink or an empty directory if VENDOR does not
+  # exist.
+  if not os.path.exists(os.path.join(OPTIONS.input_tmp, "vendor")):
+    if os.path.exists(os.path.join(OPTIONS.input_tmp, "VENDOR")):
+      os.symlink(os.path.join(OPTIONS.input_tmp, "VENDOR"),
+                 os.path.join(OPTIONS.input_tmp, "vendor"))
+    else:
+      os.mkdir(os.path.join(OPTIONS.input_tmp, "vendor"))
+
+  img = tempfile.NamedTemporaryFile()
+
+  fstab = OPTIONS.info_dict["fstab"]
+  if fstab:
+    image_props["fs_type" ] = fstab["/vendor"].fs_type
+  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "vendor"),
+                                image_props, img.name)
+  assert succ, "build vendor.img image failed"
+
+  common.CheckSize(img.name, "vendor.img", OPTIONS.info_dict)
+  output_zip.write(img.name, "vendor.img")
+  img.close()
+
+
+def AddUserdata(output_zip):
+  """Create an empty userdata image and store it in output_zip."""
+
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "data")
+  # If no userdata_size is provided for extfs, skip userdata.img.
+  if (image_props.get("fs_type", "").startswith("ext") and
+      not image_props.get("partition_size")):
+    return
+
+  print "creating userdata.img..."
+
+  # The name of the directory it is making an image out of matters to
+  # mkyaffs2image.  So we create a temp dir, and within it we create an
+  # empty dir named "data", and build the image from that.
+  temp_dir = tempfile.mkdtemp()
+  user_dir = os.path.join(temp_dir, "data")
+  os.mkdir(user_dir)
+  img = tempfile.NamedTemporaryFile()
+
+  fstab = OPTIONS.info_dict["fstab"]
+  if fstab:
+    image_props["fs_type" ] = fstab["/data"].fs_type
+  succ = build_image.BuildImage(user_dir, image_props, img.name)
+  assert succ, "build userdata.img image failed"
+
+  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
+  output_zip.write(img.name, "userdata.img")
+  img.close()
+  os.rmdir(user_dir)
+  os.rmdir(temp_dir)
+
+
+def AddCache(output_zip):
+  """Create an empty cache image and store it in output_zip."""
+
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "cache")
+  # The build system has to explicitly request for cache.img.
+  if "fs_type" not in image_props:
+    return
+
+  print "creating cache.img..."
+
+  # The name of the directory it is making an image out of matters to
+  # mkyaffs2image.  So we create a temp dir, and within it we create an
+  # empty dir named "cache", and build the image from that.
+  temp_dir = tempfile.mkdtemp()
+  user_dir = os.path.join(temp_dir, "cache")
+  os.mkdir(user_dir)
+  img = tempfile.NamedTemporaryFile()
+
+  fstab = OPTIONS.info_dict["fstab"]
+  if fstab:
+    image_props["fs_type" ] = fstab["/cache"].fs_type
+  succ = build_image.BuildImage(user_dir, image_props, img.name)
+  assert succ, "build cache.img image failed"
+
+  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
+  output_zip.write(img.name, "cache.img")
+  img.close()
+  os.rmdir(user_dir)
+  os.rmdir(temp_dir)
+
+
+def CopyInfo(output_zip):
+  """Copy the android-info.txt file from the input to the output."""
+  output_zip.write(os.path.join(OPTIONS.input_tmp, "OTA", "android-info.txt"),
+                   "android-info.txt")
+
+
+def main(argv):
+  bootable_only = [False]
+
+  def option_handler(o, a):
+    if o in ("-b", "--board_config"):
+      pass       # deprecated
+    if o in ("-z", "--bootable_zip"):
+      bootable_only[0] = True
+    else:
+      return False
+    return True
+
+  args = common.ParseOptions(argv, __doc__,
+                             extra_opts="b:z",
+                             extra_long_opts=["board_config=",
+                                              "bootable_zip"],
+                             extra_option_handler=option_handler)
+
+  bootable_only = bootable_only[0]
+
+  if len(args) != 2:
+    common.Usage(__doc__)
+    sys.exit(1)
+
+  OPTIONS.input_tmp, input_zip = common.UnzipTemp(args[0])
+  OPTIONS.info_dict = common.LoadInfoDict(input_zip)
+
+  # If this image was originally labelled with SELinux contexts, make sure we
+  # also apply the labels in our new image. During building, the "file_contexts"
+  # is in the out/ directory tree, but for repacking from target-files.zip it's
+  # in the root directory of the ramdisk.
+  if "selinux_fc" in OPTIONS.info_dict:
+    OPTIONS.info_dict["selinux_fc"] = os.path.join(OPTIONS.input_tmp, "BOOT", "RAMDISK",
+        "file_contexts")
+
+  output_zip = zipfile.ZipFile(args[1], "w", compression=zipfile.ZIP_DEFLATED)
+
+  common.GetBootableImage(
+      "boot.img", "boot.img", OPTIONS.input_tmp, "BOOT").AddToZip(output_zip)
+  common.GetBootableImage(
+      "recovery.img", "recovery.img", OPTIONS.input_tmp,
+      "RECOVERY").AddToZip(output_zip)
+
+  if not bootable_only:
+    AddSystem(output_zip)
+    AddVendor(output_zip)
+    AddUserdata(output_zip)
+    AddCache(output_zip)
+    CopyInfo(output_zip)
+
+  print "cleaning up..."
+  output_zip.close()
+  shutil.rmtree(OPTIONS.input_tmp)
+
+  print "done."
+
+
+if __name__ == '__main__':
+  try:
+    common.CloseInheritedPipes()
+    main(sys.argv[1:])
+  except common.ExternalError, e:
+    print
+    print "   ERROR: %s" % (e,)
+    print
+    sys.exit(1)

tools/releasetools/ota_from_target_files

@@ -80,6 +80,7 @@ except ImportError:
   from sha import sha as sha1
 
 import common
+import img_from_target_files
 import edify_generator
 
 OPTIONS = common.OPTIONS
@@ -434,22 +435,29 @@ else if get_stage("%(bcb_dev)s", "stage") == "3/3" then
 
   device_specific.FullOTA_InstallBegin()
 
-  script.ShowProgress(0.5, 0)
+  system_progress = 0.75
 
   if OPTIONS.wipe_user_data:
+    system_progress -= 0.1
+    script.ShowProgress(0.1, 10)
     script.FormatPartition("/data")
 
   if "selinux_fc" in OPTIONS.info_dict:
     WritePolicyConfig(OPTIONS.info_dict["selinux_fc"], output_zip)
 
-  script.FormatPartition("/system")
+  script.ShowProgress(system_progress, 30)
-  script.Mount("/system")
+  if has_recovery_patch:
-  if not has_recovery_patch:
+    img_from_target_files.AddSystem(output_zip, sparse=False)
-    script.UnpackPackageDir("recovery", "/system")
+    script.WriteRawImage("/system", "system.img")
-  script.UnpackPackageDir("system", "/system")
+  else:
+    script.FormatPartition("/system")
+    script.Mount("/system")
+    if not has_recovery_patch:
+      script.UnpackPackageDir("recovery", "/system")
+      script.UnpackPackageDir("system", "/system")
 
-  symlinks = CopySystemFiles(input_zip, output_zip)
+    symlinks = CopySystemFiles(input_zip, output_zip)
-  script.MakeSymlinks(symlinks)
+    script.MakeSymlinks(symlinks)
 
   boot_img = common.GetBootableImage("boot.img", "boot.img",
                                      OPTIONS.input_tmp, "BOOT")
@@ -462,17 +470,16 @@ else if get_stage("%(bcb_dev)s", "stage") == "3/3" then
     common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink,
                              recovery_img, boot_img)
 
-  Item.GetMetadata(input_zip)
+    Item.GetMetadata(input_zip)
-  Item.Get("system").SetPermissions(script)
+    Item.Get("system").SetPermissions(script)
 
   common.CheckSize(boot_img.data, "boot.img", OPTIONS.info_dict)
   common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
-  script.ShowProgress(0.2, 0)
 
-  script.ShowProgress(0.2, 10)
+  script.ShowProgress(0.05, 5)
   script.WriteRawImage("/boot", "boot.img")
 
-  script.ShowProgress(0.1, 0)
+  script.ShowProgress(0.2, 10)
   device_specific.FullOTA_InstallEnd()
 
   if OPTIONS.extra_script is not None: