Unify the user image building code

Bug: 5398808

Before this change we have duplicate code in the Makefile and the
python-based releasetool. That's a real headache to maintain.

Change-Id: I8ddf04a79c6886540e89d990f723d8f77c9dd883

core/Makefile

@@ -604,21 +604,19 @@ endif
 ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true)
 INTERNAL_USERIMAGES_DEPS := $(MKEXTUSERIMG) $(MAKE_EXT4FS)
 INTERNAL_USERIMAGES_BINARY_PATHS := $(sort $(dir $(INTERNAL_USERIMAGES_DEPS)))
-
-# $(1): src directory
-# $(2): output file
-# $(3): mount point
-# $(4): ext variant (ext2, ext3, ext4)
-# $(5): size of the partition
-define build-userimage-ext-target
-  @mkdir -p $(dir $(2))
-  $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \
-	  $(MKEXTUSERIMG) $(INTERNAL_USERIMAGES_SPARSE_EXT_FLAG) $(1) $(2) $(4) $(3) $(5)
-endef
 else
 INTERNAL_USERIMAGES_DEPS := $(MKYAFFS2)
 endif
 
+# $(1): the path of the output dictionary file
+define generate-userimage-prop-dictionary
+$(if $(INTERNAL_USERIMAGES_EXT_VARIANT),$(hide) echo "fs_type=$(INTERNAL_USERIMAGES_EXT_VARIANT)" >> $(1))
+$(if $(BOARD_SYSTEMIMAGE_PARTITION_SIZE),$(hide) echo "system_size=$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)" >> $(1))
+$(if $(BOARD_USERDATAIMAGE_PARTITION_SIZE),$(hide) echo "userdata_size=$(BOARD_USERDATAIMAGE_PARTITION_SIZE)" >> $(1))
+$(if $(INTERNAL_USERIMAGES_SPARSE_EXT_FLAG),$(hide) echo "extfs_sparse_flag=$(INTERNAL_USERIMAGES_SPARSE_EXT_FLAG)" >> $(1))
+$(if $(mkyaffs2_extra_flags),$(hide) echo "mkyaffs2_extra_flags=$(mkyaffs2_extra_flags)" >> $(1))
+endef
+
 # -----------------------------------------------------------------
 # Recovery image
 
@@ -772,25 +770,16 @@ systemimage_intermediates := \
 	$(call intermediates-dir-for,PACKAGING,systemimage)
 BUILT_SYSTEMIMAGE := $(systemimage_intermediates)/system.img
 
-ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true)
-## generate an ext image
 # $(1): output file
 define build-systemimage-target
-    @echo "Target system fs image: $(1)"
+  @echo "Target system fs image: $(1)"
-    $(call build-userimage-ext-target,$(TARGET_OUT),$(1),system,$(INTERNAL_USERIMAGES_EXT_VARIANT),$(BOARD_SYSTEMIMAGE_PARTITION_SIZE))
+  @mkdir -p $(dir $(1)) $(systemimage_intermediates) && rm -rf $(systemimage_intermediates)/system_image_info.txt
+  $(call generate-userimage-prop-dictionary, $(systemimage_intermediates)/system_image_info.txt)
+  $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \
+      ./build/tools/releasetools/build_image.py \
+      $(TARGET_OUT) $(systemimage_intermediates)/system_image_info.txt $(1)
 endef
 
-else # INTERNAL_USERIMAGES_USE_EXT != true
-
-## generate a yaffs2 image
-# $(1): output file
-define build-systemimage-target
-    @echo "Target system fs image: $(1)"
-    @mkdir -p $(dir $(1))
-    $(hide) $(MKYAFFS2) -f $(mkyaffs2_extra_flags) $(TARGET_OUT) $(1)
-endef
-endif # INTERNAL_USERIMAGES_USE_EXT
-
 $(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE)
 	$(call build-systemimage-target,$@)
 
@@ -890,28 +879,21 @@ boottarball-nodeps btnod: $(FS_GET_STATS) \
 INTERNAL_USERDATAIMAGE_FILES := \
 	$(filter $(TARGET_OUT_DATA)/%,$(ALL_DEFAULT_INSTALLED_MODULES))
 
-ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true)
+userdataimage_intermediates := \
-## Generate an ext2 image
+	$(call intermediates-dir-for,PACKAGING,userdata)
-define build-userdataimage-target
-    $(call pretty,"Target userdata fs image: $(INSTALLED_USERDATAIMAGE_TARGET)")
-    @mkdir -p $(TARGET_OUT_DATA)
-    $(call build-userimage-ext-target,$(TARGET_OUT_DATA),$(INSTALLED_USERDATAIMAGE_TARGET),data,$(INTERNAL_USERIMAGES_EXT_VARIANT),$(BOARD_USERDATAIMAGE_PARTITION_SIZE))
-    $(hide) $(call assert-max-image-size,$(INSTALLED_USERDATAIMAGE_TARGET),$(BOARD_USERDATAIMAGE_PARTITION_SIZE),yaffs)
-endef
-
-else # INTERNAL_USERIMAGES_USE_EXT != true
-
-## Generate a yaffs2 image
-define build-userdataimage-target
-    $(call pretty,"Target userdata fs image: $(INSTALLED_USERDATAIMAGE_TARGET)")
-    @mkdir -p $(TARGET_OUT_DATA)
-    $(hide) $(MKYAFFS2) -f $(mkyaffs2_extra_flags) $(TARGET_OUT_DATA) $(INSTALLED_USERDATAIMAGE_TARGET)
-    $(hide) $(call assert-max-image-size,$(INSTALLED_USERDATAIMAGE_TARGET),$(BOARD_USERDATAIMAGE_PARTITION_SIZE),yaffs)
-endef
-endif # INTERNAL_USERIMAGES_USE_EXT
-
 BUILT_USERDATAIMAGE_TARGET := $(PRODUCT_OUT)/userdata.img
 
+define build-userdataimage-target
+  $(call pretty,"Target userdata fs image: $(INSTALLED_USERDATAIMAGE_TARGET)")
+  @mkdir -p $(TARGET_OUT_DATA)
+  @mkdir -p $(userdataimage_intermediates) && rm -rf $(userdataimage_intermediates)/userdata_image_info.txt
+  $(call generate-userimage-prop-dictionary, $(userdataimage_intermediates)/userdata_image_info.txt)
+  $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH \
+      ./build/tools/releasetools/build_image.py \
+      $(TARGET_OUT_DATA) $(userdataimage_intermediates)/userdata_image_info.txt $@
+  $(hide) $(call assert-max-image-size,$(INSTALLED_USERDATAIMAGE_TARGET),$(BOARD_USERDATAIMAGE_PARTITION_SIZE),yaffs)
+endef
+
 # We just build this directly to the install location.
 INSTALLED_USERDATAIMAGE_TARGET := $(BUILT_USERDATAIMAGE_TARGET)
 $(INSTALLED_USERDATAIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) \
@@ -1099,24 +1081,13 @@ ifdef BOARD_BOOTIMAGE_PARTITION_SIZE
 endif
 ifdef BOARD_RECOVERYIMAGE_PARTITION_SIZE
 	$(hide) echo "recovery_size=$(BOARD_RECOVERYIMAGE_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt
-endif
-ifdef BOARD_SYSTEMIMAGE_PARTITION_SIZE
-	$(hide) echo "system_size=$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt
-endif
-ifdef BOARD_USERDATAIMAGE_PARTITION_SIZE
-	$(hide) echo "userdata_size=$(BOARD_USERDATAIMAGE_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt
 endif
 	$(hide) echo "tool_extensions=$(tool_extensions)" >> $(zip_root)/META/misc_info.txt
-ifdef mkyaffs2_extra_flags
-	$(hide) echo "mkyaffs2_extra_flags=$(mkyaffs2_extra_flags)" >> $(zip_root)/META/misc_info.txt
-endif
-ifdef INTERNAL_USERIMAGES_SPARSE_EXT_FLAG
-	$(hide) echo "extfs_sparse_flag=$(INTERNAL_USERIMAGES_SPARSE_EXT_FLAG)" >> $(zip_root)/META/misc_info.txt
-endif
 	$(hide) echo "default_system_dev_certificate=$(DEFAULT_SYSTEM_DEV_CERTIFICATE)" >> $(zip_root)/META/misc_info.txt
 ifdef PRODUCT_EXTRA_RECOVERY_KEYS
 	$(hide) echo "extra_recovery_keys=$(PRODUCT_EXTRA_RECOVERY_KEYS)" >> $(zip_root)/META/misc_info.txt
 endif
+	$(call generate-userimage-prop-dictionary, $(zip_root)/META/misc_info.txt)
 	@# Zip everything up, preserving symlinks
 	$(hide) (cd $(zip_root) && zip -qry ../$(notdir $@) .)
 	@# Run fs_config on all the system, boot ramdisk, and recovery ramdisk files in the zip, and save the output

tools/releasetools/build_image.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Build image output_image_file from input_directory and properties_file.
+
+Usage:  build_image input_directory properties_file output_image_file
+
+"""
+import os
+import subprocess
+import sys
+
+
+def BuildImage(in_dir, prop_dict, out_file):
+  """Build an image to out_file from in_dir with property prop_dict.
+
+  Args:
+    in_dir: path of input directory.
+    prop_dict: property dictionary.
+    out_file: path of the output image file.
+
+  Returns:
+    True iff the image is built successfully.
+  """
+  build_command = []
+  fs_type = prop_dict.get("fs_type", "")
+  if fs_type.startswith("ext"):
+    build_command = ["mkuserimg.sh"]
+    if "extfs_sparse_flag" in prop_dict:
+      build_command.append(prop_dict["extfs_sparse_flag"])
+    build_command.extend([in_dir, out_file, fs_type,
+                          prop_dict["mount_point"]])
+    if "partition_size" in prop_dict:
+      build_command.append(prop_dict["partition_size"])
+  else:
+    build_command = ["mkyaffs2image", "-f"]
+    if prop_dict.get("mkyaffs2_extra_flags", None):
+      build_command.extend(prop_dict["mkyaffs2_extra_flags"].split())
+    build_command.append(in_dir)
+    build_command.append(out_file)
+
+  print "Running: ", " ".join(build_command)
+  p = subprocess.Popen(build_command);
+  p.communicate()
+  return p.returncode == 0
+
+
+def ImagePropFromGlobalDict(glob_dict, mount_point):
+  """Build an image property dictionary from the global dictionary.
+
+  Args:
+    glob_dict: the global dictionary from the build system.
+    mount_point: such as "system", "data" etc.
+  """
+  d = {}
+  common_props = (
+      "fs_type",
+      "extfs_sparse_flag",
+      "mkyaffs2_extra_flags",
+      )
+  for p in common_props:
+    if p in glob_dict:
+      d[p] = glob_dict[p]
+
+  d["mount_point"] = mount_point
+  if mount_point == "system":
+    if "system_size" in glob_dict:
+      d["partition_size"] = str(glob_dict["system_size"])
+  elif mount_point == "data":
+    if "userdata_size" in glob_dict:
+      d["partition_size"] = str(glob_dict["userdata_size"])
+
+  return d
+
+
+def LoadGlobalDict(filename):
+  """Load "name=value" pairs from filename"""
+  d = {}
+  f = open(filename)
+  for line in f:
+    line = line.strip()
+    if not line or line.startswith("#"):
+      continue
+    k, v = line.split("=", 1)
+    d[k] = v
+  f.close()
+  return d
+
+
+def main(argv):
+  if len(argv) != 3:
+    print __doc__
+    sys.exit(1)
+
+  in_dir = argv[0]
+  glob_dict_file = argv[1]
+  out_file = argv[2]
+
+  glob_dict = LoadGlobalDict(glob_dict_file)
+  image_filename = os.path.basename(out_file)
+  mount_point = ""
+  if image_filename == "system.img":
+    mount_point = "system"
+  elif image_filename == "userdata.img":
+    mount_point = "data"
+
+  image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
+  if not BuildImage(in_dir, image_properties, out_file):
+    print >> sys.stderr, "error: failed to build %s from %s" % (out_file, in_dir)
+    exit(1)
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

tools/releasetools/img_from_target_files

@@ -47,6 +47,7 @@ import zipfile
 if not hasattr(os, "SEEK_SET"):
   os.SEEK_SET = 0
 
+import build_image
 import common
 
 OPTIONS = common.OPTIONS
@@ -64,27 +65,13 @@ def AddUserdata(output_zip):
   os.mkdir(user_dir)
   img = tempfile.NamedTemporaryFile()
 
-  build_command = []
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "data")
   fstab = OPTIONS.info_dict["fstab"]
-  if fstab and fstab["/data"].fs_type.startswith("ext"):
+  if fstab:
-    build_command = ["mkuserimg.sh"]
+    image_props["fs_type" ] = fstab["/data"].fs_type
-    if "extfs_sparse_flag" in OPTIONS.info_dict:
+  succ = build_image.BuildImage(user_dir, image_props, img.name)
-      build_command.append(OPTIONS.info_dict["extfs_sparse_flag"])
+  assert succ, "build userdata.img image failed"
-    build_command.extend([user_dir, img.name,
-                     fstab["/data"].fs_type, "data"])
-    if "userdata_size" in OPTIONS.info_dict:
-      build_command.append(str(OPTIONS.info_dict["userdata_size"]))
-  else:
-    build_command = ["mkyaffs2image", "-f"]
-    extra = OPTIONS.info_dict.get("mkyaffs2_extra_flags", None)
-    if extra:
-      build_command.extend(extra.split())
-    build_command.append(user_dir)
-    build_command.append(img.name)
-
-  p = common.Run(build_command);
-  p.communicate()
-  assert p.returncode == 0, "build userdata.img image failed"
 
   common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
   output_zip.write(img.name, "userdata.img")
@@ -115,28 +102,14 @@ def AddSystem(output_zip):
     if (e.errno == errno.EEXIST):
       pass
 
-  build_command = []
+  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
+                                                    "system")
   fstab = OPTIONS.info_dict["fstab"]
-  if fstab and fstab["/system"].fs_type.startswith("ext"):
+  if fstab:
-
+    image_props["fs_type" ] = fstab["/system"].fs_type
-    build_command = ["mkuserimg.sh"]
+  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "system"),
-    if "extfs_sparse_flag" in OPTIONS.info_dict:
+                                image_props, img.name)
-      build_command.append(OPTIONS.info_dict["extfs_sparse_flag"])
+  assert succ, "build system.img image failed"
-    build_command.extend([os.path.join(OPTIONS.input_tmp, "system"), img.name,
-                     fstab["/system"].fs_type, "system"])
-    if "system_size" in OPTIONS.info_dict:
-      build_command.append(str(OPTIONS.info_dict["system_size"]))
-  else:
-    build_command = ["mkyaffs2image", "-f"]
-    extra = OPTIONS.info_dict.get("mkyaffs2_extra_flags", None)
-    if extra:
-      build_command.extend(extra.split())
-    build_command.append(os.path.join(OPTIONS.input_tmp, "system"))
-    build_command.append(img.name)
-
-  p = common.Run(build_command)
-  p.communicate()
-  assert p.returncode == 0, "build system.img image failed"
 
   img.seek(os.SEEK_SET, 0)
   data = img.read()