Retry adjusting the size computation for reserved blocks.

Due to the change in https://lwn.net/Articles/546473/, kernel reserves a
few extra blocks (lesser of 2% and 4096 blocks) on ext4 FS which leads to
OTA update failures. Adjust the size computation if the device has
BOARD_HAS_EXT4_RESERVED_BLOCKS := true.

It amends the last attemp in [1]. Now it computes the used blocks from the
make_ext4fs output, instead of altering its argument.

[1]: commit efbb5d2e69.

Bug: 21522719
Bug: 22023465
Bug: 22174684
Change-Id: I9783a51abe6581ff5c75db81e78ac606d0f32c4c

core/Makefile

@@ -733,6 +733,7 @@ $(if $(INTERNAL_USERIMAGES_EXT_VARIANT),$(hide) echo "fs_type=$(INTERNAL_USERIMA
 $(if $(BOARD_SYSTEMIMAGE_PARTITION_SIZE),$(hide) echo "system_size=$(BOARD_SYSTEMIMAGE_PARTITION_SIZE)" >> $(1))
 $(if $(BOARD_SYSTEMIMAGE_FILE_SYSTEM_TYPE),$(hide) echo "system_fs_type=$(BOARD_SYSTEMIMAGE_FILE_SYSTEM_TYPE)" >> $(1))
 $(if $(BOARD_SYSTEMIMAGE_JOURNAL_SIZE),$(hide) echo "system_journal_size=$(BOARD_SYSTEMIMAGE_JOURNAL_SIZE)" >> $(1))
+$(if $(BOARD_HAS_EXT4_RESERVED_BLOCKS),$(hide) echo "has_ext4_reserved_blocks=$(BOARD_HAS_EXT4_RESERVED_BLOCKS)" >> $(1))
 $(if $(BOARD_SYSTEMIMAGE_SQUASHFS_COMPRESSOR),$(hide) echo "system_squashfs_compressor=$(BOARD_SYSTEMIMAGE_SQUASHFS_COMPRESSOR)" >> $(1))
 $(if $(BOARD_SYSTEMIMAGE_SQUASHFS_COMPRESSOR_OPT),$(hide) echo "system_squashfs_compressor_opt=$(BOARD_SYSTEMIMAGE_SQUASHFS_COMPRESSOR_OPT)" >> $(1))
 $(if $(BOARD_USERDATAIMAGE_FILE_SYSTEM_TYPE),$(hide) echo "userdata_fs_type=$(BOARD_USERDATAIMAGE_FILE_SYSTEM_TYPE)" >> $(1))
@@ -997,7 +998,15 @@ define build-systemimage-target
       $(TARGET_OUT) $(systemimage_intermediates)/system_image_info.txt $(1) \
       || ( echo "Out of space? the tree size of $(TARGET_OUT) is (MB): " 1>&2 ;\
            du -sm $(TARGET_OUT) 1>&2;\
-           echo "The max is $$(( $(BOARD_SYSTEMIMAGE_PARTITION_SIZE) / 1048576 )) MB." 1>&2 ;\
+           if [ "$(INTERNAL_USERIMAGES_EXT_VARIANT)" == "ext4" ]; then \
+               maxsize=$(BOARD_SYSTEMIMAGE_PARTITION_SIZE); \
+               if [ "$(BOARD_HAS_EXT4_RESERVED_BLOCKS)" == "true" ]; then \
+                   maxsize=$$((maxsize - 4096 * 4096)); \
+               fi; \
+               echo "The max is $$(( maxsize / 1048576 )) MB." 1>&2 ;\
+           else \
+               echo "The max is $$(( $(BOARD_SYSTEMIMAGE_PARTITION_SIZE) / 1048576 )) MB." 1>&2 ;\
+           fi; \
            mkdir -p $(DIST_DIR); cp $(INSTALLED_FILES_FILE) $(DIST_DIR)/installed-files-rescued.txt; \
            exit 1 )
 endef
@@ -1480,6 +1489,9 @@ endif
 ifdef BOARD_RECOVERYIMAGE_PARTITION_SIZE
 	$(hide) echo "recovery_size=$(BOARD_RECOVERYIMAGE_PARTITION_SIZE)" >> $(zip_root)/META/misc_info.txt
 endif
+ifdef BOARD_HAS_EXT4_RESERVED_BLOCKS
+	$(hide) echo "has_ext4_reserved_blocks=$(BOARD_HAS_EXT4_RESERVED_BLOCKS)" >> $(zip_root)/META/misc_info.txt
+endif
 ifdef TARGET_RECOVERY_FSTYPE_MOUNT_OPTIONS
 	@# TARGET_RECOVERY_FSTYPE_MOUNT_OPTIONS can be empty to indicate that nothing but defaults should be used.
 	$(hide) echo "recovery_mount_options=$(TARGET_RECOVERY_FSTYPE_MOUNT_OPTIONS)" >> $(zip_root)/META/misc_info.txt

tools/releasetools/build_image.py

@@ -22,6 +22,7 @@ Usage:  build_image input_directory properties_file output_image_file
 """
 import os
 import os.path
+import re
 import subprocess
 import sys
 import commands
@@ -34,17 +35,18 @@ OPTIONS = common.OPTIONS
 FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7"
 
 def RunCommand(cmd):
-  """ Echo and run the given command
+  """Echo and run the given command.
 
   Args:
     cmd: the command represented as a list of strings.
   Returns:
-    The exit code.
+    A tuple of the output and the exit code.
   """
   print "Running: ", " ".join(cmd)
-  p = subprocess.Popen(cmd)
+  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  p.communicate()
+  output, _ = p.communicate()
-  return p.returncode
+  print "%s" % (output.rstrip(),)
+  return (output, p.returncode)
 
 def GetVerityTreeSize(partition_size):
   cmd = "build_verity_tree -s %d"
@@ -146,7 +148,7 @@ def UnsparseImage(sparse_image_path, replace=True):
     else:
       return True, unsparse_image_path
   inflate_command = ["simg2img", sparse_image_path, unsparse_image_path]
-  exit_code = RunCommand(inflate_command)
+  (_, exit_code) = RunCommand(inflate_command)
   if exit_code != 0:
     os.remove(unsparse_image_path)
     return False, None
@@ -241,11 +243,12 @@ def BuildImage(in_dir, prop_dict, out_file):
 
   fs_spans_partition = True
   if fs_type.startswith("squash"):
-      fs_spans_partition = False
+    fs_spans_partition = False
 
   is_verity_partition = "verity_block_device" in prop_dict
   verity_supported = prop_dict.get("verity") == "true"
-  # adjust the partition size to make room for the hashes if this is to be verified
+  # Adjust the partition size to make room for the hashes if this is to be
+  # verified.
   if verity_supported and is_verity_partition and fs_spans_partition:
     partition_size = int(prop_dict.get("partition_size"))
 
@@ -307,8 +310,15 @@ def BuildImage(in_dir, prop_dict, out_file):
     staging_system = os.path.join(in_dir, "system")
     shutil.rmtree(staging_system, ignore_errors=True)
     shutil.copytree(origin_in, staging_system, symlinks=True)
+
+  reserved_blocks = prop_dict.get("has_ext4_reserved_blocks") == "true"
+  ext4fs_output = None
+
   try:
-    exit_code = RunCommand(build_command)
+    if reserved_blocks and fs_type.startswith("ext4"):
+      (ext4fs_output, exit_code) = RunCommand(build_command)
+    else:
+      (_, exit_code) = RunCommand(build_command)
   finally:
     if in_dir != origin_in:
       # Clean up temporary directories and files.
@@ -318,17 +328,42 @@ def BuildImage(in_dir, prop_dict, out_file):
   if exit_code != 0:
     return False
 
+  # Bug: 21522719, 22023465
+  # There are some reserved blocks on ext4 FS (lesser of 4096 blocks and 2%).
+  # We need to deduct those blocks from the available space, since they are
+  # not writable even with root privilege. It only affects devices using
+  # file-based OTA and a kernel version of 3.10 or greater (currently just
+  # sprout).
+  if reserved_blocks and fs_type.startswith("ext4"):
+    assert ext4fs_output is not None
+    ext4fs_stats = re.compile(
+        r'Created filesystem with .* (?P<used_blocks>[0-9]+)/'
+        r'(?P<total_blocks>[0-9]+) blocks')
+    m = ext4fs_stats.match(ext4fs_output.strip().split('\n')[-1])
+    used_blocks = int(m.groupdict().get('used_blocks'))
+    total_blocks = int(m.groupdict().get('total_blocks'))
+    reserved_blocks = min(4096, int(total_blocks * 0.02))
+    adjusted_blocks = total_blocks - reserved_blocks
+    if used_blocks > adjusted_blocks:
+      mount_point = prop_dict.get("mount_point")
+      print("Error: Not enough room on %s (total: %d blocks, used: %d blocks, "
+            "reserved: %d blocks, available: %d blocks)" % (
+                mount_point, total_blocks, used_blocks, reserved_blocks,
+                adjusted_blocks))
+      return False
+
   if not fs_spans_partition:
     mount_point = prop_dict.get("mount_point")
     partition_size = int(prop_dict.get("partition_size"))
     image_size = os.stat(out_file).st_size
     if image_size > partition_size:
-        print "Error: %s image size of %d is larger than partition size of %d" % (mount_point, image_size, partition_size)
+      print("Error: %s image size of %d is larger than partition size of "
-        return False
+            "%d" % (mount_point, image_size, partition_size))
+      return False
     if verity_supported and is_verity_partition:
-        if 2 * image_size - AdjustPartitionSizeForVerity(image_size) > partition_size:
+      if 2 * image_size - AdjustPartitionSizeForVerity(image_size) > partition_size:
-            print "Error: No more room on %s to fit verity data" % mount_point
+        print "Error: No more room on %s to fit verity data" % mount_point
-            return False
+        return False
     prop_dict["original_partition_size"] = prop_dict["partition_size"]
     prop_dict["partition_size"] = str(image_size)
 
@@ -344,7 +379,7 @@ def BuildImage(in_dir, prop_dict, out_file):
 
     # Run e2fsck on the inflated image file
     e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image]
-    exit_code = RunCommand(e2fsck_command)
+    (_, exit_code) = RunCommand(e2fsck_command)
 
     os.remove(unsparse_image)
 
@@ -391,6 +426,7 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
     copy_prop("system_verity_block_device", "verity_block_device")
     copy_prop("system_root_image", "system_root_image")
     copy_prop("ramdisk_dir", "ramdisk_dir")
+    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
     copy_prop("system_squashfs_compressor", "squashfs_compressor")
     copy_prop("system_squashfs_compressor_opt", "squashfs_compressor_opt")
   elif mount_point == "data":
@@ -406,10 +442,12 @@ def ImagePropFromGlobalDict(glob_dict, mount_point):
     copy_prop("vendor_size", "partition_size")
     copy_prop("vendor_journal_size", "journal_size")
     copy_prop("vendor_verity_block_device", "verity_block_device")
+    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
   elif mount_point == "oem":
     copy_prop("fs_type", "fs_type")
     copy_prop("oem_size", "partition_size")
     copy_prop("oem_journal_size", "journal_size")
+    copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
 
   return d
 
@@ -439,7 +477,8 @@ def main(argv):
 
   glob_dict = LoadGlobalDict(glob_dict_file)
   if "mount_point" in glob_dict:
-    # The caller knows the mount point and provides a dictionay needed by BuildImage().
+    # The caller knows the mount point and provides a dictionay needed by
+    # BuildImage().
     image_properties = glob_dict
   else:
     image_filename = os.path.basename(out_file)