Merge "Add bloaty_merger" am: 999314fd98

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1581728

MUST ONLY BE SUBMITTED BY AUTOMERGER

Change-Id: I4e1274090bd735b60a5ddbf8645f6c78b795b3a7

bloaty/Android.bp

@@ -0,0 +1,27 @@
+python_test_host {
+    name: "bloaty_merger_test",
+    srcs: [
+        "bloaty_merger_test.py",
+        "bloaty_merger.py",
+        "file_sections.proto",
+    ],
+    proto: {
+        canonical_path_from_root: false,
+    },
+    libs: [
+        "pyfakefs",
+        "ninja_rsp",
+    ],
+}
+
+python_binary_host {
+    name: "bloaty_merger",
+    srcs: [
+        "bloaty_merger.py",
+        "file_sections.proto",
+    ],
+    proto: {
+        canonical_path_from_root: false,
+    },
+    libs: ["ninja_rsp"],
+}

bloaty/bloaty_merger.py

@@ -0,0 +1,79 @@
+# Copyright 2021 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bloaty CSV Merger
+
+Merges a list of .csv files from Bloaty into a protobuf.  It takes the list as
+a first argument and the output as second. For instance:
+
+    $ bloaty_merger binary_sizes.lst binary_sizes.pb
+
+"""
+
+import argparse
+import csv
+
+import ninja_rsp
+
+import file_sections_pb2
+
+BLOATY_EXTENSION = ".bloaty.csv"
+
+def parse_csv(path):
+  """Parses a Bloaty-generated CSV file into a protobuf.
+
+  Args:
+    path: The filepath to the CSV file, relative to $ANDROID_TOP.
+
+  Returns:
+    A file_sections_pb2.File if the file was found; None otherwise.
+  """
+  file_proto = None
+  with open(path, newline='') as csv_file:
+    file_proto = file_sections_pb2.File()
+    if path.endswith(BLOATY_EXTENSION):
+      file_proto.path = path[:-len(BLOATY_EXTENSION)]
+    section_reader = csv.DictReader(csv_file)
+    for row in section_reader:
+      section = file_proto.sections.add()
+      section.name = row["sections"]
+      section.vm_size = int(row["vmsize"])
+      section.file_size = int(row["filesize"])
+  return file_proto
+
+def create_file_size_metrics(input_list, output_proto):
+  """Creates a FileSizeMetrics proto from a list of CSV files.
+
+  Args:
+    input_list: The path to the file which contains the list of CSV files. Each
+        filepath is separated by a space.
+    output_proto: The path for the output protobuf.
+  """
+  metrics = file_sections_pb2.FileSizeMetrics()
+  reader = ninja_rsp.NinjaRspFileReader(input_list)
+  for csv_path in reader:
+    file_proto = parse_csv(csv_path)
+    if file_proto:
+      metrics.files.append(file_proto)
+  with open(output_proto, "wb") as output:
+    output.write(metrics.SerializeToString())
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument("input_list_file", help="List of bloaty csv files.")
+  parser.add_argument("output_proto", help="Output proto.")
+  args = parser.parse_args()
+  create_file_size_metrics(args.input_list_file, args.output_proto)
+
+if __name__ == '__main__':
+  main()

bloaty/bloaty_merger_test.py

@@ -0,0 +1,65 @@
+# Copyright 2021 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from pyfakefs import fake_filesystem_unittest
+
+import bloaty_merger
+import file_sections_pb2
+
+
+class BloatyMergerTestCase(fake_filesystem_unittest.TestCase):
+  def setUp(self):
+    self.setUpPyfakefs()
+
+  def test_parse_csv(self):
+    csv_content = "sections,vmsize,filesize\nsection1,2,3\n"
+    self.fs.create_file("file1.bloaty.csv", contents=csv_content)
+    pb = bloaty_merger.parse_csv("file1.bloaty.csv")
+    self.assertEqual(pb.path, "file1")
+    self.assertEqual(len(pb.sections), 1)
+    s = pb.sections[0]
+    self.assertEqual(s.name, "section1")
+    self.assertEqual(s.vm_size, 2)
+    self.assertEqual(s.file_size, 3)
+
+  def test_missing_file(self):
+    with self.assertRaises(FileNotFoundError):
+      bloaty_merger.parse_csv("missing.bloaty.csv")
+
+  def test_malformed_csv(self):
+    csv_content = "header1,heaVder2,header3\n4,5,6\n"
+    self.fs.create_file("file1.bloaty.csv", contents=csv_content)
+    with self.assertRaises(KeyError):
+      bloaty_merger.parse_csv("file1.bloaty.csv")
+
+  def test_create_file_metrics(self):
+    file_list = "file1.bloaty.csv file2.bloaty.csv"
+    file1_content = "sections,vmsize,filesize\nsection1,2,3\nsection2,7,8"
+    file2_content = "sections,vmsize,filesize\nsection1,4,5\n"
+
+    self.fs.create_file("files.lst", contents=file_list)
+    self.fs.create_file("file1.bloaty.csv", contents=file1_content)
+    self.fs.create_file("file2.bloaty.csv", contents=file2_content)
+
+    bloaty_merger.create_file_size_metrics("files.lst", "output.pb")
+
+    metrics = file_sections_pb2.FileSizeMetrics()
+    with open("output.pb", "rb") as output:
+      metrics.ParseFromString(output.read())
+
+
+if __name__ == '__main__':
+  suite = unittest.TestLoader().loadTestsFromTestCase(BloatyMergerTestCase)
+  unittest.TextTestRunner(verbosity=2).run(suite)

bloaty/file_sections.proto

@@ -0,0 +1,39 @@
+// Copyright 2021 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+syntax = "proto2";
+
+package file_sections;
+
+message SectionDescriptior {
+  // Name of the section (e.g. .rodata)
+  optional string name = 1;
+
+  // Size of that section as part of the file.
+  optional uint64 file_size = 2;
+
+  // Size of that section when loaded in memory.
+  optional uint64 vm_size = 3;
+}
+
+message File {
+  // Relative path from $OUT_DIR.
+  optional string path = 1;
+
+  // File sections.
+  repeated SectionDescriptior sections = 2;
+}
+
+message FileSizeMetrics {
+  repeated File files = 1;
+}

scripts/Android.bp

@@ -209,13 +209,18 @@ python_test_host {
     test_suites: ["general-tests"],
 }
 
+python_library_host {
+    name: "ninja_rsp",
+    srcs: ["ninja_rsp.py"],
+}
+
 python_binary_host {
     name: "lint-project-xml",
     main: "lint-project-xml.py",
     srcs: [
         "lint-project-xml.py",
-        "ninja_rsp.py",
     ],
+    libs: ["ninja_rsp"],
 }
 
 python_binary_host {
@@ -223,8 +228,8 @@ python_binary_host {
     main: "gen-kotlin-build-file.py",
     srcs: [
         "gen-kotlin-build-file.py",
-        "ninja_rsp.py",
     ],
+    libs: ["ninja_rsp"],
 }
 
 python_binary_host {