From 02fb89a4d7d4837a725cf375fc3f30833906e8c8 Mon Sep 17 00:00:00 2001 From: Joe Onorato Date: Sat, 27 Jun 2020 00:10:23 -0700 Subject: [PATCH] Add mk2bp_catalog.py that outputs more data about makefiles to be converted to soong. - Adds makefile and which files are installed to the SOONG_CONV CSV file - Updates soong_to_convert.py to be able to parse that - Adds new script that is more detailed. - Outputs that file as part of the droidcore build to $(OUT_DIR)/target/product/$(TARGET_DEVICE)/mk2bp_remaining.html Test: m out/target/product/$(get_build_var TARGET_DEVICE)/mk2bp_remaining.html Change-Id: I7c380b6070754f4329bf3965595751e4dac794a0 --- core/Makefile | 15 +- core/binary.mk | 4 + core/java_common.mk | 4 + tools/mk2bp_catalog.py | 892 ++++++++++++++++++++++++++++++++++++++ tools/soong_to_convert.py | 2 +- 5 files changed, 915 insertions(+), 2 deletions(-) create mode 100755 tools/mk2bp_catalog.py diff --git a/core/Makefile b/core/Makefile index 2550c0e5e..759f54172 100644 --- a/core/Makefile +++ b/core/Makefile @@ -476,7 +476,7 @@ SOONG_CONV := $(sort $(SOONG_CONV)) SOONG_CONV_DATA := $(call intermediates-dir-for,PACKAGING,soong_conversion)/soong_conv_data $(SOONG_CONV_DATA): @rm -f $@ - @$(foreach s,$(SOONG_CONV),echo "$(s),$(SOONG_CONV.$(s).TYPE),$(sort $(SOONG_CONV.$(s).PROBLEMS)),$(sort $(filter-out $(SOONG_ALREADY_CONV),$(SOONG_CONV.$(s).DEPS)))" >>$@;) + @$(foreach s,$(SOONG_CONV),echo "$(s),$(SOONG_CONV.$(s).TYPE),$(sort $(SOONG_CONV.$(s).PROBLEMS)),$(sort $(filter-out $(SOONG_ALREADY_CONV),$(SOONG_CONV.$(s).DEPS))),$(sort $(SOONG_CONV.$(s).MAKEFILES)),$(sort $(SOONG_CONV.$(s).INSTALLED))" >>$@;) SOONG_TO_CONVERT_SCRIPT := build/make/tools/soong_to_convert.py SOONG_TO_CONVERT := $(PRODUCT_OUT)/soong_to_convert.txt @@ -485,6 +485,19 @@ $(SOONG_TO_CONVERT): $(SOONG_CONV_DATA) $(SOONG_TO_CONVERT_SCRIPT) $(hide) $(SOONG_TO_CONVERT_SCRIPT) $< >$@ $(call dist-for-goals,droidcore,$(SOONG_TO_CONVERT)) +MK2BP_CATALOG_SCRIPT := build/make/tools/mk2bp_catalog.py +MK2BP_REMAINING_HTML := $(PRODUCT_OUT)/mk2bp_remaining.html +$(MK2BP_REMAINING_HTML): PRIVATE_CODE_SEARCH_BASE_URL := "https://cs.android.com/android/platform/superproject/+/master:" +$(MK2BP_REMAINING_HTML): $(SOONG_CONV_DATA) $(MK2BP_CATALOG_SCRIPT) + @rm -f $@ + $(hide) $(MK2BP_CATALOG_SCRIPT) \ + --device=$(TARGET_DEVICE) \ + --title="Remaining Android.mk files for $(TARGET_DEVICE)-$(TARGET_BUILD_VARIANT)" \ + --codesearch=$(PRIVATE_CODE_SEARCH_BASE_URL) \ + --out_dir="$(OUT_DIR)" \ + > $@ +$(call dist-for-goals,droidcore,$(MK2BP_REMAINING_HTML)) + # ----------------------------------------------------------------- # Modules use -Wno-error, or added default -Wall -Werror WALL_WERROR := $(PRODUCT_OUT)/wall_werror.txt diff --git a/core/binary.mk b/core/binary.mk index 6c1c4db81..29a78a398 100644 --- a/core/binary.mk +++ b/core/binary.mk @@ -1818,6 +1818,10 @@ SOONG_CONV.$(LOCAL_MODULE).DEPS := \ $(my_shared_libraries) \ $(my_system_shared_libraries)) SOONG_CONV.$(LOCAL_MODULE).TYPE := native +SOONG_CONV.$(LOCAL_MODULE).MAKEFILES := \ + $(SOONG_CONV.$(LOCAL_MODULE).MAKEFILES) $(LOCAL_MODULE_MAKEFILE) +SOONG_CONV.$(LOCAL_MODULE).INSTALLED:= \ + $(SOONG_CONV.$(LOCAL_MODULE).INSTALLED) $(LOCAL_INSTALLED_MODULE) SOONG_CONV := $(SOONG_CONV) $(LOCAL_MODULE) ########################################################### diff --git a/core/java_common.mk b/core/java_common.mk index 658296d10..b2534e194 100644 --- a/core/java_common.mk +++ b/core/java_common.mk @@ -546,6 +546,10 @@ SOONG_CONV.$(LOCAL_MODULE).DEPS := \ $(LOCAL_JAVA_LIBRARIES) \ $(LOCAL_JNI_SHARED_LIBRARIES) SOONG_CONV.$(LOCAL_MODULE).TYPE := java +SOONG_CONV.$(LOCAL_MODULE).MAKEFILES := \ + $(SOONG_CONV.$(LOCAL_MODULE).MAKEFILES) $(LOCAL_MODULE_MAKEFILE) +SOONG_CONV.$(LOCAL_MODULE).INSTALLED := \ + $(SOONG_CONV.$(LOCAL_MODULE).INSTALLED) $(LOCAL_INSTALLED_MODULE) SOONG_CONV := $(SOONG_CONV) $(LOCAL_MODULE) endif diff --git a/tools/mk2bp_catalog.py b/tools/mk2bp_catalog.py new file mode 100755 index 000000000..83abd6251 --- /dev/null +++ b/tools/mk2bp_catalog.py @@ -0,0 +1,892 @@ +#!/usr/bin/env python3 + +""" +Command to print info about makefiles remaining to be converted to soong. + +See usage / argument parsing below for commandline options. +""" + +import argparse +import csv +import itertools +import json +import os +import re +import sys + +DIRECTORY_PATTERNS = [x.split("/") for x in ( + "device/*", + "frameworks/*", + "hardware/*", + "packages/*", + "vendor/*", + "*", +)] + +def match_directory_group(pattern, filename): + match = [] + filename = filename.split("/") + if len(filename) < len(pattern): + return None + for i in range(len(pattern)): + pattern_segment = pattern[i] + filename_segment = filename[i] + if pattern_segment == "*" or pattern_segment == filename_segment: + match.append(filename_segment) + else: + return None + if match: + return os.path.sep.join(match) + else: + return None + +def directory_group(filename): + for pattern in DIRECTORY_PATTERNS: + match = match_directory_group(pattern, filename) + if match: + return match + return os.path.dirname(filename) + +class Analysis(object): + def __init__(self, filename, line_matches): + self.filename = filename; + self.line_matches = line_matches + +def analyze_lines(filename, lines, func): + line_matches = [] + for i in range(len(lines)): + line = lines[i] + stripped = line.strip() + if stripped.startswith("#"): + continue + if func(stripped): + line_matches.append((i+1, line)) + if line_matches: + return Analysis(filename, line_matches); + +def analyze_has_conditional(line): + return (line.startswith("ifeq") or line.startswith("ifneq") + or line.startswith("ifdef") or line.startswith("ifndef")) + +NORMAL_INCLUDES = [re.compile(pattern) for pattern in ( + "include \$+\(CLEAR_VARS\)", # These are in defines which are tagged separately + "include \$+\(BUILD_.*\)", + "include \$\(call first-makefiles-under, *\$\(LOCAL_PATH\)\)", + "include \$\(call all-subdir-makefiles\)", + "include \$\(all-subdir-makefiles\)", + "include \$\(call all-makefiles-under, *\$\(LOCAL_PATH\)\)", + "include \$\(call all-makefiles-under, *\$\(call my-dir\).*\)", + "include \$\(BUILD_SYSTEM\)/base_rules.mk", # called out separately + "include \$\(call all-named-subdir-makefiles,.*\)", + "include \$\(subdirs\)", +)] +def analyze_has_wacky_include(line): + if not (line.startswith("include") or line.startswith("-include") + or line.startswith("sinclude")): + return False + for matcher in NORMAL_INCLUDES: + if matcher.fullmatch(line): + return False + return True + +BASE_RULES_RE = re.compile("include \$\(BUILD_SYSTEM\)/base_rules.mk") + +class Analyzer(object): + def __init__(self, title, func): + self.title = title; + self.func = func + + +ANALYZERS = ( + Analyzer("ifeq / ifneq", analyze_has_conditional), + Analyzer("Wacky Includes", analyze_has_wacky_include), + Analyzer("Calls base_rules", lambda line: BASE_RULES_RE.fullmatch(line)), + Analyzer("Calls define", lambda line: line.startswith("define ")), + Analyzer("Has ../", lambda line: "../" in line), + Analyzer("dist-for-​goals", lambda line: "dist-for-goals" in line), + Analyzer(".PHONY", lambda line: ".PHONY" in line), + Analyzer("render-​script", lambda line: ".rscript" in line), + Analyzer("vts src", lambda line: ".vts" in line), + Analyzer("COPY_​HEADERS", lambda line: "LOCAL_COPY_HEADERS" in line), +) + +class Summary(object): + def __init__(self): + self.makefiles = dict() + self.directories = dict() + + def Add(self, makefile): + self.makefiles[makefile.filename] = makefile + self.directories.setdefault(directory_group(makefile.filename), []).append(makefile) + +class Makefile(object): + def __init__(self, filename): + self.filename = filename + + # Analyze the file + with open(filename, "r", errors="ignore") as f: + try: + lines = f.readlines() + except UnicodeDecodeError as ex: + sys.stderr.write("Filename: %s\n" % filename) + raise ex + lines = [line.strip() for line in lines] + + self.analyses = dict([(analyzer, analyze_lines(filename, lines, analyzer.func)) for analyzer + in ANALYZERS]) + +def find_android_mk(): + cwd = os.getcwd() + for root, dirs, files in os.walk(cwd): + for filename in files: + if filename == "Android.mk": + yield os.path.join(root, filename)[len(cwd) + 1:] + for ignore in (".git", ".repo"): + if ignore in dirs: + dirs.remove(ignore) + +def is_aosp(dirname): + for d in ("device/sample", "hardware/interfaces", "hardware/libhardware", + "hardware/ril"): + if dirname.startswith(d): + return True + for d in ("device/", "hardware/", "vendor/"): + if dirname.startswith(d): + return False + return True + +def is_google(dirname): + for d in ("device/google", + "hardware/google", + "test/sts", + "vendor/auto", + "vendor/google", + "vendor/unbundled_google", + "vendor/widevine", + "vendor/xts"): + if dirname.startswith(d): + return True + return False + +def make_annotation_link(annotations, analysis, modules): + if analysis: + return "%s" % ( + annotations.Add(analysis, modules), + len(analysis) + ) + else: + return ""; + + +def is_clean(makefile): + for analysis in makefile.analyses.values(): + if analysis: + return False + return True + +class Annotations(object): + def __init__(self): + self.entries = [] + self.count = 0 + + def Add(self, makefiles, modules): + self.entries.append((makefiles, modules)) + self.count += 1 + return self.count-1 + +class SoongData(object): + def __init__(self, reader): + """Read the input file and store the modules and dependency mappings. + """ + self.problems = dict() + self.deps = dict() + self.reverse_deps = dict() + self.module_types = dict() + self.makefiles = dict() + self.reverse_makefiles = dict() + self.installed = dict() + self.modules = set() + + for (module, module_type, problem, dependencies, makefiles, installed) in reader: + self.modules.add(module) + makefiles = [f for f in makefiles.strip().split(' ') if f != ""] + self.module_types[module] = module_type + self.problems[module] = problem + self.deps[module] = [d for d in dependencies.strip().split(' ') if d != ""] + for dep in self.deps[module]: + if not dep in self.reverse_deps: + self.reverse_deps[dep] = [] + self.reverse_deps[dep].append(module) + self.makefiles[module] = makefiles + for f in makefiles: + self.reverse_makefiles.setdefault(f, []).append(module) + for f in installed.strip().split(' '): + self.installed[f] = module + +def count_deps(depsdb, module, seen): + """Based on the depsdb, count the number of transitive dependencies. + + You can pass in an reversed dependency graph to count the number of + modules that depend on the module.""" + count = 0 + seen.append(module) + if module in depsdb: + for dep in depsdb[module]: + if dep in seen: + continue + count += 1 + count_deps(depsdb, dep, seen) + return count + +def contains_unblocked_modules(soong, modules): + for m in modules: + if len(soong.deps[m]) == 0: + return True + return False + +def contains_blocked_modules(soong, modules): + for m in modules: + if len(soong.deps[m]) > 0: + return True + return False + +OTHER_PARTITON = "_other" +HOST_PARTITON = "_host" + +def get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, filename): + host_prefix = HOST_OUT_ROOT + "/" + device_prefix = PRODUCT_OUT + "/" + + if filename.startswith(host_prefix): + return HOST_PARTITON + + elif filename.startswith(device_prefix): + index = filename.find("/", len(device_prefix)) + if index < 0: + return OTHER_PARTITON + return filename[len(device_prefix):index] + + return OTHER_PARTITON + +def format_module_link(module): + return "%s" % (module, module) + +def format_module_list(modules): + return "".join(["
%s
" % format_module_link(m) for m in modules]) + +def main(): + parser = argparse.ArgumentParser(description="Info about remaining Android.mk files.") + parser.add_argument("--device", type=str, required=True, + help="TARGET_DEVICE") + parser.add_argument("--title", type=str, + help="page title") + parser.add_argument("--codesearch", type=str, + default="https://cs.android.com/android/platform/superproject/+/master:", + help="page title") + parser.add_argument("--out_dir", type=str, + default=None, + help="Equivalent of $OUT_DIR, which will also be checked if" + + " --out_dir is unset. If neither is set, default is" + + " 'out'.") + + args = parser.parse_args() + + # Guess out directory name + if not args.out_dir: + args.out_dir = os.getenv("OUT_DIR", "out") + while args.out_dir.endswith("/") and len(args.out_dir) > 1: + args.out_dir = args.out_dir[:-1] + + TARGET_DEVICE = args.device + HOST_OUT_ROOT = args.out_dir + "host" + PRODUCT_OUT = args.out_dir + "/target/product/%s" % TARGET_DEVICE + + if args.title: + page_title = args.title + else: + page_title = "Remaining Android.mk files" + + # Read target information + # TODO: Pull from configurable location. This is also slightly different because it's + # only a single build, where as the tree scanning we do below is all Android.mk files. + with open("%s/obj/PACKAGING/soong_conversion_intermediates/soong_conv_data" + % PRODUCT_OUT, "r", errors="ignore") as csvfile: + soong = SoongData(csv.reader(csvfile)) + + # Which modules are installed where + modules_by_partition = dict() + partitions = set() + for installed, module in soong.installed.items(): + partition = get_partition_from_installed(HOST_OUT_ROOT, PRODUCT_OUT, installed) + modules_by_partition.setdefault(partition, []).append(module) + partitions.add(partition) + + print(""" + + + %(page_title)s + + + +

%(page_title)s

+
+ Help + + Partitions: + """ % { + "page_title": page_title, + }) + for partition in sorted(partitions): + print("%s" % (partition, partition)) + + print(""" + +
+
+
+ +
+

+ This page analyzes the remaining Android.mk files in the Android Source tree. +

+ The modules are first broken down by which of the device filesystem partitions + they are installed to. This also includes host tools and testcases which don't + actually reside in their own partition but convenitely group together. +

+ The makefiles for each partition are further are grouped into a set of directories + aritrarily picked to break down the problem size by owners. +

    +
  • AOSP directories are colored green.
    • +
  • Google directories are colored blue.
    • +
  • Other partner directories are colored red.
    • +
+ Each of the makefiles are scanned for issues that are likely to come up during + conversion to soong. Clicking the number in each cell shows additional information, + including the line that triggered the warning. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TotalThe total number of makefiles in this each directory.
UnblockedMakefiles containing one or more modules that don't have any + additional dependencies pending before conversion.
BlockedMakefiles containiong one or more modules which do have + additional prerequesite depenedencies that are not yet converted.
CleanThe number of makefiles that have none of the following warnings.
ifeq / ifneqMakefiles that use ifeq or ifneq. i.e. + conditionals.
Wacky IncludesMakefiles that include files other than the standard build-system + defined template and macros.
Calls base_rulesMakefiles that include base_rules.mk directly.
Calls defineMakefiles that define their own macros. Some of these are easy to convert + to soong defaults, but others are complex.
Has ../Makefiles containing the string "../" outside of a comment. These likely + access files outside their directories.
dist-for-goalsMakefiles that call dist-for-goals directly.
.PHONYMakefiles that declare .PHONY targets.
renderscriptMakefiles defining targets that depend on .rscript source files.
vts srcMakefiles defining targets that depend on .vts source files.
COPY_HEADERSMakefiles using LOCAL_COPY_HEADERS.
+

+ Following the list of directories is a list of the modules that are installed on + each partition. Potential issues from their makefiles are listed, as well as the + total number of dependencies (both blocking that module and blocked by that module) + and the list of direct dependencies. Note: The number is the number of all transitive + dependencies and the list of modules is only the direct dependencies. +

+ """) + + annotations = Annotations() + + # For each partition + makefiles_for_partitions = dict() + for partition in sorted(partitions): + modules = modules_by_partition[partition] + + makefiles = set(itertools.chain.from_iterable( + [soong.makefiles[module] for module in modules])) + + # Read makefiles + summary = Summary() + for filename in makefiles: + if not filename.startswith(args.out_dir + "/"): + summary.Add(Makefile(filename)) + + # Categorize directories by who is responsible + aosp_dirs = [] + google_dirs = [] + partner_dirs = [] + for dirname in sorted(summary.directories.keys()): + if is_aosp(dirname): + aosp_dirs.append(dirname) + elif is_google(dirname): + google_dirs.append(dirname) + else: + partner_dirs.append(dirname) + + print(""" + +

%(partition)s

+ + + + + + + + """ % { + "partition": partition + }) + + for analyzer in ANALYZERS: + print("""""" % analyzer.title) + + print(" ") + for dirgroup, rowclass in [(aosp_dirs, "AospDir"), + (google_dirs, "GoogleDir"), + (partner_dirs, "PartnerDir"),]: + for dirname in dirgroup: + makefiles = summary.directories[dirname] + + all_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles] + clean_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if is_clean(makefile)] + unblocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if contains_unblocked_modules(soong, + soong.reverse_makefiles[makefile.filename])] + blocked_makefiles = [Analysis(makefile.filename, []) for makefile in makefiles + if contains_blocked_modules(soong, + soong.reverse_makefiles[makefile.filename])] + + print(""" + + + + + + + """ % { + "rowclass": rowclass, + "dirname": dirname, + "makefiles": make_annotation_link(annotations, all_makefiles, modules), + "unblocked": make_annotation_link(annotations, unblocked_makefiles, modules), + "blocked": make_annotation_link(annotations, blocked_makefiles, modules), + "clean": make_annotation_link(annotations, clean_makefiles, modules), + }) + for analyzer in ANALYZERS: + analyses = [m.analyses.get(analyzer) for m in makefiles if m.analyses.get(analyzer)] + print("""""" + % make_annotation_link(annotations, analyses, modules)) + + print(" ") + print(""" +
DirectoryTotalUnblockedBlockedClean%s
%(dirname)s%(makefiles)s%(unblocked)s%(blocked)s%(clean)s%s
+ """) + + module_details = [(count_deps(soong.deps, m, []), -count_deps(soong.reverse_deps, m, []), m) + for m in modules] + module_details.sort() + module_details = [m[2] for m in module_details] + print(""" + """) + print("") + print(" ") + print(" ") + print(" ") + print(" ") + print("") + altRow = True + for module in module_details: + analyses = set() + for filename in soong.makefiles[module]: + makefile = summary.makefiles.get(filename) + if makefile: + for analyzer, analysis in makefile.analyses.items(): + if analysis: + analyses.add(analyzer.title) + + altRow = not altRow + print("" % ("Alt" if altRow else "",)) + print(" " % (module, module)) + print(" " % " ".join(["%s" % title + for title in analyses])) + print(" " % count_deps(soong.deps, module, [])) + print(" " % format_module_list(soong.deps.get(module, []))) + print(" " % count_deps(soong.reverse_deps, module, [])) + print(" " % format_module_list(soong.reverse_deps.get(module, []))) + print("") + print("""
Module NameIssuesBlocked ByBlocking
%s%s%s%s%s%s
""") + + print(""" + + + """) + + print(""" +
+
+
+ + + +
+
+
+ + + """) + +if __name__ == "__main__": + main() + diff --git a/tools/soong_to_convert.py b/tools/soong_to_convert.py index 083f6f78d..949131b1c 100755 --- a/tools/soong_to_convert.py +++ b/tools/soong_to_convert.py @@ -78,7 +78,7 @@ def process(reader): reverse_deps = dict() module_types = dict() - for (module, module_type, problem, dependencies) in reader: + for (module, module_type, problem, dependencies, makefiles, installed) in reader: module_types[module] = module_type problems[module] = problem deps[module] = [d for d in dependencies.strip().split(' ') if d != ""]