From 9a89a2a0eae3920e617e50b61df3f2b1139b634d Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Wed, 28 Oct 2020 19:20:06 +0000 Subject: [PATCH] Move boot jars package check from make Adds a singleton that traverses the module variants finding the ones that are in the list (updatable and non-updatable) of boot jars and add a ninja rule to ensure that they only contain packages from an allowed list. Replaces a hack that ignored any prebuilt boot jars supplied as dex file with an equivalent one to ensure that they are still ignored. A follow up change that switches to checking dex jars will allow the hack to be removed. The boot jars check can be strict or lax. If strict then all the boot jars listed in the configuration must be found, otherwise it will only check the ones it finds. It is strict by default unless TARGET_BUILD_UNBUNDLED=true or ALLOW_MISSING_DEPENDENCIES=true. Moves the script and data file from build/make. Test: m check-boot-jars - for failing and passing cases SKIP_BOOT_JARS_CHECK=true - no check-boot-jars target created ALLOW_MISSING_DEPENDENCIES=true - not strict TARGET_BUILD_UNBUNDLED=true - not strict verified manually that apart from path differences the same files (same check sum) were checked in both old make checks and the new Soong ones EMMA_INSTRUMENT=true EMMA_INSTRUMENT_FRAMEWORK=true m check-boot-jars Bug: 171479578 Change-Id: I9d81d6650ba64fc0d48d2dab4ba5a3ba8dd03dec --- android/apex.go | 9 + android/config.go | 18 ++ android/variable.go | 1 + java/Android.bp | 1 + java/androidmk.go | 4 - java/boot_jars.go | 123 +++++++++ scripts/Android.bp | 16 ++ scripts/check_boot_jars/check_boot_jars.py | 89 +++++++ .../check_boot_jars/package_allowed_list.txt | 248 ++++++++++++++++++ 9 files changed, 505 insertions(+), 4 deletions(-) create mode 100644 java/boot_jars.go create mode 100755 scripts/check_boot_jars/check_boot_jars.py create mode 100644 scripts/check_boot_jars/package_allowed_list.txt diff --git a/android/apex.go b/android/apex.go index 3039e7944..e70ec4f3c 100644 --- a/android/apex.go +++ b/android/apex.go @@ -67,6 +67,15 @@ func (i ApexInfo) IsForPlatform() bool { return i.ApexVariationName == "" } +func (i ApexInfo) InApex(apex string) bool { + for _, a := range i.InApexes { + if a == apex { + return true + } + } + return false +} + // ApexTestForInfo stores the contents of APEXes for which this module is a test and thus has // access to APEX internals. type ApexTestForInfo struct { diff --git a/android/config.go b/android/config.go index dbae7f78c..a4990576e 100644 --- a/android/config.go +++ b/android/config.go @@ -793,6 +793,11 @@ func (c *config) AlwaysUsePrebuiltSdks() bool { return Bool(c.productVariables.Always_use_prebuilt_sdks) } +// Returns true if the boot jars check should be skipped. +func (c *config) SkipBootJarsCheck() bool { + return Bool(c.productVariables.Skip_boot_jars_check) +} + func (c *config) Fuchsia() bool { return Bool(c.productVariables.Fuchsia) } @@ -1341,6 +1346,11 @@ func (l *ConfiguredJarList) Jar(idx int) string { return l.jars[idx] } +// Apex component of idx-th pair on the list. +func (l *ConfiguredJarList) Apex(idx int) string { + return l.apexes[idx] +} + // If the list contains a pair with the given jar. func (l *ConfiguredJarList) ContainsJar(jar string) bool { return InList(jar, l.jars) @@ -1538,3 +1548,11 @@ func (c *config) BootJars() []string { return list }).([]string) } + +func (c *config) NonUpdatableBootJars() ConfiguredJarList { + return c.productVariables.BootJars +} + +func (c *config) UpdatableBootJars() ConfiguredJarList { + return c.productVariables.UpdatableBootJars +} diff --git a/android/variable.go b/android/variable.go index 7999f0fb0..a9a9c87c2 100644 --- a/android/variable.go +++ b/android/variable.go @@ -224,6 +224,7 @@ type productVariables struct { Unbundled_build *bool `json:",omitempty"` Unbundled_build_apps *bool `json:",omitempty"` Always_use_prebuilt_sdks *bool `json:",omitempty"` + Skip_boot_jars_check *bool `json:",omitempty"` Malloc_not_svelte *bool `json:",omitempty"` Malloc_zero_contents *bool `json:",omitempty"` Malloc_pattern_fill_contents *bool `json:",omitempty"` diff --git a/java/Android.bp b/java/Android.bp index 92e8ca458..9e8dc786b 100644 --- a/java/Android.bp +++ b/java/Android.bp @@ -22,6 +22,7 @@ bootstrap_go_package { "androidmk.go", "app_builder.go", "app.go", + "boot_jars.go", "builder.go", "device_host_converter.go", "dex.go", diff --git a/java/androidmk.go b/java/androidmk.go index c21c83ab5..e1a661fc1 100644 --- a/java/androidmk.go +++ b/java/androidmk.go @@ -217,10 +217,6 @@ func (prebuilt *DexImport) AndroidMkEntries() []android.AndroidMkEntries { func(entries *android.AndroidMkEntries) { if prebuilt.dexJarFile != nil { entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile) - // TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until - // boot_jars_package_check.mk can check dex jars. - entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile) - entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile) } if len(prebuilt.dexpreopter.builtInstalled) > 0 { entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled) diff --git a/java/boot_jars.go b/java/boot_jars.go new file mode 100644 index 000000000..900eb7adf --- /dev/null +++ b/java/boot_jars.go @@ -0,0 +1,123 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "android/soong/android" +) + +func init() { + android.RegisterSingletonType("boot_jars", bootJarsSingletonFactory) +} + +func bootJarsSingletonFactory() android.Singleton { + return &bootJarsSingleton{} +} + +type bootJarsSingleton struct{} + +func populateMapFromConfiguredJarList(ctx android.SingletonContext, moduleToApex map[string]string, list android.ConfiguredJarList, name string) bool { + for i := 0; i < list.Len(); i++ { + module := list.Jar(i) + // Ignore jacocoagent it is only added when instrumenting and so has no impact on + // app compatibility. + if module == "jacocoagent" { + continue + } + apex := list.Apex(i) + if existing, ok := moduleToApex[module]; ok { + ctx.Errorf("Configuration property %q is invalid as it contains multiple references to module (%s) in APEXes (%s and %s)", + module, existing, apex) + return false + } + + moduleToApex[module] = apex + } + + return true +} + +func (b *bootJarsSingleton) GenerateBuildActions(ctx android.SingletonContext) { + config := ctx.Config() + if config.SkipBootJarsCheck() { + return + } + + // Populate a map from module name to APEX from the boot jars. If there is a problem + // such as duplicate modules then fail and return immediately. + moduleToApex := make(map[string]string) + if !populateMapFromConfiguredJarList(ctx, moduleToApex, config.NonUpdatableBootJars(), "BootJars") || + !populateMapFromConfiguredJarList(ctx, moduleToApex, config.UpdatableBootJars(), "UpdatableBootJars") { + return + } + + // Map from module name to the correct apex variant. + nameToApexVariant := make(map[string]android.Module) + + // Scan all the modules looking for the module/apex variants corresponding to the + // boot jars. + ctx.VisitAllModules(func(module android.Module) { + name := ctx.ModuleName(module) + if apex, ok := moduleToApex[name]; ok { + apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo) + if (apex == "platform" && apexInfo.IsForPlatform()) || apexInfo.InApex(apex) { + // The module name/apex variant should be unique in the system but double check + // just in case something has gone wrong. + if existing, ok := nameToApexVariant[name]; ok { + ctx.Errorf("found multiple variants matching %s:%s: %q and %q", apex, name, existing, module) + } + nameToApexVariant[name] = module + } + } + }) + + timestamp := android.PathForOutput(ctx, "boot-jars-package-check/stamp") + + rule := android.NewRuleBuilder() + checkBootJars := rule.Command().BuiltTool(ctx, "check_boot_jars"). + Input(android.PathForSource(ctx, "build/soong/scripts/check_boot_jars/package_allowed_list.txt")) + + // If this is not an unbundled build and missing dependencies are not allowed + // then all the boot jars listed must have been found. + strict := !config.UnbundledBuild() && !config.AllowMissingDependencies() + + // Iterate over the module names on the boot classpath in order + for _, name := range android.SortedStringKeys(moduleToApex) { + if apexVariant, ok := nameToApexVariant[name]; ok { + if dep, ok := apexVariant.(Dependency); ok { + // Add the implementation jars for the module to be checked. This uses implementation + // and resources jar as that is what the previous make based check uses. + for _, jar := range dep.ImplementationAndResourcesJars() { + checkBootJars.Input(jar) + } + } else if _, ok := apexVariant.(*DexImport); ok { + // TODO(b/171479578): ignore deximport when doing package check until boot_jars.go can check dex jars. + } else { + ctx.Errorf("module %q is of type %q which is not supported as a boot jar", name, ctx.ModuleType(apexVariant)) + } + } else if strict { + ctx.Errorf("boot jars package check failed as it could not find module %q for apex %q", name, moduleToApex[name]) + } + } + + checkBootJars.Text("&& touch").Output(timestamp) + rule.Build(pctx, ctx, "boot_jars_package_check", "check boot jar packages") + + // The check-boot-jars phony target depends on the timestamp created if the check succeeds. + ctx.Phony("check-boot-jars", timestamp) + + // The droidcore phony target depends on the check-boot-jars phony target + ctx.Phony("droidcore", android.PathForPhony(ctx, "check-boot-jars")) +} diff --git a/scripts/Android.bp b/scripts/Android.bp index 92f5c5335..dd03f283a 100644 --- a/scripts/Android.bp +++ b/scripts/Android.bp @@ -1,3 +1,19 @@ +python_binary_host { + name: "check_boot_jars", + main: "check_boot_jars/check_boot_jars.py", + srcs: [ + "check_boot_jars/check_boot_jars.py", + ], + version: { + py2: { + enabled: true, + }, + py3: { + enabled: false, + }, + }, +} + python_binary_host { name: "manifest_fixer", main: "manifest_fixer.py", diff --git a/scripts/check_boot_jars/check_boot_jars.py b/scripts/check_boot_jars/check_boot_jars.py new file mode 100755 index 000000000..cf4ef2782 --- /dev/null +++ b/scripts/check_boot_jars/check_boot_jars.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +""" +Check boot jars. + +Usage: check_boot_jars.py ... +""" +import logging +import os.path +import re +import subprocess +import sys + + +# The compiled allow list RE. +allow_list_re = None + + +def LoadAllowList(filename): + """ Load and compile allow list regular expressions from filename. + """ + lines = [] + with open(filename, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + lines.append(line) + combined_re = r'^(%s)$' % '|'.join(lines) + global allow_list_re + try: + allow_list_re = re.compile(combined_re) + except re.error: + logging.exception( + 'Cannot compile package allow list regular expression: %r', + combined_re) + allow_list_re = None + return False + return True + + +def CheckJar(allow_list_path, jar): + """Check a jar file. + """ + # Get the list of files inside the jar file. + p = subprocess.Popen(args='jar tf %s' % jar, + stdout=subprocess.PIPE, shell=True) + stdout, _ = p.communicate() + if p.returncode != 0: + return False + items = stdout.split() + classes = 0 + for f in items: + if f.endswith('.class'): + classes += 1 + package_name = os.path.dirname(f) + package_name = package_name.replace('/', '.') + if not package_name or not allow_list_re.match(package_name): + print >> sys.stderr, ('Error: %s contains class file %s, whose package name %s is empty or' + ' not in the allow list %s of packages allowed on the bootclasspath.' + % (jar, f, package_name, allow_list_path)) + return False + if classes == 0: + print >> sys.stderr, ('Error: %s does not contain any class files.' % jar) + return False + return True + + +def main(argv): + if len(argv) < 2: + print __doc__ + return 1 + allow_list_path = argv[0] + + if not LoadAllowList(allow_list_path): + return 1 + + passed = True + for jar in argv[1:]: + if not CheckJar(allow_list_path, jar): + passed = False + if not passed: + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt new file mode 100644 index 000000000..18ab427b5 --- /dev/null +++ b/scripts/check_boot_jars/package_allowed_list.txt @@ -0,0 +1,248 @@ +# Boot jar package name allowed list. +# Each line is interpreted as a regular expression. + +################################################### +# core-libart.jar & core-oj.jar +java\.awt\.font +java\.beans +java\.io +java\.lang +java\.lang\.annotation +java\.lang\.invoke +java\.lang\.ref +java\.lang\.reflect +java\.math +java\.net +java\.nio +java\.nio\.file +java\.nio\.file\.spi +java\.nio\.file\.attribute +java\.nio\.channels +java\.nio\.channels\.spi +java\.nio\.charset +java\.nio\.charset\.spi +java\.security +java\.security\.acl +java\.security\.cert +java\.security\.interfaces +java\.security\.spec +java\.sql +java\.text +java\.text\.spi +java\.time +java\.time\.chrono +java\.time\.format +java\.time\.temporal +java\.time\.zone +java\.util +java\.util\.concurrent +java\.util\.concurrent\.atomic +java\.util\.concurrent\.locks +java\.util\.function +java\.util\.jar +java\.util\.logging +java\.util\.prefs +java\.util\.regex +java\.util\.spi +java\.util\.stream +java\.util\.zip +# TODO: Remove javax.annotation.processing if possible, see http://b/132338110: +javax\.annotation\.processing +javax\.crypto +javax\.crypto\.interfaces +javax\.crypto\.spec +javax\.net +javax\.net\.ssl +javax\.security\.auth +javax\.security\.auth\.callback +javax\.security\.auth\.login +javax\.security\.auth\.x500 +javax\.security\.cert +javax\.sql +javax\.xml +javax\.xml\.datatype +javax\.xml\.namespace +javax\.xml\.parsers +javax\.xml\.transform +javax\.xml\.transform\.dom +javax\.xml\.transform\.sax +javax\.xml\.transform\.stream +javax\.xml\.validation +javax\.xml\.xpath +jdk\.internal\.util +jdk\.internal\.vm\.annotation +jdk\.net +org\.w3c\.dom +org\.w3c\.dom\.ls +org\.w3c\.dom\.traversal +# OpenJdk internal implementation. +sun\.invoke\.util +sun\.invoke\.empty +sun\.misc +sun\.util.* +sun\.text.* +sun\.security.* +sun\.reflect.* +sun\.nio.* +sun\.net.* +com\.sun\..* + +# TODO: Move these internal org.apache.harmony classes to libcore.* +org\.apache\.harmony\.crypto\.internal +org\.apache\.harmony\.dalvik +org\.apache\.harmony\.dalvik\.ddmc +org\.apache\.harmony\.luni\.internal\.util +org\.apache\.harmony\.security +org\.apache\.harmony\.security\.asn1 +org\.apache\.harmony\.security\.fortress +org\.apache\.harmony\.security\.pkcs10 +org\.apache\.harmony\.security\.pkcs7 +org\.apache\.harmony\.security\.pkcs8 +org\.apache\.harmony\.security\.provider\.crypto +org\.apache\.harmony\.security\.utils +org\.apache\.harmony\.security\.x501 +org\.apache\.harmony\.security\.x509 +org\.apache\.harmony\.security\.x509\.tsp +org\.apache\.harmony\.xml +org\.apache\.harmony\.xml\.dom +org\.apache\.harmony\.xml\.parsers + +org\.json +org\.xmlpull\.v1 +org\.xmlpull\.v1\.sax2 + +# TODO: jarjar org.kxml2.io to com.android org\.kxml2\.io +org\.kxml2\.io +org\.xml +org\.xml\.sax +org\.xml\.sax\.ext +org\.xml\.sax\.helpers + +dalvik\..* +libcore\..* +android\..* +com\.android\..* +################################################### +# android.test.base.jar +junit\.extensions +junit\.framework +android\.test +android\.test\.suitebuilder\.annotation + + +################################################### +# ext.jar +# TODO: jarjar javax.sip to com.android +javax\.sip +javax\.sip\.address +javax\.sip\.header +javax\.sip\.message + +# TODO: jarjar org.apache.commons to com.android +org\.apache\.commons\.codec +org\.apache\.commons\.codec\.binary +org\.apache\.commons\.codec\.language +org\.apache\.commons\.codec\.net +org\.apache\.commons\.logging +org\.apache\.commons\.logging\.impl +org\.apache\.http +org\.apache\.http\.auth +org\.apache\.http\.auth\.params +org\.apache\.http\.client +org\.apache\.http\.client\.entity +org\.apache\.http\.client\.methods +org\.apache\.http\.client\.params +org\.apache\.http\.client\.protocol +org\.apache\.http\.client\.utils +org\.apache\.http\.conn +org\.apache\.http\.conn\.params +org\.apache\.http\.conn\.routing +org\.apache\.http\.conn\.scheme +org\.apache\.http\.conn\.ssl +org\.apache\.http\.conn\.util +org\.apache\.http\.cookie +org\.apache\.http\.cookie\.params +org\.apache\.http\.entity +org\.apache\.http\.impl +org\.apache\.http\.impl\.auth +org\.apache\.http\.impl\.client +org\.apache\.http\.impl\.client +org\.apache\.http\.impl\.conn +org\.apache\.http\.impl\.conn\.tsccm +org\.apache\.http\.impl\.cookie +org\.apache\.http\.impl\.entity +org\.apache\.http\.impl\.io +org\.apache\.http\.impl\.io +org\.apache\.http\.io +org\.apache\.http\.message +org\.apache\.http\.params +org\.apache\.http\.protocol +org\.apache\.http\.util + +# TODO: jarjar gov.nist to com.android +gov\.nist\.core +gov\.nist\.core\.net +gov\.nist\.javax\.sip +gov\.nist\.javax\.sip\.address +gov\.nist\.javax\.sip\.clientauthutils +gov\.nist\.javax\.sip\.header +gov\.nist\.javax\.sip\.header\.extensions +gov\.nist\.javax\.sip\.header\.ims +gov\.nist\.javax\.sip\.message +gov\.nist\.javax\.sip\.parser +gov\.nist\.javax\.sip\.parser\.extensions +gov\.nist\.javax\.sip\.parser\.ims +gov\.nist\.javax\.sip\.stack + +org\.ccil\.cowan\.tagsoup +org\.ccil\.cowan\.tagsoup\.jaxp + +################################################### +# framework.jar +javax\.microedition\.khronos\.opengles +javax\.microedition\.khronos\.egl + +android + +################################################### +# apache-xml.jar +org\.apache\.xml\.res +org\.apache\.xml\.utils +org\.apache\.xml\.utils\.res +org\.apache\.xml\.dtm +org\.apache\.xml\.dtm\.ref +org\.apache\.xml\.dtm\.ref\.dom2dtm +org\.apache\.xml\.dtm\.ref\.sax2dtm +org\.apache\.xml\.serializer +org\.apache\.xml\.serializer\.utils +org\.apache\.xml\.serializer\.dom3 +org\.apache\.xpath +org\.apache\.xpath\.operations +org\.apache\.xpath\.domapi +org\.apache\.xpath\.functions +org\.apache\.xpath\.res +org\.apache\.xpath\.axes +org\.apache\.xpath\.objects +org\.apache\.xpath\.patterns +org\.apache\.xpath\.jaxp +org\.apache\.xpath\.compiler +org\.apache\.xalan +org\.apache\.xalan\.res +org\.apache\.xalan\.templates +org\.apache\.xalan\.serialize +org\.apache\.xalan\.extensions +org\.apache\.xalan\.processor +org\.apache\.xalan\.transformer +org\.apache\.xalan\.xslt + +################################################### +# Packages in the google namespace across all bootclasspath jars. +com\.google\.android\..* +com\.google\.vr\.platform.* +com\.google\.i18n\.phonenumbers\..* +com\.google\.i18n\.phonenumbers + +################################################### +# Packages used for Android in Chrome OS +org\.chromium\.arc +org\.chromium\.arc\..*