Move boot jars package check from make

Adds a singleton that traverses the module variants finding the ones
that are in the list (updatable and non-updatable) of boot jars and
add a ninja rule to ensure that they only contain packages from an
allowed list.

Replaces a hack that ignored any prebuilt boot jars supplied as dex
file with an equivalent one to ensure that they are still ignored.
A follow up change that switches to checking dex jars will allow the
hack to be removed.

The boot jars check can be strict or lax. If strict then all the boot
jars listed in the configuration must be found, otherwise it will only
check the ones it finds. It is strict by default unless
TARGET_BUILD_UNBUNDLED=true or ALLOW_MISSING_DEPENDENCIES=true.

Moves the script and data file from build/make.

Test: m check-boot-jars - for failing and passing cases
      SKIP_BOOT_JARS_CHECK=true - no check-boot-jars target created
	  ALLOW_MISSING_DEPENDENCIES=true - not strict
	  TARGET_BUILD_UNBUNDLED=true - not strict
      verified manually that apart from path differences the same
      files (same check sum) were checked in both old make checks and
      the new Soong ones
      EMMA_INSTRUMENT=true EMMA_INSTRUMENT_FRAMEWORK=true m check-boot-jars
Bug: 171479578
Change-Id: I9d81d6650ba64fc0d48d2dab4ba5a3ba8dd03dec
This commit is contained in:
Paul Duffin 2020-10-28 19:20:06 +00:00
parent e430ac338a
commit 9a89a2a0ea
9 changed files with 505 additions and 4 deletions

View File

@ -67,6 +67,15 @@ func (i ApexInfo) IsForPlatform() bool {
return i.ApexVariationName == ""
}
func (i ApexInfo) InApex(apex string) bool {
for _, a := range i.InApexes {
if a == apex {
return true
}
}
return false
}
// ApexTestForInfo stores the contents of APEXes for which this module is a test and thus has
// access to APEX internals.
type ApexTestForInfo struct {

View File

@ -793,6 +793,11 @@ func (c *config) AlwaysUsePrebuiltSdks() bool {
return Bool(c.productVariables.Always_use_prebuilt_sdks)
}
// Returns true if the boot jars check should be skipped.
func (c *config) SkipBootJarsCheck() bool {
return Bool(c.productVariables.Skip_boot_jars_check)
}
func (c *config) Fuchsia() bool {
return Bool(c.productVariables.Fuchsia)
}
@ -1341,6 +1346,11 @@ func (l *ConfiguredJarList) Jar(idx int) string {
return l.jars[idx]
}
// Apex component of idx-th pair on the list.
func (l *ConfiguredJarList) Apex(idx int) string {
return l.apexes[idx]
}
// If the list contains a pair with the given jar.
func (l *ConfiguredJarList) ContainsJar(jar string) bool {
return InList(jar, l.jars)
@ -1538,3 +1548,11 @@ func (c *config) BootJars() []string {
return list
}).([]string)
}
func (c *config) NonUpdatableBootJars() ConfiguredJarList {
return c.productVariables.BootJars
}
func (c *config) UpdatableBootJars() ConfiguredJarList {
return c.productVariables.UpdatableBootJars
}

View File

@ -224,6 +224,7 @@ type productVariables struct {
Unbundled_build *bool `json:",omitempty"`
Unbundled_build_apps *bool `json:",omitempty"`
Always_use_prebuilt_sdks *bool `json:",omitempty"`
Skip_boot_jars_check *bool `json:",omitempty"`
Malloc_not_svelte *bool `json:",omitempty"`
Malloc_zero_contents *bool `json:",omitempty"`
Malloc_pattern_fill_contents *bool `json:",omitempty"`

View File

@ -22,6 +22,7 @@ bootstrap_go_package {
"androidmk.go",
"app_builder.go",
"app.go",
"boot_jars.go",
"builder.go",
"device_host_converter.go",
"dex.go",

View File

@ -217,10 +217,6 @@ func (prebuilt *DexImport) AndroidMkEntries() []android.AndroidMkEntries {
func(entries *android.AndroidMkEntries) {
if prebuilt.dexJarFile != nil {
entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile)
// TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until
// boot_jars_package_check.mk can check dex jars.
entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile)
entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile)
}
if len(prebuilt.dexpreopter.builtInstalled) > 0 {
entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled)

123
java/boot_jars.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package java
import (
"android/soong/android"
)
func init() {
android.RegisterSingletonType("boot_jars", bootJarsSingletonFactory)
}
func bootJarsSingletonFactory() android.Singleton {
return &bootJarsSingleton{}
}
type bootJarsSingleton struct{}
func populateMapFromConfiguredJarList(ctx android.SingletonContext, moduleToApex map[string]string, list android.ConfiguredJarList, name string) bool {
for i := 0; i < list.Len(); i++ {
module := list.Jar(i)
// Ignore jacocoagent it is only added when instrumenting and so has no impact on
// app compatibility.
if module == "jacocoagent" {
continue
}
apex := list.Apex(i)
if existing, ok := moduleToApex[module]; ok {
ctx.Errorf("Configuration property %q is invalid as it contains multiple references to module (%s) in APEXes (%s and %s)",
module, existing, apex)
return false
}
moduleToApex[module] = apex
}
return true
}
func (b *bootJarsSingleton) GenerateBuildActions(ctx android.SingletonContext) {
config := ctx.Config()
if config.SkipBootJarsCheck() {
return
}
// Populate a map from module name to APEX from the boot jars. If there is a problem
// such as duplicate modules then fail and return immediately.
moduleToApex := make(map[string]string)
if !populateMapFromConfiguredJarList(ctx, moduleToApex, config.NonUpdatableBootJars(), "BootJars") ||
!populateMapFromConfiguredJarList(ctx, moduleToApex, config.UpdatableBootJars(), "UpdatableBootJars") {
return
}
// Map from module name to the correct apex variant.
nameToApexVariant := make(map[string]android.Module)
// Scan all the modules looking for the module/apex variants corresponding to the
// boot jars.
ctx.VisitAllModules(func(module android.Module) {
name := ctx.ModuleName(module)
if apex, ok := moduleToApex[name]; ok {
apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo)
if (apex == "platform" && apexInfo.IsForPlatform()) || apexInfo.InApex(apex) {
// The module name/apex variant should be unique in the system but double check
// just in case something has gone wrong.
if existing, ok := nameToApexVariant[name]; ok {
ctx.Errorf("found multiple variants matching %s:%s: %q and %q", apex, name, existing, module)
}
nameToApexVariant[name] = module
}
}
})
timestamp := android.PathForOutput(ctx, "boot-jars-package-check/stamp")
rule := android.NewRuleBuilder()
checkBootJars := rule.Command().BuiltTool(ctx, "check_boot_jars").
Input(android.PathForSource(ctx, "build/soong/scripts/check_boot_jars/package_allowed_list.txt"))
// If this is not an unbundled build and missing dependencies are not allowed
// then all the boot jars listed must have been found.
strict := !config.UnbundledBuild() && !config.AllowMissingDependencies()
// Iterate over the module names on the boot classpath in order
for _, name := range android.SortedStringKeys(moduleToApex) {
if apexVariant, ok := nameToApexVariant[name]; ok {
if dep, ok := apexVariant.(Dependency); ok {
// Add the implementation jars for the module to be checked. This uses implementation
// and resources jar as that is what the previous make based check uses.
for _, jar := range dep.ImplementationAndResourcesJars() {
checkBootJars.Input(jar)
}
} else if _, ok := apexVariant.(*DexImport); ok {
// TODO(b/171479578): ignore deximport when doing package check until boot_jars.go can check dex jars.
} else {
ctx.Errorf("module %q is of type %q which is not supported as a boot jar", name, ctx.ModuleType(apexVariant))
}
} else if strict {
ctx.Errorf("boot jars package check failed as it could not find module %q for apex %q", name, moduleToApex[name])
}
}
checkBootJars.Text("&& touch").Output(timestamp)
rule.Build(pctx, ctx, "boot_jars_package_check", "check boot jar packages")
// The check-boot-jars phony target depends on the timestamp created if the check succeeds.
ctx.Phony("check-boot-jars", timestamp)
// The droidcore phony target depends on the check-boot-jars phony target
ctx.Phony("droidcore", android.PathForPhony(ctx, "check-boot-jars"))
}

View File

@ -1,3 +1,19 @@
python_binary_host {
name: "check_boot_jars",
main: "check_boot_jars/check_boot_jars.py",
srcs: [
"check_boot_jars/check_boot_jars.py",
],
version: {
py2: {
enabled: true,
},
py3: {
enabled: false,
},
},
}
python_binary_host {
name: "manifest_fixer",
main: "manifest_fixer.py",

View File

@ -0,0 +1,89 @@
#!/usr/bin/env python
"""
Check boot jars.
Usage: check_boot_jars.py <package_allow_list_file> <jar1> <jar2> ...
"""
import logging
import os.path
import re
import subprocess
import sys
# The compiled allow list RE.
allow_list_re = None
def LoadAllowList(filename):
""" Load and compile allow list regular expressions from filename.
"""
lines = []
with open(filename, 'r') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
lines.append(line)
combined_re = r'^(%s)$' % '|'.join(lines)
global allow_list_re
try:
allow_list_re = re.compile(combined_re)
except re.error:
logging.exception(
'Cannot compile package allow list regular expression: %r',
combined_re)
allow_list_re = None
return False
return True
def CheckJar(allow_list_path, jar):
"""Check a jar file.
"""
# Get the list of files inside the jar file.
p = subprocess.Popen(args='jar tf %s' % jar,
stdout=subprocess.PIPE, shell=True)
stdout, _ = p.communicate()
if p.returncode != 0:
return False
items = stdout.split()
classes = 0
for f in items:
if f.endswith('.class'):
classes += 1
package_name = os.path.dirname(f)
package_name = package_name.replace('/', '.')
if not package_name or not allow_list_re.match(package_name):
print >> sys.stderr, ('Error: %s contains class file %s, whose package name %s is empty or'
' not in the allow list %s of packages allowed on the bootclasspath.'
% (jar, f, package_name, allow_list_path))
return False
if classes == 0:
print >> sys.stderr, ('Error: %s does not contain any class files.' % jar)
return False
return True
def main(argv):
if len(argv) < 2:
print __doc__
return 1
allow_list_path = argv[0]
if not LoadAllowList(allow_list_path):
return 1
passed = True
for jar in argv[1:]:
if not CheckJar(allow_list_path, jar):
passed = False
if not passed:
return 1
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))

View File

@ -0,0 +1,248 @@
# Boot jar package name allowed list.
# Each line is interpreted as a regular expression.
###################################################
# core-libart.jar & core-oj.jar
java\.awt\.font
java\.beans
java\.io
java\.lang
java\.lang\.annotation
java\.lang\.invoke
java\.lang\.ref
java\.lang\.reflect
java\.math
java\.net
java\.nio
java\.nio\.file
java\.nio\.file\.spi
java\.nio\.file\.attribute
java\.nio\.channels
java\.nio\.channels\.spi
java\.nio\.charset
java\.nio\.charset\.spi
java\.security
java\.security\.acl
java\.security\.cert
java\.security\.interfaces
java\.security\.spec
java\.sql
java\.text
java\.text\.spi
java\.time
java\.time\.chrono
java\.time\.format
java\.time\.temporal
java\.time\.zone
java\.util
java\.util\.concurrent
java\.util\.concurrent\.atomic
java\.util\.concurrent\.locks
java\.util\.function
java\.util\.jar
java\.util\.logging
java\.util\.prefs
java\.util\.regex
java\.util\.spi
java\.util\.stream
java\.util\.zip
# TODO: Remove javax.annotation.processing if possible, see http://b/132338110:
javax\.annotation\.processing
javax\.crypto
javax\.crypto\.interfaces
javax\.crypto\.spec
javax\.net
javax\.net\.ssl
javax\.security\.auth
javax\.security\.auth\.callback
javax\.security\.auth\.login
javax\.security\.auth\.x500
javax\.security\.cert
javax\.sql
javax\.xml
javax\.xml\.datatype
javax\.xml\.namespace
javax\.xml\.parsers
javax\.xml\.transform
javax\.xml\.transform\.dom
javax\.xml\.transform\.sax
javax\.xml\.transform\.stream
javax\.xml\.validation
javax\.xml\.xpath
jdk\.internal\.util
jdk\.internal\.vm\.annotation
jdk\.net
org\.w3c\.dom
org\.w3c\.dom\.ls
org\.w3c\.dom\.traversal
# OpenJdk internal implementation.
sun\.invoke\.util
sun\.invoke\.empty
sun\.misc
sun\.util.*
sun\.text.*
sun\.security.*
sun\.reflect.*
sun\.nio.*
sun\.net.*
com\.sun\..*
# TODO: Move these internal org.apache.harmony classes to libcore.*
org\.apache\.harmony\.crypto\.internal
org\.apache\.harmony\.dalvik
org\.apache\.harmony\.dalvik\.ddmc
org\.apache\.harmony\.luni\.internal\.util
org\.apache\.harmony\.security
org\.apache\.harmony\.security\.asn1
org\.apache\.harmony\.security\.fortress
org\.apache\.harmony\.security\.pkcs10
org\.apache\.harmony\.security\.pkcs7
org\.apache\.harmony\.security\.pkcs8
org\.apache\.harmony\.security\.provider\.crypto
org\.apache\.harmony\.security\.utils
org\.apache\.harmony\.security\.x501
org\.apache\.harmony\.security\.x509
org\.apache\.harmony\.security\.x509\.tsp
org\.apache\.harmony\.xml
org\.apache\.harmony\.xml\.dom
org\.apache\.harmony\.xml\.parsers
org\.json
org\.xmlpull\.v1
org\.xmlpull\.v1\.sax2
# TODO: jarjar org.kxml2.io to com.android org\.kxml2\.io
org\.kxml2\.io
org\.xml
org\.xml\.sax
org\.xml\.sax\.ext
org\.xml\.sax\.helpers
dalvik\..*
libcore\..*
android\..*
com\.android\..*
###################################################
# android.test.base.jar
junit\.extensions
junit\.framework
android\.test
android\.test\.suitebuilder\.annotation
###################################################
# ext.jar
# TODO: jarjar javax.sip to com.android
javax\.sip
javax\.sip\.address
javax\.sip\.header
javax\.sip\.message
# TODO: jarjar org.apache.commons to com.android
org\.apache\.commons\.codec
org\.apache\.commons\.codec\.binary
org\.apache\.commons\.codec\.language
org\.apache\.commons\.codec\.net
org\.apache\.commons\.logging
org\.apache\.commons\.logging\.impl
org\.apache\.http
org\.apache\.http\.auth
org\.apache\.http\.auth\.params
org\.apache\.http\.client
org\.apache\.http\.client\.entity
org\.apache\.http\.client\.methods
org\.apache\.http\.client\.params
org\.apache\.http\.client\.protocol
org\.apache\.http\.client\.utils
org\.apache\.http\.conn
org\.apache\.http\.conn\.params
org\.apache\.http\.conn\.routing
org\.apache\.http\.conn\.scheme
org\.apache\.http\.conn\.ssl
org\.apache\.http\.conn\.util
org\.apache\.http\.cookie
org\.apache\.http\.cookie\.params
org\.apache\.http\.entity
org\.apache\.http\.impl
org\.apache\.http\.impl\.auth
org\.apache\.http\.impl\.client
org\.apache\.http\.impl\.client
org\.apache\.http\.impl\.conn
org\.apache\.http\.impl\.conn\.tsccm
org\.apache\.http\.impl\.cookie
org\.apache\.http\.impl\.entity
org\.apache\.http\.impl\.io
org\.apache\.http\.impl\.io
org\.apache\.http\.io
org\.apache\.http\.message
org\.apache\.http\.params
org\.apache\.http\.protocol
org\.apache\.http\.util
# TODO: jarjar gov.nist to com.android
gov\.nist\.core
gov\.nist\.core\.net
gov\.nist\.javax\.sip
gov\.nist\.javax\.sip\.address
gov\.nist\.javax\.sip\.clientauthutils
gov\.nist\.javax\.sip\.header
gov\.nist\.javax\.sip\.header\.extensions
gov\.nist\.javax\.sip\.header\.ims
gov\.nist\.javax\.sip\.message
gov\.nist\.javax\.sip\.parser
gov\.nist\.javax\.sip\.parser\.extensions
gov\.nist\.javax\.sip\.parser\.ims
gov\.nist\.javax\.sip\.stack
org\.ccil\.cowan\.tagsoup
org\.ccil\.cowan\.tagsoup\.jaxp
###################################################
# framework.jar
javax\.microedition\.khronos\.opengles
javax\.microedition\.khronos\.egl
android
###################################################
# apache-xml.jar
org\.apache\.xml\.res
org\.apache\.xml\.utils
org\.apache\.xml\.utils\.res
org\.apache\.xml\.dtm
org\.apache\.xml\.dtm\.ref
org\.apache\.xml\.dtm\.ref\.dom2dtm
org\.apache\.xml\.dtm\.ref\.sax2dtm
org\.apache\.xml\.serializer
org\.apache\.xml\.serializer\.utils
org\.apache\.xml\.serializer\.dom3
org\.apache\.xpath
org\.apache\.xpath\.operations
org\.apache\.xpath\.domapi
org\.apache\.xpath\.functions
org\.apache\.xpath\.res
org\.apache\.xpath\.axes
org\.apache\.xpath\.objects
org\.apache\.xpath\.patterns
org\.apache\.xpath\.jaxp
org\.apache\.xpath\.compiler
org\.apache\.xalan
org\.apache\.xalan\.res
org\.apache\.xalan\.templates
org\.apache\.xalan\.serialize
org\.apache\.xalan\.extensions
org\.apache\.xalan\.processor
org\.apache\.xalan\.transformer
org\.apache\.xalan\.xslt
###################################################
# Packages in the google namespace across all bootclasspath jars.
com\.google\.android\..*
com\.google\.vr\.platform.*
com\.google\.i18n\.phonenumbers\..*
com\.google\.i18n\.phonenumbers
###################################################
# Packages used for Android in Chrome OS
org\.chromium\.arc
org\.chromium\.arc\..*