From 558cb6c5ac3dbe9942a78a28ebe7741ff01ca2a6 Mon Sep 17 00:00:00 2001 From: Ulya Trafimovich Date: Wed, 6 Jan 2021 17:42:14 +0000 Subject: [PATCH] Merge dependency configs into dexpreopt.config files. Since Make does not visit modules in topological order of their dependencies, information from dependencies to the dexpreopted module has to be passed via dexpreopt.config files. A build rule for a dexpreopt.config file depends on dexpreopt.config files for dependencies, and dex_preopt_config_merger.py script extracts the necessary information from dependency configs and patches the module's config. Bug: 132357300 Test: lunch aosp_cf_x86_phone-userdebug && m Change-Id: Id0b71170a4d2ab1d33059de0e9ad9d7e61f2345e --- core/clear_vars.mk | 1 + core/definitions.mk | 8 --- core/dex_preopt_config_merger.py | 100 +++++++++++++++++++++++++++++++ core/dex_preopt_odex_install.mk | 65 ++++++++++++++------ core/soong_java_prebuilt.mk | 7 +++ 5 files changed, 156 insertions(+), 25 deletions(-) create mode 100755 core/dex_preopt_config_merger.py diff --git a/core/clear_vars.mk b/core/clear_vars.mk index 5effac7f7..019892e5e 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -277,6 +277,7 @@ LOCAL_SOONG_BUILT_INSTALLED := LOCAL_SOONG_BUNDLE := LOCAL_SOONG_CLASSES_JAR := LOCAL_SOONG_DEX_JAR := +LOCAL_SOONG_DEXPREOPT_CONFIG := LOCAL_SOONG_EXPORT_PROGUARD_FLAGS := LOCAL_SOONG_HEADER_JAR := LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR := diff --git a/core/definitions.mk b/core/definitions.mk index 4300efe17..033ab30fd 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -631,14 +631,6 @@ $(strip \ ) endef -########################################################### -## Convert install path to on-device path. -########################################################### -# $(1): install path -define install-path-to-on-device-path -$(patsubst $(PRODUCT_OUT)%,%,$(1)) -endef - ########################################################### ## The intermediates directory. Where object files go for ## a given target. We could technically get away without diff --git a/core/dex_preopt_config_merger.py b/core/dex_preopt_config_merger.py new file mode 100755 index 000000000..ebb99e196 --- /dev/null +++ b/core/dex_preopt_config_merger.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +A tool for merging dexpreopt.config files for dependencies into +the dexpreopt.config file of the library/app that uses them. This is needed to +generate class loader context (CLC) for dexpreopt. + +In Make there is no topological order when processing different modules, so a + dependency module may have not been processed yet by the time the +dependent module is processed. Therefore makefiles communicate the information +from dependencies via dexpreopt.config files and add file-level dependencies +from a module dexpreopt.config to its dependency configs. The actual patching +of configs is done by this script, which is called from the makefiles. +""" + +from __future__ import print_function + +import json +from collections import OrderedDict +import sys + + +def main(): + """Program entry point.""" + if len(sys.argv) < 2: + raise SystemExit('usage: %s [dep-config ...]' % sys.argv[0]) + + # Read all JSON configs. + cfgs = [] + for arg in sys.argv[1:]: + with open(arg, 'r') as f: + cfgs.append(json.load(f, object_pairs_hook=OrderedDict)) + + # The first config is the dexpreopted library/app, the rest are its + # dependencies. + cfg0 = cfgs[0] + + # Put dependency configs in a map keyed on module name (for easier lookup). + uses_libs = {} + for cfg in cfgs[1:]: + uses_libs[cfg['Name']] = cfg + + # Load the original CLC map. + clc_map = cfg0['ClassLoaderContexts'] + + # Create a new CLC map that will be a copy of the original one with patched + # fields from dependency dexpreopt.config files. + clc_map2 = OrderedDict() + + # Patch CLC for each SDK version. Although this should not be necessary for + # compatibility libraries (so-called "conditional CLC"), because they all have + # known names, known paths in system/framework, and no subcontext. But keep + # the loop in case this changes in the future. + for sdk_ver in clc_map: + clcs = clc_map[sdk_ver] + clcs2 = OrderedDict() + for lib in clcs: + clc = clcs[lib] + if lib in uses_libs: + ulib = uses_libs[lib] + # On-host (build) path to the dependency DEX jar file. + clc['Host'] = ulib['BuildPath'] + # On-device (install) path to the dependency DEX jar file. + clc['Device'] = ulib['DexLocation'] + # CLC of the dependency becomes a subcontext. We only need sub-CLC for + # 'any' version because all other versions are for compatibility + # libraries, which exist only for apps and not for libraries. + clc['Subcontexts'] = ulib['ClassLoaderContexts'].get('any') + # Patch the library name in the CLC as well. + clcs2[ulib['ProvidesUsesLibrary']] = clc + else: + # dexpreopt.config for this is not among the script + # arguments, which may be the case with compatibility libraries that + # don't need patching anyway. Just use the original CLC. + clcs2[lib] = clc + clc_map2[sdk_ver] = clcs2 + + # Overwrite the original class loader context with the patched one. + cfg0['ClassLoaderContexts'] = clc_map2 + + # Update dexpreopt.config file. + with open(sys.argv[1], 'w') as f: + f.write(json.dumps(cfgs[0], indent=4, separators=(',', ': '))) + +if __name__ == '__main__': + main() diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk index b74e04746..add3c45bd 100644 --- a/core/dex_preopt_odex_install.mk +++ b/core/dex_preopt_odex_install.mk @@ -189,24 +189,28 @@ ifdef LOCAL_DEX_PREOPT my_filtered_optional_uses_libraries := $(filter-out $(INTERNAL_PLATFORM_MISSING_USES_LIBRARIES), \ $(LOCAL_OPTIONAL_USES_LIBRARIES)) - # compatibility libraries are added to class loader context of an app only if - # targetSdkVersion in the app's manifest is lower than the given SDK version + ifeq ($(LOCAL_MODULE_CLASS),APPS) + # compatibility libraries are added to class loader context of an app only if + # targetSdkVersion in the app's manifest is lower than the given SDK version - my_dexpreopt_libs_compat_28 := \ - org.apache.http.legacy + my_dexpreopt_libs_compat_28 := \ + org.apache.http.legacy - my_dexpreopt_libs_compat_29 := \ - android.hidl.base-V1.0-java \ - android.hidl.manager-V1.0-java + my_dexpreopt_libs_compat_29 := \ + android.hidl.base-V1.0-java \ + android.hidl.manager-V1.0-java - my_dexpreopt_libs_compat_30 := \ - android.test.base \ - android.test.mock + my_dexpreopt_libs_compat_30 := \ + android.test.base \ + android.test.mock - my_dexpreopt_libs_compat := \ - $(my_dexpreopt_libs_compat_28) \ - $(my_dexpreopt_libs_compat_29) \ - $(my_dexpreopt_libs_compat_30) + my_dexpreopt_libs_compat := \ + $(my_dexpreopt_libs_compat_28) \ + $(my_dexpreopt_libs_compat_29) \ + $(my_dexpreopt_libs_compat_30) + else + my_extra_dexpreopt_libs := + endif my_dexpreopt_libs := $(sort \ $(LOCAL_USES_LIBRARIES) \ @@ -215,13 +219,25 @@ ifdef LOCAL_DEX_PREOPT # 1: SDK version # 2: list of libraries + # + # Make does not process modules in topological order wrt. + # dependencies, therefore we cannot rely on variables to get the information + # about dependencies (in particular, their on-device path and class loader + # context). This information is communicated via dexpreopt.config files: each + # config depends on configs for dependencies of this module, + # and the dex_preopt_config_merger.py script reads all configs and inserts the + # missing bits from dependency configs into the module config. + # + # By default on-device path is /system/framework/*.jar, and class loader + # subcontext is empty. These values are correct for compatibility libraries, + # which are special and not handled by dex_preopt_config_merger.py. + # add_json_class_loader_context = \ $(call add_json_map, $(1)) \ $(foreach lib, $(2),\ $(call add_json_map, $(lib)) \ - $(eval file := $(filter %/$(lib).jar, $(call module-installed-files,$(lib)))) \ - $(call add_json_str, Host, $(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,COMMON)/javalib.jar) \ - $(call add_json_str, Device, $(call install-path-to-on-device-path,$(file))) \ + $(call add_json_str, Host, $(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,COMMON)/javalib.jar) \ + $(call add_json_str, Device, /system/framework/$(lib).jar) \ $(call add_json_map, Subcontexts, ${$}) $(call end_json_map) \ $(call end_json_map)) \ $(call end_json_map) @@ -275,12 +291,27 @@ ifdef LOCAL_DEX_PREOPT my_dexpreopt_config := $(intermediates)/dexpreopt.config my_dexpreopt_script := $(intermediates)/dexpreopt.sh my_dexpreopt_zip := $(intermediates)/dexpreopt.zip + my_dexpreopt_config_merger := $(BUILD_SYSTEM)/dex_preopt_config_merger.py + # Module dexpreopt.config depends on dexpreopt.config files of each + # dependency, because these libraries may be processed after + # the current module by Make (there's no topological order), so the dependency + # information (paths, class loader context) may not be ready yet by the time + # this dexpreopt.config is generated. So it's necessary to add file-level + # dependencies between dexpreopt.config files. + my_dexpreopt_dep_configs := $(foreach lib, \ + $(filter-out $(my_dexpreopt_libs_compat),$(LOCAL_USES_LIBRARIES) $(my_filtered_optional_uses_libraries)), \ + $(call intermediates-dir-for,JAVA_LIBRARIES,$(lib),,)/dexpreopt.config) + + $(my_dexpreopt_config): $(my_dexpreopt_dep_configs) $(my_dexpreopt_config_merger) $(my_dexpreopt_config): PRIVATE_MODULE := $(LOCAL_MODULE) $(my_dexpreopt_config): PRIVATE_CONTENTS := $(json_contents) + $(my_dexpreopt_config): PRIVATE_DEP_CONFIGS := $(my_dexpreopt_dep_configs) + $(my_dexpreopt_config): PRIVATE_CONFIG_MERGER := $(my_dexpreopt_config_merger) $(my_dexpreopt_config): @echo "$(PRIVATE_MODULE) dexpreopt.config" echo -e -n '$(subst $(newline),\n,$(subst ','\'',$(subst \,\\,$(PRIVATE_CONTENTS))))' > $@ + $(PRIVATE_CONFIG_MERGER) $@ $(PRIVATE_DEP_CONFIGS) .KATI_RESTAT: $(my_dexpreopt_script) $(my_dexpreopt_script): PRIVATE_MODULE := $(LOCAL_MODULE) diff --git a/core/soong_java_prebuilt.mk b/core/soong_java_prebuilt.mk index 5444d960c..c60017834 100644 --- a/core/soong_java_prebuilt.mk +++ b/core/soong_java_prebuilt.mk @@ -5,6 +5,7 @@ # LOCAL_SOONG_HEADER_JAR # LOCAL_SOONG_DEX_JAR # LOCAL_SOONG_JACOCO_REPORT_CLASSES_JAR +# LOCAL_SOONG_DEXPREOPT_CONFIG ifneq ($(LOCAL_MODULE_MAKEFILE),$(SOONG_ANDROID_MK)) $(call pretty-error,soong_java_prebuilt.mk may only be used from Soong) @@ -146,6 +147,12 @@ ifdef LOCAL_SOONG_AAR ALL_MODULES.$(my_register_name).AAR := $(LOCAL_SOONG_AAR) endif +# Copy dexpreopt.config files from Soong libraries to the location where Make +# modules can find them. +ifdef LOCAL_SOONG_DEXPREOPT_CONFIG + $(eval $(call copy-one-file,$(LOCAL_SOONG_DEXPREOPT_CONFIG), $(call local-intermediates-dir,)/dexpreopt.config)) +endif + javac-check : $(full_classes_jar) javac-check-$(LOCAL_MODULE) : $(full_classes_jar) .PHONY: javac-check-$(LOCAL_MODULE)