openkylin
/
platform_build
mirror of https://gitee.com/openkylin/platform_build.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Support java compilation sharding for target side. 

Add "LOCAL_JAVAC_SHARD_SIZE" to represent the number of java
source path entries in each shard.

Sharding is not allowed when "LOCAL_JAVAC_SHARD_SIZE" and
"LOCAL_JAR_PROCESSORS" are both enabled.

Limitation:
1. 0 <= LOCAL_JAVAC_SHARD_SIZE <= 8192
1. 0 < NUM(java_sources) <= 8192
2. 0 <= NUM(shards) <= 100

Performance Compare:
<Unsharded Build------------------------------------------------>
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar

real	1m2.720s user	5m26.604s sys	0m39.552s

2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar

real	0m37.586s user	5m47.804s sys	0m50.388s

<Sharded Build-------------------------------------------------->
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar

Javac Shard Size: 50----real	1m10.163s user	25m59.008s sys	1m58.460s
Javac Shard Size: 100---real	1m2.115s user	21m3.600s sys	1m15.964s
Javac Shard Size: 150---real	0m59.520s user	18m10.544s sys	1m12.628s
Javac Shard Size: 200---real	0m56.894s user	15m39.244s sys	1m11.608s
Javac Shard Size: 250---real	0m55.991s user	14m38.716s sys	1m2.292s
Javac Shard Size: 300---real	0m55.114s user	13m6.568s sys	1m8.200s
Javac Shard Size: 350---real	0m53.144s user	12m7.740s sys	1m3.836s
Javac Shard Size: 400---real	0m54.929s user	12m9.324s sys	1m4.340s
Javac Shard Size: 450---real	1m30.194s user	25m31.468s sys	1m52.416s
Javac Shard Size: 500---real	0m53.976s user	10m35.500s sys	0m55.160s

2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar

Javac Shard Size: 50-----real	0m16.322s user	1m8.648s sys	0m31.700s
Javac Shard Size: 100----real	0m16.163s user	1m22.932s sys	0m29.440s
Javac Shard Size: 150----real	0m16.611s user	1m37.828s sys	0m21.168s
Javac Shard Size: 200----real	0m16.936s user	1m49.248s sys	0m28.636s
Javac Shard Size: 250----real	0m17.509s user	1m54.944s sys	0m32.768s
Javac Shard Size: 300----real	0m18.868s user	1m54.088s sys	0m28.824s
Javac Shard Size: 350----real	0m17.629s user	1m54.108s sys	0m31.056s
Javac Shard Size: 400----real	0m18.658s user	2m7.712s sys	0m30.636s
Javac Shard Size: 450----real	0m18.874s user	2m8.808s sys	0m33.540s
Javac Shard Size: 500----real	0m19.432s user	2m24.400s sys	0m30.368s

time m nothing:
real 0m5.799s user 0m7.236s sys 0m3.068s

Test: m clean && m -j checkbuild
Bug: b/67424047
Change-Id: Id0766d2b7de7c4546d29bbc7f8a0dd0e4b9ad45b
This commit is contained in:
Nan Zhang 2017-10-04 09:26:06 -07:00
parent f398cc78fe
commit ad818dcbea
6 changed files with 186 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/clear_vars.mk
View File

@ -120,6 +120,7 @@ LOCAL_JAR_PACKAGES:=
LOCAL_JAR_PROCESSOR:=
LOCAL_JAR_PROCESSOR_ARGS:=
LOCAL_JAVACFLAGS:=
LOCAL_JAVAC_SHARD_SIZE:=
LOCAL_JAVA_LANGUAGE_VERSION:=
LOCAL_JAVA_LAYERS_FILE:=
LOCAL_JAVA_LIBRARIES:=

59
core/definitions.mk
View File

@ -2205,6 +2205,17 @@ define jar-args-sorted-files-in-directory
@<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .")
endef
# append additional Java sources(resources/Proto sources, and etc) to $(1).
define fetch-additional-java-source
$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \
fi
$(if $(PRIVATE_HAS_PROTO_SOURCES), \
$(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
$(if $(PRIVATE_HAS_RS_SOURCES), \
$(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
endef
# Some historical notes:
# - below we write the list of java files to java-source-list to avoid argument
# list length problems with Cygwin
@ -2214,16 +2225,20 @@ define write-java-source-list
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)"
$(hide) rm -f $@
$(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp)
$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \
fi
$(if $(PRIVATE_HAS_PROTO_SOURCES), \
$(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
$(if $(PRIVATE_HAS_RS_SOURCES), \
$(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
$(call fetch-additional-java-source,$@.tmp)
$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
endef
# $(1): sharding number.
# $(2): Java source files paths.
define save-sharded-java-source-list
$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH)
@echo "shard java source list: $$@"
rm -f $$@
$$(call dump-words-to-file,$(2),$$@.tmp)
$(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@
endef
# Common definition to invoke javac on the host and target.
#
# $(1): javac
@ -2265,9 +2280,33 @@ $(hide) $(JAR) -cf $@ $(call jar-args-sorted-files-in-directory,$(PRIVATE_CLASS_
$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))
endef
define transform-java-to-classes.jar
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))"
$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
# $(1): Javac output jar name.
# $(2): Java source list file.
# $(3): Java header libs.
# $(4): Javac sharding number.
# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#')
define create-classes-full-debug.jar
$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags)
$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES)
$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES)
$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES)
$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF)
$(1): PRIVATE_JAVA_SOURCE_LIST := $(2)
$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3)
$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4)
$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4)
$(1): \
$(2) \
$(3) \
$(5) \
$$(full_java_bootclasspath_libs) \
$$(layers_file) \
$$(annotation_processor_deps) \
$$(NORMALIZE_PATH) \
$$(JAR_ARGS) \
| $$(SOONG_JAVAC_WRAPPER)
@echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))"
$$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
endef
define transform-java-to-header.jar

89
core/java.mk
View File

@ -352,7 +352,36 @@ endif
##########################################
java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \
$(filter %.java,$(LOCAL_GENERATED_SOURCES))
all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
all_java_sources := $(java_sources) $(java_intermediate_sources)
enable_sharding :=
ifneq ($(TURBINE_ENABLED),false)
ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)
ifneq ($(LOCAL_JAR_PROCESSOR),)
$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)
endif # LOCAL_JAR_PROCESSOR is not empty
enable_sharding := true
num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))
ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))
# increment number of shards by 1.
num_shards := $(call int_plus,$(num_shards),1)
endif
shard_idx_list := $(call int_range_list,1,$(num_shards))
sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))
sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))
# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
# increment number of shards by 1.
num_shards := $(call int_plus,$(num_shards),1)
sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)
sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)
LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)
LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)
endif # LOCAL_JAVAC_SHARD_SIZE is not empty
endif # TURBINE_ENABLED != false
include $(BUILD_SYSTEM)/java_common.mk
@ -419,23 +448,47 @@ java_sources_deps := \
$(java_source_list_file): $(java_sources_deps)
$(write-java-source-list)
$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)
$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)
$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)
$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)
$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)
$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)
$(full_classes_compiled_jar): \
$(java_source_list_file) \
$(java_sources_deps) \
$(full_java_header_libs) \
$(full_java_bootclasspath_libs) \
$(layers_file) \
$(annotation_processor_deps) \
$(NORMALIZE_PATH) \
$(JAR_ARGS) \
| $(SOONG_JAVAC_WRAPPER)
$(transform-java-to-classes.jar)
ifdef enable_sharding
$(foreach x,$(shard_idx_list),\
$(eval $(call save-sharded-java-source-list,$(x),\
$(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)
$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \
$(RenderScript_file_stamp) \
$(proto_java_sources_file_stamp) \
$(LOCAL_ADDITIONAL_DEPENDENCIES) \
$(NORMALIZE_PATH)
$(hide) rm -f $@
$(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)
$(call fetch-additional-java-source,$@.tmp)
$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
# Javac sharding with header libs including its own header jar as one of dependency.
$(foreach x,$(shard_idx_list),\
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\
$(java_source_list_file).shard.$(x),\
$(full_java_header_libs) $(full_classes_header_jar),$(x),\
$(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
# Javac sharding for last shard with additional Java dependencies.
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\
$(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \
$(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \
$$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))
$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)
$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)
$(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)
else
# we can't use single $ for java_sources_deps since it may contain hash '#' sign.
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\
$(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))
endif # ifdef enable_sharding
ifneq ($(TURBINE_ENABLED),false)

62
core/math.mk
View File

@ -42,6 +42,11 @@ define _math_check_valid
$(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1))))
endef
# return a list containing integers ranging from [$(1),$(2)]
define int_range_list
$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS))
endef
#$(call _math_check_valid,0)
#$(call _math_check_valid,1)
#$(call _math_check_valid,100)
@ -75,3 +80,60 @@ endef
define inc_and_print
$(strip $(eval $(1) := $($(1)) .)$(words $($(1))))
endef
_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\
$(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x)))
define _int_encode
$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\
$(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\
$(wordlist 1,$(1),$(_INT_LIMIT_WORDS)))
endef
# _int_max returns the maximum of the two arguments
# input: two (x) lists; output: one (x) list
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_max
$(subst xx,x,$(join $(1),$(2)))
endef
# first argument is greater than second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_greater-than
$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2))))
endef
# first argument equals to second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_equal
$(filter $(words $(1)),$(words $(2)))
endef
# first argument is greater than or equal to second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_greater-or-equal
$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2))
endef
define int_plus
$(words $(call _int_encode,$(1)) $(call _int_encode,$(2)))
endef
define int_subtract
$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\
$(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\
$(call pretty-error,$(1) subtract underflow $(2)))
endef
define int_multiply
$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2))))
endef
define int_divide
$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \
$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \
$(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0)))
endef

2
core/package_internal.mk
View File

@ -333,6 +333,7 @@ $(data_binding_stamp) : $(all_res_assets) $(full_android_manifest) \
# Make sure the data-binding process happens before javac and generation of R.java.
$(R_file_stamp): $(data_binding_stamp)
$(java_source_list_file): $(data_binding_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp)))
$(full_classes_compiled_jar): $(data_binding_stamp)
endif # LOCAL_DATA_BINDING
@ -426,6 +427,7 @@ $(LOCAL_BUILT_MODULE): $(R_file_stamp)
# The R.java file must exist by the time the java source
# list is generated
$(java_source_list_file): $(R_file_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
endif # need_compile_res

1
core/static_java_library.mk
View File

@ -168,6 +168,7 @@ endif # LOCAL_USE_AAPT2
$(LOCAL_BUILT_MODULE): $(R_file_stamp)
$(java_source_list_file): $(R_file_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
$(full_classes_compiled_jar): $(R_file_stamp)
$(full_classes_turbine_jar): $(R_file_stamp)