diff --git a/core/clear_vars.mk b/core/clear_vars.mk index c38059c15..b5bb963b6 100644 --- a/core/clear_vars.mk +++ b/core/clear_vars.mk @@ -120,6 +120,7 @@ LOCAL_JAR_PACKAGES:= LOCAL_JAR_PROCESSOR:= LOCAL_JAR_PROCESSOR_ARGS:= LOCAL_JAVACFLAGS:= +LOCAL_JAVAC_SHARD_SIZE:= LOCAL_JAVA_LANGUAGE_VERSION:= LOCAL_JAVA_LAYERS_FILE:= LOCAL_JAVA_LIBRARIES:= diff --git a/core/definitions.mk b/core/definitions.mk index cb0a6bb22..1f9891ee4 100644 --- a/core/definitions.mk +++ b/core/definitions.mk @@ -2205,6 +2205,17 @@ define jar-args-sorted-files-in-directory @<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .") endef +# append additional Java sources(resources/Proto sources, and etc) to $(1). +define fetch-additional-java-source +$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \ + find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \ +fi +$(if $(PRIVATE_HAS_PROTO_SOURCES), \ + $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1)) +$(if $(PRIVATE_HAS_RS_SOURCES), \ + $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1)) +endef + # Some historical notes: # - below we write the list of java files to java-source-list to avoid argument # list length problems with Cygwin @@ -2214,16 +2225,20 @@ define write-java-source-list @echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)" $(hide) rm -f $@ $(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp) -$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \ - find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \ -fi -$(if $(PRIVATE_HAS_PROTO_SOURCES), \ - $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp) -$(if $(PRIVATE_HAS_RS_SOURCES), \ - $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp) +$(call fetch-additional-java-source,$@.tmp) $(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@ endef +# $(1): sharding number. +# $(2): Java source files paths. +define save-sharded-java-source-list +$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH) + @echo "shard java source list: $$@" + rm -f $$@ + $$(call dump-words-to-file,$(2),$$@.tmp) + $(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@ +endef + # Common definition to invoke javac on the host and target. # # $(1): javac @@ -2265,9 +2280,33 @@ $(hide) $(JAR) -cf $@ $(call jar-args-sorted-files-in-directory,$(PRIVATE_CLASS_ $(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@)) endef -define transform-java-to-classes.jar -@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))" -$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES)) +# $(1): Javac output jar name. +# $(2): Java source list file. +# $(3): Java header libs. +# $(4): Javac sharding number. +# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#') +define create-classes-full-debug.jar +$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags) +$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES) +$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES) +$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES) +$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF) +$(1): PRIVATE_JAVA_SOURCE_LIST := $(2) +$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3) +$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4) +$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4) +$(1): \ + $(2) \ + $(3) \ + $(5) \ + $$(full_java_bootclasspath_libs) \ + $$(layers_file) \ + $$(annotation_processor_deps) \ + $$(NORMALIZE_PATH) \ + $$(JAR_ARGS) \ + | $$(SOONG_JAVAC_WRAPPER) + @echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))" + $$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES)) endef define transform-java-to-header.jar diff --git a/core/java.mk b/core/java.mk index 5eddb0e37..cf4999490 100644 --- a/core/java.mk +++ b/core/java.mk @@ -352,7 +352,36 @@ endif ########################################## java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \ $(filter %.java,$(LOCAL_GENERATED_SOURCES)) -all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES))) +java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES))) +all_java_sources := $(java_sources) $(java_intermediate_sources) + +enable_sharding := +ifneq ($(TURBINE_ENABLED),false) +ifneq ($(LOCAL_JAVAC_SHARD_SIZE),) +ifneq ($(LOCAL_JAR_PROCESSOR),) +$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!) +endif # LOCAL_JAR_PROCESSOR is not empty +enable_sharding := true + +num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE)) +ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards))) +# increment number of shards by 1. +num_shards := $(call int_plus,$(num_shards),1) +endif + +shard_idx_list := $(call int_range_list,1,$(num_shards)) +sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x)) +sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x)) + +# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard. +# increment number of shards by 1. +num_shards := $(call int_plus,$(num_shards),1) +sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards) +sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards) +LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files) +LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list) +endif # LOCAL_JAVAC_SHARD_SIZE is not empty +endif # TURBINE_ENABLED != false include $(BUILD_SYSTEM)/java_common.mk @@ -419,23 +448,47 @@ java_sources_deps := \ $(java_source_list_file): $(java_sources_deps) $(write-java-source-list) -$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags) -$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES) -$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES) -$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES) -$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF) -$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file) -$(full_classes_compiled_jar): \ - $(java_source_list_file) \ - $(java_sources_deps) \ - $(full_java_header_libs) \ - $(full_java_bootclasspath_libs) \ - $(layers_file) \ - $(annotation_processor_deps) \ - $(NORMALIZE_PATH) \ - $(JAR_ARGS) \ - | $(SOONG_JAVAC_WRAPPER) - $(transform-java-to-classes.jar) +ifdef enable_sharding +$(foreach x,$(shard_idx_list),\ + $(eval $(call save-sharded-java-source-list,$(x),\ + $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\ + $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources)))))) + +# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard. +$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources) +$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \ + $(RenderScript_file_stamp) \ + $(proto_java_sources_file_stamp) \ + $(LOCAL_ADDITIONAL_DEPENDENCIES) \ + $(NORMALIZE_PATH) + $(hide) rm -f $@ + $(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp) + $(call fetch-additional-java-source,$@.tmp) + $(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@ + +# Javac sharding with header libs including its own header jar as one of dependency. +$(foreach x,$(shard_idx_list),\ + $(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\ + $(java_source_list_file).shard.$(x),\ + $(full_java_header_libs) $(full_classes_header_jar),$(x),\ + $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\ + $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources)))))) + +# Javac sharding for last shard with additional Java dependencies. +$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\ + $(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \ + $(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \ + $$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES))) + +$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list) +$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS) + $(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST) +else +# we can't use single $ for java_sources_deps since it may contain hash '#' sign. +$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\ + $(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps))) + +endif # ifdef enable_sharding ifneq ($(TURBINE_ENABLED),false) diff --git a/core/math.mk b/core/math.mk index 047d046b4..57cb68181 100644 --- a/core/math.mk +++ b/core/math.mk @@ -42,6 +42,11 @@ define _math_check_valid $(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1)))) endef +# return a list containing integers ranging from [$(1),$(2)] +define int_range_list +$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS)) +endef + #$(call _math_check_valid,0) #$(call _math_check_valid,1) #$(call _math_check_valid,100) @@ -75,3 +80,60 @@ endef define inc_and_print $(strip $(eval $(1) := $($(1)) .)$(words $($(1)))) endef + +_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\ + $(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x))) + +define _int_encode +$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\ + $(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\ + $(wordlist 1,$(1),$(_INT_LIMIT_WORDS))) +endef + +# _int_max returns the maximum of the two arguments +# input: two (x) lists; output: one (x) list +# integer cannot be passed in directly. It has to be converted using _int_encode. +define _int_max +$(subst xx,x,$(join $(1),$(2))) +endef + +# first argument is greater than second argument +# output: non-empty if true +# integer cannot be passed in directly. It has to be converted using _int_encode. +define _int_greater-than +$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2)))) +endef + +# first argument equals to second argument +# output: non-empty if true +# integer cannot be passed in directly. It has to be converted using _int_encode. +define _int_equal +$(filter $(words $(1)),$(words $(2))) +endef + +# first argument is greater than or equal to second argument +# output: non-empty if true +# integer cannot be passed in directly. It has to be converted using _int_encode. +define _int_greater-or-equal +$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2)) +endef + +define int_plus +$(words $(call _int_encode,$(1)) $(call _int_encode,$(2))) +endef + +define int_subtract +$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\ + $(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\ + $(call pretty-error,$(1) subtract underflow $(2))) +endef + +define int_multiply +$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2)))) +endef + +define int_divide +$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \ + $(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \ + $(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0))) +endef diff --git a/core/package_internal.mk b/core/package_internal.mk index 5a1ef051b..01e246366 100644 --- a/core/package_internal.mk +++ b/core/package_internal.mk @@ -333,6 +333,7 @@ $(data_binding_stamp) : $(all_res_assets) $(full_android_manifest) \ # Make sure the data-binding process happens before javac and generation of R.java. $(R_file_stamp): $(data_binding_stamp) $(java_source_list_file): $(data_binding_stamp) +$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp))) $(full_classes_compiled_jar): $(data_binding_stamp) endif # LOCAL_DATA_BINDING @@ -426,6 +427,7 @@ $(LOCAL_BUILT_MODULE): $(R_file_stamp) # The R.java file must exist by the time the java source # list is generated $(java_source_list_file): $(R_file_stamp) +$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp))) endif # need_compile_res diff --git a/core/static_java_library.mk b/core/static_java_library.mk index 8952df34d..69cf95595 100644 --- a/core/static_java_library.mk +++ b/core/static_java_library.mk @@ -168,6 +168,7 @@ endif # LOCAL_USE_AAPT2 $(LOCAL_BUILT_MODULE): $(R_file_stamp) $(java_source_list_file): $(R_file_stamp) +$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp))) $(full_classes_compiled_jar): $(R_file_stamp) $(full_classes_turbine_jar): $(R_file_stamp)