From 78c40be5606d13520af1e4b9c2f8dc78a6691511 Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Wed, 17 Oct 2018 16:50:49 -0700 Subject: [PATCH] Create a new kati packaging step; move dist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of looking at `dist` and DIST_DIR directly in the Kati Build step, always write out information about every call to dist, then create the rules in another ckati run. So instead of having: dist: -> -> \______________↑ nodist: -----------> Always use another phony target in the Kati Build step: ---> \----> _dist_ Then in the packaging step (which is much faster), choose between dist and no dist: dist: _dist_ -> -> nodist: _dist_ Bug: 117463001 Test: m dist Change-Id: Ic96bb6356740300dd3113f6ed699e6a619360c40 --- Changes.md | 26 +++++++++++++++++ CleanSpec.mk | 3 ++ core/config.mk | 21 ++++++++++---- core/distdir.mk | 66 ++++++++++++++++++++++---------------------- core/main.mk | 2 ++ core/ninja_config.mk | 1 - packaging/distdir.mk | 45 ++++++++++++++++++++++++++++++ packaging/main.mk | 37 +++++++++++++++++++++++++ 8 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 packaging/distdir.mk create mode 100644 packaging/main.mk diff --git a/Changes.md b/Changes.md index baa5e6e7a..4aa7ea2ab 100644 --- a/Changes.md +++ b/Changes.md @@ -1,5 +1,31 @@ # Build System Changes for Android.mk Writers +## `DIST_DIR`, `dist_goal`, and `dist-for-goals` {#dist} + +`DIST_DIR` and `dist_goal` are no longer available when reading Android.mk +files (or other build tasks). Always use `dist-for-goals` instead, which takes +a PHONY goal, and a list of files to copy to `$DIST_DIR`. Whenever `dist` is +specified, and the goal would be built (either explicitly on the command line, +or as a dependency of something on the command line), that file will be copied +into `$DIST_DIR`. For example, + +``` make +$(call dist-for-goals,foo,bar/baz) +``` + +will copy `bar/baz` into `$DIST_DIR/baz` when `m foo dist` is run. + +#### Renames during copy + +Instead of specifying just a file, a destination name can be specified, +including subdirectories: + +``` make +$(call dist-for-goals,foo,bar/baz:logs/foo.log) +``` + +will copy `bar/baz` into `$DIST_DIR/logs/foo.log` when `m foo dist` is run. + ## `.PHONY` rule enforcement {#phony_targets} There are several new warnings/errors meant to ensure the proper use of diff --git a/CleanSpec.mk b/CleanSpec.mk index a9093d224..39441e108 100644 --- a/CleanSpec.mk +++ b/CleanSpec.mk @@ -512,6 +512,9 @@ $(call add-clean-step, rm -rf $(addsuffix /lib,\ # Remove strip.sh intermediates to save space $(call add-clean-step, find $(OUT_DIR) \( -name "*.so.debug" -o -name "*.so.dynsyms" -o -name "*.so.funcsyms" -o -name "*.so.keep_symbols" -o -name "*.so.mini_debuginfo.xz" \) -print0 | xargs -0 rm -f) +# Clean up old ninja files +$(call add-clean-step, rm -f $(OUT_DIR)/build-*-dist*.ninja) + # ************************************************ # NEWER CLEAN STEPS MUST BE AT THE END OF THE LIST # ************************************************ diff --git a/core/config.mk b/core/config.mk index 5491234e6..fae53bbd2 100644 --- a/core/config.mk +++ b/core/config.mk @@ -24,7 +24,13 @@ include $(BUILD_SYSTEM_COMMON)/core.mk # Mark variables that should be coming as environment variables from soong_ui # as readonly -.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE DIST_DIR +.KATI_READONLY := OUT_DIR TMPDIR BUILD_DATETIME_FILE +ifdef CALLED_FROM_SETUP + .KATI_READONLY := CALLED_FROM_SETUP DIST_DIR +endif +ifdef KATI_PACKAGE_MK_DIR + .KATI_READONLY := KATI_PACKAGE_MK_DIR +endif # Mark variables deprecated/obsolete CHANGES_URL := https://android.googlesource.com/platform/build/+/master/Changes.md @@ -87,16 +93,21 @@ $(KATI_obsolete_var \ # This is marked as obsolete in envsetup.mk after reading the BoardConfig.mk $(KATI_deprecate_export It is a global setting. See $(CHANGES_URL)#export_keyword) -CHANGES_URL := - # Used to force goals to build. Only use for conditionally defined goals. .PHONY: FORCE FORCE: ORIGINAL_MAKECMDGOALS := $(MAKECMDGOALS) -dist_goal := $(strip $(filter dist,$(MAKECMDGOALS))) -MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS))) +ifdef CALLED_FROM_SETUP + dist_goal := $(strip $(filter dist,$(MAKECMDGOALS))) + MAKECMDGOALS := $(strip $(filter-out dist,$(MAKECMDGOALS))) + .KATI_READONLY := dist_goal +else + $(KATI_obsolete_var DIST_DIR dist_goal,Use dist-for-goals instead. See $(CHANGES_URL)#dist) +endif + +CHANGES_URL := UNAME := $(shell uname -sm) diff --git a/core/distdir.mk b/core/distdir.mk index c074186b8..a2eabd2b3 100644 --- a/core/distdir.mk +++ b/core/distdir.mk @@ -17,52 +17,52 @@ # When specifying "dist", the user has asked that we copy the important # files from this build into DIST_DIR. -ifdef dist_goal - -# $(1): source file -# $(2): destination file -# $(3): goals that should copy the file -# -define copy-one-dist-file -$(3): $(2) -$(2): $(1) - @echo "Dist: $$@" - $$(copy-file-to-new-target-with-cp) -endef - -# A global variable to remember all dist'ed src:dst pairs. -# So if a src:dst is already dist'ed by another goal, -# we should just establish the dependency and don't really call the -# copy-one-dist-file to avoid multiple rules for the same target. +# list of all goals that depend on any dist files +_all_dist_goals := +# pairs of goal:distfile +_all_dist_goal_output_pairs := +# pairs of srcfile:distfile _all_dist_src_dst_pairs := + # Other parts of the system should use this function to associate # certain files with certain goals. When those goals are built # and "dist" is specified, the marked files will be copied to DIST_DIR. # -# $(1): a list of goals (e.g. droid, sdk, pdk, ndk) +# $(1): a list of goals (e.g. droid, sdk, pdk, ndk). These must be PHONY # $(2): the dist files to add to those goals. If the file contains ':', # the text following the colon is the name that the file is copied # to under the dist directory. Subdirs are ok, and will be created # at copy time if necessary. define dist-for-goals +$(eval _all_dist_goals += $$(1)) \ $(foreach file,$(2), \ - $(eval fw := $(subst :,$(space),$(file))) \ - $(eval src := $(word 1,$(fw))) \ - $(eval dst := $(word 2,$(fw))) \ - $(eval dst := $(if $(dst),$(dst),$(notdir $(src)))) \ - $(if $(filter $(_all_dist_src_dst_pairs),$(src):$(dst)),\ - $(eval $(call add-dependency,$(1),$(DIST_DIR)/$(dst))),\ - $(eval $(call copy-one-dist-file,\ - $(src),$(DIST_DIR)/$(dst),$(1)))\ - $(eval _all_dist_src_dst_pairs += $(src):$(dst))\ - )\ -) + $(eval src := $(call word-colon,1,$(file))) \ + $(eval dst := $(call word-colon,2,$(file))) \ + $(if $(dst),,$(eval dst := $$(notdir $$(src)))) \ + $(eval _all_dist_src_dst_pairs += $$(src):$$(dst)) \ + $(foreach goal,$(1), \ + $(eval _all_dist_goal_output_pairs += $$(goal):$$(dst)))) endef -else # !dist_goal +#------------------------------------------------------------------ +# To be used at the end of the build to collect all the uses of +# dist-for-goals, and write them into a file for the packaging step to use. -# empty definition when not building dist -define dist-for-goals +# $(1): The file to write +define dist-write-file +$(strip \ + $(KATI_obsolete_var dist-for-goals,Cannot be used after dist-write-file) \ + $(foreach goal,$(sort $(_all_dist_goals)), \ + $(eval $$(goal): _dist_$$(goal))) \ + $(shell mkdir -p $(dir $(1))) \ + $(file >$(1).tmp, \ + DIST_GOAL_OUTPUT_PAIRS := $(sort $(_all_dist_goal_output_pairs)) \ + $(newline)DIST_SRC_DST_PAIRS := $(sort $(_all_dist_src_dst_pairs))) \ + $(shell if ! cmp -s $(1).tmp $(1); then \ + mv $(1).tmp $(1); \ + else \ + rm $(1).tmp; \ + fi)) endef -endif # !dist_goal +.KATI_READONLY := dist-for-goals dist-write-file diff --git a/core/main.mk b/core/main.mk index 967f52bbf..3c4dd9ab7 100644 --- a/core/main.mk +++ b/core/main.mk @@ -1490,6 +1490,8 @@ tidy_only: ndk: $(SOONG_OUT_DIR)/ndk.timestamp .PHONY: ndk +$(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk) + $(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] writing build rules ...) endif # KATI diff --git a/core/ninja_config.mk b/core/ninja_config.mk index ca2dceece..2d44d8fc6 100644 --- a/core/ninja_config.mk +++ b/core/ninja_config.mk @@ -28,7 +28,6 @@ PARSE_TIME_MAKE_GOALS := \ custom_images \ deps-license \ dicttool_aosp \ - dist \ dump-products \ eng \ fusion \ diff --git a/packaging/distdir.mk b/packaging/distdir.mk new file mode 100644 index 000000000..f1ef33631 --- /dev/null +++ b/packaging/distdir.mk @@ -0,0 +1,45 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# From the Android.mk pass: +DIST_GOAL_OUTPUT_PAIRS := +DIST_SRC_DST_PAIRS := +include $(KATI_PACKAGE_MK_DIR)/dist.mk + +$(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS), \ + $(eval goal := $(call word-colon,1,$(pair))) \ + $(eval output := $(call word-colon,2,$(pair))) \ + $(eval .PHONY: _dist_$$(goal)) \ + $(if $(call streq,$(DIST),true),\ + $(eval _dist_$$(goal): $$(DIST_DIR)/$$(output)))) + +define copy-one-dist-file +$(2): $(1) + @echo "Dist: $$@" + rm -f $$@ + cp $$< $$@ +endef + +ifeq ($(DIST),true) + $(foreach pair,$(DIST_SRC_DST_PAIRS), \ + $(eval src := $(call word-colon,1,$(pair))) \ + $(eval dst := $(DIST_DIR)/$(call word-colon,2,$(pair))) \ + $(eval $(call copy-one-dist-file,$(src),$(dst)))) +endif + +copy-one-dist-file := +DIST_GOAL_OUTPUT_PAIRS := +DIST_SRC_DST_PAIRS := diff --git a/packaging/main.mk b/packaging/main.mk new file mode 100644 index 000000000..0b746a8f8 --- /dev/null +++ b/packaging/main.mk @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Create a default rule. This is unused currently, as the real default rule is +# still in the Kati build step. +.PHONY: _packaging_default_rule_ +_packaging_default_rule_: + +ifndef KATI +$(error Only Kati is supported.) +endif + +$(info [1/3] initializing packaging system ...) + +.KATI_READONLY := KATI_PACKAGE_MK_DIR + +include build/make/common/core.mk +include build/make/common/strings.mk + +$(info [2/3] including distdir.mk ...) + +include build/make/packaging/distdir.mk + +$(info [3/3] writing packaging rules ...)