diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index a4554fc34..7a4cb29f9 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -23,29 +23,42 @@ import ( "strings" "time" + "android/soong/bp2build" "android/soong/shared" "github.com/google/blueprint/bootstrap" "android/soong/android" - "android/soong/bp2build" ) var ( - topDir string - outDir string + topDir string + outDir string + availableEnvFile string + usedEnvFile string + + delveListen string + delvePath string + docFile string bazelQueryViewDir string - delveListen string - delvePath string + bp2buildMarker string ) func init() { + // Flags that make sense in every mode flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)") + flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables") + flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables") + + // Debug flags flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging") flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set") + + // Flags representing various modes soong_build can run in flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") + flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") } func newNameResolver(config android.Config) *android.NameResolver { @@ -147,8 +160,8 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir()) } -func doChosenActivity(configuration android.Config, extraNinjaDeps []string) { - bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") +func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { + bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") || bp2buildMarker != "" mixedModeBuild := configuration.BazelContext.BazelEnabled() generateQueryView := bazelQueryViewDir != "" jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH") @@ -159,7 +172,11 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. runBp2Build(configuration, extraNinjaDeps) - return + if bp2buildMarker != "" { + return bp2buildMarker + } else { + return bootstrap.CmdlineOutFile() + } } ctx := newContext(configuration, prepareBuildActions) @@ -172,15 +189,44 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) { // Convert the Soong module graph into Bazel BUILD files. if generateQueryView { runQueryView(configuration, ctx) - return + return bootstrap.CmdlineOutFile() // TODO: This is a lie } if jsonModuleFile != "" { writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps) - return + return bootstrap.CmdlineOutFile() // TODO: This is a lie } writeMetrics(configuration) + return bootstrap.CmdlineOutFile() +} + +// soong_ui dumps the available environment variables to +// soong.environment.available . Then soong_build itself is run with an empty +// environment so that the only way environment variables can be accessed is +// using Config, which tracks access to them. + +// At the end of the build, a file called soong.environment.used is written +// containing the current value of all used environment variables. The next +// time soong_ui is run, it checks whether any environment variables that was +// used had changed and if so, it deletes soong.environment.used to cause a +// rebuild. +// +// The dependency of build.ninja on soong.environment.used is declared in +// build.ninja.d +func parseAvailableEnv() map[string]string { + if availableEnvFile == "" { + fmt.Fprintf(os.Stderr, "--available_env not set\n") + os.Exit(1) + } + + result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile)) + if err != nil { + fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err) + os.Exit(1) + } + + return result } func main() { @@ -189,26 +235,7 @@ func main() { shared.ReexecWithDelveMaybe(delveListen, delvePath) android.InitSandbox(topDir) - // soong_ui dumps the available environment variables to - // soong.environment.available . Then soong_build itself is run with an empty - // environment so that the only way environment variables can be accessed is - // using Config, which tracks access to them. - - // At the end of the build, a file called soong.environment.used is written - // containing the current value of all used environment variables. The next - // time soong_ui is run, it checks whether any environment variables that was - // used had changed and if so, it deletes soong.environment.used to cause a - // rebuild. - // - // The dependency of build.ninja on soong.environment.used is declared in - // build.ninja.d - availableEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.available") - usedEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.used") - availableEnv, err := shared.EnvFromFile(availableEnvFile) - if err != nil { - fmt.Fprintf(os.Stderr, "error reading available environment file %s: %s\n", availableEnvFile, err) - os.Exit(1) - } + availableEnv := parseAvailableEnv() // The top-level Blueprints file is passed as the first argument. srcDir := filepath.Dir(flag.Arg(0)) @@ -233,37 +260,37 @@ func main() { // because that is done from within the actual builds as a Ninja action and // thus it would overwrite the actual used variables file so this is // special-cased. + // TODO: Fix this by not passing --used_env to the soong_docs invocation runSoongDocs(configuration, extraNinjaDeps) return } - doChosenActivity(configuration, extraNinjaDeps) - writeUsedEnvironmentFile(usedEnvFile, configuration) + finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) + writeUsedEnvironmentFile(configuration, finalOutputFile) } -func writeUsedEnvironmentFile(path string, configuration android.Config) { +func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) { + if usedEnvFile == "" { + return + } + + path := shared.JoinPath(topDir, usedEnvFile) data, err := shared.EnvFileContents(configuration.EnvDeps()) if err != nil { - fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err) + fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) os.Exit(1) } err = ioutil.WriteFile(path, data, 0666) if err != nil { - fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err) + fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) os.Exit(1) } - // Touch the output Ninja file so that it's not older than the file we just + // Touch the output file so that it's not older than the file we just // wrote. We can't write the environment file earlier because one an access // new environment variables while writing it. - outputNinjaFile := shared.JoinPath(topDir, bootstrap.CmdlineOutFile()) - currentTime := time.Now().Local() - err = os.Chtimes(outputNinjaFile, currentTime, currentTime) - if err != nil { - fmt.Fprintf(os.Stderr, "error touching output file %s: %s\n", outputNinjaFile, err) - os.Exit(1) - } + touch(shared.JoinPath(topDir, finalOutputFile)) } // Workarounds to support running bp2build in a clean AOSP checkout with no @@ -289,6 +316,27 @@ func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) { 0666) } +func touch(path string) { + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) + os.Exit(1) + } + + err = f.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) + os.Exit(1) + } + + currentTime := time.Now().Local() + err = os.Chtimes(path, currentTime, currentTime) + if err != nil { + fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err) + os.Exit(1) + } +} + // Run Soong in the bp2build mode. This creates a standalone context that registers // an alternate pipeline of mutators and singletons specifically for generating // Bazel BUILD files instead of Ninja files. @@ -296,6 +344,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { // Register an alternate set of singletons and mutators for bazel // conversion for Bazel conversion. bp2buildCtx := android.NewContext(configuration) + bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) bp2buildCtx.RegisterForBazelConversion() // No need to generate Ninja build rules/statements from Modules and Singletons. @@ -330,5 +379,9 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { metrics.Print() extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...) - writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir()) + if bp2buildMarker != "" { + touch(shared.JoinPath(topDir, bp2buildMarker)) + } else { + writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir()) + } } diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 5271f8d9e..f85af1aa2 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -114,7 +114,9 @@ EOF rm a/Android.bp run_soong - grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output" + if grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja; then + fail "Old module in output" + fi } function test_add_file_to_glob() { @@ -404,7 +406,9 @@ EOF grep -q "Engage" out/soong/build.ninja || fail "New action not present" - grep -q "Make it so" out/soong/build.ninja && fail "Original action still present" + if grep -q "Make it so" out/soong/build.ninja; then + fail "Original action still present" + fi } function test_null_build_after_docs { @@ -421,6 +425,27 @@ function test_null_build_after_docs { fi } +function test_integrated_bp2build_smoke { + setup + INTEGRATED_BP2BUILD=1 run_soong + if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then + fail "b2build marker file not created" + fi +} + +function test_integrated_bp2build_null_build { + setup + INTEGRATED_BP2BUILD=1 run_soong + local mtime1=$(stat -c "%y" out/soong/build.ninja) + + INTEGRATED_BP2BUILD=1 run_soong + local mtime2=$(stat -c "%y" out/soong/build.ninja) + + if [[ "$mtime1" != "$mtime2" ]]; then + fail "Output Ninja file changed on null build" + fi +} + function test_dump_json_module_graph() { setup SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong @@ -441,3 +466,5 @@ test_add_file_to_soong_build test_glob_during_bootstrapping test_soong_build_rerun_iff_environment_changes test_dump_json_module_graph +test_integrated_bp2build_smoke +test_integrated_bp2build_null_build diff --git a/ui/build/soong.go b/ui/build/soong.go index 0089075a4..d77a089ab 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -33,6 +33,11 @@ import ( "android/soong/ui/status" ) +const ( + availableEnvFile = "soong.environment.available" + usedEnvFile = "soong.environment.used" +) + func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error { data, err := shared.EnvFileContents(envDeps) if err != nil { @@ -87,12 +92,22 @@ func (c BlueprintConfig) DebugCompilation() bool { return c.debugCompilation } -func bootstrapBlueprint(ctx Context, config Config) { +func environmentArgs(config Config, suffix string) []string { + return []string{ + "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile), + "--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix), + } +} +func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") defer ctx.EndTrace() var args bootstrap.Args + mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja") + globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja") + bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") + args.RunGoTests = !config.skipSoongTests args.UseValidations = true // Use validations to depend on tests args.BuildDir = config.SoongOutDir() @@ -101,7 +116,7 @@ func bootstrapBlueprint(ctx Context, config Config) { args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") - args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja") + args.GlobFile = globFile args.GeneratingPrimaryBuilder = true args.DelveListen = os.Getenv("SOONG_DELVE") @@ -109,6 +124,44 @@ func bootstrapBlueprint(ctx Context, config Config) { args.DelvePath = shared.ResolveDelveBinary() } + commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile) + bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker") + mainSoongBuildInputs := []string{"Android.bp"} + + if integratedBp2Build { + mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile) + } + + soongBuildArgs := make([]string, 0) + soongBuildArgs = append(soongBuildArgs, commonArgs...) + soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...) + soongBuildArgs = append(soongBuildArgs, "Android.bp") + + mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{ + Inputs: mainSoongBuildInputs, + Outputs: []string{mainNinjaFile}, + Args: soongBuildArgs, + } + + if integratedBp2Build { + bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile} + bp2buildArgs = append(bp2buildArgs, commonArgs...) + bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...) + bp2buildArgs = append(bp2buildArgs, "Android.bp") + + bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{ + Inputs: []string{"Android.bp"}, + Outputs: []string{bp2BuildMarkerFile}, + Args: bp2buildArgs, + } + args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{ + bp2buildInvocation, + mainSoongBuildInvocation, + } + } else { + args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation} + } + blueprintCtx := blueprint.NewContext() blueprintCtx.SetIgnoreUnknownModuleTypes(true) blueprintConfig := BlueprintConfig{ @@ -121,6 +174,16 @@ func bootstrapBlueprint(ctx Context, config Config) { bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig) } +func checkEnvironmentFile(currentEnv *Environment, envFile string) { + getenv := func(k string) string { + v, _ := currentEnv.Get(k) + return v + } + if stale, _ := shared.StaleEnvFile(envFile, getenv); stale { + os.Remove(envFile) + } +} + func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "soong") defer ctx.EndTrace() @@ -129,7 +192,7 @@ func runSoong(ctx Context, config Config) { // .used with the ones that were actually used. The latter is used to // determine whether Soong needs to be re-run since why re-run it if only // unused variables were changed? - envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available") + envFile := filepath.Join(config.SoongOutDir(), availableEnvFile) for _, n := range []string{".bootstrap", ".minibootstrap"} { dir := filepath.Join(config.SoongOutDir(), n) @@ -138,8 +201,10 @@ func runSoong(ctx Context, config Config) { } } + integratedBp2Build := config.Environment().IsEnvTrue("INTEGRATED_BP2BUILD") + // This is done unconditionally, but does not take a measurable amount of time - bootstrapBlueprint(ctx, config) + bootstrapBlueprint(ctx, config, integratedBp2Build) soongBuildEnv := config.Environment().Copy() soongBuildEnv.Set("TOP", os.Getenv("TOP")) @@ -164,13 +229,12 @@ func runSoong(ctx Context, config Config) { ctx.BeginTrace(metrics.RunSoong, "environment check") defer ctx.EndTrace() - envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used") - getenv := func(k string) string { - v, _ := soongBuildEnv.Get(k) - return v - } - if stale, _ := shared.StaleEnvFile(envFile, getenv); stale { - os.Remove(envFile) + soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile) + checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile) + + if integratedBp2Build { + bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build") + checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile) } }()