From d518e1a4077cbe89c1903e181b6fc02aa2c268a8 Mon Sep 17 00:00:00 2001 From: "Lukacs T. Berki" Date: Wed, 14 Apr 2021 13:49:50 +0200 Subject: [PATCH] Make bp2build be more correct. It now handles adding .bp files and changing globs. In order to do this, depfiles are now written separately from RunBlueprint. This is necessary due to the confluence of a number of seemingly unrelated factors: 1. The glob filelist dependencies are discovered in globSingleton 2. Singletons need to be registered because otherwise singleton module types panic 3. Singletons don't work because they require mutators bp2build does not run Due to (1), we would need to run the glob singleton. However, due to (2) and (3), we can't run singletons and have to run Blueprint with StopBeforeGeneratingBuildActions, which is when the build actions writing glob files would be generated. So what happens is: 1. When bp2build is run, the glob singleton is disabled 2. At the end of bp2build, the list of glob files is artifically added to the depfile of the workspace marker file 3. When build.ninja is generated, the Ninja file containing the glob list file is written by the now-active glob singleton Test: Presubmits. Change-Id: I3c5898d8c57c554a93520276c64a952afc912dbe --- android/bazel_handler.go | 2 +- cmd/soong_build/main.go | 70 +++++++++++++++++++++++++++++----------- tests/bootstrap_test.sh | 59 ++++++++++++++++++++++++++++++++- ui/build/soong.go | 9 ++++-- 4 files changed, 117 insertions(+), 23 deletions(-) diff --git a/android/bazel_handler.go b/android/bazel_handler.go index 2697007a5..8d561d277 100644 --- a/android/bazel_handler.go +++ b/android/bazel_handler.go @@ -737,7 +737,7 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) { // Add ninja file dependencies for files which all bazel invocations require. bazelBuildList := absolutePath(filepath.Join( - filepath.Dir(bootstrap.CmdlineModuleListFile()), "bazel.list")) + filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list")) ctx.AddNinjaFileDeps(bazelBuildList) data, err := ioutil.ReadFile(bazelBuildList) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 7a4cb29f9..1e796ecca 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -26,6 +26,7 @@ import ( "android/soong/bp2build" "android/soong/shared" "github.com/google/blueprint/bootstrap" + "github.com/google/blueprint/deptools" "android/soong/android" ) @@ -89,7 +90,7 @@ func newContext(configuration android.Config, prepareBuildActions bool) *android } func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Config { - configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineModuleListFile(), availableEnv) + configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineArgs.ModuleListFile, availableEnv) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -103,21 +104,31 @@ func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Co // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite // the incorrect results from the first pass, and file I/O is expensive. func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { + var firstArgs, secondArgs bootstrap.Args + + firstArgs = bootstrap.CmdlineArgs configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja) - bootstrap.Main(firstCtx.Context, configuration, false, extraNinjaDeps...) + bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration, extraNinjaDeps...) + // Invoke bazel commands and save results for second pass. if err := configuration.BazelContext.InvokeBazel(); err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } // Second pass: Full analysis, using the bazel command results. Output ninja file. - secondPassConfig, err := android.ConfigForAdditionalRun(configuration) + secondConfig, err := android.ConfigForAdditionalRun(configuration) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } - secondCtx := newContext(secondPassConfig, true) - bootstrap.Main(secondCtx.Context, secondPassConfig, false, extraNinjaDeps...) + secondCtx := newContext(secondConfig, true) + secondArgs = bootstrap.CmdlineArgs + ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig, extraNinjaDeps...) + err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err) + os.Exit(1) + } } // Run the code-generation phase to convert BazelTargetModules to BUILD files. @@ -132,7 +143,8 @@ func runQueryView(configuration android.Config, ctx *android.Context) { func runSoongDocs(configuration android.Config, extraNinjaDeps []string) { ctx := newContext(configuration, false) - bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...) + soongDocsArgs := bootstrap.CmdlineArgs + bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration, extraNinjaDeps...) if err := writeDocs(ctx, configuration, docFile); err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -166,8 +178,8 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str generateQueryView := bazelQueryViewDir != "" jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH") + blueprintArgs := bootstrap.CmdlineArgs prepareBuildActions := !generateQueryView && jsonModuleFile == "" - if bazelConversionRequested { // Run the alternate pipeline of bp2build mutators and singleton to convert // Blueprint to BUILD files before everything else. @@ -175,7 +187,7 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str if bp2buildMarker != "" { return bp2buildMarker } else { - return bootstrap.CmdlineOutFile() + return bootstrap.CmdlineArgs.OutFile } } @@ -183,22 +195,27 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str if mixedModeBuild { runMixedModeBuild(configuration, ctx, extraNinjaDeps) } else { - bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...) + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration, extraNinjaDeps...) + err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err) + os.Exit(1) + } } // Convert the Soong module graph into Bazel BUILD files. if generateQueryView { runQueryView(configuration, ctx) - return bootstrap.CmdlineOutFile() // TODO: This is a lie + return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie } if jsonModuleFile != "" { writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps) - return bootstrap.CmdlineOutFile() // TODO: This is a lie + return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie } writeMetrics(configuration) - return bootstrap.CmdlineOutFile() + return bootstrap.CmdlineArgs.OutFile } // soong_ui dumps the available environment variables to @@ -242,7 +259,7 @@ func main() { configuration := newConfig(srcDir, outDir, availableEnv) extraNinjaDeps := []string{ configuration.ProductVariablesFileName, - shared.JoinPath(outDir, "soong.environment.used"), + usedEnvFile, } if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { @@ -344,18 +361,18 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { // Register an alternate set of singletons and mutators for bazel // conversion for Bazel conversion. bp2buildCtx := android.NewContext(configuration) - bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) - bp2buildCtx.RegisterForBazelConversion() - // No need to generate Ninja build rules/statements from Modules and Singletons. - configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) + // Propagate "allow misssing dependencies" bit. This is normally set in + // newContext(), but we create bp2buildCtx without calling that method. + bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) bp2buildCtx.SetNameInterface(newNameResolver(configuration)) + bp2buildCtx.RegisterForBazelConversion() // The bp2build process is a purely functional process that only depends on // Android.bp files. It must not depend on the values of per-build product // configurations or variables, since those will generate different BUILD // files based on how the user has configured their tree. - bp2buildCtx.SetModuleListFile(bootstrap.CmdlineModuleListFile()) + bp2buildCtx.SetModuleListFile(bootstrap.CmdlineArgs.ModuleListFile) modulePaths, err := bp2buildCtx.ListModulePaths(configuration.SrcDir()) if err != nil { panic(err) @@ -363,10 +380,25 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { extraNinjaDeps = append(extraNinjaDeps, modulePaths...) + // No need to generate Ninja build rules/statements from Modules and Singletons. + configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) + // Run the loading and analysis pipeline to prepare the graph of regular // Modules parsed from Android.bp files, and the BazelTargetModules mapped // from the regular Modules. - bootstrap.Main(bp2buildCtx.Context, configuration, false, extraNinjaDeps...) + blueprintArgs := bootstrap.CmdlineArgs + ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration, extraNinjaDeps...) + + for _, globPath := range bp2buildCtx.Globs() { + ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir())) + } + + depFile := bp2buildMarker + ".d" + err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err) + os.Exit(1) + } // Run the code-generation phase to convert BazelTargetModules to BUILD files // and print conversion metrics to the user. diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index f85af1aa2..f1ebca296 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -429,12 +429,47 @@ function test_integrated_bp2build_smoke { setup INTEGRATED_BP2BUILD=1 run_soong if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then - fail "b2build marker file not created" + fail "bp2build marker file not created" + fi +} + +function test_integrated_bp2build_add_android_bp { + setup + + mkdir -p a + touch a/a.txt + cat > a/Android.bp <<'EOF' +filegroup { + name: "a", + srcs: ["a.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + INTEGRATED_BP2BUILD=1 run_soong + if [[ ! -e out/soong/bp2build/a/BUILD ]]; then + fail "a/BUILD not created"; + fi + + mkdir -p b + touch b/b.txt + cat > b/Android.bp <<'EOF' +filegroup { + name: "b", + srcs: ["b.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + INTEGRATED_BP2BUILD=1 run_soong + if [[ ! -e out/soong/bp2build/b/BUILD ]]; then + fail "b/BUILD not created"; fi } function test_integrated_bp2build_null_build { setup + INTEGRATED_BP2BUILD=1 run_soong local mtime1=$(stat -c "%y" out/soong/build.ninja) @@ -446,6 +481,27 @@ function test_integrated_bp2build_null_build { fi } +function test_integrated_bp2build_add_to_glob { + setup + + mkdir -p a + touch a/a1.txt + cat > a/Android.bp <<'EOF' +filegroup { + name: "a", + srcs: ["*.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + INTEGRATED_BP2BUILD=1 run_soong + grep -q a1.txt out/soong/bp2build/a/BUILD || fail "a1.txt not in BUILD file" + + touch a/a2.txt + INTEGRATED_BP2BUILD=1 run_soong + grep -q a2.txt out/soong/bp2build/a/BUILD || fail "a2.txt not in BUILD file" +} + function test_dump_json_module_graph() { setup SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong @@ -468,3 +524,4 @@ test_soong_build_rerun_iff_environment_changes test_dump_json_module_graph test_integrated_bp2build_smoke test_integrated_bp2build_null_build +test_integrated_bp2build_add_to_glob diff --git a/ui/build/soong.go b/ui/build/soong.go index d77a089ab..7e94b2589 100644 --- a/ui/build/soong.go +++ b/ui/build/soong.go @@ -21,6 +21,7 @@ import ( "strconv" "android/soong/shared" + "github.com/google/blueprint/deptools" soong_metrics_proto "android/soong/ui/metrics/metrics_proto" "github.com/google/blueprint" @@ -107,6 +108,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja") globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja") bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja") + bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") args.RunGoTests = !config.skipSoongTests args.UseValidations = true // Use validations to depend on tests @@ -115,7 +117,6 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { args.TopFile = "Android.bp" args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list") args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja") - args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d") args.GlobFile = globFile args.GeneratingPrimaryBuilder = true @@ -171,7 +172,11 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) { debugCompilation: os.Getenv("SOONG_DELVE") != "", } - bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig) + bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig) + err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps) + if err != nil { + ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err) + } } func checkEnvironmentFile(currentEnv *Environment, envFile string) {