Make bp2build be more correct.
It now handles adding .bp files and changing globs. In order to do this, depfiles are now written separately from RunBlueprint. This is necessary due to the confluence of a number of seemingly unrelated factors: 1. The glob filelist dependencies are discovered in globSingleton 2. Singletons need to be registered because otherwise singleton module types panic 3. Singletons don't work because they require mutators bp2build does not run Due to (1), we would need to run the glob singleton. However, due to (2) and (3), we can't run singletons and have to run Blueprint with StopBeforeGeneratingBuildActions, which is when the build actions writing glob files would be generated. So what happens is: 1. When bp2build is run, the glob singleton is disabled 2. At the end of bp2build, the list of glob files is artifically added to the depfile of the workspace marker file 3. When build.ninja is generated, the Ninja file containing the glob list file is written by the now-active glob singleton Test: Presubmits. Change-Id: I3c5898d8c57c554a93520276c64a952afc912dbe
This commit is contained in:
Lukacs T. Berki
parent
53d69eb028
commit
d518e1a407
|
|
@ -737,7 +737,7 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
|
|||
|
||||
// Add ninja file dependencies for files which all bazel invocations require.
|
||||
bazelBuildList := absolutePath(filepath.Join(
|
||||
filepath.Dir(bootstrap.CmdlineModuleListFile()), "bazel.list"))
|
||||
filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list"))
|
||||
ctx.AddNinjaFileDeps(bazelBuildList)
|
||||
|
||||
data, err := ioutil.ReadFile(bazelBuildList)
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
"android/soong/bp2build"
|
||||
"android/soong/shared"
|
||||
"github.com/google/blueprint/bootstrap"
|
||||
"github.com/google/blueprint/deptools"
|
||||
|
||||
"android/soong/android"
|
||||
)
|
||||
|
|
@ -89,7 +90,7 @@ func newContext(configuration android.Config, prepareBuildActions bool) *android
|
|||
}
|
||||
|
||||
func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Config {
|
||||
configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineModuleListFile(), availableEnv)
|
||||
configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineArgs.ModuleListFile, availableEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -103,21 +104,31 @@ func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Co
|
|||
// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
|
||||
// the incorrect results from the first pass, and file I/O is expensive.
|
||||
func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
|
||||
var firstArgs, secondArgs bootstrap.Args
|
||||
|
||||
firstArgs = bootstrap.CmdlineArgs
|
||||
configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja)
|
||||
bootstrap.Main(firstCtx.Context, configuration, false, extraNinjaDeps...)
|
||||
bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration, extraNinjaDeps...)
|
||||
|
||||
// Invoke bazel commands and save results for second pass.
|
||||
if err := configuration.BazelContext.InvokeBazel(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Second pass: Full analysis, using the bazel command results. Output ninja file.
|
||||
secondPassConfig, err := android.ConfigForAdditionalRun(configuration)
|
||||
secondConfig, err := android.ConfigForAdditionalRun(configuration)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
secondCtx := newContext(secondPassConfig, true)
|
||||
bootstrap.Main(secondCtx.Context, secondPassConfig, false, extraNinjaDeps...)
|
||||
secondCtx := newContext(secondConfig, true)
|
||||
secondArgs = bootstrap.CmdlineArgs
|
||||
ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig, extraNinjaDeps...)
|
||||
err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
|
||||
|
|
@ -132,7 +143,8 @@ func runQueryView(configuration android.Config, ctx *android.Context) {
|
|||
|
||||
func runSoongDocs(configuration android.Config, extraNinjaDeps []string) {
|
||||
ctx := newContext(configuration, false)
|
||||
bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...)
|
||||
soongDocsArgs := bootstrap.CmdlineArgs
|
||||
bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration, extraNinjaDeps...)
|
||||
if err := writeDocs(ctx, configuration, docFile); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -166,8 +178,8 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
|
|||
generateQueryView := bazelQueryViewDir != ""
|
||||
jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
|
||||
|
||||
blueprintArgs := bootstrap.CmdlineArgs
|
||||
prepareBuildActions := !generateQueryView && jsonModuleFile == ""
|
||||
|
||||
if bazelConversionRequested {
|
||||
// Run the alternate pipeline of bp2build mutators and singleton to convert
|
||||
// Blueprint to BUILD files before everything else.
|
||||
|
|
@ -175,7 +187,7 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
|
|||
if bp2buildMarker != "" {
|
||||
return bp2buildMarker
|
||||
} else {
|
||||
return bootstrap.CmdlineOutFile()
|
||||
return bootstrap.CmdlineArgs.OutFile
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -183,22 +195,27 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) str
|
|||
if mixedModeBuild {
|
||||
runMixedModeBuild(configuration, ctx, extraNinjaDeps)
|
||||
} else {
|
||||
bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...)
|
||||
ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration, extraNinjaDeps...)
|
||||
err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert the Soong module graph into Bazel BUILD files.
|
||||
if generateQueryView {
|
||||
runQueryView(configuration, ctx)
|
||||
return bootstrap.CmdlineOutFile() // TODO: This is a lie
|
||||
return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie
|
||||
}
|
||||
|
||||
if jsonModuleFile != "" {
|
||||
writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
|
||||
return bootstrap.CmdlineOutFile() // TODO: This is a lie
|
||||
return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie
|
||||
}
|
||||
|
||||
writeMetrics(configuration)
|
||||
return bootstrap.CmdlineOutFile()
|
||||
return bootstrap.CmdlineArgs.OutFile
|
||||
}
|
||||
|
||||
// soong_ui dumps the available environment variables to
|
||||
|
|
@ -242,7 +259,7 @@ func main() {
|
|||
configuration := newConfig(srcDir, outDir, availableEnv)
|
||||
extraNinjaDeps := []string{
|
||||
configuration.ProductVariablesFileName,
|
||||
shared.JoinPath(outDir, "soong.environment.used"),
|
||||
usedEnvFile,
|
||||
}
|
||||
|
||||
if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
|
||||
|
|
@ -344,18 +361,18 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
|
|||
// Register an alternate set of singletons and mutators for bazel
|
||||
// conversion for Bazel conversion.
|
||||
bp2buildCtx := android.NewContext(configuration)
|
||||
bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
|
||||
bp2buildCtx.RegisterForBazelConversion()
|
||||
|
||||
// No need to generate Ninja build rules/statements from Modules and Singletons.
|
||||
configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
|
||||
// Propagate "allow misssing dependencies" bit. This is normally set in
|
||||
// newContext(), but we create bp2buildCtx without calling that method.
|
||||
bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
|
||||
bp2buildCtx.SetNameInterface(newNameResolver(configuration))
|
||||
bp2buildCtx.RegisterForBazelConversion()
|
||||
|
||||
// The bp2build process is a purely functional process that only depends on
|
||||
// Android.bp files. It must not depend on the values of per-build product
|
||||
// configurations or variables, since those will generate different BUILD
|
||||
// files based on how the user has configured their tree.
|
||||
bp2buildCtx.SetModuleListFile(bootstrap.CmdlineModuleListFile())
|
||||
bp2buildCtx.SetModuleListFile(bootstrap.CmdlineArgs.ModuleListFile)
|
||||
modulePaths, err := bp2buildCtx.ListModulePaths(configuration.SrcDir())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -363,10 +380,25 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
|
|||
|
||||
extraNinjaDeps = append(extraNinjaDeps, modulePaths...)
|
||||
|
||||
// No need to generate Ninja build rules/statements from Modules and Singletons.
|
||||
configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
|
||||
|
||||
// Run the loading and analysis pipeline to prepare the graph of regular
|
||||
// Modules parsed from Android.bp files, and the BazelTargetModules mapped
|
||||
// from the regular Modules.
|
||||
bootstrap.Main(bp2buildCtx.Context, configuration, false, extraNinjaDeps...)
|
||||
blueprintArgs := bootstrap.CmdlineArgs
|
||||
ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration, extraNinjaDeps...)
|
||||
|
||||
for _, globPath := range bp2buildCtx.Globs() {
|
||||
ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir()))
|
||||
}
|
||||
|
||||
depFile := bp2buildMarker + ".d"
|
||||
err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Run the code-generation phase to convert BazelTargetModules to BUILD files
|
||||
// and print conversion metrics to the user.
|
||||
|
|
|
|||
|
|
@ -429,12 +429,47 @@ function test_integrated_bp2build_smoke {
|
|||
setup
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
|
||||
fail "b2build marker file not created"
|
||||
fail "bp2build marker file not created"
|
||||
fi
|
||||
}
|
||||
|
||||
function test_integrated_bp2build_add_android_bp {
|
||||
setup
|
||||
|
||||
mkdir -p a
|
||||
touch a/a.txt
|
||||
cat > a/Android.bp <<'EOF'
|
||||
filegroup {
|
||||
name: "a",
|
||||
srcs: ["a.txt"],
|
||||
bazel_module: { bp2build_available: true },
|
||||
}
|
||||
EOF
|
||||
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
if [[ ! -e out/soong/bp2build/a/BUILD ]]; then
|
||||
fail "a/BUILD not created";
|
||||
fi
|
||||
|
||||
mkdir -p b
|
||||
touch b/b.txt
|
||||
cat > b/Android.bp <<'EOF'
|
||||
filegroup {
|
||||
name: "b",
|
||||
srcs: ["b.txt"],
|
||||
bazel_module: { bp2build_available: true },
|
||||
}
|
||||
EOF
|
||||
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
if [[ ! -e out/soong/bp2build/b/BUILD ]]; then
|
||||
fail "b/BUILD not created";
|
||||
fi
|
||||
}
|
||||
|
||||
function test_integrated_bp2build_null_build {
|
||||
setup
|
||||
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
local mtime1=$(stat -c "%y" out/soong/build.ninja)
|
||||
|
||||
|
|
@ -446,6 +481,27 @@ function test_integrated_bp2build_null_build {
|
|||
fi
|
||||
}
|
||||
|
||||
function test_integrated_bp2build_add_to_glob {
|
||||
setup
|
||||
|
||||
mkdir -p a
|
||||
touch a/a1.txt
|
||||
cat > a/Android.bp <<'EOF'
|
||||
filegroup {
|
||||
name: "a",
|
||||
srcs: ["*.txt"],
|
||||
bazel_module: { bp2build_available: true },
|
||||
}
|
||||
EOF
|
||||
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
grep -q a1.txt out/soong/bp2build/a/BUILD || fail "a1.txt not in BUILD file"
|
||||
|
||||
touch a/a2.txt
|
||||
INTEGRATED_BP2BUILD=1 run_soong
|
||||
grep -q a2.txt out/soong/bp2build/a/BUILD || fail "a2.txt not in BUILD file"
|
||||
}
|
||||
|
||||
function test_dump_json_module_graph() {
|
||||
setup
|
||||
SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
|
||||
|
|
@ -468,3 +524,4 @@ test_soong_build_rerun_iff_environment_changes
|
|||
test_dump_json_module_graph
|
||||
test_integrated_bp2build_smoke
|
||||
test_integrated_bp2build_null_build
|
||||
test_integrated_bp2build_add_to_glob
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
"strconv"
|
||||
|
||||
"android/soong/shared"
|
||||
"github.com/google/blueprint/deptools"
|
||||
|
||||
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
|
||||
"github.com/google/blueprint"
|
||||
|
|
@ -107,6 +108,7 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
|
|||
mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
|
||||
globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
|
||||
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
|
||||
bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
|
||||
|
||||
args.RunGoTests = !config.skipSoongTests
|
||||
args.UseValidations = true // Use validations to depend on tests
|
||||
|
|
@ -115,7 +117,6 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
|
|||
args.TopFile = "Android.bp"
|
||||
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
|
||||
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
|
||||
args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
|
||||
args.GlobFile = globFile
|
||||
args.GeneratingPrimaryBuilder = true
|
||||
|
||||
|
|
@ -171,7 +172,11 @@ func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
|
|||
debugCompilation: os.Getenv("SOONG_DELVE") != "",
|
||||
}
|
||||
|
||||
bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
|
||||
bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
|
||||
err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
|
||||
if err != nil {
|
||||
ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkEnvironmentFile(currentEnv *Environment, envFile string) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue