openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Allow running bp2build as part of a regular build. 

This is done by setting the INTEGRATED_BP2BUILD environment variable
when invoking the build.

Even though the name of the marker file insinuates that a Bazel
workspace is already created, this is not the case yet.

An issue that remains is that a .d file is not written for the marker
file so it won't be rebuilt if a .bp file changes. Fixing this requires
delicate surgery because writing the .d file is the result of delicate
interplay between Soong and Blueprint.

There are also a number of semi-related fixes:

- The name of soong.environment.{used,available} is now on the command
  line of soong_build (soong_docs is still special cased because its
  command line in the Ninja file is taken from the os.Args of
  soong_build so it's not trivial to remove the --{available,used}_env
  from it
- bp2build writes a separate soong.environment.used file
- I had to call SetAllowMissingDependencies() separately when creating
  the android.Context for bp2build so that bp2build runs in the
  integration tests (it was not obvious how not to do this)
- Fixed a number of integration tests where a command with an expected
  exit code of 1 was used as the last one in a test case, thereby
  breaking the test suite

Test: Presubmits.
Change-Id: Ibeb61c26022cf801dcb98505b4039151b3409873
This commit is contained in:
Lukacs T. Berki 2021-04-14 10:31:00 +02:00
parent 017a1bb7c9
commit f8e2428c5d
3 changed files with 201 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
cmd/soong_build/main.go
View File

@ -23,29 +23,42 @@ import (
"strings"
"time"
"android/soong/bp2build"
"android/soong/shared"
"github.com/google/blueprint/bootstrap"
"android/soong/android"
"android/soong/bp2build"
)
var (
topDir string
outDir string
topDir string
outDir string
availableEnvFile string
usedEnvFile string
delveListen string
delvePath string
docFile string
bazelQueryViewDir string
delveListen string
delvePath string
bp2buildMarker string
)
func init() {
// Flags that make sense in every mode
flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
// Debug flags
flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
// Flags representing various modes soong_build can run in
flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
}
func newNameResolver(config android.Config) *android.NameResolver {
@ -147,8 +160,8 @@ func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, pa
writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir())
}
func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES")
func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string {
bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") || bp2buildMarker != ""
mixedModeBuild := configuration.BazelContext.BazelEnabled()
generateQueryView := bazelQueryViewDir != ""
jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
@ -159,7 +172,11 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
runBp2Build(configuration, extraNinjaDeps)
return
if bp2buildMarker != "" {
return bp2buildMarker
} else {
return bootstrap.CmdlineOutFile()
}
}
ctx := newContext(configuration, prepareBuildActions)
@ -172,15 +189,44 @@ func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
runQueryView(configuration, ctx)
return
return bootstrap.CmdlineOutFile() // TODO: This is a lie
}
if jsonModuleFile != "" {
writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
return
return bootstrap.CmdlineOutFile() // TODO: This is a lie
}
writeMetrics(configuration)
return bootstrap.CmdlineOutFile()
}
// soong_ui dumps the available environment variables to
// soong.environment.available . Then soong_build itself is run with an empty
// environment so that the only way environment variables can be accessed is
// using Config, which tracks access to them.
// At the end of the build, a file called soong.environment.used is written
// containing the current value of all used environment variables. The next
// time soong_ui is run, it checks whether any environment variables that was
// used had changed and if so, it deletes soong.environment.used to cause a
// rebuild.
//
// The dependency of build.ninja on soong.environment.used is declared in
// build.ninja.d
func parseAvailableEnv() map[string]string {
if availableEnvFile == "" {
fmt.Fprintf(os.Stderr, "--available_env not set\n")
os.Exit(1)
}
result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
if err != nil {
fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
os.Exit(1)
}
return result
}
func main() {
@ -189,26 +235,7 @@ func main() {
shared.ReexecWithDelveMaybe(delveListen, delvePath)
android.InitSandbox(topDir)
// soong_ui dumps the available environment variables to
// soong.environment.available . Then soong_build itself is run with an empty
// environment so that the only way environment variables can be accessed is
// using Config, which tracks access to them.
// At the end of the build, a file called soong.environment.used is written
// containing the current value of all used environment variables. The next
// time soong_ui is run, it checks whether any environment variables that was
// used had changed and if so, it deletes soong.environment.used to cause a
// rebuild.
//
// The dependency of build.ninja on soong.environment.used is declared in
// build.ninja.d
availableEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.available")
usedEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.used")
availableEnv, err := shared.EnvFromFile(availableEnvFile)
if err != nil {
fmt.Fprintf(os.Stderr, "error reading available environment file %s: %s\n", availableEnvFile, err)
os.Exit(1)
}
availableEnv := parseAvailableEnv()
// The top-level Blueprints file is passed as the first argument.
srcDir := filepath.Dir(flag.Arg(0))
@ -233,37 +260,37 @@ func main() {
// because that is done from within the actual builds as a Ninja action and
// thus it would overwrite the actual used variables file so this is
// special-cased.
// TODO: Fix this by not passing --used_env to the soong_docs invocation
runSoongDocs(configuration, extraNinjaDeps)
return
}
doChosenActivity(configuration, extraNinjaDeps)
writeUsedEnvironmentFile(usedEnvFile, configuration)
finalOutputFile := doChosenActivity(configuration, extraNinjaDeps)
writeUsedEnvironmentFile(configuration, finalOutputFile)
}
func writeUsedEnvironmentFile(path string, configuration android.Config) {
func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
if usedEnvFile == "" {
return
}
path := shared.JoinPath(topDir, usedEnvFile)
data, err := shared.EnvFileContents(configuration.EnvDeps())
if err != nil {
fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
os.Exit(1)
}
err = ioutil.WriteFile(path, data, 0666)
if err != nil {
fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
os.Exit(1)
}
// Touch the output Ninja file so that it's not older than the file we just
// Touch the output file so that it's not older than the file we just
// wrote. We can't write the environment file earlier because one an access
// new environment variables while writing it.
outputNinjaFile := shared.JoinPath(topDir, bootstrap.CmdlineOutFile())
currentTime := time.Now().Local()
err = os.Chtimes(outputNinjaFile, currentTime, currentTime)
if err != nil {
fmt.Fprintf(os.Stderr, "error touching output file %s: %s\n", outputNinjaFile, err)
os.Exit(1)
}
touch(shared.JoinPath(topDir, finalOutputFile))
}
// Workarounds to support running bp2build in a clean AOSP checkout with no
@ -289,6 +316,27 @@ func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) {
0666)
}
func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
os.Exit(1)
}
err = f.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
os.Exit(1)
}
currentTime := time.Now().Local()
err = os.Chtimes(path, currentTime, currentTime)
if err != nil {
fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
os.Exit(1)
}
}
// Run Soong in the bp2build mode. This creates a standalone context that registers
// an alternate pipeline of mutators and singletons specifically for generating
// Bazel BUILD files instead of Ninja files.
@ -296,6 +344,7 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
// Register an alternate set of singletons and mutators for bazel
// conversion for Bazel conversion.
bp2buildCtx := android.NewContext(configuration)
bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
bp2buildCtx.RegisterForBazelConversion()
// No need to generate Ninja build rules/statements from Modules and Singletons.
@ -330,5 +379,9 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
metrics.Print()
extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
if bp2buildMarker != "" {
touch(shared.JoinPath(topDir, bp2buildMarker))
} else {
writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
}
}

31
tests/bootstrap_test.sh
View File

@ -114,7 +114,9 @@ EOF
rm a/Android.bp
run_soong
grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output"
if grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja; then
fail "Old module in output"
fi
}
function test_add_file_to_glob() {
@ -404,7 +406,9 @@ EOF
grep -q "Engage" out/soong/build.ninja || fail "New action not present"
grep -q "Make it so" out/soong/build.ninja && fail "Original action still present"
if grep -q "Make it so" out/soong/build.ninja; then
fail "Original action still present"
fi
}
function test_null_build_after_docs {
@ -421,6 +425,27 @@ function test_null_build_after_docs {
fi
}
function test_integrated_bp2build_smoke {
setup
INTEGRATED_BP2BUILD=1 run_soong
if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
fail "b2build marker file not created"
fi
}
function test_integrated_bp2build_null_build {
setup
INTEGRATED_BP2BUILD=1 run_soong
local mtime1=$(stat -c "%y" out/soong/build.ninja)
INTEGRATED_BP2BUILD=1 run_soong
local mtime2=$(stat -c "%y" out/soong/build.ninja)
if [[ "$mtime1" != "$mtime2" ]]; then
fail "Output Ninja file changed on null build"
fi
}
function test_dump_json_module_graph() {
setup
SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
@ -441,3 +466,5 @@ test_add_file_to_soong_build
test_glob_during_bootstrapping
test_soong_build_rerun_iff_environment_changes
test_dump_json_module_graph
test_integrated_bp2build_smoke
test_integrated_bp2build_null_build

86
ui/build/soong.go
View File

@ -33,6 +33,11 @@ import (
"android/soong/ui/status"
)
const (
availableEnvFile = "soong.environment.available"
usedEnvFile = "soong.environment.used"
)
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
data, err := shared.EnvFileContents(envDeps)
if err != nil {
@ -87,12 +92,22 @@ func (c BlueprintConfig) DebugCompilation() bool {
return c.debugCompilation
}
func bootstrapBlueprint(ctx Context, config Config) {
func environmentArgs(config Config, suffix string) []string {
return []string{
"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
}
}
func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
var args bootstrap.Args
mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
args.RunGoTests = !config.skipSoongTests
args.UseValidations = true // Use validations to depend on tests
args.BuildDir = config.SoongOutDir()
@ -101,7 +116,7 @@ func bootstrapBlueprint(ctx Context, config Config) {
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
args.GlobFile = globFile
args.GeneratingPrimaryBuilder = true
args.DelveListen = os.Getenv("SOONG_DELVE")
@ -109,6 +124,44 @@ func bootstrapBlueprint(ctx Context, config Config) {
args.DelvePath = shared.ResolveDelveBinary()
}
commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
mainSoongBuildInputs := []string{"Android.bp"}
if integratedBp2Build {
mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
}
soongBuildArgs := make([]string, 0)
soongBuildArgs = append(soongBuildArgs, commonArgs...)
soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
soongBuildArgs = append(soongBuildArgs, "Android.bp")
mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: mainSoongBuildInputs,
Outputs: []string{mainNinjaFile},
Args: soongBuildArgs,
}
if integratedBp2Build {
bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
bp2buildArgs = append(bp2buildArgs, commonArgs...)
bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
bp2buildArgs = append(bp2buildArgs, "Android.bp")
bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: []string{"Android.bp"},
Outputs: []string{bp2BuildMarkerFile},
Args: bp2buildArgs,
}
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
bp2buildInvocation,
mainSoongBuildInvocation,
}
} else {
args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
}
blueprintCtx := blueprint.NewContext()
blueprintCtx.SetIgnoreUnknownModuleTypes(true)
blueprintConfig := BlueprintConfig{
@ -121,6 +174,16 @@ func bootstrapBlueprint(ctx Context, config Config) {
bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
}
func checkEnvironmentFile(currentEnv *Environment, envFile string) {
getenv := func(k string) string {
v, _ := currentEnv.Get(k)
return v
}
if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
os.Remove(envFile)
}
}
func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "soong")
defer ctx.EndTrace()
@ -129,7 +192,7 @@ func runSoong(ctx Context, config Config) {
// .used with the ones that were actually used. The latter is used to
// determine whether Soong needs to be re-run since why re-run it if only
// unused variables were changed?
envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
for _, n := range []string{".bootstrap", ".minibootstrap"} {
dir := filepath.Join(config.SoongOutDir(), n)
@ -138,8 +201,10 @@ func runSoong(ctx Context, config Config) {
}
}
integratedBp2Build := config.Environment().IsEnvTrue("INTEGRATED_BP2BUILD")
// This is done unconditionally, but does not take a measurable amount of time
bootstrapBlueprint(ctx, config)
bootstrapBlueprint(ctx, config, integratedBp2Build)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@ -164,13 +229,12 @@ func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "environment check")
defer ctx.EndTrace()
envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used")
getenv := func(k string) string {
v, _ := soongBuildEnv.Get(k)
return v
}
if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
os.Remove(envFile)
soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
if integratedBp2Build {
bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
}
}()