From daa54bcbba9dbea37ddf28feb948a112e3f4c95d Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Mon, 14 Dec 2020 02:58:54 -0500 Subject: [PATCH] Implement code-generation step for bp2build. Implement bp2build codegen as a discrete step that runs after an alternatively registered pipeline of mutators, instead of a presingleton. bp2build codegen requires a Context that supports VisitAllModules and PathContext, so this CL also makes a BpToBuildWrapperContext that conforms to PathContext by adding two method implementations. Test: GENERATE_BAZEL_FILES=true m nothing && bazel query //... --config=bp2build | wc -l # 31433 Test: m queryview && bazel query //... --config=queryview # 63638 Change-Id: I0dd359746584b228046d2d0ff00895f28f9bdfc3 --- android/register.go | 20 +++----------- bp2build/bp2build.go | 42 +++++++++-------------------- bp2build/build_conversion.go | 26 +++++++++++++++--- bp2build/build_conversion_test.go | 6 +---- bp2build/testing.go | 34 ------------------------ cmd/soong_build/main.go | 44 ++++++++++++++++++++++--------- cmd/soong_build/queryview.go | 39 +-------------------------- 7 files changed, 72 insertions(+), 139 deletions(-) diff --git a/android/register.go b/android/register.go index ca658f5ff..f84acad3b 100644 --- a/android/register.go +++ b/android/register.go @@ -37,9 +37,6 @@ type singleton struct { var singletons []singleton var preSingletons []singleton -var bazelConverterSingletons []singleton -var bazelConverterPreSingletons []singleton - type mutator struct { name string bottomUpMutator blueprint.BottomUpMutator @@ -94,14 +91,6 @@ func RegisterPreSingletonType(name string, factory SingletonFactory) { preSingletons = append(preSingletons, singleton{name, factory}) } -func RegisterBazelConverterSingletonType(name string, factory SingletonFactory) { - bazelConverterSingletons = append(bazelConverterSingletons, singleton{name, factory}) -} - -func RegisterBazelConverterPreSingletonType(name string, factory SingletonFactory) { - bazelConverterPreSingletons = append(bazelConverterPreSingletons, singleton{name, factory}) -} - type Context struct { *blueprint.Context config Config @@ -117,21 +106,20 @@ func NewContext(config Config) *Context { // singletons, module types and mutators to register for converting Blueprint // files to semantically equivalent BUILD files. func (ctx *Context) RegisterForBazelConversion() { - for _, t := range bazelConverterPreSingletons { - ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) - } - for _, t := range moduleTypes { ctx.RegisterModuleType(t.name, ModuleFactoryAdaptor(t.factory)) } - for _, t := range bazelConverterSingletons { + // Required for SingletonModule types, even though we are not using them. + for _, t := range singletons { ctx.RegisterSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) } registerMutatorsForBazelConversion(ctx.Context) } +// Register the pipeline of singletons, module types, and mutators for +// generating build.ninja and other files for Kati, from Android.bp files. func (ctx *Context) Register() { for _, t := range preSingletons { ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 30f298a0c..49729e02a 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -16,52 +16,34 @@ package bp2build import ( "android/soong/android" + "fmt" "os" ) -// The Bazel bp2build singleton is responsible for writing .bzl files that are equivalent to +// The Bazel bp2build code generator is responsible for writing .bzl files that are equivalent to // Android.bp files that are capable of being built with Bazel. -func init() { - android.RegisterBazelConverterPreSingletonType("androidbp_to_build", AndroidBpToBuildSingleton) -} - -func AndroidBpToBuildSingleton() android.Singleton { - return &androidBpToBuildSingleton{ - name: "bp2build", - } -} - -type androidBpToBuildSingleton struct { - name string - outputDir android.OutputPath -} - -func (s *androidBpToBuildSingleton) GenerateBuildActions(ctx android.SingletonContext) { - s.outputDir = android.PathForOutput(ctx, s.name) - android.RemoveAllOutputDir(s.outputDir) - - if !ctx.Config().IsEnvTrue("CONVERT_TO_BAZEL") { - return - } +func Codegen(ctx CodegenContext) { + outputDir := android.PathForOutput(ctx, "bp2build") + android.RemoveAllOutputDir(outputDir) ruleShims := CreateRuleShims(android.ModuleTypeFactories()) - buildToTargets := GenerateSoongModuleTargets(ctx) + buildToTargets := GenerateSoongModuleTargets(ctx.Context()) filesToWrite := CreateBazelFiles(ruleShims, buildToTargets) for _, f := range filesToWrite { - if err := s.writeFile(ctx, f); err != nil { - ctx.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err) + if err := writeFile(outputDir, ctx, f); err != nil { + fmt.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err) } } } -func (s *androidBpToBuildSingleton) getOutputPath(ctx android.PathContext, dir string) android.OutputPath { - return s.outputDir.Join(ctx, dir) +func writeFile(outputDir android.OutputPath, ctx android.PathContext, f BazelFile) error { + return writeReadOnlyFile(ctx, getOutputPath(outputDir, ctx, f.Dir), f.Basename, f.Contents) } -func (s *androidBpToBuildSingleton) writeFile(ctx android.PathContext, f BazelFile) error { - return writeReadOnlyFile(ctx, s.getOutputPath(ctx, f.Dir), f.Basename, f.Contents) +func getOutputPath(outputDir android.OutputPath, ctx android.PathContext, dir string) android.OutputPath { + return outputDir.Join(ctx, dir) } // The auto-conversion directory should be read-only, sufficient for bazel query. The files diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index 03296852f..bece8f65c 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -39,8 +39,26 @@ type bpToBuildContext interface { ModuleSubDir(module blueprint.Module) string ModuleType(module blueprint.Module) string - VisitAllModulesBlueprint(visit func(blueprint.Module)) - VisitDirectDeps(module android.Module, visit func(android.Module)) + VisitAllModules(visit func(blueprint.Module)) + VisitDirectDeps(module blueprint.Module, visit func(blueprint.Module)) +} + +type CodegenContext struct { + config android.Config + context android.Context +} + +func (ctx CodegenContext) AddNinjaFileDeps(...string) {} +func (ctx CodegenContext) Config() android.Config { return ctx.config } +func (ctx CodegenContext) Context() android.Context { return ctx.context } + +// NewCodegenContext creates a wrapper context that conforms to PathContext for +// writing BUILD files in the output directory. +func NewCodegenContext(config android.Config, context android.Context) CodegenContext { + return CodegenContext{ + context: context, + config: config, + } } // props is an unsorted map. This function ensures that @@ -57,7 +75,7 @@ func propsToAttributes(props map[string]string) string { func GenerateSoongModuleTargets(ctx bpToBuildContext) map[string][]BazelTarget { buildFileToTargets := make(map[string][]BazelTarget) - ctx.VisitAllModulesBlueprint(func(m blueprint.Module) { + ctx.VisitAllModules(func(m blueprint.Module) { dir := ctx.ModuleDir(m) t := generateSoongModuleTarget(ctx, m) buildFileToTargets[ctx.ModuleDir(m)] = append(buildFileToTargets[dir], t) @@ -75,7 +93,7 @@ func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) BazelTa // out the implications of that. depLabels := map[string]bool{} if aModule, ok := m.(android.Module); ok { - ctx.VisitDirectDeps(aModule, func(depModule android.Module) { + ctx.VisitDirectDeps(aModule, func(depModule blueprint.Module) { depLabels[qualifiedTargetLabel(ctx, depModule)] = true }) } diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go index 8230ad8c0..4e31aa78a 100644 --- a/bp2build/build_conversion_test.go +++ b/bp2build/build_conversion_test.go @@ -200,11 +200,7 @@ func TestGenerateSoongModuleTargets(t *testing.T) { _, errs = ctx.PrepareBuildActions(config) android.FailIfErrored(t, errs) - bp2BuildCtx := bp2buildBlueprintWrapContext{ - bpCtx: ctx.Context.Context, - } - - bazelTargets := GenerateSoongModuleTargets(&bp2BuildCtx)[dir] + bazelTargets := GenerateSoongModuleTargets(ctx.Context.Context)[dir] if g, w := len(bazelTargets), 1; g != w { t.Fatalf("Expected %d bazel target, got %d", w, g) } diff --git a/bp2build/testing.go b/bp2build/testing.go index 160412de8..2da32c660 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -2,8 +2,6 @@ package bp2build import ( "android/soong/android" - - "github.com/google/blueprint" ) type nestedProps struct { @@ -102,35 +100,3 @@ func customDefaultsModuleFactory() android.Module { android.InitDefaultsModule(m) return m } - -type bp2buildBlueprintWrapContext struct { - bpCtx *blueprint.Context -} - -func (ctx *bp2buildBlueprintWrapContext) ModuleName(module blueprint.Module) string { - return ctx.bpCtx.ModuleName(module) -} - -func (ctx *bp2buildBlueprintWrapContext) ModuleDir(module blueprint.Module) string { - return ctx.bpCtx.ModuleDir(module) -} - -func (ctx *bp2buildBlueprintWrapContext) ModuleSubDir(module blueprint.Module) string { - return ctx.bpCtx.ModuleSubDir(module) -} - -func (ctx *bp2buildBlueprintWrapContext) ModuleType(module blueprint.Module) string { - return ctx.bpCtx.ModuleType(module) -} - -func (ctx *bp2buildBlueprintWrapContext) VisitAllModulesBlueprint(visit func(blueprint.Module)) { - ctx.bpCtx.VisitAllModules(visit) -} - -func (ctx *bp2buildBlueprintWrapContext) VisitDirectDeps(module android.Module, visit func(android.Module)) { - ctx.bpCtx.VisitDirectDeps(module, func(m blueprint.Module) { - if aModule, ok := m.(android.Module); ok { - visit(aModule) - } - }) -} diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 907bed3c7..1d5e7b3af 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -23,6 +23,7 @@ import ( "github.com/google/blueprint/bootstrap" "android/soong/android" + "android/soong/bp2build" ) var ( @@ -54,18 +55,12 @@ func newNameResolver(config android.Config) *android.NameResolver { // bazelConversionRequested checks that the user is intending to convert // Blueprint to Bazel BUILD files. func bazelConversionRequested(configuration android.Config) bool { - return configuration.IsEnvTrue("CONVERT_TO_BAZEL") + return configuration.IsEnvTrue("GENERATE_BAZEL_FILES") } -func newContext(srcDir string, configuration android.Config) *android.Context { +func newContext(configuration android.Config) *android.Context { ctx := android.NewContext(configuration) - if bazelConversionRequested(configuration) { - // Register an alternate set of singletons and mutators for bazel - // conversion for Bazel conversion. - ctx.RegisterForBazelConversion() - } else { - ctx.Register() - } + ctx.Register() if !shouldPrepareBuildActions(configuration) { configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) } @@ -100,13 +95,22 @@ func main() { // enabled even if it completed successfully. extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve")) } + + if bazelConversionRequested(configuration) { + // Run the alternate pipeline of bp2build mutators and singleton to convert Blueprint to BUILD files + // before everything else. + runBp2Build(configuration, extraNinjaDeps) + // Short-circuit and return. + return + } + if configuration.BazelContext.BazelEnabled() { // Bazel-enabled mode. Soong runs in two passes. // First pass: Analyze the build tree, but only store all bazel commands // needed to correctly evaluate the tree in the second pass. // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite // the incorrect results from the first pass, and file I/O is expensive. - firstCtx := newContext(srcDir, configuration) + firstCtx := newContext(configuration) configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja) bootstrap.Main(firstCtx.Context, configuration, extraNinjaDeps...) // Invoke bazel commands and save results for second pass. @@ -120,10 +124,10 @@ func main() { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) } - ctx = newContext(srcDir, secondPassConfig) + ctx = newContext(secondPassConfig) bootstrap.Main(ctx.Context, secondPassConfig, extraNinjaDeps...) } else { - ctx = newContext(srcDir, configuration) + ctx = newContext(configuration) bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) } @@ -154,6 +158,22 @@ func main() { } } +// Run Soong in the bp2build mode. This creates a standalone context that registers +// an alternate pipeline of mutators and singletons specifically for generating +// Bazel BUILD files instead of Ninja files. +func runBp2Build(configuration android.Config, extraNinjaDeps []string) { + // Register an alternate set of singletons and mutators for bazel + // conversion for Bazel conversion. + bp2buildCtx := android.NewContext(configuration) + bp2buildCtx.RegisterForBazelConversion() + configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) + bp2buildCtx.SetNameInterface(newNameResolver(configuration)) + bootstrap.Main(bp2buildCtx.Context, configuration, extraNinjaDeps...) + + codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx) + bp2build.Codegen(codegenContext) +} + // shouldPrepareBuildActions reads configuration and flags if build actions // should be generated. func shouldPrepareBuildActions(configuration android.Config) bool { diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go index 3657ea8ef..79ea94af7 100644 --- a/cmd/soong_build/queryview.go +++ b/cmd/soong_build/queryview.go @@ -20,48 +20,11 @@ import ( "io/ioutil" "os" "path/filepath" - - "github.com/google/blueprint" ) -type queryviewContext struct { - bpCtx *blueprint.Context -} - -func (ctx *queryviewContext) ModuleName(module blueprint.Module) string { - return ctx.bpCtx.ModuleName(module) -} - -func (ctx *queryviewContext) ModuleDir(module blueprint.Module) string { - return ctx.bpCtx.ModuleDir(module) -} - -func (ctx *queryviewContext) ModuleSubDir(module blueprint.Module) string { - return ctx.bpCtx.ModuleSubDir(module) -} - -func (ctx *queryviewContext) ModuleType(module blueprint.Module) string { - return ctx.bpCtx.ModuleType(module) -} - -func (ctx *queryviewContext) VisitAllModulesBlueprint(visit func(blueprint.Module)) { - ctx.bpCtx.VisitAllModules(visit) -} - -func (ctx *queryviewContext) VisitDirectDeps(module android.Module, visit func(android.Module)) { - ctx.bpCtx.VisitDirectDeps(module, func(m blueprint.Module) { - if aModule, ok := m.(android.Module); ok { - visit(aModule) - } - }) -} - func createBazelQueryView(ctx *android.Context, bazelQueryViewDir string) error { - qvCtx := queryviewContext{ - bpCtx: ctx.Context, - } ruleShims := bp2build.CreateRuleShims(android.ModuleTypeFactories()) - buildToTargets := bp2build.GenerateSoongModuleTargets(&qvCtx) + buildToTargets := bp2build.GenerateSoongModuleTargets(*ctx) filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets) for _, f := range filesToWrite {