From 4133ce6912fcdc8ef364b5c08f9acfb66d94fbfb Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Wed, 2 Dec 2020 04:34:15 -0500 Subject: [PATCH] Add bp2build mode to soong_build. This CL adds a new CONVERT_TO_BAZEL env var, and a bp2build goal. It reuses the queryview architecture, but registers no mutators before converting to BUILD files. This gives us a blank slate of the module graph to work with, and create a shadow/alternate pipeline of mutators to converge the Bazel BUILD graph to be semantically equivalent with the Android.bp graph (after apply its current set of mutators). The command to do so is: $ CONVERT_TO_BAZEL=true m bp2build To not clobber with queryview, the generated files are in out/soong/bp2build. Test: CONVERT_TO_BAZEL=true m bp2build && bazel query --config=bp2build //... Test: m queryview && bazel query --config=queryview //.. Test: soong tests Test: TH presubmit Fixes: 174465461 Signed-off-by: Jingwen Chen Change-Id: I4dd0ccc73abc345d70a50ca2803d6f400cd8c863 --- android/mutator.go | 5 +++++ android/queryview.go | 48 +++++++++++++++++++++++++++++++++++------ android/register.go | 15 +++++++++++++ cmd/soong_build/main.go | 39 +++++++++++++++++++++++++++------ 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/android/mutator.go b/android/mutator.go index 7a104772f..31edea331 100644 --- a/android/mutator.go +++ b/android/mutator.go @@ -44,6 +44,11 @@ func registerMutatorsToContext(ctx *blueprint.Context, mutators []*mutator) { } } +func registerMutatorsForBazelConversion(ctx *blueprint.Context) { + // FIXME(b/171263886): Start bringing in mutators to make the Bionic + // module subgraph suitable for automated conversion. +} + func registerMutators(ctx *blueprint.Context, preArch, preDeps, postDeps, finalDeps []RegisterMutatorFunc) { mctx := ®isterMutatorsContext{} diff --git a/android/queryview.go b/android/queryview.go index 970ae0116..1b7e77dd6 100644 --- a/android/queryview.go +++ b/android/queryview.go @@ -26,15 +26,40 @@ import ( // for calling the soong_build primary builder in the main build.ninja file. func init() { RegisterSingletonType("bazel_queryview", BazelQueryViewSingleton) + RegisterSingletonType("bazel_converter", BazelConverterSingleton) } +// BazelQueryViewSingleton is the singleton responsible for registering the +// soong_build build statement that will convert the Soong module graph after +// applying *all* mutators, enabing the feature to query the final state of the +// Soong graph. This mode is meant for querying the build graph state, and not meant +// for generating BUILD files to be checked in. func BazelQueryViewSingleton() Singleton { return &bazelQueryViewSingleton{} } -type bazelQueryViewSingleton struct{} +// BazelConverterSingleton is the singleton responsible for registering the soong_build +// build statement that will convert the Soong module graph by applying an alternate +// pipeline of mutators, with the goal of reaching semantic equivalence between the original +// Blueprint and final BUILD files. Using this mode, the goal is to be able to +// build with these BUILD files directly in the source tree. +func BazelConverterSingleton() Singleton { + return &bazelConverterSingleton{} +} + +type bazelQueryViewSingleton struct{} +type bazelConverterSingleton struct{} + +func generateBuildActionsForBazelConversion(ctx SingletonContext, converterMode bool) { + name := "queryview" + additionalEnvVars := "" + descriptionTemplate := "[EXPERIMENTAL, PRE-PRODUCTION] Creating the Bazel QueryView workspace with %s at $outDir" + if converterMode { + name = "bp2build" + additionalEnvVars = "CONVERT_TO_BAZEL=true" + descriptionTemplate = "[EXPERIMENTAL, PRE-PRODUCTION] Converting all Android.bp to Bazel BUILD files with %s at $outDir" + } -func (c *bazelQueryViewSingleton) GenerateBuildActions(ctx SingletonContext) { // Create a build and rule statement, using the Bazel QueryView's WORKSPACE // file as the output file marker. var deps Paths @@ -42,22 +67,23 @@ func (c *bazelQueryViewSingleton) GenerateBuildActions(ctx SingletonContext) { deps = append(deps, moduleListFilePath) deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName)) - bazelQueryViewDirectory := PathForOutput(ctx, "queryview") + bazelQueryViewDirectory := PathForOutput(ctx, name) bazelQueryViewWorkspaceFile := bazelQueryViewDirectory.Join(ctx, "WORKSPACE") primaryBuilder := primaryBuilderPath(ctx) bazelQueryView := ctx.Rule(pctx, "bazelQueryView", blueprint.RuleParams{ Command: fmt.Sprintf( "rm -rf ${outDir}/* && "+ - "%s --bazel_queryview_dir ${outDir} %s && "+ + "%s %s --bazel_queryview_dir ${outDir} %s && "+ "echo WORKSPACE: `cat %s` > ${outDir}/.queryview-depfile.d", + additionalEnvVars, primaryBuilder.String(), strings.Join(os.Args[1:], " "), moduleListFilePath.String(), // Use the contents of Android.bp.list as the depfile. ), CommandDeps: []string{primaryBuilder.String()}, Description: fmt.Sprintf( - "[EXPERIMENTAL, PRE-PRODUCTION] Creating the Bazel QueryView workspace with %s at $outDir", + descriptionTemplate, primaryBuilder.Base()), Deps: blueprint.DepsGCC, Depfile: "${outDir}/.queryview-depfile.d", @@ -73,6 +99,14 @@ func (c *bazelQueryViewSingleton) GenerateBuildActions(ctx SingletonContext) { }, }) - // Add a phony target for building the Bazel QueryView - ctx.Phony("queryview", bazelQueryViewWorkspaceFile) + // Add a phony target for generating the workspace + ctx.Phony(name, bazelQueryViewWorkspaceFile) +} + +func (c *bazelQueryViewSingleton) GenerateBuildActions(ctx SingletonContext) { + generateBuildActionsForBazelConversion(ctx, false) +} + +func (c *bazelConverterSingleton) GenerateBuildActions(ctx SingletonContext) { + generateBuildActionsForBazelConversion(ctx, true) } diff --git a/android/register.go b/android/register.go index 08e47b330..b26f9b97a 100644 --- a/android/register.go +++ b/android/register.go @@ -90,6 +90,21 @@ func NewContext(config Config) *Context { return ctx } +// RegisterForBazelConversion registers an alternate shadow pipeline of +// singletons, module types and mutators to register for converting Blueprint +// files to semantically equivalent BUILD files. +func (ctx *Context) RegisterForBazelConversion() { + for _, t := range moduleTypes { + ctx.RegisterModuleType(t.name, ModuleFactoryAdaptor(t.factory)) + } + + bazelConverterSingleton := singleton{"bp2build", BazelConverterSingleton} + ctx.RegisterSingletonType(bazelConverterSingleton.name, + SingletonFactoryAdaptor(ctx, bazelConverterSingleton.factory)) + + registerMutatorsForBazelConversion(ctx.Context) +} + func (ctx *Context) Register() { for _, t := range preSingletons { ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory)) diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index b88803a72..eeee3a8c8 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -51,10 +51,22 @@ func newNameResolver(config android.Config) *android.NameResolver { return android.NewNameResolver(exportFilter) } +// bazelConversionRequested checks that the user is intending to convert +// Blueprint to Bazel BUILD files. +func bazelConversionRequested(configuration android.Config) bool { + return configuration.IsEnvTrue("CONVERT_TO_BAZEL") +} + func newContext(srcDir string, configuration android.Config) *android.Context { ctx := android.NewContext(configuration) - ctx.Register() - if !shouldPrepareBuildActions() { + if bazelConversionRequested(configuration) { + // Register an alternate set of singletons and mutators for bazel + // conversion for Bazel conversion. + ctx.RegisterForBazelConversion() + } else { + ctx.Register() + } + if !shouldPrepareBuildActions(configuration) { configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) } ctx.SetNameInterface(newNameResolver(configuration)) @@ -114,6 +126,8 @@ func main() { ctx = newContext(srcDir, configuration) bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) } + + // Convert the Soong module graph into Bazel BUILD files. if bazelQueryViewDir != "" { if err := createBazelQueryView(ctx, bazelQueryViewDir); err != nil { fmt.Fprintf(os.Stderr, "%s", err) @@ -130,7 +144,7 @@ func main() { // TODO(ccross): make this a command line argument. Requires plumbing through blueprint // to affect the command line of the primary builder. - if shouldPrepareBuildActions() { + if shouldPrepareBuildActions(configuration) { metricsFile := filepath.Join(bootstrap.BuildDir, "soong_build_metrics.pb") err := android.WriteMetrics(configuration, metricsFile) if err != nil { @@ -140,8 +154,19 @@ func main() { } } -func shouldPrepareBuildActions() bool { - // If we're writing soong_docs or queryview, don't write build.ninja or - // collect metrics. - return docFile == "" && bazelQueryViewDir == "" +// shouldPrepareBuildActions reads configuration and flags if build actions +// should be generated. +func shouldPrepareBuildActions(configuration android.Config) bool { + // Generating Soong docs + if docFile != "" { + return false + } + + // Generating a directory for Soong query (queryview) + if bazelQueryViewDir != "" { + return false + } + + // Generating a directory for converted Bazel BUILD files + return !bazelConversionRequested(configuration) }