diff --git a/android/Android.bp b/android/Android.bp index a1b515940..7aa228ee7 100644 --- a/android/Android.bp +++ b/android/Android.bp @@ -15,6 +15,7 @@ bootstrap_go_package { "apex.go", "api_levels.go", "arch.go", + "bazel_handler.go", "bazel_overlay.go", "config.go", "csuite_config.go", diff --git a/android/bazel_handler.go b/android/bazel_handler.go new file mode 100644 index 000000000..d4f6e4c9e --- /dev/null +++ b/android/bazel_handler.go @@ -0,0 +1,239 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "sync" +) + +// Map key to describe bazel cquery requests. +type cqueryKey struct { + label string + starlarkExpr string +} + +type BazelContext interface { + // The below methods involve queuing cquery requests to be later invoked + // by bazel. If any of these methods return (_, false), then the request + // has been queued to be run later. + + // Returns result files built by building the given bazel target label. + GetAllFiles(label string) ([]string, bool) + + // TODO(cparsons): Other cquery-related methods should be added here. + // ** End cquery methods + + // Issues commands to Bazel to receive results for all cquery requests + // queued in the BazelContext. + InvokeBazel() error + + // Returns true if bazel is enabled for the given configuration. + BazelEnabled() bool +} + +// A context object which tracks queued requests that need to be made to Bazel, +// and their results after the requests have been made. +type bazelContext struct { + homeDir string + bazelPath string + outputBase string + workspaceDir string + + requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel + requestMutex sync.Mutex // requests can be written in parallel + + results map[cqueryKey]string // Results of cquery requests after Bazel invocations +} + +var _ BazelContext = &bazelContext{} + +// A bazel context to use when Bazel is disabled. +type noopBazelContext struct{} + +var _ BazelContext = noopBazelContext{} + +// A bazel context to use for tests. +type MockBazelContext struct { + AllFiles map[string][]string +} + +func (m MockBazelContext) GetAllFiles(label string) ([]string, bool) { + result, ok := m.AllFiles[label] + return result, ok +} + +func (m MockBazelContext) InvokeBazel() error { + panic("unimplemented") +} + +func (m MockBazelContext) BazelEnabled() bool { + return true +} + +var _ BazelContext = MockBazelContext{} + +func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) { + starlarkExpr := "', '.join([f.path for f in target.files.to_list()])" + result, ok := bazelCtx.cquery(label, starlarkExpr) + if ok { + bazelOutput := strings.TrimSpace(result) + return strings.Split(bazelOutput, ", "), true + } else { + return nil, false + } +} + +func (n noopBazelContext) GetAllFiles(label string) ([]string, bool) { + panic("unimplemented") +} + +func (n noopBazelContext) InvokeBazel() error { + panic("unimplemented") +} + +func (n noopBazelContext) BazelEnabled() bool { + return false +} + +func NewBazelContext(c *config) (BazelContext, error) { + if c.Getenv("USE_BAZEL") != "1" { + return noopBazelContext{}, nil + } + + bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)} + missingEnvVars := []string{} + if len(c.Getenv("BAZEL_HOME")) > 1 { + bazelCtx.homeDir = c.Getenv("BAZEL_HOME") + } else { + missingEnvVars = append(missingEnvVars, "BAZEL_HOME") + } + if len(c.Getenv("BAZEL_PATH")) > 1 { + bazelCtx.bazelPath = c.Getenv("BAZEL_PATH") + } else { + missingEnvVars = append(missingEnvVars, "BAZEL_PATH") + } + if len(c.Getenv("BAZEL_OUTPUT_BASE")) > 1 { + bazelCtx.outputBase = c.Getenv("BAZEL_OUTPUT_BASE") + } else { + missingEnvVars = append(missingEnvVars, "BAZEL_OUTPUT_BASE") + } + if len(c.Getenv("BAZEL_WORKSPACE")) > 1 { + bazelCtx.workspaceDir = c.Getenv("BAZEL_WORKSPACE") + } else { + missingEnvVars = append(missingEnvVars, "BAZEL_WORKSPACE") + } + if len(missingEnvVars) > 0 { + return nil, errors.New(fmt.Sprintf("missing required env vars to use bazel: %s", missingEnvVars)) + } else { + return &bazelCtx, nil + } +} + +func (context *bazelContext) BazelEnabled() bool { + return true +} + +// Adds a cquery request to the Bazel request queue, to be later invoked, or +// returns the result of the given request if the request was already made. +// If the given request was already made (and the results are available), then +// returns (result, true). If the request is queued but no results are available, +// then returns ("", false). +func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) { + key := cqueryKey{label, starlarkExpr} + if result, ok := context.results[key]; ok { + return result, true + } else { + context.requestMutex.Lock() + defer context.requestMutex.Unlock() + context.requests[key] = true + return "", false + } +} + +func pwdPrefix() string { + // Darwin doesn't have /proc + if runtime.GOOS != "darwin" { + return "PWD=/proc/self/cwd" + } + return "" +} + +func (context *bazelContext) issueBazelCommand(command string, labels []string, + extraFlags ...string) (string, error) { + + cmdFlags := []string{"--output_base=" + context.outputBase, command} + cmdFlags = append(cmdFlags, labels...) + cmdFlags = append(cmdFlags, extraFlags...) + + bazelCmd := exec.Command(context.bazelPath, cmdFlags...) + bazelCmd.Dir = context.workspaceDir + bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix()) + + var stderr bytes.Buffer + bazelCmd.Stderr = &stderr + + if output, err := bazelCmd.Output(); err != nil { + return "", fmt.Errorf("bazel command failed. command: [%s], error [%s]", bazelCmd, stderr) + } else { + return string(output), nil + } +} + +// Issues commands to Bazel to receive results for all cquery requests +// queued in the BazelContext. +func (context *bazelContext) InvokeBazel() error { + context.results = make(map[cqueryKey]string) + + var labels []string + var cqueryOutput string + var err error + for val, _ := range context.requests { + labels = append(labels, val.label) + + // TODO(cparsons): Combine requests into a batch cquery request. + // TODO(cparsons): Use --query_file to avoid command line limits. + cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label}, + "--output=starlark", + "--starlark:expr="+val.starlarkExpr) + + if err != nil { + return err + } else { + context.results[val] = string(cqueryOutput) + } + } + + // Issue a build command. + // TODO(cparsons): Invoking bazel execution during soong_build should be avoided; + // bazel actions should either be added to the Ninja file and executed later, + // or bazel should handle execution. + // TODO(cparsons): Use --target_pattern_file to avoid command line limits. + _, err = context.issueBazelCommand("build", labels) + + if err != nil { + return err + } + + // Clear requests. + context.requests = map[cqueryKey]bool{} + return nil +} diff --git a/android/config.go b/android/config.go index 8df65f720..345f26ede 100644 --- a/android/config.go +++ b/android/config.go @@ -85,6 +85,8 @@ type config struct { // Only available on configs created by TestConfig TestProductVariables *productVariables + BazelContext BazelContext + PrimaryBuilder string ConfigFileName string ProductVariablesFileName string @@ -248,6 +250,8 @@ func TestConfig(buildDir string, env map[string]string, bp string, fs map[string // Set testAllowNonExistentPaths so that test contexts don't need to specify every path // passed to PathForSource or PathForModuleSrc. testAllowNonExistentPaths: true, + + BazelContext: noopBazelContext{}, } config.deviceConfig = &deviceConfig{ config: config, @@ -324,6 +328,20 @@ func TestArchConfig(buildDir string, env map[string]string, bp string, fs map[st return testConfig } +// Returns a config object which is "reset" for another bootstrap run. +// Only per-run data is reset. Data which needs to persist across multiple +// runs in the same program execution is carried over (such as Bazel context +// or environment deps). +func ConfigForAdditionalRun(c Config) (Config, error) { + newConfig, err := NewConfig(c.srcDir, c.buildDir, c.moduleListFile) + if err != nil { + return Config{}, err + } + newConfig.BazelContext = c.BazelContext + newConfig.envDeps = c.envDeps + return newConfig, nil +} + // New creates a new Config object. The srcDir argument specifies the path to // the root source directory. It also loads the config file, if found. func NewConfig(srcDir, buildDir string, moduleListFile string) (Config, error) { @@ -425,6 +443,10 @@ func NewConfig(srcDir, buildDir string, moduleListFile string) (Config, error) { Bool(config.productVariables.GcovCoverage) || Bool(config.productVariables.ClangCoverage)) + config.BazelContext, err = NewBazelContext(config) + if err != nil { + return Config{}, err + } return Config{config}, nil } diff --git a/android/makevars.go b/android/makevars.go index 374986e84..3ca7792d4 100644 --- a/android/makevars.go +++ b/android/makevars.go @@ -128,7 +128,7 @@ var _ PathContext = MakeVarsContext(nil) type MakeVarsProvider func(ctx MakeVarsContext) func RegisterMakeVarsProvider(pctx PackageContext, provider MakeVarsProvider) { - makeVarsProviders = append(makeVarsProviders, makeVarsProvider{pctx, provider}) + makeVarsInitProviders = append(makeVarsInitProviders, makeVarsProvider{pctx, provider}) } // SingletonMakeVarsProvider is a Singleton with an extra method to provide extra values to be exported to Make. @@ -142,7 +142,8 @@ type SingletonMakeVarsProvider interface { // registerSingletonMakeVarsProvider adds a singleton that implements SingletonMakeVarsProvider to the list of // MakeVarsProviders to run. func registerSingletonMakeVarsProvider(singleton SingletonMakeVarsProvider) { - makeVarsProviders = append(makeVarsProviders, makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)}) + singletonMakeVarsProviders = append(singletonMakeVarsProviders, + makeVarsProvider{pctx, SingletonmakeVarsProviderAdapter(singleton)}) } // SingletonmakeVarsProviderAdapter converts a SingletonMakeVarsProvider to a MakeVarsProvider. @@ -171,7 +172,11 @@ type makeVarsProvider struct { call MakeVarsProvider } -var makeVarsProviders []makeVarsProvider +// Collection of makevars providers that are registered in init() methods. +var makeVarsInitProviders []makeVarsProvider + +// Collection of singleton makevars providers that are not registered as part of init() methods. +var singletonMakeVarsProviders []makeVarsProvider type makeVarsContext struct { SingletonContext @@ -219,7 +224,7 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { var vars []makeVarsVariable var dists []dist var phonies []phony - for _, provider := range makeVarsProviders { + for _, provider := range append(makeVarsInitProviders) { mctx := &makeVarsContext{ SingletonContext: ctx, pctx: provider.pctx, @@ -232,6 +237,25 @@ func (s *makeVarsSingleton) GenerateBuildActions(ctx SingletonContext) { dists = append(dists, mctx.dists...) } + for _, provider := range append(singletonMakeVarsProviders) { + mctx := &makeVarsContext{ + SingletonContext: ctx, + pctx: provider.pctx, + } + + provider.call(mctx) + + vars = append(vars, mctx.vars...) + phonies = append(phonies, mctx.phonies...) + dists = append(dists, mctx.dists...) + } + + // Clear singleton makevars providers after use. Since these are in-memory + // singletons, this ensures state is reset if the build tree is processed + // multiple times. + // TODO(cparsons): Clean up makeVarsProviders to be part of the context. + singletonMakeVarsProviders = nil + ctx.VisitAllModules(func(m Module) { if provider, ok := m.(ModuleMakeVarsProvider); ok && m.Enabled() { mctx := &makeVarsContext{ diff --git a/bazel/bazelenv.sh b/bazel/bazelenv.sh new file mode 100755 index 000000000..2ca8baf98 --- /dev/null +++ b/bazel/bazelenv.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script for setting environment variables required for Bazel/Soong +# mixed builds prototype. For development use only. +# +# Usage: +# export BAZEL_PATH=[some_bazel_path] && source bazelenv.sh +# +# If BAZEL_PATH is not set, `which bazel` will be used +# to locate the appropriate bazel to use. + + +# Function to find top of the source tree (if $TOP isn't set) by walking up the +# tree. +function gettop +{ + local TOPFILE=build/soong/root.bp + if [ -n "${TOP-}" -a -f "${TOP-}/${TOPFILE}" ] ; then + # The following circumlocution ensures we remove symlinks from TOP. + (cd $TOP; PWD= /bin/pwd) + else + if [ -f $TOPFILE ] ; then + # The following circumlocution (repeated below as well) ensures + # that we record the true directory name and not one that is + # faked up with symlink names. + PWD= /bin/pwd + else + local HERE=$PWD + T= + while [ \( ! \( -f $TOPFILE \) \) -a \( $PWD != "/" \) ]; do + \cd .. + T=`PWD= /bin/pwd -P` + done + \cd $HERE + if [ -f "$T/$TOPFILE" ]; then + echo $T + fi + fi + fi +} + +BASE_DIR="$(mktemp -d)" + +if [ -z "$BAZEL_PATH" ] ; then + export BAZEL_PATH="$(which bazel)" +fi + +export USE_BAZEL=1 +export BAZEL_HOME="$BASE_DIR/bazelhome" +export BAZEL_OUTPUT_BASE="$BASE_DIR/output" +export BAZEL_WORKSPACE="$(gettop)" + +echo "USE_BAZEL=${USE_BAZEL}" +echo "BAZEL_PATH=${BAZEL_PATH}" +echo "BAZEL_HOME=${BAZEL_HOME}" +echo "BAZEL_OUTPUT_BASE=${BAZEL_OUTPUT_BASE}" +echo "BAZEL_WORKSPACE=${BAZEL_WORKSPACE}" + +mkdir -p $BAZEL_HOME +mkdir -p $BAZEL_OUTPUT_BASE diff --git a/bazel/master.WORKSPACE.bazel b/bazel/master.WORKSPACE.bazel new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index 01a39a216..7ae1c3738 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -51,30 +51,34 @@ func newNameResolver(config android.Config) *android.NameResolver { return android.NewNameResolver(exportFilter) } +func newContext(srcDir string, configuration android.Config) *android.Context { + ctx := android.NewContext() + ctx.Register() + if !shouldPrepareBuildActions() { + configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) + } + ctx.SetNameInterface(newNameResolver(configuration)) + ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) + return ctx +} + +func newConfig(srcDir string) android.Config { + configuration, err := android.NewConfig(srcDir, bootstrap.BuildDir, bootstrap.ModuleListFile) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + return configuration +} + func main() { android.ReexecWithDelveMaybe() flag.Parse() // The top-level Blueprints file is passed as the first argument. srcDir := filepath.Dir(flag.Arg(0)) - - ctx := android.NewContext() - ctx.Register() - - configuration, err := android.NewConfig(srcDir, bootstrap.BuildDir, bootstrap.ModuleListFile) - if err != nil { - fmt.Fprintf(os.Stderr, "%s", err) - os.Exit(1) - } - - if !shouldPrepareBuildActions() { - configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) - } - - ctx.SetNameInterface(newNameResolver(configuration)) - - ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) - + var ctx *android.Context + configuration := newConfig(srcDir) extraNinjaDeps := []string{configuration.ConfigFileName, configuration.ProductVariablesFileName} // Read the SOONG_DELVE again through configuration so that there is a dependency on the environment variable @@ -84,9 +88,31 @@ func main() { // enabled even if it completed successfully. extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve")) } - - bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) - + if configuration.BazelContext.BazelEnabled() { + // Bazel-enabled mode. Soong runs in two passes. + // First pass: Analyze the build tree, but only store all bazel commands + // needed to correctly evaluate the tree in the second pass. + // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite + // the incorrect results from the first pass, and file I/O is expensive. + firstCtx := newContext(srcDir, configuration) + bootstrap.Main(firstCtx.Context, configuration, extraNinjaDeps...) + // Invoke bazel commands and save results for second pass. + if err := configuration.BazelContext.InvokeBazel(); err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + // Second pass: Full analysis, using the bazel command results. Output ninja file. + secondPassConfig, err := android.ConfigForAdditionalRun(configuration) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + ctx = newContext(srcDir, secondPassConfig) + bootstrap.Main(ctx.Context, secondPassConfig, extraNinjaDeps...) + } else { + ctx = newContext(srcDir, configuration) + bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...) + } if bazelOverlayDir != "" { if err := createBazelOverlay(ctx, bazelOverlayDir); err != nil { fmt.Fprintf(os.Stderr, "%s", err) @@ -105,7 +131,7 @@ func main() { // to affect the command line of the primary builder. if shouldPrepareBuildActions() { metricsFile := filepath.Join(bootstrap.BuildDir, "soong_build_metrics.pb") - err = android.WriteMetrics(configuration, metricsFile) + err := android.WriteMetrics(configuration, metricsFile) if err != nil { fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err) os.Exit(1) diff --git a/genrule/genrule.go b/genrule/genrule.go index 4a2f81073..178587ad5 100644 --- a/genrule/genrule.go +++ b/genrule/genrule.go @@ -113,8 +113,10 @@ type generatorProperties struct { // input files to exclude Exclude_srcs []string `android:"path,arch_variant"` -} + // in bazel-enabled mode, the bazel label to evaluate instead of this module + Bazel_module string +} type Module struct { android.ModuleBase android.DefaultableModuleBase @@ -186,6 +188,20 @@ func toolDepsMutator(ctx android.BottomUpMutatorContext) { } } +// Returns true if information was available from Bazel, false if bazel invocation still needs to occur. +func (c *Module) generateBazelBuildActions(ctx android.ModuleContext, label string) bool { + bazelCtx := ctx.Config().BazelContext + filePaths, ok := bazelCtx.GetAllFiles(label) + if ok { + var bazelOutputFiles android.Paths + for _, bazelOutputFile := range filePaths { + bazelOutputFiles = append(bazelOutputFiles, android.PathForSource(ctx, bazelOutputFile)) + } + c.outputFiles = bazelOutputFiles + c.outputDeps = bazelOutputFiles + } + return ok +} func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { g.subName = ctx.ModuleSubDir() @@ -456,26 +472,29 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) { g.outputFiles = outputFiles.Paths() - // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of - // the genrules on AOSP. That will make things simpler to look at the graph in the common - // case. For larger sets of outputs, inject a phony target in between to limit ninja file - // growth. - if len(g.outputFiles) <= 6 { - g.outputDeps = g.outputFiles - } else { - phonyFile := android.PathForModuleGen(ctx, "genrule-phony") - - ctx.Build(pctx, android.BuildParams{ - Rule: blueprint.Phony, - Output: phonyFile, - Inputs: g.outputFiles, - }) - - g.outputDeps = android.Paths{phonyFile} + bazelModuleLabel := g.properties.Bazel_module + bazelActionsUsed := false + if ctx.Config().BazelContext.BazelEnabled() && len(bazelModuleLabel) > 0 { + bazelActionsUsed = g.generateBazelBuildActions(ctx, bazelModuleLabel) + } + if !bazelActionsUsed { + // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of + // the genrules on AOSP. That will make things simpler to look at the graph in the common + // case. For larger sets of outputs, inject a phony target in between to limit ninja file + // growth. + if len(g.outputFiles) <= 6 { + g.outputDeps = g.outputFiles + } else { + phonyFile := android.PathForModuleGen(ctx, "genrule-phony") + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: phonyFile, + Inputs: g.outputFiles, + }) + g.outputDeps = android.Paths{phonyFile} + } } - } - func hashSrcFiles(srcFiles android.Paths) string { h := sha256.New() for _, src := range srcFiles { diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go index 4b36600a6..c692019c3 100644 --- a/genrule/genrule_test.go +++ b/genrule/genrule_test.go @@ -721,6 +721,39 @@ func TestGenruleDefaults(t *testing.T) { } } +func TestGenruleWithBazel(t *testing.T) { + bp := ` + genrule { + name: "foo", + out: ["one.txt", "two.txt"], + bazel_module: "//foo/bar:bar", + } + ` + + config := testConfig(bp, nil) + config.BazelContext = android.MockBazelContext{ + AllFiles: map[string][]string{ + "//foo/bar:bar": []string{"bazelone.txt", "bazeltwo.txt"}}} + + ctx := testContext(config) + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + if errs == nil { + _, errs = ctx.PrepareBuildActions(config) + } + if errs != nil { + t.Fatal(errs) + } + gen := ctx.ModuleForTests("foo", "").Module().(*Module) + + expectedOutputFiles := []string{"bazelone.txt", "bazeltwo.txt"} + if !reflect.DeepEqual(gen.outputFiles.Strings(), expectedOutputFiles) { + t.Errorf("Expected output files: %q, actual: %q", expectedOutputFiles, gen.outputFiles) + } + if !reflect.DeepEqual(gen.outputDeps.Strings(), expectedOutputFiles) { + t.Errorf("Expected output deps: %q, actual: %q", expectedOutputFiles, gen.outputDeps) + } +} + type testTool struct { android.ModuleBase outputFile android.Path