Create a AOSP Bazel overlay workspace with Soong

The Bazel overlay is a directory at out/soong/bazel_overlay that
replicates the layout of the AOSP Soong module tree, but as a Bazel
workspace. Each Soong module variant is represented as a BUILD target
created with the `soong_module` rule.

To create this overlay, run `m bazel_overlay`.

A `soong_module` target can depend on other `soong_module` targets.
These dependencies replicate each module's `directDeps` in the Blueprint
graph, just before `PrepareBuildActions`.

This enables users to use bazel query as a way to introspect the Soong
module graph. For example,

- Direct reverse dependencies of //bionic/libc:generated_android_ids in
//bionic/libc/...:

$ bazel query 'rdeps(//bionic/libc/...,
//bionic/libc:generated_android_ids, 1)'
//bionic/libc:libc_bionic_ndk--android_recovery_arm_armv7-a-neon_static
//bionic/libc:libc_bionic_ndk--android_ramdisk_arm_armv7-a-neon_static
//bionic/libc:libc_bionic_ndk--android_arm_armv7-a-neon_static_com.android.runtime
//bionic/libc:libc_bionic_ndk--android_arm_armv7-a-neon_static
//bionic/libc:generated_android_ids

- Why does com.android.runtime depend on lzma?

$ bazel query
'somepath(//bionic/apex:com.android.runtime--android_common_com.android.runtime_image,
//external/lzma/...)'
//bionic/apex:com.android.runtime--android_common_com.android.runtime_image
//bionic/libc/malloc_debug:libc_malloc_debug--android_arm_armv7-a-neon_shared_com.android.runtime
//system/core/libunwindstack:libunwindstack--android_arm_armv7-a-neon_shared_com.android.runtime
//external/lzma/C:liblzma--android_arm_armv7-a-neon_shared_com.android.runtime

- What does the dep graph of //bionic/libc:crtbegin_so look like?

$ bazel query
'deps(//bionic/libc:crtbegin_so--android_arm_armv7-a-neon)'
--output=graph > graph.in && dot -Tpng < graph.in > graph.png

https://photos.app.goo.gl/DfsdoFRNsRjGwTmy8

Test:  croot && m bazel_overlay && cd out/soong/bazel_overlay && bazel
query //... && bazel query 'rdeps(//bionic/libc/...,
//bionic/libc:generated_android_ids, 1)'

Signed-off-by: Jingwen Chen <jingwen@google.com>
Change-Id: I3bf40309bfb2d963bb8a688706385a57ee304c37#

android/Android.bp

@@ -15,6 +15,7 @@ bootstrap_go_package {
         "apex.go",
         "api_levels.go",
         "arch.go",
+        "bazel_overlay.go",
         "config.go",
         "csuite_config.go",
         "defaults.go",

android/bazel_overlay.go

@@ -0,0 +1,76 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/google/blueprint"
+)
+
+// The Bazel Overlay singleton is responsible for generating the Ninja actions
+// for calling the soong_build primary builder in the main build.ninja file.
+func init() {
+	RegisterSingletonType("bazel_overlay", BazelOverlaySingleton)
+}
+
+func BazelOverlaySingleton() Singleton {
+	return &bazelOverlaySingleton{}
+}
+
+type bazelOverlaySingleton struct{}
+
+func (c *bazelOverlaySingleton) GenerateBuildActions(ctx SingletonContext) {
+	// Create a build and rule statement, using the Bazel overlay's WORKSPACE
+	// file as the output file marker.
+	var deps Paths
+	moduleListFilePath := pathForBuildToolDep(ctx, ctx.Config().moduleListFile)
+	deps = append(deps, moduleListFilePath)
+	deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName))
+
+	bazelOverlayDirectory := PathForOutput(ctx, "bazel_overlay")
+	bazelOverlayWorkspaceFile := bazelOverlayDirectory.Join(ctx, "WORKSPACE")
+	primaryBuilder := primaryBuilderPath(ctx)
+	bazelOverlay := ctx.Rule(pctx, "bazelOverlay",
+		blueprint.RuleParams{
+			Command: fmt.Sprintf(
+				"rm -rf ${outDir}/* && %s --bazel_overlay_dir ${outDir} %s && echo WORKSPACE: `cat %s` > ${outDir}/.overlay-depfile.d",
+				primaryBuilder.String(),
+				strings.Join(os.Args[1:], " "),
+				moduleListFilePath.String(), // Use the contents of Android.bp.list as the depfile.
+			),
+			CommandDeps: []string{primaryBuilder.String()},
+			Description: fmt.Sprintf(
+				"Creating the Bazel overlay workspace with %s at $outDir",
+				primaryBuilder.Base()),
+			Deps:    blueprint.DepsGCC,
+			Depfile: "${outDir}/.overlay-depfile.d",
+		},
+		"outDir")
+
+	ctx.Build(pctx, BuildParams{
+		Rule:   bazelOverlay,
+		Output: bazelOverlayWorkspaceFile,
+		Inputs: deps,
+		Args: map[string]string{
+			"outDir": bazelOverlayDirectory.String(),
+		},
+	})
+
+	// Add a phony target for building the bazel overlay
+	ctx.Phony("bazel_overlay", bazelOverlayWorkspaceFile)
+}

cmd/soong_build/Android.bp

@@ -26,6 +26,7 @@ bootstrap_go_binary {
     srcs: [
         "main.go",
         "writedocs.go",
+        "bazel_overlay.go",
     ],
     primaryBuilder: true,
 }

cmd/soong_build/bazel_overlay.go

@@ -0,0 +1,173 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"android/soong/android"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/blueprint"
+)
+
+const (
+	soongModuleLoad = `package(default_visibility = ["//visibility:public"])
+load("//:soong_module.bzl", "soong_module")
+`
+
+	// A BUILD file target snippet representing a Soong module
+	soongModuleTarget = `soong_module(
+    name = "%s",
+    module_name = "%s",
+    module_type = "%s",
+    module_variant = "%s",
+    deps = [
+        %s
+    ],
+)
+`
+
+	// The soong_module rule implementation in a .bzl file
+	soongModuleBzl = `
+SoongModuleInfo = provider(
+    fields = {
+        "name": "Name of module",
+        "type": "Type of module",
+        "variant": "Variant of module",
+    },
+)
+
+def _soong_module_impl(ctx):
+    return [
+        SoongModuleInfo(
+            name = ctx.attr.module_name,
+            type = ctx.attr.module_type,
+            variant = ctx.attr.module_variant,
+        ),
+    ]
+
+soong_module = rule(
+    implementation = _soong_module_impl,
+    attrs = {
+        "module_name": attr.string(mandatory = True),
+        "module_type": attr.string(mandatory = True),
+        "module_variant": attr.string(),
+        "deps": attr.label_list(providers = [SoongModuleInfo]),
+    },
+)
+`
+)
+
+func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string {
+	name := ""
+	if c.ModuleSubDir(logicModule) != "" {
+		name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
+	} else {
+		name = c.ModuleName(logicModule)
+	}
+
+	return strings.Replace(name, "//", "", 1)
+}
+
+func qualifiedTargetLabel(c *blueprint.Context, logicModule blueprint.Module) string {
+	return "//" +
+		packagePath(c, logicModule) +
+		":" +
+		targetNameWithVariant(c, logicModule)
+}
+
+func packagePath(c *blueprint.Context, logicModule blueprint.Module) string {
+	return filepath.Dir(c.BlueprintFile(logicModule))
+}
+
+func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
+	blueprintCtx := ctx.Context
+	blueprintCtx.VisitAllModules(func(module blueprint.Module) {
+		buildFile, err := buildFileForModule(blueprintCtx, module)
+		if err != nil {
+			panic(err)
+		}
+
+		// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
+		// items, if the modules are added using different DependencyTag. Figure
+		// out the implications of that.
+		depLabels := map[string]bool{}
+		blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) {
+			depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true
+		})
+
+		var depLabelList string
+		for depLabel, _ := range depLabels {
+			depLabelList += "\"" + depLabel + "\",\n        "
+		}
+		buildFile.Write([]byte(
+			fmt.Sprintf(
+				soongModuleTarget,
+				targetNameWithVariant(blueprintCtx, module),
+				blueprintCtx.ModuleName(module),
+				blueprintCtx.ModuleType(module),
+				// misleading name, this actually returns the variant.
+				blueprintCtx.ModuleSubDir(module),
+				depLabelList)))
+		buildFile.Close()
+	})
+
+	if err := writeReadOnlyFile(bazelOverlayDir, "WORKSPACE", ""); err != nil {
+		return err
+	}
+
+	if err := writeReadOnlyFile(bazelOverlayDir, "BUILD", ""); err != nil {
+		return err
+	}
+
+	return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl)
+}
+
+func buildFileForModule(ctx *blueprint.Context, module blueprint.Module) (*os.File, error) {
+	// Create nested directories for the BUILD file
+	dirPath := filepath.Join(bazelOverlayDir, packagePath(ctx, module))
+	if _, err := os.Stat(dirPath); os.IsNotExist(err) {
+		os.MkdirAll(dirPath, os.ModePerm)
+	}
+	// Open the file for appending, and create it if it doesn't exist
+	f, err := os.OpenFile(
+		filepath.Join(dirPath, "BUILD.bazel"),
+		os.O_APPEND|os.O_CREATE|os.O_WRONLY,
+		0644)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the file is empty, add the load statement for the `soong_module` rule
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	if fi.Size() == 0 {
+		f.Write([]byte(soongModuleLoad + "\n"))
+	}
+
+	return f, nil
+}
+
+// The overlay directory should be read-only, sufficient for bazel query.
+func writeReadOnlyFile(dir string, baseName string, content string) error {
+	workspaceFile := filepath.Join(bazelOverlayDir, baseName)
+	// 0444 is read-only
+	return ioutil.WriteFile(workspaceFile, []byte(content), 0444)
+}

cmd/soong_build/main.go

@@ -27,10 +27,12 @@ import (
 
 var (
 	docFile         string
+	bazelOverlayDir string
 )
 
 func init() {
 	flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
+	flag.StringVar(&bazelOverlayDir, "bazel_overlay_dir", "", "path to the bazel overlay directory")
 }
 
 func newNameResolver(config android.Config) *android.NameResolver {
@@ -65,7 +67,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	if docFile != "" {
+	if !shouldPrepareBuildActions() {
 		configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
 	}
 
@@ -85,6 +87,13 @@ func main() {
 
 	bootstrap.Main(ctx.Context, configuration, extraNinjaDeps...)
 
+	if bazelOverlayDir != "" {
+		if err := createBazelOverlay(ctx, bazelOverlayDir); err != nil {
+			fmt.Fprintf(os.Stderr, "%s", err)
+			os.Exit(1)
+		}
+	}
+
 	if docFile != "" {
 		if err := writeDocs(ctx, docFile); err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
@@ -94,7 +103,7 @@ func main() {
 
 	// TODO(ccross): make this a command line argument.  Requires plumbing through blueprint
 	//  to affect the command line of the primary builder.
-	if docFile == "" {
+	if shouldPrepareBuildActions() {
 		metricsFile := filepath.Join(bootstrap.BuildDir, "soong_build_metrics.pb")
 		err = android.WriteMetrics(configuration, metricsFile)
 		if err != nil {
@@ -103,3 +112,9 @@ func main() {
 		}
 	}
 }
+
+func shouldPrepareBuildActions() bool {
+	// If we're writing soong_docs or bazel_overlay, don't write build.ninja or
+	// collect metrics.
+	return docFile == "" && bazelOverlayDir == ""
+}