Merge "Generate BUILD files for every directory that has an Android.bp file." am: 4949557d50 am: de3feaefd4

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1677620

Change-Id: I6cf5d1a5940e9b9eaa3adb778a66b2e0cbbdda63

android/bazel.go

@@ -126,6 +126,42 @@ const (
 )
 
 var (
+	// Do not write BUILD files for these directories
+	// NOTE: this is not recursive
+	bp2buildDoNotWriteBuildFileList = []string{
+		// Don't generate these BUILD files - because external BUILD files already exist
+		"external/boringssl",
+		"external/brotli",
+		"external/dagger2",
+		"external/flatbuffers",
+		"external/gflags",
+		"external/google-fruit",
+		"external/grpc-grpc",
+		"external/grpc-grpc/test/core/util",
+		"external/grpc-grpc/test/cpp/common",
+		"external/grpc-grpc/third_party/address_sorting",
+		"external/nanopb-c",
+		"external/nos/host/generic",
+		"external/nos/host/generic/libnos",
+		"external/nos/host/generic/libnos/generator",
+		"external/nos/host/generic/libnos_datagram",
+		"external/nos/host/generic/libnos_transport",
+		"external/nos/host/generic/nugget/proto",
+		"external/perfetto",
+		"external/protobuf",
+		"external/rust/cxx",
+		"external/rust/cxx/demo",
+		"external/ruy",
+		"external/tensorflow",
+		"external/tensorflow/tensorflow/lite",
+		"external/tensorflow/tensorflow/lite/java",
+		"external/tensorflow/tensorflow/lite/kernels",
+		"external/tflite-support",
+		"external/tinyalsa_new",
+		"external/wycheproof",
+		"external/libyuv",
+	}
+
 	// Configure modules in these directories to enable bp2build_available: true or false by default.
 	bp2buildDefaultConfig = Bp2BuildConfig{
 		"bionic":                Bp2BuildDefaultTrueRecursively,
@@ -190,11 +226,16 @@ var (
 	}
 
 	// Used for quicker lookups
-	bp2buildModuleDoNotConvert = map[string]bool{}
+	bp2buildDoNotWriteBuildFile = map[string]bool{}
-	mixedBuildsDisabled        = map[string]bool{}
+	bp2buildModuleDoNotConvert  = map[string]bool{}
+	mixedBuildsDisabled         = map[string]bool{}
 )
 
 func init() {
+	for _, moduleName := range bp2buildDoNotWriteBuildFileList {
+		bp2buildDoNotWriteBuildFile[moduleName] = true
+	}
+
 	for _, moduleName := range bp2buildModuleDoNotConvertList {
 		bp2buildModuleDoNotConvert[moduleName] = true
 	}
@@ -204,6 +245,14 @@ func init() {
 	}
 }
 
+func ShouldWriteBuildFileForDir(dir string) bool {
+	if _, ok := bp2buildDoNotWriteBuildFile[dir]; ok {
+		return false
+	} else {
+		return true
+	}
+}
+
 // MixedBuildsEnabled checks that a module is ready to be replaced by a
 // converted or handcrafted Bazel target.
 func (b *BazelModuleBase) MixedBuildsEnabled(ctx BazelConversionPathContext) bool {

bp2build/bp2build.go

@@ -28,7 +28,7 @@ func Codegen(ctx *CodegenContext) CodegenMetrics {
 	outputDir := android.PathForOutput(ctx, "bp2build")
 	android.RemoveAllOutputDir(outputDir)
 
-	buildToTargets, metrics := GenerateBazelTargets(ctx)
+	buildToTargets, metrics := GenerateBazelTargets(ctx, true)
 
 	filesToWrite := CreateBazelFiles(nil, buildToTargets, ctx.mode)
 

bp2build/build_conversion.go

@@ -176,7 +176,7 @@ func propsToAttributes(props map[string]string) string {
 	return attributes
 }
 
-func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, CodegenMetrics) {
+func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (map[string]BazelTargets, CodegenMetrics) {
 	buildFileToTargets := make(map[string]BazelTargets)
 	buildFileToAppend := make(map[string]bool)
 
@@ -185,9 +185,13 @@ func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, Codegen
 		RuleClassCount: make(map[string]int),
 	}
 
+	dirs := make(map[string]bool)
+
 	bpCtx := ctx.Context()
 	bpCtx.VisitAllModules(func(m blueprint.Module) {
 		dir := bpCtx.ModuleDir(m)
+		dirs[dir] = true
+
 		var t BazelTarget
 
 		switch ctx.Mode() {
@@ -230,6 +234,17 @@ func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, Codegen
 
 		buildFileToTargets[dir] = append(buildFileToTargets[dir], t)
 	})
+	if generateFilegroups {
+		// Add a filegroup target that exposes all sources in the subtree of this package
+		// NOTE: This also means we generate a BUILD file for every Android.bp file (as long as it has at least one module)
+		for dir, _ := range dirs {
+			buildFileToTargets[dir] = append(buildFileToTargets[dir], BazelTarget{
+				name:      "bp2build_all_srcs",
+				content:   `filegroup(name = "bp2build_all_srcs", srcs = glob(["**/*"]))`,
+				ruleClass: "filegroup",
+			})
+		}
+	}
 
 	return buildFileToTargets, metrics
 }

bp2build/conversion.go

@@ -2,6 +2,7 @@ package bp2build
 
 import (
 	"android/soong/android"
+	"fmt"
 	"reflect"
 	"sort"
 	"strings"
@@ -48,6 +49,10 @@ func CreateBazelFiles(
 func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {
 	files := make([]BazelFile, 0, len(buildToTargets))
 	for _, dir := range android.SortedStringKeys(buildToTargets) {
+		if !android.ShouldWriteBuildFileForDir(dir) {
+			fmt.Printf("[bp2build] Not writing generated BUILD file for dir: '%s'\n", dir)
+			continue
+		}
 		targets := buildToTargets[dir]
 		sort.Slice(targets, func(i, j int) bool {
 			// this will cover all bp2build generated targets

bp2build/testing.go

@@ -183,6 +183,7 @@ func customBp2BuildMutatorFromStarlark(ctx android.TopDownMutatorContext) {
 
 // Helper method for tests to easily access the targets in a dir.
 func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) BazelTargets {
-	buildFileToTargets, _ := GenerateBazelTargets(codegenCtx)
+	// TODO: Set generateFilegroups to true and/or remove the generateFilegroups argument completely
+	buildFileToTargets, _ := GenerateBazelTargets(codegenCtx, false)
 	return buildFileToTargets[dir]
 }

cmd/soong_build/queryview.go

@@ -27,7 +27,7 @@ func createBazelQueryView(ctx *bp2build.CodegenContext, bazelQueryViewDir string
 
 	// Ignore metrics reporting for queryview, since queryview is already a full-repo
 	// conversion and can use data from bazel query directly.
-	buildToTargets, _ := bp2build.GenerateBazelTargets(ctx)
+	buildToTargets, _ := bp2build.GenerateBazelTargets(ctx, true)
 
 	filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets, bp2build.QueryView)
 	for _, f := range filesToWrite {

tests/bootstrap_test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -eu
 
+set -o pipefail
+
 # This test exercises the bootstrapping process of the build system
 # in a source tree that only contains enough files for Bazel and Soong to work.
 

tests/bp2build_bazel_test.sh

@@ -0,0 +1,75 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+# Test that bp2build and Bazel can play nicely together
+
+source "$(dirname "$0")/lib.sh"
+
+function test_bp2build_generates_all_buildfiles {
+  setup
+  create_mock_bazel
+
+  mkdir -p foo/convertible_soong_module
+  cat > foo/convertible_soong_module/Android.bp <<'EOF'
+genrule {
+    name: "the_answer",
+    cmd: "echo '42' > $(out)",
+    out: [
+        "the_answer.txt",
+    ],
+    bazel_module: {
+        bp2build_available: true,
+    },
+  }
+EOF
+
+  mkdir -p foo/unconvertible_soong_module
+  cat > foo/unconvertible_soong_module/Android.bp <<'EOF'
+genrule {
+    name: "not_the_answer",
+    cmd: "echo '43' > $(out)",
+    out: [
+        "not_the_answer.txt",
+    ],
+    bazel_module: {
+        bp2build_available: false,
+    },
+  }
+EOF
+
+  run_bp2build
+
+  if [[ ! -f "./out/soong/workspace/foo/convertible_soong_module/BUILD" ]]; then
+    fail "./out/soong/workspace/foo/convertible_soong_module/BUILD was not generated"
+  fi
+
+  if [[ ! -f "./out/soong/workspace/foo/unconvertible_soong_module/BUILD" ]]; then
+    fail "./out/soong/workspace/foo/unconvertible_soong_module/BUILD was not generated"
+  fi
+
+  if ! grep "the_answer" "./out/soong/workspace/foo/convertible_soong_module/BUILD"; then
+    fail "missing BUILD target the_answer in convertible_soong_module/BUILD"
+  fi
+
+  if grep "not_the_answer" "./out/soong/workspace/foo/unconvertible_soong_module/BUILD"; then
+    fail "found unexpected BUILD target not_the_answer in unconvertible_soong_module/BUILD"
+  fi
+
+  if ! grep "filegroup" "./out/soong/workspace/foo/unconvertible_soong_module/BUILD"; then
+    fail "missing filegroup in unconvertible_soong_module/BUILD"
+  fi
+
+  # NOTE: We don't actually use the extra BUILD file for anything here
+  run_bazel build --package_path=out/soong/workspace //foo/...
+
+  local the_answer_file="bazel-out/k8-fastbuild/bin/foo/convertible_soong_module/the_answer.txt"
+  if [[ ! -f "${the_answer_file}" ]]; then
+    fail "Expected '${the_answer_file}' to be generated, but was missing"
+  fi
+  if ! grep 42 "${the_answer_file}"; then
+    fail "Expected to find 42 in '${the_answer_file}'"
+  fi
+}
+
+test_bp2build_generates_all_buildfiles

tests/lib.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -eu
 
+set -o pipefail
+
 HARDWIRED_MOCK_TOP=
 # Uncomment this to be able to view the source tree after a test is run
 # HARDWIRED_MOCK_TOP=/tmp/td
@@ -102,7 +104,25 @@ function setup() {
 }
 
 function run_soong() {
-  build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests
+  build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests "$@"
+}
+
+function create_mock_bazel() {
+  copy_directory build/bazel
+
+  symlink_directory prebuilts/bazel
+  symlink_directory prebuilts/jdk
+
+  symlink_file WORKSPACE
+  symlink_file tools/bazel
+}
+
+run_bazel() {
+  tools/bazel "$@"
+}
+
+run_bp2build() {
+  GENERATE_BAZEL_FILES=true build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests nothing
 }
 
 info "Starting Soong integration test suite $(basename $0)"

tests/mixed_mode_test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -eu
 
+set -o pipefail
+
 # This test exercises mixed builds where Soong and Bazel cooperate in building
 # Android.
 #
@@ -8,21 +10,11 @@
 
 source "$(dirname "$0")/lib.sh"
 
-function create_mock_bazel() {
-  copy_directory build/bazel
-
-  symlink_directory prebuilts/bazel
-  symlink_directory prebuilts/jdk
-
-  symlink_file WORKSPACE
-  symlink_file tools/bazel
-}
-
 function test_bazel_smoke {
   setup
   create_mock_bazel
 
-  tools/bazel info
+  run_bazel info
 }
 
 test_bazel_smoke

tests/run_integration_tests.sh

@@ -1,6 +1,8 @@
 #!/bin/bash -eu
 
+set -o pipefail
+
 TOP="$(readlink -f "$(dirname "$0")"/../../..)"
 "$TOP/build/soong/tests/bootstrap_test.sh"
 "$TOP/build/soong/tests/mixed_mode_test.sh"
-
+"$TOP/build/soong/tests/bp2build_bazel_test.sh"