From 2a4fc3ecdcb19ac0f40e1d39434a7fcbac55c005 Mon Sep 17 00:00:00 2001 From: Rupert Shuttleworth Date: Wed, 21 Apr 2021 07:10:09 -0400 Subject: [PATCH] Generate BUILD files for every directory that has an Android.bp file. Test: Added an integration test Test: bazel build --package_path=out/soong/workspace //bionic/... Change-Id: Ie34bd23ab3c5428e6c9c9919e5fb6fcb4e709adc --- android/bazel.go | 53 +++++++++++++++++++++++- bp2build/bp2build.go | 2 +- bp2build/build_conversion.go | 17 +++++++- bp2build/conversion.go | 5 +++ bp2build/testing.go | 3 +- cmd/soong_build/queryview.go | 2 +- tests/bootstrap_test.sh | 2 + tests/bp2build_bazel_test.sh | 75 ++++++++++++++++++++++++++++++++++ tests/lib.sh | 22 +++++++++- tests/mixed_mode_test.sh | 14 ++----- tests/run_integration_tests.sh | 4 +- 11 files changed, 180 insertions(+), 19 deletions(-) create mode 100755 tests/bp2build_bazel_test.sh diff --git a/android/bazel.go b/android/bazel.go index 1f7f7e6b3..6e87d5770 100644 --- a/android/bazel.go +++ b/android/bazel.go @@ -126,6 +126,42 @@ const ( ) var ( + // Do not write BUILD files for these directories + // NOTE: this is not recursive + bp2buildDoNotWriteBuildFileList = []string{ + // Don't generate these BUILD files - because external BUILD files already exist + "external/boringssl", + "external/brotli", + "external/dagger2", + "external/flatbuffers", + "external/gflags", + "external/google-fruit", + "external/grpc-grpc", + "external/grpc-grpc/test/core/util", + "external/grpc-grpc/test/cpp/common", + "external/grpc-grpc/third_party/address_sorting", + "external/nanopb-c", + "external/nos/host/generic", + "external/nos/host/generic/libnos", + "external/nos/host/generic/libnos/generator", + "external/nos/host/generic/libnos_datagram", + "external/nos/host/generic/libnos_transport", + "external/nos/host/generic/nugget/proto", + "external/perfetto", + "external/protobuf", + "external/rust/cxx", + "external/rust/cxx/demo", + "external/ruy", + "external/tensorflow", + "external/tensorflow/tensorflow/lite", + "external/tensorflow/tensorflow/lite/java", + "external/tensorflow/tensorflow/lite/kernels", + "external/tflite-support", + "external/tinyalsa_new", + "external/wycheproof", + "external/libyuv", + } + // Configure modules in these directories to enable bp2build_available: true or false by default. bp2buildDefaultConfig = Bp2BuildConfig{ "bionic": Bp2BuildDefaultTrueRecursively, @@ -190,11 +226,16 @@ var ( } // Used for quicker lookups - bp2buildModuleDoNotConvert = map[string]bool{} - mixedBuildsDisabled = map[string]bool{} + bp2buildDoNotWriteBuildFile = map[string]bool{} + bp2buildModuleDoNotConvert = map[string]bool{} + mixedBuildsDisabled = map[string]bool{} ) func init() { + for _, moduleName := range bp2buildDoNotWriteBuildFileList { + bp2buildDoNotWriteBuildFile[moduleName] = true + } + for _, moduleName := range bp2buildModuleDoNotConvertList { bp2buildModuleDoNotConvert[moduleName] = true } @@ -204,6 +245,14 @@ func init() { } } +func ShouldWriteBuildFileForDir(dir string) bool { + if _, ok := bp2buildDoNotWriteBuildFile[dir]; ok { + return false + } else { + return true + } +} + // MixedBuildsEnabled checks that a module is ready to be replaced by a // converted or handcrafted Bazel target. func (b *BazelModuleBase) MixedBuildsEnabled(ctx BazelConversionPathContext) bool { diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go index 007d6d8b0..f1bf64820 100644 --- a/bp2build/bp2build.go +++ b/bp2build/bp2build.go @@ -28,7 +28,7 @@ func Codegen(ctx *CodegenContext) CodegenMetrics { outputDir := android.PathForOutput(ctx, "bp2build") android.RemoveAllOutputDir(outputDir) - buildToTargets, metrics := GenerateBazelTargets(ctx) + buildToTargets, metrics := GenerateBazelTargets(ctx, true) filesToWrite := CreateBazelFiles(nil, buildToTargets, ctx.mode) diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go index b7a28100b..08790d13e 100644 --- a/bp2build/build_conversion.go +++ b/bp2build/build_conversion.go @@ -176,7 +176,7 @@ func propsToAttributes(props map[string]string) string { return attributes } -func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, CodegenMetrics) { +func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (map[string]BazelTargets, CodegenMetrics) { buildFileToTargets := make(map[string]BazelTargets) buildFileToAppend := make(map[string]bool) @@ -185,9 +185,13 @@ func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, Codegen RuleClassCount: make(map[string]int), } + dirs := make(map[string]bool) + bpCtx := ctx.Context() bpCtx.VisitAllModules(func(m blueprint.Module) { dir := bpCtx.ModuleDir(m) + dirs[dir] = true + var t BazelTarget switch ctx.Mode() { @@ -230,6 +234,17 @@ func GenerateBazelTargets(ctx *CodegenContext) (map[string]BazelTargets, Codegen buildFileToTargets[dir] = append(buildFileToTargets[dir], t) }) + if generateFilegroups { + // Add a filegroup target that exposes all sources in the subtree of this package + // NOTE: This also means we generate a BUILD file for every Android.bp file (as long as it has at least one module) + for dir, _ := range dirs { + buildFileToTargets[dir] = append(buildFileToTargets[dir], BazelTarget{ + name: "bp2build_all_srcs", + content: `filegroup(name = "bp2build_all_srcs", srcs = glob(["**/*"]))`, + ruleClass: "filegroup", + }) + } + } return buildFileToTargets, metrics } diff --git a/bp2build/conversion.go b/bp2build/conversion.go index 6b47cd1f1..114b66823 100644 --- a/bp2build/conversion.go +++ b/bp2build/conversion.go @@ -2,6 +2,7 @@ package bp2build import ( "android/soong/android" + "fmt" "reflect" "sort" "strings" @@ -48,6 +49,10 @@ func CreateBazelFiles( func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile { files := make([]BazelFile, 0, len(buildToTargets)) for _, dir := range android.SortedStringKeys(buildToTargets) { + if !android.ShouldWriteBuildFileForDir(dir) { + fmt.Printf("[bp2build] Not writing generated BUILD file for dir: '%s'\n", dir) + continue + } targets := buildToTargets[dir] sort.Slice(targets, func(i, j int) bool { // this will cover all bp2build generated targets diff --git a/bp2build/testing.go b/bp2build/testing.go index ef3a78fd3..c4661eaca 100644 --- a/bp2build/testing.go +++ b/bp2build/testing.go @@ -183,6 +183,7 @@ func customBp2BuildMutatorFromStarlark(ctx android.TopDownMutatorContext) { // Helper method for tests to easily access the targets in a dir. func generateBazelTargetsForDir(codegenCtx *CodegenContext, dir string) BazelTargets { - buildFileToTargets, _ := GenerateBazelTargets(codegenCtx) + // TODO: Set generateFilegroups to true and/or remove the generateFilegroups argument completely + buildFileToTargets, _ := GenerateBazelTargets(codegenCtx, false) return buildFileToTargets[dir] } diff --git a/cmd/soong_build/queryview.go b/cmd/soong_build/queryview.go index edc8a422d..e2ce77202 100644 --- a/cmd/soong_build/queryview.go +++ b/cmd/soong_build/queryview.go @@ -27,7 +27,7 @@ func createBazelQueryView(ctx *bp2build.CodegenContext, bazelQueryViewDir string // Ignore metrics reporting for queryview, since queryview is already a full-repo // conversion and can use data from bazel query directly. - buildToTargets, _ := bp2build.GenerateBazelTargets(ctx) + buildToTargets, _ := bp2build.GenerateBazelTargets(ctx, true) filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets, bp2build.QueryView) for _, f := range filesToWrite { diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index a3429ddca..52a4a7cbe 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +set -o pipefail + # This test exercises the bootstrapping process of the build system # in a source tree that only contains enough files for Bazel and Soong to work. diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh new file mode 100755 index 000000000..082cd0671 --- /dev/null +++ b/tests/bp2build_bazel_test.sh @@ -0,0 +1,75 @@ +#!/bin/bash -eu + +set -o pipefail + +# Test that bp2build and Bazel can play nicely together + +source "$(dirname "$0")/lib.sh" + +function test_bp2build_generates_all_buildfiles { + setup + create_mock_bazel + + mkdir -p foo/convertible_soong_module + cat > foo/convertible_soong_module/Android.bp <<'EOF' +genrule { + name: "the_answer", + cmd: "echo '42' > $(out)", + out: [ + "the_answer.txt", + ], + bazel_module: { + bp2build_available: true, + }, + } +EOF + + mkdir -p foo/unconvertible_soong_module + cat > foo/unconvertible_soong_module/Android.bp <<'EOF' +genrule { + name: "not_the_answer", + cmd: "echo '43' > $(out)", + out: [ + "not_the_answer.txt", + ], + bazel_module: { + bp2build_available: false, + }, + } +EOF + + run_bp2build + + if [[ ! -f "./out/soong/workspace/foo/convertible_soong_module/BUILD" ]]; then + fail "./out/soong/workspace/foo/convertible_soong_module/BUILD was not generated" + fi + + if [[ ! -f "./out/soong/workspace/foo/unconvertible_soong_module/BUILD" ]]; then + fail "./out/soong/workspace/foo/unconvertible_soong_module/BUILD was not generated" + fi + + if ! grep "the_answer" "./out/soong/workspace/foo/convertible_soong_module/BUILD"; then + fail "missing BUILD target the_answer in convertible_soong_module/BUILD" + fi + + if grep "not_the_answer" "./out/soong/workspace/foo/unconvertible_soong_module/BUILD"; then + fail "found unexpected BUILD target not_the_answer in unconvertible_soong_module/BUILD" + fi + + if ! grep "filegroup" "./out/soong/workspace/foo/unconvertible_soong_module/BUILD"; then + fail "missing filegroup in unconvertible_soong_module/BUILD" + fi + + # NOTE: We don't actually use the extra BUILD file for anything here + run_bazel build --package_path=out/soong/workspace //foo/... + + local the_answer_file="bazel-out/k8-fastbuild/bin/foo/convertible_soong_module/the_answer.txt" + if [[ ! -f "${the_answer_file}" ]]; then + fail "Expected '${the_answer_file}' to be generated, but was missing" + fi + if ! grep 42 "${the_answer_file}"; then + fail "Expected to find 42 in '${the_answer_file}'" + fi +} + +test_bp2build_generates_all_buildfiles diff --git a/tests/lib.sh b/tests/lib.sh index 3795dfc5d..e561a3d49 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +set -o pipefail + HARDWIRED_MOCK_TOP= # Uncomment this to be able to view the source tree after a test is run # HARDWIRED_MOCK_TOP=/tmp/td @@ -102,7 +104,25 @@ function setup() { } function run_soong() { - build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests + build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests "$@" +} + +function create_mock_bazel() { + copy_directory build/bazel + + symlink_directory prebuilts/bazel + symlink_directory prebuilts/jdk + + symlink_file WORKSPACE + symlink_file tools/bazel +} + +run_bazel() { + tools/bazel "$@" +} + +run_bp2build() { + GENERATE_BAZEL_FILES=true build/soong/soong_ui.bash --make-mode --skip-ninja --skip-make --skip-soong-tests nothing } info "Starting Soong integration test suite $(basename $0)" diff --git a/tests/mixed_mode_test.sh b/tests/mixed_mode_test.sh index 7dbafea01..80774bf0f 100755 --- a/tests/mixed_mode_test.sh +++ b/tests/mixed_mode_test.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +set -o pipefail + # This test exercises mixed builds where Soong and Bazel cooperate in building # Android. # @@ -8,21 +10,11 @@ source "$(dirname "$0")/lib.sh" -function create_mock_bazel() { - copy_directory build/bazel - - symlink_directory prebuilts/bazel - symlink_directory prebuilts/jdk - - symlink_file WORKSPACE - symlink_file tools/bazel -} - function test_bazel_smoke { setup create_mock_bazel - tools/bazel info + run_bazel info } test_bazel_smoke diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 76b324b63..839957348 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -1,6 +1,8 @@ #!/bin/bash -eu +set -o pipefail + TOP="$(readlink -f "$(dirname "$0")"/../../..)" "$TOP/build/soong/tests/bootstrap_test.sh" "$TOP/build/soong/tests/mixed_mode_test.sh" - +"$TOP/build/soong/tests/bp2build_bazel_test.sh"