diff --git a/bp2build/Android.bp b/bp2build/Android.bp index d2a872976..abd79f56f 100644 --- a/bp2build/Android.bp +++ b/bp2build/Android.bp @@ -14,6 +14,7 @@ bootstrap_go_package { "constants.go", "conversion.go", "metrics.go", + "symlink_forest.go", ], deps: [ "soong-android", diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go index 9fd68172c..262a488f3 100644 --- a/bp2build/conversion_test.go +++ b/bp2build/conversion_test.go @@ -19,14 +19,14 @@ import ( "testing" ) -type filepath struct { +type bazelFilepath struct { dir string basename string } func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) { files := CreateBazelFiles(map[string]RuleShim{}, map[string]BazelTargets{}, QueryView) - expectedFilePaths := []filepath{ + expectedFilePaths := []bazelFilepath{ { dir: "", basename: "BUILD", diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go new file mode 100644 index 000000000..80ad3b6ec --- /dev/null +++ b/bp2build/symlink_forest.go @@ -0,0 +1,189 @@ +package bp2build + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "android/soong/shared" +) + +// A tree structure that describes what to do at each directory in the created +// symlink tree. Currently it is used to enumerate which files/directories +// should be excluded from symlinking. Each instance of "node" represents a file +// or a directory. If excluded is true, then that file/directory should be +// excluded from symlinking. Otherwise, the node is not excluded, but one of its +// descendants is (otherwise the node in question would not exist) +type node struct { + name string + excluded bool // If false, this is just an intermediate node + children map[string]*node +} + +// Ensures that the a node for the given path exists in the tree and returns it. +func ensureNodeExists(root *node, path string) *node { + if path == "" { + return root + } + + if path[len(path)-1] == '/' { + path = path[:len(path)-1] // filepath.Split() leaves a trailing slash + } + + dir, base := filepath.Split(path) + + // First compute the parent node... + dn := ensureNodeExists(root, dir) + + // then create the requested node as its direct child, if needed. + if child, ok := dn.children[base]; ok { + return child + } else { + dn.children[base] = &node{base, false, make(map[string]*node)} + return dn.children[base] + } +} + +// Turns a list of paths to be excluded into a tree made of "node" objects where +// the specified paths are marked as excluded. +func treeFromExcludePathList(paths []string) *node { + result := &node{"", false, make(map[string]*node)} + + for _, p := range paths { + ensureNodeExists(result, p).excluded = true + } + + return result +} + +// Calls readdir() and returns it as a map from the basename of the files in dir +// to os.FileInfo. +func readdirToMap(dir string) map[string]os.FileInfo { + entryList, err := ioutil.ReadDir(dir) + result := make(map[string]os.FileInfo) + + if err != nil { + if os.IsNotExist(err) { + // It's okay if a directory doesn't exist; it just means that one of the + // trees to be merged contains parts the other doesn't + return result + } else { + fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err) + os.Exit(1) + } + } + + for _, fi := range entryList { + result[fi.Name()] = fi + } + + return result +} + +// Creates a symbolic link at dst pointing to src +func symlinkIntoForest(topdir, dst, src string) { + err := os.Symlink(shared.JoinPath(topdir, src), shared.JoinPath(topdir, dst)) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err) + os.Exit(1) + } +} + +// Recursively plants a symlink forest at forestDir. The symlink tree will +// contain every file in buildFilesDir and srcDir excluding the files in +// exclude. Collects every directory encountered during the traversal of srcDir +// into acc. +func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir string, srcDir string, exclude *node, acc *[]string) { + if exclude != nil && exclude.excluded { + // This directory is not needed, bail out + return + } + + *acc = append(*acc, srcDir) + srcDirMap := readdirToMap(shared.JoinPath(topdir, srcDir)) + buildFilesMap := readdirToMap(shared.JoinPath(topdir, buildFilesDir)) + + allEntries := make(map[string]bool) + for n, _ := range srcDirMap { + allEntries[n] = true + } + + for n, _ := range buildFilesMap { + allEntries[n] = true + } + + err := os.MkdirAll(shared.JoinPath(topdir, forestDir), 0777) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err) + os.Exit(1) + } + + for f, _ := range allEntries { + if f[0] == '.' { + continue // Ignore dotfiles + } + + // The full paths of children in the input trees and in the output tree + fp := shared.JoinPath(forestDir, f) + sp := shared.JoinPath(srcDir, f) + bp := shared.JoinPath(buildFilesDir, f) + + // Descend in the exclusion tree, if there are any excludes left + var ce *node + if exclude == nil { + ce = nil + } else { + ce = exclude.children[f] + } + + sf, sExists := srcDirMap[f] + bf, bExists := buildFilesMap[f] + excluded := ce != nil && ce.excluded + + if excluded { + continue + } + + if !sExists { + if bf.IsDir() && ce != nil { + // Not in the source tree, but we have to exclude something from under + // this subtree, so descend + plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc) + } else { + // Not in the source tree, symlink BUILD file + symlinkIntoForest(topdir, fp, bp) + } + } else if !bExists { + if sf.IsDir() && ce != nil { + // Not in the build file tree, but we have to exclude something from + // under this subtree, so descend + plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc) + } else { + // Not in the build file tree, symlink source tree, carry on + symlinkIntoForest(topdir, fp, sp) + } + } else if sf.IsDir() && bf.IsDir() { + // Both are directories. Descend. + plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc) + } else { + // Both exist and one is a file. This is an error. + fmt.Fprintf(os.Stderr, + "Conflict in workspace symlink tree creation: both '%s' and '%s' exist and at least one of them is a file\n", + sp, bp) + os.Exit(1) + } + } +} + +// Creates a symlink forest by merging the directory tree at "buildFiles" and +// "srcDir" while excluding paths listed in "exclude". Returns the set of paths +// under srcDir on which readdir() had to be called to produce the symlink +// forest. +func PlantSymlinkForest(topdir string, forest string, buildFiles string, srcDir string, exclude []string) []string { + deps := make([]string, 0) + os.RemoveAll(shared.JoinPath(topdir, forest)) + excludeTree := treeFromExcludePathList(exclude) + plantSymlinkForestRecursive(topdir, forest, buildFiles, srcDir, excludeTree, &deps) + return deps +} diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go index c9cab13f5..5bccf58c2 100644 --- a/cmd/soong_build/main.go +++ b/cmd/soong_build/main.go @@ -396,6 +396,37 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir())) } + // Run the code-generation phase to convert BazelTargetModules to BUILD files + // and print conversion metrics to the user. + codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) + metrics := bp2build.Codegen(codegenContext) + + generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build") + workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace") + + excludes := []string{ + "bazel-bin", + "bazel-genfiles", + "bazel-out", + "bazel-testlogs", + "bazel-" + filepath.Base(topDir), + } + + if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' { + excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir) + } + + symlinkForestDeps := bp2build.PlantSymlinkForest( + topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes) + + // Only report metrics when in bp2build mode. The metrics aren't relevant + // for queryview, since that's a total repo-wide conversion and there's a + // 1:1 mapping for each module. + metrics.Print() + + ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) + ninjaDeps = append(ninjaDeps, symlinkForestDeps...) + depFile := bp2buildMarker + ".d" err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps) if err != nil { @@ -403,17 +434,6 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) { os.Exit(1) } - // Run the code-generation phase to convert BazelTargetModules to BUILD files - // and print conversion metrics to the user. - codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) - metrics := bp2build.Codegen(codegenContext) - - // Only report metrics when in bp2build mode. The metrics aren't relevant - // for queryview, since that's a total repo-wide conversion and there's a - // 1:1 mapping for each module. - metrics.Print() - - extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...) if bp2buildMarker != "" { touch(shared.JoinPath(topDir, bp2buildMarker)) } else { diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh index 7265ac5ae..e45df061b 100755 --- a/tests/bootstrap_test.sh +++ b/tests/bootstrap_test.sh @@ -485,9 +485,8 @@ function test_null_build_after_docs { function test_integrated_bp2build_smoke { setup INTEGRATED_BP2BUILD=1 run_soong - if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then - fail "bp2build marker file not created" - fi + [[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created" + [[ -e out/soong/workspace ]] || fail "Bazel workspace not created" } function test_integrated_bp2build_add_android_bp { @@ -504,9 +503,8 @@ filegroup { EOF INTEGRATED_BP2BUILD=1 run_soong - if [[ ! -e out/soong/bp2build/a/BUILD ]]; then - fail "a/BUILD not created"; - fi + [[ -e out/soong/bp2build/a/BUILD ]] || fail "a/BUILD not created" + [[ -L out/soong/workspace/a/BUILD ]] || fail "a/BUILD not symlinked" mkdir -p b touch b/b.txt @@ -519,9 +517,8 @@ filegroup { EOF INTEGRATED_BP2BUILD=1 run_soong - if [[ ! -e out/soong/bp2build/b/BUILD ]]; then - fail "b/BUILD not created"; - fi + [[ -e out/soong/bp2build/b/BUILD ]] || fail "a/BUILD not created" + [[ -L out/soong/workspace/b/BUILD ]] || fail "a/BUILD not symlinked" } function test_integrated_bp2build_null_build { @@ -567,11 +564,56 @@ function test_dump_json_module_graph() { fi } +function test_integrated_bp2build_bazel_workspace_structure { + setup + + mkdir -p a/b + touch a/a.txt + touch a/b/b.txt + cat > a/b/Android.bp <<'EOF' +filegroup { + name: "b", + srcs: ["b.txt"], + bazel_module: { bp2build_available: true }, +} +EOF + + INTEGRATED_BP2BUILD=1 run_soong + [[ -e out/soong/workspace ]] || fail "Bazel workspace not created" + [[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory" + [[ -L out/soong/workspace/a/b/BUILD ]] || fail "BUILD file not symlinked" + [[ "$(readlink -f out/soong/workspace/a/b/BUILD)" =~ bp2build/a/b/BUILD$ ]] \ + || fail "BUILD files symlinked at the wrong place" + [[ -L out/soong/workspace/a/b/b.txt ]] || fail "a/b/b.txt not symlinked" + [[ -L out/soong/workspace/a/a.txt ]] || fail "a/b/a.txt not symlinked" + [[ ! -e out/soong/workspace/out ]] || fail "out directory symlinked" +} + +function test_integrated_bp2build_bazel_workspace_add_file { + setup + + mkdir -p a + touch a/a.txt + cat > a/Android.bp <