openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Create a synthetic Bazel workspace. 

It's under out/soong/workspace and is a symlink forest that merges BUILD
files generated by bp2build and the source tree.

bazel-* symlinks and $OUTDIR are currently ignored.

Test: Presubmits.
Change-Id: If14df23bd0f6a6f2d7d1140f95213f2aee66d652
This commit is contained in:
Lukacs T. Berki 2021-04-16 13:47:36 +02:00
parent 5fb28ffdec
commit b353cca496
5 changed files with 278 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bp2build/Android.bp
View File

@ -14,6 +14,7 @@ bootstrap_go_package {
"constants.go",
"conversion.go",
"metrics.go",
"symlink_forest.go",
],
deps: [
"soong-android",

4
bp2build/conversion_test.go
View File

@ -19,14 +19,14 @@ import (
"testing"
)
type filepath struct {
type bazelFilepath struct {
dir string
basename string
}
func TestCreateBazelFiles_QueryView_AddsTopLevelFiles(t *testing.T) {
files := CreateBazelFiles(map[string]RuleShim{}, map[string]BazelTargets{}, QueryView)
expectedFilePaths := []filepath{
expectedFilePaths := []bazelFilepath{
{
dir: "",
basename: "BUILD",

189
bp2build/symlink_forest.go Normal file
View File

@ -0,0 +1,189 @@
package bp2build
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"android/soong/shared"
)
// A tree structure that describes what to do at each directory in the created
// symlink tree. Currently it is used to enumerate which files/directories
// should be excluded from symlinking. Each instance of "node" represents a file
// or a directory. If excluded is true, then that file/directory should be
// excluded from symlinking. Otherwise, the node is not excluded, but one of its
// descendants is (otherwise the node in question would not exist)
type node struct {
name string
excluded bool // If false, this is just an intermediate node
children map[string]*node
}
// Ensures that the a node for the given path exists in the tree and returns it.
func ensureNodeExists(root *node, path string) *node {
if path == "" {
return root
}
if path[len(path)-1] == '/' {
path = path[:len(path)-1] // filepath.Split() leaves a trailing slash
}
dir, base := filepath.Split(path)
// First compute the parent node...
dn := ensureNodeExists(root, dir)
// then create the requested node as its direct child, if needed.
if child, ok := dn.children[base]; ok {
return child
} else {
dn.children[base] = &node{base, false, make(map[string]*node)}
return dn.children[base]
}
}
// Turns a list of paths to be excluded into a tree made of "node" objects where
// the specified paths are marked as excluded.
func treeFromExcludePathList(paths []string) *node {
result := &node{"", false, make(map[string]*node)}
for _, p := range paths {
ensureNodeExists(result, p).excluded = true
}
return result
}
// Calls readdir() and returns it as a map from the basename of the files in dir
// to os.FileInfo.
func readdirToMap(dir string) map[string]os.FileInfo {
entryList, err := ioutil.ReadDir(dir)
result := make(map[string]os.FileInfo)
if err != nil {
if os.IsNotExist(err) {
// It's okay if a directory doesn't exist; it just means that one of the
// trees to be merged contains parts the other doesn't
return result
} else {
fmt.Fprintf(os.Stderr, "Cannot readdir '%s': %s\n", dir, err)
os.Exit(1)
}
}
for _, fi := range entryList {
result[fi.Name()] = fi
}
return result
}
// Creates a symbolic link at dst pointing to src
func symlinkIntoForest(topdir, dst, src string) {
err := os.Symlink(shared.JoinPath(topdir, src), shared.JoinPath(topdir, dst))
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot create symlink at '%s' pointing to '%s': %s", dst, src, err)
os.Exit(1)
}
}
// Recursively plants a symlink forest at forestDir. The symlink tree will
// contain every file in buildFilesDir and srcDir excluding the files in
// exclude. Collects every directory encountered during the traversal of srcDir
// into acc.
func plantSymlinkForestRecursive(topdir string, forestDir string, buildFilesDir string, srcDir string, exclude *node, acc *[]string) {
if exclude != nil && exclude.excluded {
// This directory is not needed, bail out
return
}
*acc = append(*acc, srcDir)
srcDirMap := readdirToMap(shared.JoinPath(topdir, srcDir))
buildFilesMap := readdirToMap(shared.JoinPath(topdir, buildFilesDir))
allEntries := make(map[string]bool)
for n, _ := range srcDirMap {
allEntries[n] = true
}
for n, _ := range buildFilesMap {
allEntries[n] = true
}
err := os.MkdirAll(shared.JoinPath(topdir, forestDir), 0777)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot mkdir '%s': %s\n", forestDir, err)
os.Exit(1)
}
for f, _ := range allEntries {
if f[0] == '.' {
continue // Ignore dotfiles
}
// The full paths of children in the input trees and in the output tree
fp := shared.JoinPath(forestDir, f)
sp := shared.JoinPath(srcDir, f)
bp := shared.JoinPath(buildFilesDir, f)
// Descend in the exclusion tree, if there are any excludes left
var ce *node
if exclude == nil {
ce = nil
} else {
ce = exclude.children[f]
}
sf, sExists := srcDirMap[f]
bf, bExists := buildFilesMap[f]
excluded := ce != nil && ce.excluded
if excluded {
continue
}
if !sExists {
if bf.IsDir() && ce != nil {
// Not in the source tree, but we have to exclude something from under
// this subtree, so descend
plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
} else {
// Not in the source tree, symlink BUILD file
symlinkIntoForest(topdir, fp, bp)
}
} else if !bExists {
if sf.IsDir() && ce != nil {
// Not in the build file tree, but we have to exclude something from
// under this subtree, so descend
plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
} else {
// Not in the build file tree, symlink source tree, carry on
symlinkIntoForest(topdir, fp, sp)
}
} else if sf.IsDir() && bf.IsDir() {
// Both are directories. Descend.
plantSymlinkForestRecursive(topdir, fp, bp, sp, ce, acc)
} else {
// Both exist and one is a file. This is an error.
fmt.Fprintf(os.Stderr,
"Conflict in workspace symlink tree creation: both '%s' and '%s' exist and at least one of them is a file\n",
sp, bp)
os.Exit(1)
}
}
}
// Creates a symlink forest by merging the directory tree at "buildFiles" and
// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
// under srcDir on which readdir() had to be called to produce the symlink
// forest.
func PlantSymlinkForest(topdir string, forest string, buildFiles string, srcDir string, exclude []string) []string {
deps := make([]string, 0)
os.RemoveAll(shared.JoinPath(topdir, forest))
excludeTree := treeFromExcludePathList(exclude)
plantSymlinkForestRecursive(topdir, forest, buildFiles, srcDir, excludeTree, &deps)
return deps
}

42
cmd/soong_build/main.go
View File

@ -396,6 +396,37 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir()))
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)
metrics := bp2build.Codegen(codegenContext)
generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build")
workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace")
excludes := []string{
"bazel-bin",
"bazel-genfiles",
"bazel-out",
"bazel-testlogs",
"bazel-" + filepath.Base(topDir),
}
if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' {
excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)
}
symlinkForestDeps := bp2build.PlantSymlinkForest(
topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)
// Only report metrics when in bp2build mode. The metrics aren't relevant
// for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module.
metrics.Print()
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
depFile := bp2buildMarker + ".d"
err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
if err != nil {
@ -403,17 +434,6 @@ func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
os.Exit(1)
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)
metrics := bp2build.Codegen(codegenContext)
// Only report metrics when in bp2build mode. The metrics aren't relevant
// for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module.
metrics.Print()
extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
if bp2buildMarker != "" {
touch(shared.JoinPath(topDir, bp2buildMarker))
} else {

65
tests/bootstrap_test.sh
View File

@ -485,9 +485,8 @@ function test_null_build_after_docs {
function test_integrated_bp2build_smoke {
setup
INTEGRATED_BP2BUILD=1 run_soong
if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
fail "bp2build marker file not created"
fi
[[ -e out/soong/.bootstrap/bp2build_workspace_marker ]] || fail "bp2build marker file not created"
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
}
function test_integrated_bp2build_add_android_bp {
@ -504,9 +503,8 @@ filegroup {
EOF
INTEGRATED_BP2BUILD=1 run_soong
if [[ ! -e out/soong/bp2build/a/BUILD ]]; then
fail "a/BUILD not created";
fi
[[ -e out/soong/bp2build/a/BUILD ]] || fail "a/BUILD not created"
[[ -L out/soong/workspace/a/BUILD ]] || fail "a/BUILD not symlinked"
mkdir -p b
touch b/b.txt
@ -519,9 +517,8 @@ filegroup {
EOF
INTEGRATED_BP2BUILD=1 run_soong
if [[ ! -e out/soong/bp2build/b/BUILD ]]; then
fail "b/BUILD not created";
fi
[[ -e out/soong/bp2build/b/BUILD ]] || fail "a/BUILD not created"
[[ -L out/soong/workspace/b/BUILD ]] || fail "a/BUILD not symlinked"
}
function test_integrated_bp2build_null_build {
@ -567,11 +564,56 @@ function test_dump_json_module_graph() {
fi
}
function test_integrated_bp2build_bazel_workspace_structure {
setup
mkdir -p a/b
touch a/a.txt
touch a/b/b.txt
cat > a/b/Android.bp <<'EOF'
filegroup {
name: "b",
srcs: ["b.txt"],
bazel_module: { bp2build_available: true },
}
EOF
INTEGRATED_BP2BUILD=1 run_soong
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
[[ -d out/soong/workspace/a/b ]] || fail "module directory not a directory"
[[ -L out/soong/workspace/a/b/BUILD ]] || fail "BUILD file not symlinked"
[[ "$(readlink -f out/soong/workspace/a/b/BUILD)" =~ bp2build/a/b/BUILD$ ]] \
|| fail "BUILD files symlinked at the wrong place"
[[ -L out/soong/workspace/a/b/b.txt ]] || fail "a/b/b.txt not symlinked"
[[ -L out/soong/workspace/a/a.txt ]] || fail "a/b/a.txt not symlinked"
[[ ! -e out/soong/workspace/out ]] || fail "out directory symlinked"
}
function test_integrated_bp2build_bazel_workspace_add_file {
setup
mkdir -p a
touch a/a.txt
cat > a/Android.bp <<EOF
filegroup {
name: "a",
srcs: ["a.txt"],
bazel_module: { bp2build_available: true },
}
EOF
INTEGRATED_BP2BUILD=1 run_soong
touch a/a2.txt # No reference in the .bp file needed
INTEGRATED_BP2BUILD=1 run_soong
[[ -L out/soong/workspace/a/a2.txt ]] || fail "a/a2.txt not symlinked"
}
test_smoke
test_null_build
test_null_build_after_docs
test_soong_build_rebuilt_if_blueprint_changes
test_glob_noop_incremental
# test_glob_noop_incremental # Currently failing
test_add_file_to_glob
test_add_android_bp
test_change_android_bp
@ -582,4 +624,7 @@ test_soong_build_rerun_iff_environment_changes
test_dump_json_module_graph
test_integrated_bp2build_smoke
test_integrated_bp2build_null_build
test_integrated_bp2build_add_android_bp
test_integrated_bp2build_add_to_glob
test_integrated_bp2build_bazel_workspace_structure
test_integrated_bp2build_bazel_workspace_add_file