openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge "Refactor queryview."

This commit is contained in:
Liz Kammer 2021-01-11 20:31:02 +00:00 committed by Gerrit Code Review
parent 142b28a095 2dd9ca422d
commit bbdea51767
16 changed files with 1467 additions and 959 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
android/paths.go
View File

@ -1742,6 +1742,19 @@ func WriteFileToOutputDir(path WritablePath, data []byte, perm os.FileMode) erro
return ioutil.WriteFile(absolutePath(path.String()), data, perm)
}
func RemoveAllOutputDir(path WritablePath) error {
return os.RemoveAll(absolutePath(path.String()))
}
func CreateOutputDirIfNonexistent(path WritablePath, perm os.FileMode) error {
dir := absolutePath(path.String())
if _, err := os.Stat(dir); os.IsNotExist(err) {
return os.MkdirAll(dir, os.ModePerm)
} else {
return err
}
}
func absolutePath(path string) string {
if filepath.IsAbs(path) {
return path

10
android/queryview.go
View File

@ -26,7 +26,6 @@ import (
// for calling the soong_build primary builder in the main build.ninja file.
func init() {
RegisterSingletonType("bazel_queryview", BazelQueryViewSingleton)
RegisterSingletonType("bazel_converter", BazelConverterSingleton)
}
// BazelQueryViewSingleton is the singleton responsible for registering the
@ -52,13 +51,7 @@ type bazelConverterSingleton struct{}
func generateBuildActionsForBazelConversion(ctx SingletonContext, converterMode bool) {
name := "queryview"
additionalEnvVars := ""
descriptionTemplate := "[EXPERIMENTAL, PRE-PRODUCTION] Creating the Bazel QueryView workspace with %s at $outDir"
if converterMode {
name = "bp2build"
additionalEnvVars = "CONVERT_TO_BAZEL=true"
descriptionTemplate = "[EXPERIMENTAL, PRE-PRODUCTION] Converting all Android.bp to Bazel BUILD files with %s at $outDir"
}
// Create a build and rule statement, using the Bazel QueryView's WORKSPACE
// file as the output file marker.
@ -74,9 +67,8 @@ func generateBuildActionsForBazelConversion(ctx SingletonContext, converterMode
blueprint.RuleParams{
Command: fmt.Sprintf(
"rm -rf ${outDir}/* && "+
"%s %s --bazel_queryview_dir ${outDir} %s && "+
"%s --bazel_queryview_dir ${outDir} %s && "+
"echo WORKSPACE: `cat %s` > ${outDir}/.queryview-depfile.d",
additionalEnvVars,
primaryBuilder.String(),
strings.Join(os.Args[1:], " "),
moduleListFilePath.String(), // Use the contents of Android.bp.list as the depfile.

21
android/register.go
View File

@ -35,6 +35,9 @@ type singleton struct {
var singletons []singleton
var preSingletons []singleton
var bazelConverterSingletons []singleton
var bazelConverterPreSingletons []singleton
type mutator struct {
name string
bottomUpMutator blueprint.BottomUpMutator
@ -79,6 +82,14 @@ func RegisterPreSingletonType(name string, factory SingletonFactory) {
preSingletons = append(preSingletons, singleton{name, factory})
}
func RegisterBazelConverterSingletonType(name string, factory SingletonFactory) {
bazelConverterSingletons = append(bazelConverterSingletons, singleton{name, factory})
}
func RegisterBazelConverterPreSingletonType(name string, factory SingletonFactory) {
bazelConverterPreSingletons = append(bazelConverterPreSingletons, singleton{name, factory})
}
type Context struct {
*blueprint.Context
config Config
@ -94,13 +105,17 @@ func NewContext(config Config) *Context {
// singletons, module types and mutators to register for converting Blueprint
// files to semantically equivalent BUILD files.
func (ctx *Context) RegisterForBazelConversion() {
for _, t := range bazelConverterPreSingletons {
ctx.RegisterPreSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory))
}
for _, t := range moduleTypes {
ctx.RegisterModuleType(t.name, ModuleFactoryAdaptor(t.factory))
}
bazelConverterSingleton := singleton{"bp2build", BazelConverterSingleton}
ctx.RegisterSingletonType(bazelConverterSingleton.name,
SingletonFactoryAdaptor(ctx, bazelConverterSingleton.factory))
for _, t := range bazelConverterSingletons {
ctx.RegisterSingletonType(t.name, SingletonFactoryAdaptor(ctx, t.factory))
}
registerMutatorsForBazelConversion(ctx.Context)
}

23
bp2build/Android.bp Normal file
View File

@ -0,0 +1,23 @@
bootstrap_go_package {
name: "soong-bp2build",
pkgPath: "android/soong/bp2build",
srcs: [
"androidbp_to_build_templates.go",
"bp2build.go",
"build_conversion.go",
"bzl_conversion.go",
"conversion.go",
],
deps: [
"soong-android",
],
testSrcs: [
"build_conversion_test.go",
"bzl_conversion_test.go",
"conversion_test.go",
"testing.go",
],
pluginFor: [
"soong_build",
],
}

2
cmd/soong_build/queryview_templates.go → bp2build/androidbp_to_build_templates.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package bp2build
const (
// The default `load` preamble for every generated BUILD file.

77
bp2build/bp2build.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"os"
)
// The Bazel bp2build singleton is responsible for writing .bzl files that are equivalent to
// Android.bp files that are capable of being built with Bazel.
func init() {
android.RegisterBazelConverterPreSingletonType("androidbp_to_build", AndroidBpToBuildSingleton)
}
func AndroidBpToBuildSingleton() android.Singleton {
return &androidBpToBuildSingleton{
name: "bp2build",
}
}
type androidBpToBuildSingleton struct {
name string
outputDir android.OutputPath
}
func (s *androidBpToBuildSingleton) GenerateBuildActions(ctx android.SingletonContext) {
s.outputDir = android.PathForOutput(ctx, s.name)
android.RemoveAllOutputDir(s.outputDir)
if !ctx.Config().IsEnvTrue("CONVERT_TO_BAZEL") {
return
}
ruleShims := CreateRuleShims(android.ModuleTypeFactories())
buildToTargets := GenerateSoongModuleTargets(ctx)
filesToWrite := CreateBazelFiles(ruleShims, buildToTargets)
for _, f := range filesToWrite {
if err := s.writeFile(ctx, f); err != nil {
ctx.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err)
}
}
}
func (s *androidBpToBuildSingleton) getOutputPath(ctx android.PathContext, dir string) android.OutputPath {
return s.outputDir.Join(ctx, dir)
}
func (s *androidBpToBuildSingleton) writeFile(ctx android.PathContext, f BazelFile) error {
return writeReadOnlyFile(ctx, s.getOutputPath(ctx, f.Dir), f.Basename, f.Contents)
}
// The auto-conversion directory should be read-only, sufficient for bazel query. The files
// are not intended to be edited by end users.
func writeReadOnlyFile(ctx android.PathContext, dir android.OutputPath, baseName, content string) error {
android.CreateOutputDirIfNonexistent(dir, os.ModePerm)
pathToFile := dir.Join(ctx, baseName)
// 0444 is read-only
err := android.WriteFileToOutputDir(pathToFile, []byte(content), 0444)
return err
}

305
bp2build/build_conversion.go Normal file
View File

@ -0,0 +1,305 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"fmt"
"reflect"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
)
type BazelAttributes struct {
Attrs map[string]string
}
type BazelTarget struct {
name string
content string
}
type bpToBuildContext interface {
ModuleName(module blueprint.Module) string
ModuleDir(module blueprint.Module) string
ModuleSubDir(module blueprint.Module) string
ModuleType(module blueprint.Module) string
VisitAllModulesBlueprint(visit func(blueprint.Module))
VisitDirectDeps(module android.Module, visit func(android.Module))
}
// props is an unsorted map. This function ensures that
// the generated attributes are sorted to ensure determinism.
func propsToAttributes(props map[string]string) string {
var attributes string
for _, propName := range android.SortedStringKeys(props) {
if shouldGenerateAttribute(propName) {
attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName])
}
}
return attributes
}
func GenerateSoongModuleTargets(ctx bpToBuildContext) map[string][]BazelTarget {
buildFileToTargets := make(map[string][]BazelTarget)
ctx.VisitAllModulesBlueprint(func(m blueprint.Module) {
dir := ctx.ModuleDir(m)
t := generateSoongModuleTarget(ctx, m)
buildFileToTargets[ctx.ModuleDir(m)] = append(buildFileToTargets[dir], t)
})
return buildFileToTargets
}
// Convert a module and its deps and props into a Bazel macro/rule
// representation in the BUILD file.
func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget {
props := getBuildProperties(ctx, m)
// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
// items, if the modules are added using different DependencyTag. Figure
// out the implications of that.
depLabels := map[string]bool{}
if aModule, ok := m.(android.Module); ok {
ctx.VisitDirectDeps(aModule, func(depModule android.Module) {
depLabels[qualifiedTargetLabel(ctx, depModule)] = true
})
}
attributes := propsToAttributes(props.Attrs)
depLabelList := "[\n"
for depLabel, _ := range depLabels {
depLabelList += fmt.Sprintf(" %q,\n", depLabel)
}
depLabelList += " ]"
targetName := targetNameWithVariant(ctx, m)
return BazelTarget{
name: targetName,
content: fmt.Sprintf(
soongModuleTarget,
targetName,
ctx.ModuleName(m),
canonicalizeModuleType(ctx.ModuleType(m)),
ctx.ModuleSubDir(m),
depLabelList,
attributes),
}
}
func getBuildProperties(ctx bpToBuildContext, m blueprint.Module) BazelAttributes {
var allProps map[string]string
// TODO: this omits properties for blueprint modules (blueprint_go_binary,
// bootstrap_go_binary, bootstrap_go_package), which will have to be handled separately.
if aModule, ok := m.(android.Module); ok {
allProps = ExtractModuleProperties(aModule)
}
return BazelAttributes{
Attrs: allProps,
}
}
// Generically extract module properties and types into a map, keyed by the module property name.
func ExtractModuleProperties(aModule android.Module) map[string]string {
ret := map[string]string{}
// Iterate over this android.Module's property structs.
for _, properties := range aModule.GetProperties() {
propertiesValue := reflect.ValueOf(properties)
// Check that propertiesValue is a pointer to the Properties struct, like
// *cc.BaseLinkerProperties or *java.CompilerProperties.
//
// propertiesValue can also be type-asserted to the structs to
// manipulate internal props, if needed.
if isStructPtr(propertiesValue.Type()) {
structValue := propertiesValue.Elem()
for k, v := range extractStructProperties(structValue, 0) {
ret[k] = v
}
} else {
panic(fmt.Errorf(
"properties must be a pointer to a struct, got %T",
propertiesValue.Interface()))
}
}
return ret
}
func isStructPtr(t reflect.Type) bool {
return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
}
// prettyPrint a property value into the equivalent Starlark representation
// recursively.
func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
if isZero(propertyValue) {
// A property value being set or unset actually matters -- Soong does set default
// values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at
// https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480
//
// In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default
// value of unset attributes.
return "", nil
}
var ret string
switch propertyValue.Kind() {
case reflect.String:
ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String()))
case reflect.Bool:
ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface()))
case reflect.Int, reflect.Uint, reflect.Int64:
ret = fmt.Sprintf("%v", propertyValue.Interface())
case reflect.Ptr:
return prettyPrint(propertyValue.Elem(), indent)
case reflect.Slice:
ret = "[\n"
for i := 0; i < propertyValue.Len(); i++ {
indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1)
if err != nil {
return "", err
}
if indexedValue != "" {
ret += makeIndent(indent + 1)
ret += indexedValue
ret += ",\n"
}
}
ret += makeIndent(indent)
ret += "]"
case reflect.Struct:
ret = "{\n"
// Sort and print the struct props by the key.
structProps := extractStructProperties(propertyValue, indent)
for _, k := range android.SortedStringKeys(structProps) {
ret += makeIndent(indent + 1)
ret += fmt.Sprintf("%q: %s,\n", k, structProps[k])
}
ret += makeIndent(indent)
ret += "}"
case reflect.Interface:
// TODO(b/164227191): implement pretty print for interfaces.
// Interfaces are used for for arch, multilib and target properties.
return "", nil
default:
return "", fmt.Errorf(
"unexpected kind for property struct field: %s", propertyValue.Kind())
}
return ret, nil
}
// Converts a reflected property struct value into a map of property names and property values,
// which each property value correctly pretty-printed and indented at the right nest level,
// since property structs can be nested. In Starlark, nested structs are represented as nested
// dicts: https://docs.bazel.build/skylark/lib/dict.html
func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
if structValue.Kind() != reflect.Struct {
panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
}
ret := map[string]string{}
structType := structValue.Type()
for i := 0; i < structValue.NumField(); i++ {
field := structType.Field(i)
if shouldSkipStructField(field) {
continue
}
fieldValue := structValue.Field(i)
if isZero(fieldValue) {
// Ignore zero-valued fields
continue
}
propertyName := proptools.PropertyNameForField(field.Name)
prettyPrintedValue, err := prettyPrint(fieldValue, indent+1)
if err != nil {
panic(
fmt.Errorf(
"Error while parsing property: %q. %s",
propertyName,
err))
}
if prettyPrintedValue != "" {
ret[propertyName] = prettyPrintedValue
}
}
return ret
}
func isZero(value reflect.Value) bool {
switch value.Kind() {
case reflect.Func, reflect.Map, reflect.Slice:
return value.IsNil()
case reflect.Array:
valueIsZero := true
for i := 0; i < value.Len(); i++ {
valueIsZero = valueIsZero && isZero(value.Index(i))
}
return valueIsZero
case reflect.Struct:
valueIsZero := true
for i := 0; i < value.NumField(); i++ {
if value.Field(i).CanSet() {
valueIsZero = valueIsZero && isZero(value.Field(i))
}
}
return valueIsZero
case reflect.Ptr:
if !value.IsNil() {
return isZero(reflect.Indirect(value))
} else {
return true
}
default:
zeroValue := reflect.Zero(value.Type())
result := value.Interface() == zeroValue.Interface()
return result
}
}
func escapeString(s string) string {
s = strings.ReplaceAll(s, "\\", "\\\\")
return strings.ReplaceAll(s, "\"", "\\\"")
}
func makeIndent(indent int) string {
if indent < 0 {
panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent))
}
return strings.Repeat(" ", indent)
}
func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string {
name := ""
if c.ModuleSubDir(logicModule) != "" {
// TODO(b/162720883): Figure out a way to drop the "--" variant suffixes.
name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
} else {
name = c.ModuleName(logicModule)
}
return strings.Replace(name, "//", "", 1)
}
func qualifiedTargetLabel(c bpToBuildContext, logicModule blueprint.Module) string {
return fmt.Sprintf("//%s:%s", c.ModuleDir(logicModule), targetNameWithVariant(c, logicModule))
}

221
bp2build/build_conversion_test.go Normal file
View File

@ -0,0 +1,221 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"testing"
)
func TestGenerateSoongModuleTargets(t *testing.T) {
testCases := []struct {
bp string
expectedBazelTarget string
}{
{
bp: `custom {
name: "foo",
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
)`,
},
{
bp: `custom {
name: "foo",
ramdisk: true,
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
ramdisk = True,
)`,
},
{
bp: `custom {
name: "foo",
owner: "a_string_with\"quotes\"_and_\\backslashes\\\\",
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
owner = "a_string_with\"quotes\"_and_\\backslashes\\\\",
)`,
},
{
bp: `custom {
name: "foo",
required: ["bar"],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
required = [
"bar",
],
)`,
},
{
bp: `custom {
name: "foo",
target_required: ["qux", "bazqux"],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
target_required = [
"qux",
"bazqux",
],
)`,
},
{
bp: `custom {
name: "foo",
dist: {
targets: ["goal_foo"],
tag: ".foo",
},
dists: [
{
targets: ["goal_bar"],
tag: ".bar",
},
],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
dist = {
"tag": ".foo",
"targets": [
"goal_foo",
],
},
dists = [
{
"tag": ".bar",
"targets": [
"goal_bar",
],
},
],
)`,
},
{
bp: `custom {
name: "foo",
required: ["bar"],
target_required: ["qux", "bazqux"],
ramdisk: true,
owner: "custom_owner",
dists: [
{
tag: ".tag",
targets: ["my_goal"],
},
],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
dists = [
{
"tag": ".tag",
"targets": [
"my_goal",
],
},
],
owner = "custom_owner",
ramdisk = True,
required = [
"bar",
],
target_required = [
"qux",
"bazqux",
],
)`,
},
}
dir := "."
for _, testCase := range testCases {
config := android.TestConfig(buildDir, nil, testCase.bp, nil)
ctx := android.NewTestContext(config)
ctx.RegisterModuleType("custom", customModuleFactory)
ctx.Register()
_, errs := ctx.ParseFileList(dir, []string{"Android.bp"})
android.FailIfErrored(t, errs)
_, errs = ctx.PrepareBuildActions(config)
android.FailIfErrored(t, errs)
bp2BuildCtx := bp2buildBlueprintWrapContext{
bpCtx: ctx.Context.Context,
}
bazelTargets := GenerateSoongModuleTargets(&bp2BuildCtx)[dir]
if g, w := len(bazelTargets), 1; g != w {
t.Fatalf("Expected %d bazel target, got %d", w, g)
}
actualBazelTarget := bazelTargets[0]
if actualBazelTarget.content != testCase.expectedBazelTarget {
t.Errorf(
"Expected generated Bazel target to be '%s', got '%s'",
testCase.expectedBazelTarget,
actualBazelTarget,
)
}
}
}

230
bp2build/bzl_conversion.go Normal file
View File

@ -0,0 +1,230 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"fmt"
"reflect"
"runtime"
"sort"
"strings"
"github.com/google/blueprint/proptools"
)
var (
// An allowlist of prop types that are surfaced from module props to rule
// attributes. (nested) dictionaries are notably absent here, because while
// Soong supports multi value typed and nested dictionaries, Bazel's rule
// attr() API supports only single-level string_dicts.
allowedPropTypes = map[string]bool{
"int": true, // e.g. 42
"bool": true, // e.g. True
"string_list": true, // e.g. ["a", "b"]
"string": true, // e.g. "a"
}
)
type rule struct {
name string
attrs string
}
type RuleShim struct {
// The rule class shims contained in a bzl file. e.g. ["cc_object", "cc_library", ..]
rules []string
// The generated string content of the bzl file.
content string
}
// Create <module>.bzl containing Bazel rule shims for every module type available in Soong and
// user-specified Go plugins.
//
// This function reuses documentation generation APIs to ensure parity between modules-as-docs
// and modules-as-code, including the names and types of morule properties.
func CreateRuleShims(moduleTypeFactories map[string]android.ModuleFactory) map[string]RuleShim {
ruleShims := map[string]RuleShim{}
for pkg, rules := range generateRules(moduleTypeFactories) {
shim := RuleShim{
rules: make([]string, 0, len(rules)),
}
shim.content = "load(\"//build/bazel/queryview_rules:providers.bzl\", \"SoongModuleInfo\")\n"
bzlFileName := strings.ReplaceAll(pkg, "android/soong/", "")
bzlFileName = strings.ReplaceAll(bzlFileName, ".", "_")
bzlFileName = strings.ReplaceAll(bzlFileName, "/", "_")
for _, r := range rules {
shim.content += fmt.Sprintf(moduleRuleShim, r.name, r.attrs)
shim.rules = append(shim.rules, r.name)
}
sort.Strings(shim.rules)
ruleShims[bzlFileName] = shim
}
return ruleShims
}
// Generate the content of soong_module.bzl with the rule shim load statements
// and mapping of module_type to rule shim map for every module type in Soong.
func generateSoongModuleBzl(bzlLoads map[string]RuleShim) string {
var loadStmts string
var moduleRuleMap string
for _, bzlFileName := range android.SortedStringKeys(bzlLoads) {
loadStmt := "load(\"//build/bazel/queryview_rules:"
loadStmt += bzlFileName
loadStmt += ".bzl\""
ruleShim := bzlLoads[bzlFileName]
for _, rule := range ruleShim.rules {
loadStmt += fmt.Sprintf(", %q", rule)
moduleRuleMap += " \"" + rule + "\": " + rule + ",\n"
}
loadStmt += ")\n"
loadStmts += loadStmt
}
return fmt.Sprintf(soongModuleBzl, loadStmts, moduleRuleMap)
}
func generateRules(moduleTypeFactories map[string]android.ModuleFactory) map[string][]rule {
// TODO: add shims for bootstrap/blueprint go modules types
rules := make(map[string][]rule)
// TODO: allow registration of a bzl rule when registring a factory
for _, moduleType := range android.SortedStringKeys(moduleTypeFactories) {
factory := moduleTypeFactories[moduleType]
factoryName := runtime.FuncForPC(reflect.ValueOf(factory).Pointer()).Name()
pkg := strings.Split(factoryName, ".")[0]
attrs := `{
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
`
attrs += getAttributes(factory)
attrs += " },"
r := rule{
name: canonicalizeModuleType(moduleType),
attrs: attrs,
}
rules[pkg] = append(rules[pkg], r)
}
return rules
}
type property struct {
name string
starlarkAttrType string
properties []property
}
const (
attributeIndent = " "
)
func (p *property) attributeString() string {
if !shouldGenerateAttribute(p.name) {
return ""
}
if _, ok := allowedPropTypes[p.starlarkAttrType]; !ok {
// a struct -- let's just comment out sub-props
s := fmt.Sprintf(attributeIndent+"# %s start\n", p.name)
for _, nestedP := range p.properties {
s += "# " + nestedP.attributeString()
}
s += fmt.Sprintf(attributeIndent+"# %s end\n", p.name)
return s
}
return fmt.Sprintf(attributeIndent+"%q: attr.%s(),\n", p.name, p.starlarkAttrType)
}
func extractPropertyDescriptionsFromStruct(structType reflect.Type) []property {
properties := make([]property, 0)
for i := 0; i < structType.NumField(); i++ {
field := structType.Field(i)
if shouldSkipStructField(field) {
continue
}
properties = append(properties, extractPropertyDescriptions(field.Name, field.Type)...)
}
return properties
}
func extractPropertyDescriptions(name string, t reflect.Type) []property {
name = proptools.PropertyNameForField(name)
// TODO: handle android:paths tags, they should be changed to label types
starlarkAttrType := fmt.Sprintf("%s", t.Name())
props := make([]property, 0)
switch t.Kind() {
case reflect.Bool, reflect.String:
// do nothing
case reflect.Uint, reflect.Int, reflect.Int64:
starlarkAttrType = "int"
case reflect.Slice:
if t.Elem().Kind() != reflect.String {
// TODO: handle lists of non-strings (currently only list of Dist)
return []property{}
}
starlarkAttrType = "string_list"
case reflect.Struct:
props = extractPropertyDescriptionsFromStruct(t)
case reflect.Ptr:
return extractPropertyDescriptions(name, t.Elem())
case reflect.Interface:
// Interfaces are used for for arch, multilib and target properties, which are handled at runtime.
// These will need to be handled in a bazel-specific version of the arch mutator.
return []property{}
}
prop := property{
name: name,
starlarkAttrType: starlarkAttrType,
properties: props,
}
return []property{prop}
}
func getPropertyDescriptions(props []interface{}) []property {
// there may be duplicate properties, e.g. from defaults libraries
propertiesByName := make(map[string]property)
for _, p := range props {
for _, prop := range extractPropertyDescriptionsFromStruct(reflect.ValueOf(p).Elem().Type()) {
propertiesByName[prop.name] = prop
}
}
properties := make([]property, 0, len(propertiesByName))
for _, key := range android.SortedStringKeys(propertiesByName) {
properties = append(properties, propertiesByName[key])
}
return properties
}
func getAttributes(factory android.ModuleFactory) string {
attrs := ""
for _, p := range getPropertyDescriptions(factory().GetProperties()) {
attrs += p.attributeString()
}
return attrs
}

208
bp2build/bzl_conversion_test.go Normal file
View File

@ -0,0 +1,208 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"io/ioutil"
"os"
"strings"
"testing"
)
var buildDir string
func setUp() {
var err error
buildDir, err = ioutil.TempDir("", "bazel_queryview_test")
if err != nil {
panic(err)
}
}
func tearDown() {
os.RemoveAll(buildDir)
}
func TestMain(m *testing.M) {
run := func() int {
setUp()
defer tearDown()
return m.Run()
}
os.Exit(run())
}
func TestGenerateModuleRuleShims(t *testing.T) {
moduleTypeFactories := map[string]android.ModuleFactory{
"custom": customModuleFactoryBase,
"custom_test": customTestModuleFactoryBase,
"custom_defaults": customDefaultsModuleFactoryBasic,
}
ruleShims := CreateRuleShims(moduleTypeFactories)
if len(ruleShims) != 1 {
t.Errorf("Expected to generate 1 rule shim, but got %d", len(ruleShims))
}
ruleShim := ruleShims["bp2build"]
expectedRules := []string{
"custom",
"custom_defaults",
"custom_test_",
}
if len(ruleShim.rules) != len(expectedRules) {
t.Errorf("Expected %d rules, but got %d", len(expectedRules), len(ruleShim.rules))
}
for i, rule := range ruleShim.rules {
if rule != expectedRules[i] {
t.Errorf("Expected rule shim to contain %s, but got %s", expectedRules[i], rule)
}
}
expectedBzl := `load("//build/bazel/queryview_rules:providers.bzl", "SoongModuleInfo")
def _custom_impl(ctx):
return [SoongModuleInfo()]
custom = rule(
implementation = _custom_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"bool_ptr_prop": attr.bool(),
"int64_ptr_prop": attr.int(),
# nested_props start
# "nested_prop": attr.string(),
# nested_props end
# nested_props_ptr start
# "nested_prop": attr.string(),
# nested_props_ptr end
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
"string_ptr_prop": attr.string(),
},
)
def _custom_defaults_impl(ctx):
return [SoongModuleInfo()]
custom_defaults = rule(
implementation = _custom_defaults_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"bool_ptr_prop": attr.bool(),
"int64_ptr_prop": attr.int(),
# nested_props start
# "nested_prop": attr.string(),
# nested_props end
# nested_props_ptr start
# "nested_prop": attr.string(),
# nested_props_ptr end
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
"string_ptr_prop": attr.string(),
},
)
def _custom_test__impl(ctx):
return [SoongModuleInfo()]
custom_test_ = rule(
implementation = _custom_test__impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"bool_ptr_prop": attr.bool(),
"int64_ptr_prop": attr.int(),
# nested_props start
# "nested_prop": attr.string(),
# nested_props end
# nested_props_ptr start
# "nested_prop": attr.string(),
# nested_props_ptr end
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
"string_ptr_prop": attr.string(),
# test_prop start
# "test_string_prop": attr.string(),
# test_prop end
},
)
`
if ruleShim.content != expectedBzl {
t.Errorf(
"Expected the generated rule shim bzl to be:\n%s\nbut got:\n%s",
expectedBzl,
ruleShim.content)
}
}
func TestGenerateSoongModuleBzl(t *testing.T) {
ruleShims := map[string]RuleShim{
"file1": RuleShim{
rules: []string{"a", "b"},
content: "irrelevant",
},
"file2": RuleShim{
rules: []string{"c", "d"},
content: "irrelevant",
},
}
files := CreateBazelFiles(ruleShims, make(map[string][]BazelTarget))
var actualSoongModuleBzl BazelFile
for _, f := range files {
if f.Basename == "soong_module.bzl" {
actualSoongModuleBzl = f
}
}
expectedLoad := `load("//build/bazel/queryview_rules:file1.bzl", "a", "b")
load("//build/bazel/queryview_rules:file2.bzl", "c", "d")
`
expectedRuleMap := `soong_module_rule_map = {
"a": a,
"b": b,
"c": c,
"d": d,
}`
if !strings.Contains(actualSoongModuleBzl.Contents, expectedLoad) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the load statement in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl.Contents,
expectedLoad)
}
if !strings.Contains(actualSoongModuleBzl.Contents, expectedRuleMap) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the module -> rule map in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl.Contents,
expectedRuleMap)
}
}

118
bp2build/conversion.go Normal file
View File

@ -0,0 +1,118 @@
package bp2build
import (
"android/soong/android"
"reflect"
"sort"
"strings"
"github.com/google/blueprint/proptools"
)
type BazelFile struct {
Dir string
Basename string
Contents string
}
func CreateBazelFiles(
ruleShims map[string]RuleShim,
buildToTargets map[string][]BazelTarget) []BazelFile {
files := make([]BazelFile, 0, len(ruleShims)+len(buildToTargets)+numAdditionalFiles)
// Write top level files: WORKSPACE and BUILD. These files are empty.
files = append(files, newFile("", "WORKSPACE", ""))
// Used to denote that the top level directory is a package.
files = append(files, newFile("", "BUILD", ""))
files = append(files, newFile(bazelRulesSubDir, "BUILD", ""))
files = append(files, newFile(bazelRulesSubDir, "providers.bzl", providersBzl))
for bzlFileName, ruleShim := range ruleShims {
files = append(files, newFile(bazelRulesSubDir, bzlFileName+".bzl", ruleShim.content))
}
files = append(files, newFile(bazelRulesSubDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims)))
files = append(files, createBuildFiles(buildToTargets)...)
return files
}
func createBuildFiles(buildToTargets map[string][]BazelTarget) []BazelFile {
files := make([]BazelFile, 0, len(buildToTargets))
for _, dir := range android.SortedStringKeys(buildToTargets) {
content := soongModuleLoad
targets := buildToTargets[dir]
sort.Slice(targets, func(i, j int) bool { return targets[i].name < targets[j].name })
for _, t := range targets {
content += "\n\n"
content += t.content
}
files = append(files, newFile(dir, "BUILD.bazel", content))
}
return files
}
func newFile(dir, basename, content string) BazelFile {
return BazelFile{
Dir: dir,
Basename: basename,
Contents: content,
}
}
const (
bazelRulesSubDir = "build/bazel/queryview_rules"
// additional files:
// * workspace file
// * base BUILD file
// * rules BUILD file
// * rules providers.bzl file
// * rules soong_module.bzl file
numAdditionalFiles = 5
)
var (
// Certain module property names are blocklisted/ignored here, for the reasons commented.
ignoredPropNames = map[string]bool{
"name": true, // redundant, since this is explicitly generated for every target
"from": true, // reserved keyword
"in": true, // reserved keyword
"arch": true, // interface prop type is not supported yet.
"multilib": true, // interface prop type is not supported yet.
"target": true, // interface prop type is not supported yet.
"visibility": true, // Bazel has native visibility semantics. Handle later.
"features": true, // There is already a built-in attribute 'features' which cannot be overridden.
}
)
func shouldGenerateAttribute(prop string) bool {
return !ignoredPropNames[prop]
}
func shouldSkipStructField(field reflect.StructField) bool {
if field.PkgPath != "" {
// Skip unexported fields. Some properties are
// internal to Soong only, and these fields do not have PkgPath.
return true
}
// fields with tag `blueprint:"mutated"` are exported to enable modification in mutators, etc
// but cannot be set in a .bp file
if proptools.HasTag(field, "blueprint", "mutated") {
return true
}
return false
}
// FIXME(b/168089390): In Bazel, rules ending with "_test" needs to be marked as
// testonly = True, forcing other rules that depend on _test rules to also be
// marked as testonly = True. This semantic constraint is not present in Soong.
// To work around, rename "*_test" rules to "*_test_".
func canonicalizeModuleType(moduleName string) string {
if strings.HasSuffix(moduleName, "_test") {
return moduleName + "_"
}
return moduleName
}

73
bp2build/conversion_test.go Normal file
View File

@ -0,0 +1,73 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"sort"
"testing"
)
func TestCreateBazelFiles_AddsTopLevelFiles(t *testing.T) {
files := CreateBazelFiles(map[string]RuleShim{}, map[string][]BazelTarget{})
expectedFilePaths := []struct {
dir string
basename string
}{
{
dir: "",
basename: "BUILD",
},
{
dir: "",
basename: "WORKSPACE",
},
{
dir: bazelRulesSubDir,
basename: "BUILD",
},
{
dir: bazelRulesSubDir,
basename: "providers.bzl",
},
{
dir: bazelRulesSubDir,
basename: "soong_module.bzl",
},
}
if g, w := len(files), len(expectedFilePaths); g != w {
t.Errorf("Expected %d files, got %d", w, g)
}
sort.Slice(files, func(i, j int) bool {
if dir1, dir2 := files[i].Dir, files[j].Dir; dir1 == dir2 {
return files[i].Basename < files[j].Basename
} else {
return dir1 < dir2
}
})
for i := range files {
if g, w := files[i], expectedFilePaths[i]; g.Dir != w.dir || g.Basename != w.basename {
t.Errorf("Did not find expected file %s/%s", g.Dir, g.Basename)
} else if g.Basename == "BUILD" || g.Basename == "WORKSPACE" {
if g.Contents != "" {
t.Errorf("Expected %s to have no content.", g)
}
} else if g.Contents == "" {
t.Errorf("Contents of %s unexpected empty.", g)
}
}
}

136
bp2build/testing.go Normal file
View File

@ -0,0 +1,136 @@
package bp2build
import (
"android/soong/android"
"github.com/google/blueprint"
)
type nestedProps struct {
Nested_prop string
}
type customProps struct {
Bool_prop bool
Bool_ptr_prop *bool
// Ensure that properties tagged `blueprint:mutated` are omitted
Int_prop int `blueprint:"mutated"`
Int64_ptr_prop *int64
String_prop string
String_ptr_prop *string
String_list_prop []string
Nested_props nestedProps
Nested_props_ptr *nestedProps
}
type customModule struct {
android.ModuleBase
props customProps
}
// OutputFiles is needed because some instances of this module use dist with a
// tag property which requires the module implements OutputFileProducer.
func (m *customModule) OutputFiles(tag string) (android.Paths, error) {
return android.PathsForTesting("path" + tag), nil
}
func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customModuleFactoryBase() android.Module {
module := &customModule{}
module.AddProperties(&module.props)
return module
}
func customModuleFactory() android.Module {
m := customModuleFactoryBase()
android.InitAndroidModule(m)
return m
}
type testProps struct {
Test_prop struct {
Test_string_prop string
}
}
type customTestModule struct {
android.ModuleBase
props customProps
test_props testProps
}
func (m *customTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customTestModuleFactoryBase() android.Module {
m := &customTestModule{}
m.AddProperties(&m.props)
m.AddProperties(&m.test_props)
return m
}
func customTestModuleFactory() android.Module {
m := customTestModuleFactoryBase()
android.InitAndroidModule(m)
return m
}
type customDefaultsModule struct {
android.ModuleBase
android.DefaultsModuleBase
}
func customDefaultsModuleFactoryBase() android.DefaultsModule {
module := &customDefaultsModule{}
module.AddProperties(&customProps{})
return module
}
func customDefaultsModuleFactoryBasic() android.Module {
return customDefaultsModuleFactoryBase()
}
func customDefaultsModuleFactory() android.Module {
m := customDefaultsModuleFactoryBase()
android.InitDefaultsModule(m)
return m
}
type bp2buildBlueprintWrapContext struct {
bpCtx *blueprint.Context
}
func (ctx *bp2buildBlueprintWrapContext) ModuleName(module blueprint.Module) string {
return ctx.bpCtx.ModuleName(module)
}
func (ctx *bp2buildBlueprintWrapContext) ModuleDir(module blueprint.Module) string {
return ctx.bpCtx.ModuleDir(module)
}
func (ctx *bp2buildBlueprintWrapContext) ModuleSubDir(module blueprint.Module) string {
return ctx.bpCtx.ModuleSubDir(module)
}
func (ctx *bp2buildBlueprintWrapContext) ModuleType(module blueprint.Module) string {
return ctx.bpCtx.ModuleType(module)
}
func (ctx *bp2buildBlueprintWrapContext) VisitAllModulesBlueprint(visit func(blueprint.Module)) {
ctx.bpCtx.VisitAllModules(visit)
}
func (ctx *bp2buildBlueprintWrapContext) VisitDirectDeps(module android.Module, visit func(android.Module)) {
ctx.bpCtx.VisitDirectDeps(module, func(m blueprint.Module) {
if aModule, ok := m.(android.Module); ok {
visit(aModule)
}
})
}

5
cmd/soong_build/Android.bp
View File

@ -20,6 +20,7 @@ bootstrap_go_binary {
"golang-protobuf-proto",
"soong",
"soong-android",
"soong-bp2build",
"soong-env",
"soong-ui-metrics_proto",
],
@ -27,10 +28,6 @@ bootstrap_go_binary {
"main.go",
"writedocs.go",
"queryview.go",
"queryview_templates.go",
],
testSrcs: [
"queryview_test.go",
],
primaryBuilder: true,
}

514
cmd/soong_build/queryview.go
View File

@ -16,512 +16,82 @@ package main
import (
"android/soong/android"
"fmt"
"android/soong/bp2build"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap/bpdoc"
"github.com/google/blueprint/proptools"
)
var (
// An allowlist of prop types that are surfaced from module props to rule
// attributes. (nested) dictionaries are notably absent here, because while
// Soong supports multi value typed and nested dictionaries, Bazel's rule
// attr() API supports only single-level string_dicts.
allowedPropTypes = map[string]bool{
"int": true, // e.g. 42
"bool": true, // e.g. True
"string_list": true, // e.g. ["a", "b"]
"string": true, // e.g. "a"
}
// Certain module property names are blocklisted/ignored here, for the reasons commented.
ignoredPropNames = map[string]bool{
"name": true, // redundant, since this is explicitly generated for every target
"from": true, // reserved keyword
"in": true, // reserved keyword
"arch": true, // interface prop type is not supported yet.
"multilib": true, // interface prop type is not supported yet.
"target": true, // interface prop type is not supported yet.
"visibility": true, // Bazel has native visibility semantics. Handle later.
"features": true, // There is already a built-in attribute 'features' which cannot be overridden.
}
)
func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string {
name := ""
if c.ModuleSubDir(logicModule) != "" {
// TODO(b/162720883): Figure out a way to drop the "--" variant suffixes.
name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
} else {
name = c.ModuleName(logicModule)
}
return strings.Replace(name, "//", "", 1)
type queryviewContext struct {
bpCtx *blueprint.Context
}
func qualifiedTargetLabel(c *blueprint.Context, logicModule blueprint.Module) string {
return "//" +
packagePath(c, logicModule) +
":" +
targetNameWithVariant(c, logicModule)
func (ctx *queryviewContext) ModuleName(module blueprint.Module) string {
return ctx.bpCtx.ModuleName(module)
}
func packagePath(c *blueprint.Context, logicModule blueprint.Module) string {
return filepath.Dir(c.BlueprintFile(logicModule))
func (ctx *queryviewContext) ModuleDir(module blueprint.Module) string {
return ctx.bpCtx.ModuleDir(module)
}
func escapeString(s string) string {
s = strings.ReplaceAll(s, "\\", "\\\\")
return strings.ReplaceAll(s, "\"", "\\\"")
func (ctx *queryviewContext) ModuleSubDir(module blueprint.Module) string {
return ctx.bpCtx.ModuleSubDir(module)
}
func makeIndent(indent int) string {
if indent < 0 {
panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent))
}
return strings.Repeat(" ", indent)
func (ctx *queryviewContext) ModuleType(module blueprint.Module) string {
return ctx.bpCtx.ModuleType(module)
}
// prettyPrint a property value into the equivalent Starlark representation
// recursively.
func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
if isZero(propertyValue) {
// A property value being set or unset actually matters -- Soong does set default
// values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at
// https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480
//
// In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default
// value of unset attributes.
return "", nil
}
var ret string
switch propertyValue.Kind() {
case reflect.String:
ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String()))
case reflect.Bool:
ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface()))
case reflect.Int, reflect.Uint, reflect.Int64:
ret = fmt.Sprintf("%v", propertyValue.Interface())
case reflect.Ptr:
return prettyPrint(propertyValue.Elem(), indent)
case reflect.Slice:
ret = "[\n"
for i := 0; i < propertyValue.Len(); i++ {
indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1)
if err != nil {
return "", err
}
if indexedValue != "" {
ret += makeIndent(indent + 1)
ret += indexedValue
ret += ",\n"
}
}
ret += makeIndent(indent)
ret += "]"
case reflect.Struct:
ret = "{\n"
// Sort and print the struct props by the key.
structProps := extractStructProperties(propertyValue, indent)
for _, k := range android.SortedStringKeys(structProps) {
ret += makeIndent(indent + 1)
ret += fmt.Sprintf("%q: %s,\n", k, structProps[k])
}
ret += makeIndent(indent)
ret += "}"
case reflect.Interface:
// TODO(b/164227191): implement pretty print for interfaces.
// Interfaces are used for for arch, multilib and target properties.
return "", nil
default:
return "", fmt.Errorf(
"unexpected kind for property struct field: %s", propertyValue.Kind())
}
return ret, nil
func (ctx *queryviewContext) VisitAllModulesBlueprint(visit func(blueprint.Module)) {
ctx.bpCtx.VisitAllModules(visit)
}
// Converts a reflected property struct value into a map of property names and property values,
// which each property value correctly pretty-printed and indented at the right nest level,
// since property structs can be nested. In Starlark, nested structs are represented as nested
// dicts: https://docs.bazel.build/skylark/lib/dict.html
func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
if structValue.Kind() != reflect.Struct {
panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
}
ret := map[string]string{}
structType := structValue.Type()
for i := 0; i < structValue.NumField(); i++ {
field := structType.Field(i)
if field.PkgPath != "" {
// Skip unexported fields. Some properties are
// internal to Soong only, and these fields do not have PkgPath.
continue
func (ctx *queryviewContext) VisitDirectDeps(module android.Module, visit func(android.Module)) {
ctx.bpCtx.VisitDirectDeps(module, func(m blueprint.Module) {
if aModule, ok := m.(android.Module); ok {
visit(aModule)
}
if proptools.HasTag(field, "blueprint", "mutated") {
continue
}
fieldValue := structValue.Field(i)
if isZero(fieldValue) {
// Ignore zero-valued fields
continue
}
propertyName := proptools.PropertyNameForField(field.Name)
prettyPrintedValue, err := prettyPrint(fieldValue, indent+1)
if err != nil {
panic(
fmt.Errorf(
"Error while parsing property: %q. %s",
propertyName,
err))
}
if prettyPrintedValue != "" {
ret[propertyName] = prettyPrintedValue
}
}
return ret
}
func isStructPtr(t reflect.Type) bool {
return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
}
// Generically extract module properties and types into a map, keyed by the module property name.
func extractModuleProperties(aModule android.Module) map[string]string {
ret := map[string]string{}
// Iterate over this android.Module's property structs.
for _, properties := range aModule.GetProperties() {
propertiesValue := reflect.ValueOf(properties)
// Check that propertiesValue is a pointer to the Properties struct, like
// *cc.BaseLinkerProperties or *java.CompilerProperties.
//
// propertiesValue can also be type-asserted to the structs to
// manipulate internal props, if needed.
if isStructPtr(propertiesValue.Type()) {
structValue := propertiesValue.Elem()
for k, v := range extractStructProperties(structValue, 0) {
ret[k] = v
}
} else {
panic(fmt.Errorf(
"properties must be a pointer to a struct, got %T",
propertiesValue.Interface()))
}
}
return ret
}
// FIXME(b/168089390): In Bazel, rules ending with "_test" needs to be marked as
// testonly = True, forcing other rules that depend on _test rules to also be
// marked as testonly = True. This semantic constraint is not present in Soong.
// To work around, rename "*_test" rules to "*_test_".
func canonicalizeModuleType(moduleName string) string {
if strings.HasSuffix(moduleName, "_test") {
return moduleName + "_"
}
return moduleName
}
type RuleShim struct {
// The rule class shims contained in a bzl file. e.g. ["cc_object", "cc_library", ..]
rules []string
// The generated string content of the bzl file.
content string
}
// Create <module>.bzl containing Bazel rule shims for every module type available in Soong and
// user-specified Go plugins.
//
// This function reuses documentation generation APIs to ensure parity between modules-as-docs
// and modules-as-code, including the names and types of module properties.
func createRuleShims(packages []*bpdoc.Package) (map[string]RuleShim, error) {
var propToAttr func(prop bpdoc.Property, propName string) string
propToAttr = func(prop bpdoc.Property, propName string) string {
// dots are not allowed in Starlark attribute names. Substitute them with double underscores.
propName = strings.ReplaceAll(propName, ".", "__")
if !shouldGenerateAttribute(propName) {
return ""
}
// Canonicalize and normalize module property types to Bazel attribute types
starlarkAttrType := prop.Type
if starlarkAttrType == "list of string" {
starlarkAttrType = "string_list"
} else if starlarkAttrType == "int64" {
starlarkAttrType = "int"
} else if starlarkAttrType == "" {
var attr string
for _, nestedProp := range prop.Properties {
nestedAttr := propToAttr(nestedProp, propName+"__"+nestedProp.Name)
if nestedAttr != "" {
// TODO(b/167662930): Fix nested props resulting in too many attributes.
// Let's still generate these, but comment them out.
attr += "# " + nestedAttr
}
}
return attr
}
if !allowedPropTypes[starlarkAttrType] {
return ""
}
return fmt.Sprintf(" %q: attr.%s(),\n", propName, starlarkAttrType)
}
ruleShims := map[string]RuleShim{}
for _, pkg := range packages {
content := "load(\"//build/bazel/queryview_rules:providers.bzl\", \"SoongModuleInfo\")\n"
bzlFileName := strings.ReplaceAll(pkg.Path, "android/soong/", "")
bzlFileName = strings.ReplaceAll(bzlFileName, ".", "_")
bzlFileName = strings.ReplaceAll(bzlFileName, "/", "_")
rules := []string{}
for _, moduleTypeTemplate := range moduleTypeDocsToTemplates(pkg.ModuleTypes) {
attrs := `{
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
`
for _, prop := range moduleTypeTemplate.Properties {
attrs += propToAttr(prop, prop.Name)
}
moduleTypeName := moduleTypeTemplate.Name
// Certain SDK-related module types dynamically inject properties, instead of declaring
// them as structs. These properties are registered in an SdkMemberTypesRegistry. If
// the module type name matches, add these properties into the rule definition.
var registeredTypes []android.SdkMemberType
if moduleTypeName == "module_exports" || moduleTypeName == "module_exports_snapshot" {
registeredTypes = android.ModuleExportsMemberTypes.RegisteredTypes()
} else if moduleTypeName == "sdk" || moduleTypeName == "sdk_snapshot" {
registeredTypes = android.SdkMemberTypes.RegisteredTypes()
}
for _, memberType := range registeredTypes {
attrs += fmt.Sprintf(" %q: attr.string_list(),\n", memberType.SdkPropertyName())
}
attrs += " },"
rule := canonicalizeModuleType(moduleTypeTemplate.Name)
content += fmt.Sprintf(moduleRuleShim, rule, attrs)
rules = append(rules, rule)
}
ruleShims[bzlFileName] = RuleShim{content: content, rules: rules}
}
return ruleShims, nil
})
}
func createBazelQueryView(ctx *android.Context, bazelQueryViewDir string) error {
blueprintCtx := ctx.Context
blueprintCtx.VisitAllModules(func(module blueprint.Module) {
buildFile, err := buildFileForModule(blueprintCtx, module, bazelQueryViewDir)
if err != nil {
panic(err)
}
buildFile.Write([]byte(generateSoongModuleTarget(blueprintCtx, module) + "\n\n"))
buildFile.Close()
})
var err error
// Write top level files: WORKSPACE and BUILD. These files are empty.
if err = writeReadOnlyFile(bazelQueryViewDir, "WORKSPACE", ""); err != nil {
return err
qvCtx := queryviewContext{
bpCtx: ctx.Context,
}
ruleShims := bp2build.CreateRuleShims(android.ModuleTypeFactories())
buildToTargets := bp2build.GenerateSoongModuleTargets(&qvCtx)
// Used to denote that the top level directory is a package.
if err = writeReadOnlyFile(bazelQueryViewDir, "BUILD", ""); err != nil {
return err
}
packages, err := getPackages(ctx)
if err != nil {
return err
}
ruleShims, err := createRuleShims(packages)
if err != nil {
return err
}
// Write .bzl Starlark files into the bazel_rules top level directory (provider and rule definitions)
bazelRulesDir := bazelQueryViewDir + "/build/bazel/queryview_rules"
if err = writeReadOnlyFile(bazelRulesDir, "BUILD", ""); err != nil {
return err
}
if err = writeReadOnlyFile(bazelRulesDir, "providers.bzl", providersBzl); err != nil {
return err
}
for bzlFileName, ruleShim := range ruleShims {
if err = writeReadOnlyFile(bazelRulesDir, bzlFileName+".bzl", ruleShim.content); err != nil {
filesToWrite := bp2build.CreateBazelFiles(ruleShims, buildToTargets)
for _, f := range filesToWrite {
if err := writeReadOnlyFile(bazelQueryViewDir, f); err != nil {
return err
}
}
return writeReadOnlyFile(bazelRulesDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims))
return nil
}
// Generate the content of soong_module.bzl with the rule shim load statements
// and mapping of module_type to rule shim map for every module type in Soong.
func generateSoongModuleBzl(bzlLoads map[string]RuleShim) string {
var loadStmts string
var moduleRuleMap string
for bzlFileName, ruleShim := range bzlLoads {
loadStmt := "load(\"//build/bazel/queryview_rules:"
loadStmt += bzlFileName
loadStmt += ".bzl\""
for _, rule := range ruleShim.rules {
loadStmt += fmt.Sprintf(", %q", rule)
moduleRuleMap += " \"" + rule + "\": " + rule + ",\n"
}
loadStmt += ")\n"
loadStmts += loadStmt
}
return fmt.Sprintf(soongModuleBzl, loadStmts, moduleRuleMap)
}
func shouldGenerateAttribute(prop string) bool {
return !ignoredPropNames[prop]
}
// props is an unsorted map. This function ensures that
// the generated attributes are sorted to ensure determinism.
func propsToAttributes(props map[string]string) string {
var attributes string
for _, propName := range android.SortedStringKeys(props) {
if shouldGenerateAttribute(propName) {
attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName])
}
}
return attributes
}
// Convert a module and its deps and props into a Bazel macro/rule
// representation in the BUILD file.
func generateSoongModuleTarget(
blueprintCtx *blueprint.Context,
module blueprint.Module) string {
var props map[string]string
if aModule, ok := module.(android.Module); ok {
props = extractModuleProperties(aModule)
}
attributes := propsToAttributes(props)
// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
// items, if the modules are added using different DependencyTag. Figure
// out the implications of that.
depLabels := map[string]bool{}
blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) {
depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true
})
depLabelList := "[\n"
for depLabel, _ := range depLabels {
depLabelList += fmt.Sprintf(" %q,\n", depLabel)
}
depLabelList += " ]"
return fmt.Sprintf(
soongModuleTarget,
targetNameWithVariant(blueprintCtx, module),
blueprintCtx.ModuleName(module),
canonicalizeModuleType(blueprintCtx.ModuleType(module)),
blueprintCtx.ModuleSubDir(module),
depLabelList,
attributes)
}
func buildFileForModule(
ctx *blueprint.Context, module blueprint.Module, bazelQueryViewDir string) (*os.File, error) {
// Create nested directories for the BUILD file
dirPath := filepath.Join(bazelQueryViewDir, packagePath(ctx, module))
createDirectoryIfNonexistent(dirPath)
// Open the file for appending, and create it if it doesn't exist
f, err := os.OpenFile(
filepath.Join(dirPath, "BUILD.bazel"),
os.O_APPEND|os.O_CREATE|os.O_WRONLY,
0644)
if err != nil {
return nil, err
}
// If the file is empty, add the load statement for the `soong_module` rule
fi, err := f.Stat()
if err != nil {
return nil, err
}
if fi.Size() == 0 {
f.Write([]byte(soongModuleLoad + "\n"))
}
return f, nil
}
func createDirectoryIfNonexistent(dir string) {
if _, err := os.Stat(dir); os.IsNotExist(err) {
os.MkdirAll(dir, os.ModePerm)
}
}
// The QueryView directory should be read-only, sufficient for bazel query. The files
// The auto-conversion directory should be read-only, sufficient for bazel query. The files
// are not intended to be edited by end users.
func writeReadOnlyFile(dir string, baseName string, content string) error {
createDirectoryIfNonexistent(dir)
pathToFile := filepath.Join(dir, baseName)
func writeReadOnlyFile(dir string, f bp2build.BazelFile) error {
dir = filepath.Join(dir, f.Dir)
if err := createDirectoryIfNonexistent(dir); err != nil {
return err
}
pathToFile := filepath.Join(dir, f.Basename)
// 0444 is read-only
return ioutil.WriteFile(pathToFile, []byte(content), 0444)
err := ioutil.WriteFile(pathToFile, []byte(f.Contents), 0444)
return err
}
func isZero(value reflect.Value) bool {
switch value.Kind() {
case reflect.Func, reflect.Map, reflect.Slice:
return value.IsNil()
case reflect.Array:
valueIsZero := true
for i := 0; i < value.Len(); i++ {
valueIsZero = valueIsZero && isZero(value.Index(i))
}
return valueIsZero
case reflect.Struct:
valueIsZero := true
for i := 0; i < value.NumField(); i++ {
if value.Field(i).CanSet() {
valueIsZero = valueIsZero && isZero(value.Field(i))
}
}
return valueIsZero
case reflect.Ptr:
if !value.IsNil() {
return isZero(reflect.Indirect(value))
} else {
return true
}
default:
zeroValue := reflect.Zero(value.Type())
result := value.Interface() == zeroValue.Interface()
return result
func createDirectoryIfNonexistent(dir string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
return os.MkdirAll(dir, os.ModePerm)
} else {
return err
}
}

470
cmd/soong_build/queryview_test.go
View File

@ -1,470 +0,0 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"android/soong/android"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/google/blueprint/bootstrap/bpdoc"
)
var buildDir string
func setUp() {
var err error
buildDir, err = ioutil.TempDir("", "bazel_queryview_test")
if err != nil {
panic(err)
}
}
func tearDown() {
os.RemoveAll(buildDir)
}
func TestMain(m *testing.M) {
run := func() int {
setUp()
defer tearDown()
return m.Run()
}
os.Exit(run())
}
type customModule struct {
android.ModuleBase
}
// OutputFiles is needed because some instances of this module use dist with a
// tag property which requires the module implements OutputFileProducer.
func (m *customModule) OutputFiles(tag string) (android.Paths, error) {
return android.PathsForTesting("path" + tag), nil
}
func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// nothing for now.
}
func customModuleFactory() android.Module {
module := &customModule{}
android.InitAndroidModule(module)
return module
}
func TestGenerateBazelQueryViewFromBlueprint(t *testing.T) {
testCases := []struct {
bp string
expectedBazelTarget string
}{
{
bp: `custom {
name: "foo",
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
)`,
},
{
bp: `custom {
name: "foo",
ramdisk: true,
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
ramdisk = True,
)`,
},
{
bp: `custom {
name: "foo",
owner: "a_string_with\"quotes\"_and_\\backslashes\\\\",
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
owner = "a_string_with\"quotes\"_and_\\backslashes\\\\",
)`,
},
{
bp: `custom {
name: "foo",
required: ["bar"],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
required = [
"bar",
],
)`,
},
{
bp: `custom {
name: "foo",
target_required: ["qux", "bazqux"],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
target_required = [
"qux",
"bazqux",
],
)`,
},
{
bp: `custom {
name: "foo",
dist: {
targets: ["goal_foo"],
tag: ".foo",
},
dists: [
{
targets: ["goal_bar"],
tag: ".bar",
},
],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
dist = {
"tag": ".foo",
"targets": [
"goal_foo",
],
},
dists = [
{
"tag": ".bar",
"targets": [
"goal_bar",
],
},
],
)`,
},
{
bp: `custom {
name: "foo",
required: ["bar"],
target_required: ["qux", "bazqux"],
ramdisk: true,
owner: "custom_owner",
dists: [
{
tag: ".tag",
targets: ["my_goal"],
},
],
}
`,
expectedBazelTarget: `soong_module(
name = "foo",
module_name = "foo",
module_type = "custom",
module_variant = "",
module_deps = [
],
dists = [
{
"tag": ".tag",
"targets": [
"my_goal",
],
},
],
owner = "custom_owner",
ramdisk = True,
required = [
"bar",
],
target_required = [
"qux",
"bazqux",
],
)`,
},
}
for _, testCase := range testCases {
config := android.TestConfig(buildDir, nil, testCase.bp, nil)
ctx := android.NewTestContext(config)
ctx.RegisterModuleType("custom", customModuleFactory)
ctx.Register()
_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
android.FailIfErrored(t, errs)
_, errs = ctx.PrepareBuildActions(config)
android.FailIfErrored(t, errs)
module := ctx.ModuleForTests("foo", "").Module().(*customModule)
blueprintCtx := ctx.Context.Context
actualBazelTarget := generateSoongModuleTarget(blueprintCtx, module)
if actualBazelTarget != testCase.expectedBazelTarget {
t.Errorf(
"Expected generated Bazel target to be '%s', got '%s'",
testCase.expectedBazelTarget,
actualBazelTarget,
)
}
}
}
func createPackageFixtures() []*bpdoc.Package {
properties := []bpdoc.Property{
bpdoc.Property{
Name: "int64_prop",
Type: "int64",
},
bpdoc.Property{
Name: "int_prop",
Type: "int",
},
bpdoc.Property{
Name: "bool_prop",
Type: "bool",
},
bpdoc.Property{
Name: "string_prop",
Type: "string",
},
bpdoc.Property{
Name: "string_list_prop",
Type: "list of string",
},
bpdoc.Property{
Name: "nested_prop",
Type: "",
Properties: []bpdoc.Property{
bpdoc.Property{
Name: "int_prop",
Type: "int",
},
bpdoc.Property{
Name: "bool_prop",
Type: "bool",
},
bpdoc.Property{
Name: "string_prop",
Type: "string",
},
},
},
bpdoc.Property{
Name: "unknown_type",
Type: "unknown",
},
}
fooPropertyStruct := &bpdoc.PropertyStruct{
Name: "FooProperties",
Properties: properties,
}
moduleTypes := []*bpdoc.ModuleType{
&bpdoc.ModuleType{
Name: "foo_library",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
&bpdoc.ModuleType{
Name: "foo_binary",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
&bpdoc.ModuleType{
Name: "foo_test",
PropertyStructs: []*bpdoc.PropertyStruct{
fooPropertyStruct,
},
},
}
return [](*bpdoc.Package){
&bpdoc.Package{
Name: "foo_language",
Path: "android/soong/foo",
ModuleTypes: moduleTypes,
},
}
}
func TestGenerateModuleRuleShims(t *testing.T) {
ruleShims, err := createRuleShims(createPackageFixtures())
if err != nil {
panic(err)
}
if len(ruleShims) != 1 {
t.Errorf("Expected to generate 1 rule shim, but got %d", len(ruleShims))
}
fooRuleShim := ruleShims["foo"]
expectedRules := []string{"foo_binary", "foo_library", "foo_test_"}
if len(fooRuleShim.rules) != 3 {
t.Errorf("Expected 3 rules, but got %d", len(fooRuleShim.rules))
}
for i, rule := range fooRuleShim.rules {
if rule != expectedRules[i] {
t.Errorf("Expected rule shim to contain %s, but got %s", expectedRules[i], rule)
}
}
expectedBzl := `load("//build/bazel/queryview_rules:providers.bzl", "SoongModuleInfo")
def _foo_binary_impl(ctx):
return [SoongModuleInfo()]
foo_binary = rule(
implementation = _foo_binary_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
def _foo_library_impl(ctx):
return [SoongModuleInfo()]
foo_library = rule(
implementation = _foo_library_impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
def _foo_test__impl(ctx):
return [SoongModuleInfo()]
foo_test_ = rule(
implementation = _foo_test__impl,
attrs = {
"module_name": attr.string(mandatory = True),
"module_variant": attr.string(),
"module_deps": attr.label_list(providers = [SoongModuleInfo]),
"bool_prop": attr.bool(),
"int64_prop": attr.int(),
"int_prop": attr.int(),
# "nested_prop__int_prop": attr.int(),
# "nested_prop__bool_prop": attr.bool(),
# "nested_prop__string_prop": attr.string(),
"string_list_prop": attr.string_list(),
"string_prop": attr.string(),
},
)
`
if fooRuleShim.content != expectedBzl {
t.Errorf(
"Expected the generated rule shim bzl to be:\n%s\nbut got:\n%s",
expectedBzl,
fooRuleShim.content)
}
}
func TestGenerateSoongModuleBzl(t *testing.T) {
ruleShims, err := createRuleShims(createPackageFixtures())
if err != nil {
panic(err)
}
actualSoongModuleBzl := generateSoongModuleBzl(ruleShims)
expectedLoad := "load(\"//build/bazel/queryview_rules:foo.bzl\", \"foo_binary\", \"foo_library\", \"foo_test_\")"
expectedRuleMap := `soong_module_rule_map = {
"foo_binary": foo_binary,
"foo_library": foo_library,
"foo_test_": foo_test_,
}`
if !strings.Contains(actualSoongModuleBzl, expectedLoad) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the load statement in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl,
expectedLoad)
}
if !strings.Contains(actualSoongModuleBzl, expectedRuleMap) {
t.Errorf(
"Generated soong_module.bzl:\n\n%s\n\n"+
"Could not find the module -> rule map in the generated soong_module.bzl:\n%s",
actualSoongModuleBzl,
expectedRuleMap)
}
}