Merge "Surface module properties as Bazel BUILD target attributes in the Bazel overlay"

cmd/soong_build/Android.bp

@@ -28,5 +28,8 @@ bootstrap_go_binary {
         "writedocs.go",
         "bazel_overlay.go",
     ],
+    testSrcs: [
+        "bazel_overlay_test.go",
+    ],
     primaryBuilder: true,
 }

cmd/soong_build/bazel_overlay.go

@@ -20,14 +20,17 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 
 	"github.com/google/blueprint"
+	"github.com/google/blueprint/proptools"
 )
 
 const (
 	soongModuleLoad = `package(default_visibility = ["//visibility:public"])
 load("//:soong_module.bzl", "soong_module")
+
 `
 
 	// A BUILD file target snippet representing a Soong module
@@ -36,15 +39,11 @@ load("//:soong_module.bzl", "soong_module")
     module_name = "%s",
     module_type = "%s",
     module_variant = "%s",
-    deps = [
+    deps = %s,
-        %s
+%s)`
-    ],
-)
-`
 
 	// The soong_module rule implementation in a .bzl file
-	soongModuleBzl = `
+	soongModuleBzl = `SoongModuleInfo = provider(
-SoongModuleInfo = provider(
     fields = {
         "name": "Name of module",
         "type": "Type of module",
@@ -52,7 +51,17 @@ SoongModuleInfo = provider(
     },
 )
 
-def _soong_module_impl(ctx):
+def _merge_dicts(*dicts):
+    """Adds a list of dictionaries into a single dictionary."""
+
+    # If keys are repeated in multiple dictionaries, the latter one "wins".
+    result = {}
+    for d in dicts:
+        result.update(d)
+
+    return result
+
+def _generic_soong_module_impl(ctx):
     return [
         SoongModuleInfo(
             name = ctx.attr.module_name,
@@ -61,21 +70,66 @@ def _soong_module_impl(ctx):
         ),
     ]
 
-soong_module = rule(
+_COMMON_ATTRS = {
-    implementation = _soong_module_impl,
+    "module_name": attr.string(mandatory = True),
-    attrs = {
+    "module_type": attr.string(mandatory = True),
-        "module_name": attr.string(mandatory = True),
+    "module_variant": attr.string(),
-        "module_type": attr.string(mandatory = True),
+    "deps": attr.label_list(providers = [SoongModuleInfo]),
-        "module_variant": attr.string(),
+}
-        "deps": attr.label_list(providers = [SoongModuleInfo]),
+
-    },
+
+generic_soong_module = rule(
+    implementation = _generic_soong_module_impl,
+    attrs = _COMMON_ATTRS,
 )
+
+# TODO(jingwen): auto generate Soong module shims
+def _soong_filegroup_impl(ctx):
+    return [SoongModuleInfo(),]
+
+soong_filegroup = rule(
+    implementation = _soong_filegroup_impl,
+    # Matches https://cs.android.com/android/platform/superproject/+/master:build/soong/android/filegroup.go;l=25-40;drc=6a6478d49e78703ba22a432c41d819c8df79ef6c
+    attrs = _merge_dicts(_COMMON_ATTRS, {
+        "srcs": attr.string_list(doc = "srcs lists files that will be included in this filegroup"),
+        "exclude_srcs": attr.string_list(),
+        "path": attr.string(doc = "The base path to the files. May be used by other modules to determine which portion of the path to use. For example, when a filegroup is used as data in a cc_test rule, the base path is stripped off the path and the remaining path is used as the installation directory."),
+        "export_to_make_var": attr.string(doc = "Create a make variable with the specified name that contains the list of files in the filegroup, relative to the root of the source tree."),
+    })
+)
+
+soong_module_rule_map = {
+    "filegroup": soong_filegroup,
+}
+
+# soong_module is a macro that supports arbitrary kwargs, and uses module_type to
+# expand to the right underlying shim.
+def soong_module(name, module_type, **kwargs):
+    soong_module_rule = soong_module_rule_map.get(module_type)
+
+    if soong_module_rule == None:
+        # This module type does not have an existing rule to map to, so use the
+        # generic_soong_module rule instead.
+        generic_soong_module(
+            name = name,
+            module_type = module_type,
+            module_name = kwargs.pop("module_name", ""),
+            module_variant = kwargs.pop("module_variant", ""),
+            deps = kwargs.pop("deps", []),
+        )
+    else:
+        soong_module_rule(
+            name = name,
+            module_type = module_type,
+            **kwargs,
+        )
 `
 )
 
 func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string {
 	name := ""
 	if c.ModuleSubDir(logicModule) != "" {
+		// TODO(b/162720883): Figure out a way to drop the "--" variant suffixes.
 		name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
 	} else {
 		name = c.ModuleName(logicModule)
@@ -95,6 +149,153 @@ func packagePath(c *blueprint.Context, logicModule blueprint.Module) string {
 	return filepath.Dir(c.BlueprintFile(logicModule))
 }
 
+func escapeString(s string) string {
+	s = strings.ReplaceAll(s, "\\", "\\\\")
+	return strings.ReplaceAll(s, "\"", "\\\"")
+}
+
+func makeIndent(indent int) string {
+	if indent < 0 {
+		panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent))
+	}
+	return strings.Repeat("    ", indent)
+}
+
+// prettyPrint a property value into the equivalent Starlark representation
+// recursively.
+func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
+	if isZero(propertyValue) {
+		// A property value being set or unset actually matters -- Soong does set default
+		// values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at
+		// https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480
+		//
+		// In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default
+		// value of unset attributes.
+		return "", nil
+	}
+
+	var ret string
+	switch propertyValue.Kind() {
+	case reflect.String:
+		ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String()))
+	case reflect.Bool:
+		ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface()))
+	case reflect.Int, reflect.Uint, reflect.Int64:
+		ret = fmt.Sprintf("%v", propertyValue.Interface())
+	case reflect.Ptr:
+		return prettyPrint(propertyValue.Elem(), indent)
+	case reflect.Slice:
+		ret = "[\n"
+		for i := 0; i < propertyValue.Len(); i++ {
+			indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1)
+			if err != nil {
+				return "", err
+			}
+
+			if indexedValue != "" {
+				ret += makeIndent(indent + 1)
+				ret += indexedValue
+				ret += ",\n"
+			}
+		}
+		ret += makeIndent(indent)
+		ret += "]"
+	case reflect.Struct:
+		ret = "{\n"
+		// Sort and print the struct props by the key.
+		structProps := extractStructProperties(propertyValue, indent)
+		for _, k := range android.SortedStringKeys(structProps) {
+			ret += makeIndent(indent + 1)
+			ret += "\"" + k + "\": "
+			ret += structProps[k]
+			ret += ",\n"
+		}
+		ret += makeIndent(indent)
+		ret += "}"
+	case reflect.Interface:
+		// TODO(b/164227191): implement pretty print for interfaces.
+		// Interfaces are used for for arch, multilib and target properties.
+		return "", nil
+	default:
+		return "", fmt.Errorf(
+			"unexpected kind for property struct field: %s", propertyValue.Kind())
+	}
+	return ret, nil
+}
+
+func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
+	if structValue.Kind() != reflect.Struct {
+		panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
+	}
+
+	ret := map[string]string{}
+	structType := structValue.Type()
+	for i := 0; i < structValue.NumField(); i++ {
+		field := structType.Field(i)
+		if field.PkgPath != "" {
+			// Skip unexported fields. Some properties are
+			// internal to Soong only, and these fields do not have PkgPath.
+			continue
+		}
+		if proptools.HasTag(field, "blueprint", "mutated") {
+			continue
+		}
+
+		fieldValue := structValue.Field(i)
+		if isZero(fieldValue) {
+			// Ignore zero-valued fields
+			continue
+		}
+
+		propertyName := proptools.PropertyNameForField(field.Name)
+		prettyPrintedValue, err := prettyPrint(fieldValue, indent+1)
+		if err != nil {
+			panic(
+				fmt.Errorf(
+					"Error while parsing property: %q. %s",
+					propertyName,
+					err))
+		}
+		if prettyPrintedValue != "" {
+			ret[propertyName] = prettyPrintedValue
+		}
+	}
+
+	return ret
+}
+
+func isStructPtr(t reflect.Type) bool {
+	return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
+}
+
+// Generically extract module properties and types into a map, keyed by the module property name.
+func extractModuleProperties(aModule android.Module) map[string]string {
+	ret := map[string]string{}
+
+	// Iterate over this android.Module's property structs.
+	for _, properties := range aModule.GetProperties() {
+		propertiesValue := reflect.ValueOf(properties)
+		// Check that propertiesValue is a pointer to the Properties struct, like
+		// *cc.BaseLinkerProperties or *java.CompilerProperties.
+		//
+		// propertiesValue can also be type-asserted to the structs to
+		// manipulate internal props, if needed.
+		if isStructPtr(propertiesValue.Type()) {
+			structValue := propertiesValue.Elem()
+			for k, v := range extractStructProperties(structValue, 0) {
+				ret[k] = v
+			}
+		} else {
+			panic(fmt.Errorf(
+				"properties must be a pointer to a struct, got %T",
+				propertiesValue.Interface()))
+		}
+
+	}
+
+	return ret
+}
+
 func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
 	blueprintCtx := ctx.Context
 	blueprintCtx.VisitAllModules(func(module blueprint.Module) {
@@ -103,27 +304,7 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
 			panic(err)
 		}
 
-		// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
+		buildFile.Write([]byte(generateSoongModuleTarget(blueprintCtx, module) + "\n\n"))
-		// items, if the modules are added using different DependencyTag. Figure
-		// out the implications of that.
-		depLabels := map[string]bool{}
-		blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) {
-			depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true
-		})
-
-		var depLabelList string
-		for depLabel, _ := range depLabels {
-			depLabelList += "\"" + depLabel + "\",\n        "
-		}
-		buildFile.Write([]byte(
-			fmt.Sprintf(
-				soongModuleTarget,
-				targetNameWithVariant(blueprintCtx, module),
-				blueprintCtx.ModuleName(module),
-				blueprintCtx.ModuleType(module),
-				// misleading name, this actually returns the variant.
-				blueprintCtx.ModuleSubDir(module),
-				depLabelList)))
 		buildFile.Close()
 	})
 
@@ -138,6 +319,70 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error {
 	return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl)
 }
 
+var ignoredProps map[string]bool = map[string]bool{
+	"name":       true, // redundant, since this is explicitly generated for every target
+	"from":       true, // reserved keyword
+	"in":         true, // reserved keyword
+	"arch":       true, // interface prop type is not supported yet.
+	"multilib":   true, // interface prop type is not supported yet.
+	"target":     true, // interface prop type is not supported yet.
+	"visibility": true, // Bazel has native visibility semantics. Handle later.
+}
+
+func shouldGenerateAttribute(prop string) bool {
+	return !ignoredProps[prop]
+}
+
+// props is an unsorted map. This function ensures that
+// the generated attributes are sorted to ensure determinism.
+func propsToAttributes(props map[string]string) string {
+	var attributes string
+	for _, propName := range android.SortedStringKeys(props) {
+		if shouldGenerateAttribute(propName) {
+			attributes += fmt.Sprintf("    %s = %s,\n", propName, props[propName])
+		}
+	}
+	return attributes
+}
+
+// Convert a module and its deps and props into a Bazel macro/rule
+// representation in the BUILD file.
+func generateSoongModuleTarget(
+	blueprintCtx *blueprint.Context,
+	module blueprint.Module) string {
+
+	var props map[string]string
+	if aModule, ok := module.(android.Module); ok {
+		props = extractModuleProperties(aModule)
+	}
+	attributes := propsToAttributes(props)
+
+	// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
+	// items, if the modules are added using different DependencyTag. Figure
+	// out the implications of that.
+	depLabels := map[string]bool{}
+	blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) {
+		depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true
+	})
+
+	depLabelList := "[\n"
+	for depLabel, _ := range depLabels {
+		depLabelList += "        \""
+		depLabelList += depLabel
+		depLabelList += "\",\n"
+	}
+	depLabelList += "    ]"
+
+	return fmt.Sprintf(
+		soongModuleTarget,
+		targetNameWithVariant(blueprintCtx, module),
+		blueprintCtx.ModuleName(module),
+		blueprintCtx.ModuleType(module),
+		blueprintCtx.ModuleSubDir(module),
+		depLabelList,
+		attributes)
+}
+
 func buildFileForModule(ctx *blueprint.Context, module blueprint.Module) (*os.File, error) {
 	// Create nested directories for the BUILD file
 	dirPath := filepath.Join(bazelOverlayDir, packagePath(ctx, module))
@@ -171,3 +416,34 @@ func writeReadOnlyFile(dir string, baseName string, content string) error {
 	// 0444 is read-only
 	return ioutil.WriteFile(workspaceFile, []byte(content), 0444)
 }
+
+func isZero(value reflect.Value) bool {
+	switch value.Kind() {
+	case reflect.Func, reflect.Map, reflect.Slice:
+		return value.IsNil()
+	case reflect.Array:
+		valueIsZero := true
+		for i := 0; i < value.Len(); i++ {
+			valueIsZero = valueIsZero && isZero(value.Index(i))
+		}
+		return valueIsZero
+	case reflect.Struct:
+		valueIsZero := true
+		for i := 0; i < value.NumField(); i++ {
+			if value.Field(i).CanSet() {
+				valueIsZero = valueIsZero && isZero(value.Field(i))
+			}
+		}
+		return valueIsZero
+	case reflect.Ptr:
+		if !value.IsNil() {
+			return isZero(reflect.Indirect(value))
+		} else {
+			return true
+		}
+	default:
+		zeroValue := reflect.Zero(value.Type())
+		result := value.Interface() == zeroValue.Interface()
+		return result
+	}
+}

cmd/soong_build/bazel_overlay_test.go

@@ -0,0 +1,255 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"android/soong/android"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+var buildDir string
+
+func setUp() {
+	var err error
+	buildDir, err = ioutil.TempDir("", "bazel_overlay_test")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func tearDown() {
+	os.RemoveAll(buildDir)
+}
+
+func TestMain(m *testing.M) {
+	run := func() int {
+		setUp()
+		defer tearDown()
+
+		return m.Run()
+	}
+
+	os.Exit(run())
+}
+
+type customModule struct {
+	android.ModuleBase
+}
+
+func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+	// nothing for now.
+}
+
+func customModuleFactory() android.Module {
+	module := &customModule{}
+	android.InitAndroidModule(module)
+	return module
+}
+
+func TestGenerateBazelOverlayFromBlueprint(t *testing.T) {
+	testCases := []struct {
+		bp                  string
+		expectedBazelTarget string
+	}{
+		{
+			bp: `custom {
+	name: "foo",
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	ramdisk: true,
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    ramdisk = True,
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	owner: "a_string_with\"quotes\"_and_\\backslashes\\\\",
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    owner = "a_string_with\"quotes\"_and_\\backslashes\\\\",
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	required: ["bar"],
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    required = [
+        "bar",
+    ],
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	target_required: ["qux", "bazqux"],
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    target_required = [
+        "qux",
+        "bazqux",
+    ],
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	dist: {
+		targets: ["goal_foo"],
+		tag: ".foo",
+	},
+	dists: [
+		{
+			targets: ["goal_bar"],
+			tag: ".bar",
+		},
+	],
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    dist = {
+        "tag": ".foo",
+        "targets": [
+            "goal_foo",
+        ],
+    },
+    dists = [
+        {
+            "tag": ".bar",
+            "targets": [
+                "goal_bar",
+            ],
+        },
+    ],
+)`,
+		},
+		{
+			bp: `custom {
+	name: "foo",
+	required: ["bar"],
+	target_required: ["qux", "bazqux"],
+	ramdisk: true,
+	owner: "custom_owner",
+	dists: [
+		{
+			tag: ".tag",
+			targets: ["my_goal"],
+		},
+	],
+}
+		`,
+			expectedBazelTarget: `soong_module(
+    name = "foo",
+    module_name = "foo",
+    module_type = "custom",
+    module_variant = "",
+    deps = [
+    ],
+    dists = [
+        {
+            "tag": ".tag",
+            "targets": [
+                "my_goal",
+            ],
+        },
+    ],
+    owner = "custom_owner",
+    ramdisk = True,
+    required = [
+        "bar",
+    ],
+    target_required = [
+        "qux",
+        "bazqux",
+    ],
+)`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		config := android.TestConfig(buildDir, nil, testCase.bp, nil)
+		ctx := android.NewTestContext()
+		ctx.RegisterModuleType("custom", customModuleFactory)
+		ctx.Register(config)
+
+		_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+		android.FailIfErrored(t, errs)
+		_, errs = ctx.PrepareBuildActions(config)
+		android.FailIfErrored(t, errs)
+
+		module := ctx.ModuleForTests("foo", "").Module().(*customModule)
+		blueprintCtx := ctx.Context.Context
+
+		actualBazelTarget := generateSoongModuleTarget(blueprintCtx, module)
+		if actualBazelTarget != testCase.expectedBazelTarget {
+			t.Errorf(
+				"Expected generated Bazel target to be '%s', got '%s'",
+				testCase.expectedBazelTarget,
+				actualBazelTarget,
+			)
+		}
+	}
+}