Annotate dependency tags for dependencies of installed files

Relands Ic22603a5c0718b5a21686672a7471f952b4d1017 with a minor
change to track libc++ dependencies for python hosts and after
a fix to an internal genrule that depended on transitively
installed java libraries (ag/13068670).

Soong currently assumes that installed files should depend on
installed files of all transitive dependencies, which results
in extra installed file dependencies through genrules, static
libs, etc.

Annotate dependency tags for dependencies for which the
installed files are necessary such as shared libraries
and JNI libraries.

This avoids extra installed files, and is also a first step
towards genrules using their own copy of tools instead of
the installed copy.

Bug: 124313442
Test: m checkbuild
Test: java.TestBinary
Test: cc.TestInstallSharedLibs
Test: deptag_test.go
Change-Id: I725871249d561428e6f67bba6a7c65b580012b72
This commit is contained in:
Colin Cross 2020-11-10 18:12:15 -08:00
parent b893f8766c
commit e9fe2949b8
15 changed files with 346 additions and 30 deletions

View File

@ -22,6 +22,7 @@ bootstrap_go_package {
"defaults.go",
"defs.go",
"depset.go",
"deptag.go",
"expand.go",
"filegroup.go",
"hooks.go",
@ -71,6 +72,7 @@ bootstrap_go_package {
"config_test.go",
"csuite_config_test.go",
"depset_test.go",
"deptag_test.go",
"expand_test.go",
"module_test.go",
"mutator_test.go",

View File

@ -271,7 +271,7 @@ func newOsType(name string, class OsClass, defDisabled bool, archTypes ...ArchTy
if _, found := commonTargetMap[name]; found {
panic(fmt.Errorf("Found Os type duplicate during OsType registration: %q", name))
} else {
commonTargetMap[name] = Target{Os: os, Arch: Arch{ArchType: Common}}
commonTargetMap[name] = Target{Os: os, Arch: CommonArch}
}
osArchTypeMap[os] = archTypes
@ -341,6 +341,10 @@ var (
// CommonOS is a pseudo OSType for a common OS variant, which is OsType agnostic and which
// has dependencies on all the OS variants.
CommonOS = newOsType("common_os", Generic, false)
// CommonArch is the Arch for all modules that are os-specific but not arch specific,
// for example most Java modules.
CommonArch = Arch{ArchType: Common}
)
// Target specifies the OS and architecture that a module is being compiled for.
@ -511,9 +515,6 @@ type archDepTag struct {
// Identifies the dependency from CommonOS variant to the os specific variants.
var commonOsToOsSpecificVariantTag = archDepTag{name: "common os to os specific"}
// Identifies the dependency from arch variant to the common variant for a "common_first" multilib.
var firstArchToCommonArchDepTag = archDepTag{name: "first arch to common arch"}
// Get the OsType specific variants for the current CommonOS variant.
//
// The returned list will only contain enabled OsType specific variants of the
@ -667,12 +668,6 @@ func archMutator(bpctx blueprint.BottomUpMutatorContext) {
addTargetProperties(m, targets[i], multiTargets, i == 0)
m.base().setArchProperties(mctx)
}
if multilib == "common_first" && len(modules) >= 2 {
for i := range modules[1:] {
mctx.AddInterVariantDependency(firstArchToCommonArchDepTag, modules[i+1], modules[0])
}
}
}
// addTargetProperties annotates a variant with the Target is is being compiled for, the list

45
android/deptag.go Normal file
View File

@ -0,0 +1,45 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import "github.com/google/blueprint"
// Dependency tags can implement this interface and return true from InstallDepNeeded to annotate
// that the installed files of the parent should depend on the installed files of the child.
type InstallNeededDependencyTag interface {
// If InstallDepNeeded returns true then the installed files of the parent will depend on the
// installed files of the child.
InstallDepNeeded() bool
}
// Dependency tags can embed this struct to annotate that the installed files of the parent should
// depend on the installed files of the child.
type InstallAlwaysNeededDependencyTag struct{}
func (i InstallAlwaysNeededDependencyTag) InstallDepNeeded() bool {
return true
}
var _ InstallNeededDependencyTag = InstallAlwaysNeededDependencyTag{}
// IsInstallDepNeeded returns true if the dependency tag implements the InstallNeededDependencyTag
// interface and the InstallDepNeeded returns true, meaning that the installed files of the parent
// should depend on the installed files of the child.
func IsInstallDepNeeded(tag blueprint.DependencyTag) bool {
if i, ok := tag.(InstallNeededDependencyTag); ok {
return i.InstallDepNeeded()
}
return false
}

135
android/deptag_test.go Normal file
View File

@ -0,0 +1,135 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package android
import (
"testing"
"github.com/google/blueprint"
)
type testInstallDependencyTagModule struct {
ModuleBase
Properties struct {
Install_deps []string
Deps []string
}
}
func (t *testInstallDependencyTagModule) GenerateAndroidBuildActions(ctx ModuleContext) {
outputFile := PathForModuleOut(ctx, "out")
ctx.Build(pctx, BuildParams{
Rule: Touch,
Output: outputFile,
})
ctx.InstallFile(PathForModuleInstall(ctx), ctx.ModuleName(), outputFile)
}
var testInstallDependencyTagAlwaysDepTag = struct {
blueprint.DependencyTag
InstallAlwaysNeededDependencyTag
}{}
var testInstallDependencyTagNeverDepTag = struct {
blueprint.DependencyTag
}{}
func (t *testInstallDependencyTagModule) DepsMutator(ctx BottomUpMutatorContext) {
ctx.AddVariationDependencies(nil, testInstallDependencyTagAlwaysDepTag, t.Properties.Install_deps...)
ctx.AddVariationDependencies(nil, testInstallDependencyTagNeverDepTag, t.Properties.Deps...)
}
func testInstallDependencyTagModuleFactory() Module {
module := &testInstallDependencyTagModule{}
InitAndroidArchModule(module, HostAndDeviceDefault, MultilibCommon)
module.AddProperties(&module.Properties)
return module
}
func TestInstallDependencyTag(t *testing.T) {
bp := `
test_module {
name: "foo",
deps: ["dep"],
install_deps: ["install_dep"],
}
test_module {
name: "install_dep",
install_deps: ["transitive"],
}
test_module {
name: "transitive",
}
test_module {
name: "dep",
}
`
config := TestArchConfig(buildDir, nil, bp, nil)
ctx := NewTestArchContext(config)
ctx.RegisterModuleType("test_module", testInstallDependencyTagModuleFactory)
ctx.Register()
_, errs := ctx.ParseFileList(".", []string{"Android.bp"})
FailIfErrored(t, errs)
_, errs = ctx.PrepareBuildActions(config)
FailIfErrored(t, errs)
hostFoo := ctx.ModuleForTests("foo", config.BuildOSCommonTarget.String()).Description("install")
hostInstallDep := ctx.ModuleForTests("install_dep", config.BuildOSCommonTarget.String()).Description("install")
hostTransitive := ctx.ModuleForTests("transitive", config.BuildOSCommonTarget.String()).Description("install")
hostDep := ctx.ModuleForTests("dep", config.BuildOSCommonTarget.String()).Description("install")
if g, w := hostFoo.Implicits.Strings(), hostInstallDep.Output.String(); !InList(w, g) {
t.Errorf("expected host dependency %q, got %q", w, g)
}
if g, w := hostFoo.Implicits.Strings(), hostTransitive.Output.String(); !InList(w, g) {
t.Errorf("expected host dependency %q, got %q", w, g)
}
if g, w := hostInstallDep.Implicits.Strings(), hostTransitive.Output.String(); !InList(w, g) {
t.Errorf("expected host dependency %q, got %q", w, g)
}
if g, w := hostFoo.Implicits.Strings(), hostDep.Output.String(); InList(w, g) {
t.Errorf("expected no host dependency %q, got %q", w, g)
}
deviceFoo := ctx.ModuleForTests("foo", "android_common").Description("install")
deviceInstallDep := ctx.ModuleForTests("install_dep", "android_common").Description("install")
deviceTransitive := ctx.ModuleForTests("transitive", "android_common").Description("install")
deviceDep := ctx.ModuleForTests("dep", "android_common").Description("install")
if g, w := deviceFoo.OrderOnly.Strings(), deviceInstallDep.Output.String(); !InList(w, g) {
t.Errorf("expected device dependency %q, got %q", w, g)
}
if g, w := deviceFoo.OrderOnly.Strings(), deviceTransitive.Output.String(); !InList(w, g) {
t.Errorf("expected device dependency %q, got %q", w, g)
}
if g, w := deviceInstallDep.OrderOnly.Strings(), deviceTransitive.Output.String(); !InList(w, g) {
t.Errorf("expected device dependency %q, got %q", w, g)
}
if g, w := deviceFoo.OrderOnly.Strings(), deviceDep.Output.String(); InList(w, g) {
t.Errorf("expected no device dependency %q, got %q", w, g)
}
}

View File

@ -1286,14 +1286,18 @@ func (m *ModuleBase) ExportedToMake() bool {
return m.commonProperties.NamespaceExportedToMake
}
// computeInstallDeps finds the installed paths of all dependencies that have a dependency
// tag that is annotated as needing installation via the IsInstallDepNeeded method.
func (m *ModuleBase) computeInstallDeps(ctx blueprint.ModuleContext) InstallPaths {
var result InstallPaths
// TODO(ccross): we need to use WalkDeps and have some way to know which dependencies require installation
ctx.VisitDepsDepthFirst(func(m blueprint.Module) {
if a, ok := m.(Module); ok {
result = append(result, a.FilesToInstall()...)
ctx.WalkDeps(func(child, parent blueprint.Module) bool {
if a, ok := child.(Module); ok {
if IsInstallDepNeeded(ctx.OtherModuleDependencyTag(child)) {
result = append(result, a.FilesToInstall()...)
return true
}
}
return false
})
return result

View File

@ -551,7 +551,15 @@ func (d libraryDependencyTag) static() bool {
return d.Kind == staticLibraryDependency
}
// dependencyTag is used for tagging miscellanous dependency types that don't fit into
// InstallDepNeeded returns true for shared libraries so that shared library dependencies of
// binaries or other shared libraries are installed as dependencies.
func (d libraryDependencyTag) InstallDepNeeded() bool {
return d.shared()
}
var _ android.InstallNeededDependencyTag = libraryDependencyTag{}
// dependencyTag is used for tagging miscellaneous dependency types that don't fit into
// libraryDependencyTag. Each tag object is created globally and reused for multiple
// dependencies (although since the object contains no references, assigning a tag to a
// variable and modifying it will not modify the original). Users can compare the tag
@ -561,6 +569,15 @@ type dependencyTag struct {
name string
}
// installDependencyTag is used for tagging miscellaneous dependency types that don't fit into
// libraryDependencyTag, but where the dependency needs to be installed when the parent is
// installed.
type installDependencyTag struct {
blueprint.BaseDependencyTag
android.InstallAlwaysNeededDependencyTag
name string
}
var (
genSourceDepTag = dependencyTag{name: "gen source"}
genHeaderDepTag = dependencyTag{name: "gen header"}
@ -572,7 +589,7 @@ var (
staticVariantTag = dependencyTag{name: "static variant"}
vndkExtDepTag = dependencyTag{name: "vndk extends"}
dataLibDepTag = dependencyTag{name: "data lib"}
runtimeDepTag = dependencyTag{name: "runtime lib"}
runtimeDepTag = installDependencyTag{name: "runtime lib"}
testPerSrcDepTag = dependencyTag{name: "test_per_src"}
stubImplDepTag = dependencyTag{name: "stub_impl"}
)
@ -599,8 +616,7 @@ func IsHeaderDepTag(depTag blueprint.DependencyTag) bool {
}
func IsRuntimeDepTag(depTag blueprint.DependencyTag) bool {
ccDepTag, ok := depTag.(dependencyTag)
return ok && ccDepTag == runtimeDepTag
return depTag == runtimeDepTag
}
func IsTestPerSrcDepTag(depTag blueprint.DependencyTag) bool {

View File

@ -4069,3 +4069,98 @@ func TestEmptyWholeStaticLibsAllowMissingDependencies(t *testing.T) {
}
}
func TestInstallSharedLibs(t *testing.T) {
bp := `
cc_binary {
name: "bin",
host_supported: true,
shared_libs: ["libshared"],
runtime_libs: ["libruntime"],
srcs: [":gen"],
}
cc_library_shared {
name: "libshared",
host_supported: true,
shared_libs: ["libtransitive"],
}
cc_library_shared {
name: "libtransitive",
host_supported: true,
}
cc_library_shared {
name: "libruntime",
host_supported: true,
}
cc_binary_host {
name: "tool",
srcs: ["foo.cpp"],
}
genrule {
name: "gen",
tools: ["tool"],
out: ["gen.cpp"],
cmd: "$(location tool) $(out)",
}
`
config := TestConfig(buildDir, android.Android, nil, bp, nil)
ctx := testCcWithConfig(t, config)
hostBin := ctx.ModuleForTests("bin", config.BuildOSTarget.String()).Description("install")
hostShared := ctx.ModuleForTests("libshared", config.BuildOSTarget.String()+"_shared").Description("install")
hostRuntime := ctx.ModuleForTests("libruntime", config.BuildOSTarget.String()+"_shared").Description("install")
hostTransitive := ctx.ModuleForTests("libtransitive", config.BuildOSTarget.String()+"_shared").Description("install")
hostTool := ctx.ModuleForTests("tool", config.BuildOSTarget.String()).Description("install")
if g, w := hostBin.Implicits.Strings(), hostShared.Output.String(); !android.InList(w, g) {
t.Errorf("expected host bin dependency %q, got %q", w, g)
}
if g, w := hostBin.Implicits.Strings(), hostTransitive.Output.String(); !android.InList(w, g) {
t.Errorf("expected host bin dependency %q, got %q", w, g)
}
if g, w := hostShared.Implicits.Strings(), hostTransitive.Output.String(); !android.InList(w, g) {
t.Errorf("expected host bin dependency %q, got %q", w, g)
}
if g, w := hostBin.Implicits.Strings(), hostRuntime.Output.String(); !android.InList(w, g) {
t.Errorf("expected host bin dependency %q, got %q", w, g)
}
if g, w := hostBin.Implicits.Strings(), hostTool.Output.String(); android.InList(w, g) {
t.Errorf("expected no host bin dependency %q, got %q", w, g)
}
deviceBin := ctx.ModuleForTests("bin", "android_arm64_armv8-a").Description("install")
deviceShared := ctx.ModuleForTests("libshared", "android_arm64_armv8-a_shared").Description("install")
deviceTransitive := ctx.ModuleForTests("libtransitive", "android_arm64_armv8-a_shared").Description("install")
deviceRuntime := ctx.ModuleForTests("libruntime", "android_arm64_armv8-a_shared").Description("install")
if g, w := deviceBin.OrderOnly.Strings(), deviceShared.Output.String(); !android.InList(w, g) {
t.Errorf("expected device bin dependency %q, got %q", w, g)
}
if g, w := deviceBin.OrderOnly.Strings(), deviceTransitive.Output.String(); !android.InList(w, g) {
t.Errorf("expected device bin dependency %q, got %q", w, g)
}
if g, w := deviceShared.OrderOnly.Strings(), deviceTransitive.Output.String(); !android.InList(w, g) {
t.Errorf("expected device bin dependency %q, got %q", w, g)
}
if g, w := deviceBin.OrderOnly.Strings(), deviceRuntime.Output.String(); !android.InList(w, g) {
t.Errorf("expected device bin dependency %q, got %q", w, g)
}
if g, w := deviceBin.OrderOnly.Strings(), hostTool.Output.String(); android.InList(w, g) {
t.Errorf("expected no device bin dependency %q, got %q", w, g)
}
}

View File

@ -16,6 +16,7 @@ package cc
import (
"android/soong/android"
"android/soong/genrule"
)
func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) {
@ -24,6 +25,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) {
RegisterBinaryBuildComponents(ctx)
RegisterLibraryBuildComponents(ctx)
RegisterLibraryHeadersBuildComponents(ctx)
genrule.RegisterGenruleBuildComponents(ctx)
ctx.RegisterModuleType("toolchain_library", ToolchainLibraryFactory)
ctx.RegisterModuleType("llndk_library", LlndkLibraryFactory)

View File

@ -29,10 +29,10 @@ import (
)
func init() {
registerGenruleBuildComponents(android.InitRegistrationContext)
RegisterGenruleBuildComponents(android.InitRegistrationContext)
}
func registerGenruleBuildComponents(ctx android.RegistrationContext) {
func RegisterGenruleBuildComponents(ctx android.RegistrationContext) {
ctx.RegisterModuleType("genrule_defaults", defaultsFactory)
ctx.RegisterModuleType("gensrcs", GenSrcsFactory)

View File

@ -57,7 +57,7 @@ func testContext(config android.Config) *android.TestContext {
ctx.RegisterModuleType("filegroup", android.FileGroupFactory)
ctx.RegisterModuleType("tool", toolFactory)
registerGenruleBuildComponents(ctx)
RegisterGenruleBuildComponents(ctx)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
ctx.Register()

View File

@ -550,6 +550,14 @@ type dependencyTag struct {
name string
}
// installDependencyTag is a dependency tag that is annotated to cause the installed files of the
// dependency to be installed when the parent module is installed.
type installDependencyTag struct {
blueprint.BaseDependencyTag
android.InstallAlwaysNeededDependencyTag
name string
}
type usesLibraryDependencyTag struct {
dependencyTag
sdkVersion int // SDK version in which the library appared as a standalone library.
@ -584,6 +592,8 @@ var (
instrumentationForTag = dependencyTag{name: "instrumentation_for"}
extraLintCheckTag = dependencyTag{name: "extra-lint-check"}
jniLibTag = dependencyTag{name: "jnilib"}
jniInstallTag = installDependencyTag{name: "jni install"}
binaryInstallTag = installDependencyTag{name: "binary install"}
usesLibTag = makeUsesLibraryDependencyTag(dexpreopt.AnySdkVersion)
usesLibCompat28Tag = makeUsesLibraryDependencyTag(28)
usesLibCompat29Tag = makeUsesLibraryDependencyTag(29)
@ -2568,9 +2578,12 @@ func (j *Binary) DepsMutator(ctx android.BottomUpMutatorContext) {
if ctx.Arch().ArchType == android.Common {
j.deps(ctx)
} else {
// This dependency ensures the host installation rules will install the jni libraries
// when the wrapper is installed.
ctx.AddVariationDependencies(nil, jniLibTag, j.binaryProperties.Jni_libs...)
// These dependencies ensure the host installation rules will install the jar file and
// the jni libraries when the wrapper is installed.
ctx.AddVariationDependencies(nil, jniInstallTag, j.binaryProperties.Jni_libs...)
ctx.AddVariationDependencies(
[]blueprint.Variation{{Mutator: "arch", Variation: android.CommonArch.String()}},
binaryInstallTag, ctx.ModuleName())
}
}

View File

@ -30,7 +30,6 @@ import (
"android/soong/android"
"android/soong/cc"
"android/soong/dexpreopt"
"android/soong/genrule"
"android/soong/python"
)
@ -81,7 +80,6 @@ func testContext(config android.Config) *android.TestContext {
RegisterSystemModulesBuildComponents(ctx)
ctx.RegisterModuleType("java_plugin", PluginFactory)
ctx.RegisterModuleType("filegroup", android.FileGroupFactory)
ctx.RegisterModuleType("genrule", genrule.GenRuleFactory)
ctx.RegisterModuleType("python_binary_host", python.PythonBinaryHostFactory)
RegisterDocsBuildComponents(ctx)
RegisterStubsBuildComponents(ctx)

View File

@ -214,10 +214,16 @@ type dependencyTag struct {
name string
}
type installDependencyTag struct {
blueprint.BaseDependencyTag
android.InstallAlwaysNeededDependencyTag
name string
}
var (
pythonLibTag = dependencyTag{name: "pythonLib"}
launcherTag = dependencyTag{name: "launcher"}
launcherSharedLibTag = dependencyTag{name: "launcherSharedLib"}
launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
pyIdentifierRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
pyExt = ".py"
protoExt = ".proto"
@ -328,6 +334,7 @@ func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
// cannot read the property at this stage and it will be too late to add
// dependencies later.
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libsqlite")
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, "libc++")
if ctx.Target().Os.Bionic() {
ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag,

View File

@ -286,6 +286,12 @@ func TestSourceProviderDeps(t *testing.T) {
srcs: ["src/any.h"],
out: ["src/any.rs"],
}
rust_binary_host {
name: "any_rust_binary",
srcs: [
"foo.rs",
],
}
rust_bindgen {
name: "libbindings",
crate_name: "bindings",

View File

@ -17,7 +17,6 @@ package rust
import (
"android/soong/android"
"android/soong/cc"
"android/soong/genrule"
)
func GatherRequiredDepsForTest() string {
@ -132,7 +131,6 @@ func CreateTestContext(config android.Config) *android.TestContext {
android.RegisterPrebuiltMutators(ctx)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
cc.RegisterRequiredBuildComponentsForTest(ctx)
ctx.RegisterModuleType("genrule", genrule.GenRuleFactory)
ctx.RegisterModuleType("rust_binary", RustBinaryFactory)
ctx.RegisterModuleType("rust_binary_host", RustBinaryHostFactory)
ctx.RegisterModuleType("rust_bindgen", RustBindgenFactory)