Add support for multiple os types

Updates the member snapshot creation code to support multiple os types.
It basically sorts the variants by os type, then applies the code to
optimize the arch properties and then it optimizes the properties that
are common across architectures and extracts any properties that are
common across os types.

The java and cc member types needed to be modified to make the location
of the generated files within the snapshot os type dependent when there
is more than one os type. That was done by adding an OsPrefix() method
to the SdkMemberPropertiesBase which returns the os prefix to use when
there is > 1 os type and otherwise returns an empty string.

Added three tests, one for cc shared libraries, one for cc binary and
one for java header libraries.

Bug: 150451422
Test: m nothing
Change-Id: I08f5fbdd7852b06c9a9a2f1cfdc364338a3d5bac
This commit is contained in:
Paul Duffin 2020-03-02 10:16:35 +00:00
parent 71754bf640
commit a04c107bfa
10 changed files with 586 additions and 74 deletions

View File

@ -596,7 +596,7 @@ var BuildOs = func() OsType {
}()
var (
osTypeList []OsType
OsTypeList []OsType
commonTargetMap = make(map[string]Target)
NoOsType OsType
@ -672,7 +672,7 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType {
DefaultDisabled: defDisabled,
}
osTypeList = append(osTypeList, os)
OsTypeList = append(OsTypeList, os)
if _, found := commonTargetMap[name]; found {
panic(fmt.Errorf("Found Os type duplicate during OsType registration: %q", name))
@ -684,7 +684,7 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType {
}
func osByName(name string) OsType {
for _, os := range osTypeList {
for _, os := range OsTypeList {
if os.Name == name {
return os
}
@ -750,7 +750,7 @@ func osMutator(mctx BottomUpMutatorContext) {
var moduleOSList []OsType
for _, os := range osTypeList {
for _, os := range OsTypeList {
supportedClass := false
for _, osClass := range osClasses {
if os.Class == osClass {
@ -1071,7 +1071,7 @@ func createArchPropTypeDesc(props reflect.Type) []archPropTypeDesc {
"Arm_on_x86",
"Arm_on_x86_64",
}
for _, os := range osTypeList {
for _, os := range OsTypeList {
targets = append(targets, os.Field)
for _, archType := range osArchTypeMap[os] {

View File

@ -448,10 +448,29 @@ func RegisterSdkMemberType(memberType SdkMemberType) {
// Base structure for all implementations of SdkMemberProperties.
//
// Contains common properties that apply across many different member types.
// Contains common properties that apply across many different member types. These
// are not affected by the optimization to extract common values.
type SdkMemberPropertiesBase struct {
// The setting to use for the compile_multilib property.
Compile_multilib string
// The number of unique os types supported by the member variants.
Os_count int
// The os type for which these properties refer.
Os OsType
}
// The os prefix to use for any file paths in the sdk.
//
// Is an empty string if the member only provides variants for a single os type, otherwise
// is the OsType.Name.
func (b *SdkMemberPropertiesBase) OsPrefix() string {
if b.Os_count == 1 {
return ""
} else {
return b.Os.Name
}
}
func (b *SdkMemberPropertiesBase) Base() *SdkMemberPropertiesBase {

View File

@ -78,7 +78,7 @@ const (
// path to the native binary. Relative to <sdk_root>/<api_dir>
func nativeBinaryPathFor(lib nativeBinaryInfoProperties) string {
return filepath.Join(lib.archType,
return filepath.Join(lib.OsPrefix(), lib.archType,
nativeBinaryDir, lib.outputFile.Base())
}

View File

@ -254,7 +254,7 @@ const (
// path to the native library. Relative to <sdk_root>/<api_dir>
func nativeLibraryPathFor(lib *nativeLibInfoProperties) string {
return filepath.Join(lib.archType,
return filepath.Join(lib.OsPrefix(), lib.archType,
nativeStubDir, lib.outputFile.Base())
}

View File

@ -34,7 +34,7 @@ func RegisterRequiredBuildComponentsForTest(ctx android.RegistrationContext) {
ctx.RegisterModuleType("ndk_prebuilt_object", NdkPrebuiltObjectFactory)
}
func GatherRequiredDepsForTest(os android.OsType) string {
func GatherRequiredDepsForTest(oses ...android.OsType) string {
ret := `
toolchain_library {
name: "libatomic",
@ -341,6 +341,7 @@ func GatherRequiredDepsForTest(os android.OsType) string {
}
`
for _, os := range oses {
if os == android.Fuchsia {
ret += `
cc_library {
@ -353,6 +354,22 @@ func GatherRequiredDepsForTest(os android.OsType) string {
}
`
}
if os == android.Windows {
ret += `
toolchain_library {
name: "libwinpthread",
host_supported: true,
enabled: false,
target: {
windows: {
enabled: true,
},
},
src: "",
}
`
}
}
return ret
}

View File

@ -1879,12 +1879,12 @@ const (
)
// path to the jar file of a java library. Relative to <sdk_root>/<api_dir>
func sdkSnapshotFilePathForJar(name string) string {
return sdkSnapshotFilePathForMember(name, jarFileSuffix)
func sdkSnapshotFilePathForJar(osPrefix, name string) string {
return sdkSnapshotFilePathForMember(osPrefix, name, jarFileSuffix)
}
func sdkSnapshotFilePathForMember(name string, suffix string) string {
return filepath.Join(javaDir, name+suffix)
func sdkSnapshotFilePathForMember(osPrefix, name string, suffix string) string {
return filepath.Join(javaDir, osPrefix, name+suffix)
}
type librarySdkMemberType struct {
@ -1931,7 +1931,7 @@ func (p *librarySdkMemberProperties) PopulateFromVariant(variant android.SdkAwar
func (p *librarySdkMemberProperties) AddToPropertySet(sdkModuleContext android.ModuleContext, builder android.SnapshotBuilder, propertySet android.BpPropertySet) {
if p.jarToExport != nil {
exportedJar := p.jarToExport
snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.library.Name())
snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.OsPrefix(), p.library.Name())
builder.CopyToSnapshot(exportedJar, snapshotRelativeJavaLibPath)
for _, dir := range p.library.AidlIncludeDirs() {
@ -2137,10 +2137,10 @@ func (p *testSdkMemberProperties) PopulateFromVariant(variant android.SdkAware)
func (p *testSdkMemberProperties) AddToPropertySet(sdkModuleContext android.ModuleContext, builder android.SnapshotBuilder, propertySet android.BpPropertySet) {
if p.jarToExport != nil {
snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.test.Name())
snapshotRelativeJavaLibPath := sdkSnapshotFilePathForJar(p.OsPrefix(), p.test.Name())
builder.CopyToSnapshot(p.jarToExport, snapshotRelativeJavaLibPath)
snapshotRelativeTestConfigPath := sdkSnapshotFilePathForMember(p.test.Name(), testConfigSuffix)
snapshotRelativeTestConfigPath := sdkSnapshotFilePathForMember(p.OsPrefix(), p.test.Name(), testConfigSuffix)
builder.CopyToSnapshot(p.test.testConfig, snapshotRelativeTestConfigPath)
propertySet.AddProperty("jars", []string{snapshotRelativeJavaLibPath})
@ -2348,7 +2348,7 @@ func BinaryHostFactory() android.Module {
//
type ImportProperties struct {
Jars []string `android:"path"`
Jars []string `android:"path,arch_variant"`
Sdk_version *string

View File

@ -27,6 +27,8 @@ func testSdkWithCc(t *testing.T, bp string) *testSdkResult {
fs := map[string][]byte{
"Test.cpp": nil,
"include/Test.h": nil,
"include-android/AndroidTest.h": nil,
"include-host/HostTest.h": nil,
"arm64/include/Arm64Test.h": nil,
"libfoo.so": nil,
"aidl/foo/bar/Test.aidl": nil,
@ -401,6 +403,108 @@ module_exports_snapshot {
)
}
func TestMultipleHostOsTypesSnapshotWithCcBinary(t *testing.T) {
// b/145598135 - Generating host snapshots for anything other than linux is not supported.
SkipIfNotLinux(t)
result := testSdkWithCc(t, `
module_exports {
name: "myexports",
device_supported: false,
host_supported: true,
native_binaries: ["mynativebinary"],
target: {
windows: {
enabled: true,
},
},
}
cc_binary {
name: "mynativebinary",
device_supported: false,
host_supported: true,
srcs: [
"Test.cpp",
],
compile_multilib: "both",
system_shared_libs: [],
stl: "none",
target: {
windows: {
enabled: true,
},
},
}
`)
result.CheckSnapshot("myexports", "",
checkAndroidBpContents(`
// This is auto-generated. DO NOT EDIT.
cc_prebuilt_binary {
name: "myexports_mynativebinary@current",
sdk_member_name: "mynativebinary",
device_supported: false,
host_supported: true,
target: {
linux_glibc: {
compile_multilib: "both",
},
linux_glibc_x86_64: {
srcs: ["linux_glibc/x86_64/bin/mynativebinary"],
},
linux_glibc_x86: {
srcs: ["linux_glibc/x86/bin/mynativebinary"],
},
windows: {
compile_multilib: "64",
},
windows_x86_64: {
srcs: ["windows/x86_64/bin/mynativebinary.exe"],
},
},
}
cc_prebuilt_binary {
name: "mynativebinary",
prefer: false,
device_supported: false,
host_supported: true,
target: {
linux_glibc: {
compile_multilib: "both",
},
linux_glibc_x86_64: {
srcs: ["linux_glibc/x86_64/bin/mynativebinary"],
},
linux_glibc_x86: {
srcs: ["linux_glibc/x86/bin/mynativebinary"],
},
windows: {
compile_multilib: "64",
},
windows_x86_64: {
srcs: ["windows/x86_64/bin/mynativebinary.exe"],
},
},
}
module_exports_snapshot {
name: "myexports@current",
device_supported: false,
host_supported: true,
native_binaries: ["myexports_mynativebinary@current"],
}
`),
checkAllCopyRules(`
.intermediates/mynativebinary/linux_glibc_x86_64/mynativebinary -> linux_glibc/x86_64/bin/mynativebinary
.intermediates/mynativebinary/linux_glibc_x86/mynativebinary -> linux_glibc/x86/bin/mynativebinary
.intermediates/mynativebinary/windows_x86_64/mynativebinary.exe -> windows/x86_64/bin/mynativebinary.exe
`),
)
}
func TestSnapshotWithCcSharedLibrary(t *testing.T) {
result := testSdkWithCc(t, `
sdk {
@ -588,6 +692,99 @@ include/Test.h -> include/include/Test.h
)
}
func TestMultipleHostOsTypesSnapshotWithCcSharedLibrary(t *testing.T) {
// b/145598135 - Generating host snapshots for anything other than linux is not supported.
SkipIfNotLinux(t)
result := testSdkWithCc(t, `
sdk {
name: "mysdk",
device_supported: false,
host_supported: true,
native_shared_libs: ["mynativelib"],
target: {
windows: {
enabled: true,
},
},
}
cc_library_shared {
name: "mynativelib",
device_supported: false,
host_supported: true,
srcs: [
"Test.cpp",
],
system_shared_libs: [],
stl: "none",
target: {
windows: {
enabled: true,
},
},
}
`)
result.CheckSnapshot("mysdk", "",
checkAndroidBpContents(`
// This is auto-generated. DO NOT EDIT.
cc_prebuilt_library_shared {
name: "mysdk_mynativelib@current",
sdk_member_name: "mynativelib",
device_supported: false,
host_supported: true,
target: {
linux_glibc_x86_64: {
srcs: ["linux_glibc/x86_64/lib/mynativelib.so"],
},
linux_glibc_x86: {
srcs: ["linux_glibc/x86/lib/mynativelib.so"],
},
windows_x86_64: {
srcs: ["windows/x86_64/lib/mynativelib.dll"],
},
},
stl: "none",
system_shared_libs: [],
}
cc_prebuilt_library_shared {
name: "mynativelib",
prefer: false,
device_supported: false,
host_supported: true,
target: {
linux_glibc_x86_64: {
srcs: ["linux_glibc/x86_64/lib/mynativelib.so"],
},
linux_glibc_x86: {
srcs: ["linux_glibc/x86/lib/mynativelib.so"],
},
windows_x86_64: {
srcs: ["windows/x86_64/lib/mynativelib.dll"],
},
},
stl: "none",
system_shared_libs: [],
}
sdk_snapshot {
name: "mysdk@current",
device_supported: false,
host_supported: true,
native_shared_libs: ["mysdk_mynativelib@current"],
}
`),
checkAllCopyRules(`
.intermediates/mynativelib/linux_glibc_x86_64_shared/mynativelib.so -> linux_glibc/x86_64/lib/mynativelib.so
.intermediates/mynativelib/linux_glibc_x86_shared/mynativelib.so -> linux_glibc/x86/lib/mynativelib.so
.intermediates/mynativelib/windows_x86_64_shared/mynativelib.dll -> windows/x86_64/lib/mynativelib.dll
`),
)
}
func TestSnapshotWithCcStaticLibrary(t *testing.T) {
result := testSdkWithCc(t, `
module_exports {
@ -958,3 +1155,83 @@ include/Test.h -> include/include/Test.h
`),
)
}
func TestDeviceAndHostSnapshotWithCcHeadersLibrary(t *testing.T) {
// b/145598135 - Generating host snapshots for anything other than linux is not supported.
SkipIfNotLinux(t)
result := testSdkWithCc(t, `
sdk {
name: "mysdk",
host_supported: true,
native_header_libs: ["mynativeheaders"],
}
cc_library_headers {
name: "mynativeheaders",
host_supported: true,
system_shared_libs: [],
stl: "none",
export_system_include_dirs: ["include"],
target: {
android: {
export_include_dirs: ["include-android"],
},
host: {
export_include_dirs: ["include-host"],
},
},
}
`)
result.CheckSnapshot("mysdk", "",
checkAndroidBpContents(`
// This is auto-generated. DO NOT EDIT.
cc_prebuilt_library_headers {
name: "mysdk_mynativeheaders@current",
sdk_member_name: "mynativeheaders",
host_supported: true,
export_system_include_dirs: ["include/include"],
target: {
android: {
export_include_dirs: ["include/include-android"],
},
linux_glibc: {
export_include_dirs: ["include/include-host"],
},
},
stl: "none",
system_shared_libs: [],
}
cc_prebuilt_library_headers {
name: "mynativeheaders",
prefer: false,
host_supported: true,
export_system_include_dirs: ["include/include"],
target: {
android: {
export_include_dirs: ["include/include-android"],
},
linux_glibc: {
export_include_dirs: ["include/include-host"],
},
},
stl: "none",
system_shared_libs: [],
}
sdk_snapshot {
name: "mysdk@current",
host_supported: true,
native_header_libs: ["mysdk_mynativeheaders@current"],
}
`),
checkAllCopyRules(`
include/Test.h -> include/include/Test.h
include-android/AndroidTest.h -> include/include-android/AndroidTest.h
include-host/HostTest.h -> include/include-host/HostTest.h
`),
)
}

View File

@ -230,6 +230,72 @@ aidl/foo/bar/Test.aidl -> aidl/aidl/foo/bar/Test.aidl
)
}
func TestDeviceAndHostSnapshotWithJavaHeaderLibrary(t *testing.T) {
// b/145598135 - Generating host snapshots for anything other than linux is not supported.
SkipIfNotLinux(t)
result := testSdkWithJava(t, `
sdk {
name: "mysdk",
host_supported: true,
java_header_libs: ["myjavalib"],
}
java_library {
name: "myjavalib",
host_supported: true,
srcs: ["Test.java"],
system_modules: "none",
sdk_version: "none",
compile_dex: true,
}
`)
result.CheckSnapshot("mysdk", "",
checkAndroidBpContents(`
// This is auto-generated. DO NOT EDIT.
java_import {
name: "mysdk_myjavalib@current",
sdk_member_name: "myjavalib",
host_supported: true,
target: {
android: {
jars: ["java/android/myjavalib.jar"],
},
linux_glibc: {
jars: ["java/linux_glibc/myjavalib.jar"],
},
},
}
java_import {
name: "myjavalib",
prefer: false,
host_supported: true,
target: {
android: {
jars: ["java/android/myjavalib.jar"],
},
linux_glibc: {
jars: ["java/linux_glibc/myjavalib.jar"],
},
},
}
sdk_snapshot {
name: "mysdk@current",
host_supported: true,
java_header_libs: ["mysdk_myjavalib@current"],
}
`),
checkAllCopyRules(`
.intermediates/myjavalib/android_common/turbine-combined/myjavalib.jar -> java/android/myjavalib.jar
.intermediates/myjavalib/linux_glibc_common/javac/myjavalib.jar -> java/linux_glibc/myjavalib.jar
`),
)
}
func TestSnapshotWithJavaImplLibrary(t *testing.T) {
result := testSdkWithJava(t, `
module_exports {

View File

@ -40,7 +40,7 @@ func testSdkContext(bp string, fs map[string][]byte) (*android.TestContext, andr
name: "myapex.cert",
certificate: "myapex",
}
` + cc.GatherRequiredDepsForTest(android.Android)
` + cc.GatherRequiredDepsForTest(android.Android, android.Windows)
mockFS := map[string][]byte{
"build/make/target/product/security": nil,

View File

@ -17,6 +17,7 @@ package sdk
import (
"fmt"
"reflect"
"sort"
"strings"
"github.com/google/blueprint"
@ -660,6 +661,9 @@ type osTypeSpecificInfo struct {
// The list of arch type specific info for this os type.
archTypes []*archTypeSpecificInfo
// True if the member has common arch variants for this os type.
commonArch bool
}
type archTypeSpecificInfo struct {
@ -672,12 +676,45 @@ func (s *sdk) createMemberSnapshot(sdkModuleContext android.ModuleContext, build
memberType := member.memberType
// Group the properties for each variant by arch type.
osInfo := &osTypeSpecificInfo{}
osInfo.Properties = memberType.CreateVariantPropertiesStruct()
// Group the variants by os type.
variantsByOsType := make(map[android.OsType][]android.SdkAware)
variants := member.Variants()
commonArch := false
for _, variant := range variants {
osType := variant.Target().Os
variantsByOsType[osType] = append(variantsByOsType[osType], variant)
}
osCount := len(variantsByOsType)
createVariantPropertiesStruct := func(os android.OsType) android.SdkMemberProperties {
properties := memberType.CreateVariantPropertiesStruct()
base := properties.Base()
base.Os_count = osCount
base.Os = os
return properties
}
osTypeToInfo := make(map[android.OsType]*osTypeSpecificInfo)
// The set of properties that are common across all architectures and os types.
commonProperties := createVariantPropertiesStruct(android.CommonOS)
// The list of property structures which are os type specific but common across
// architectures within that os type.
var osSpecificPropertiesList []android.SdkMemberProperties
for osType, osTypeVariants := range variantsByOsType {
// Group the properties for each variant by arch type within the os.
osInfo := &osTypeSpecificInfo{}
osTypeToInfo[osType] = osInfo
// Create a structure into which properties common across the architectures in
// this os type will be stored. Add it to the list of os type specific yet
// architecture independent properties structs.
osInfo.Properties = createVariantPropertiesStruct(osType)
osSpecificPropertiesList = append(osSpecificPropertiesList, osInfo.Properties)
commonArch := false
for _, variant := range osTypeVariants {
var properties android.SdkMemberProperties
// Get the info associated with the arch type inside the os info.
@ -690,7 +727,7 @@ func (s *sdk) createMemberSnapshot(sdkModuleContext android.ModuleContext, build
commonArch = true
} else {
archInfo := &archTypeSpecificInfo{archType: archType}
properties = memberType.CreateVariantPropertiesStruct()
properties = createVariantPropertiesStruct(osType)
archInfo.Properties = properties
osInfo.archTypes = append(osInfo.archTypes, archInfo)
@ -700,16 +737,16 @@ func (s *sdk) createMemberSnapshot(sdkModuleContext android.ModuleContext, build
}
if commonArch {
if len(variants) != 1 {
if len(osTypeVariants) != 1 {
panic("Expected to only have 1 variant when arch type is common but found " + string(len(variants)))
}
} else {
var archProperties []android.SdkMemberProperties
var archPropertiesList []android.SdkMemberProperties
for _, archInfo := range osInfo.archTypes {
archProperties = append(archProperties, archInfo.Properties)
archPropertiesList = append(archPropertiesList, archInfo.Properties)
}
extractCommonProperties(osInfo.Properties, archProperties)
extractCommonProperties(osInfo.Properties, archPropertiesList)
// Choose setting for compile_multilib that is appropriate for the arch variants supplied.
var multilib string
@ -724,23 +761,114 @@ func (s *sdk) createMemberSnapshot(sdkModuleContext android.ModuleContext, build
}
}
osInfo.commonArch = commonArch
osInfo.Properties.Base().Compile_multilib = multilib
}
}
osInfo.Properties.AddToPropertySet(sdkModuleContext, builder, bpModule)
// Extract properties which are common across all architectures and os types.
extractCommonProperties(commonProperties, osSpecificPropertiesList)
if !commonArch {
archPropertySet := bpModule.AddPropertySet("arch")
// Add the common properties to the module.
commonProperties.AddToPropertySet(sdkModuleContext, builder, bpModule)
// Create a target property set into which target specific properties can be
// added.
targetPropertySet := bpModule.AddPropertySet("target")
// Iterate over the os types in a fixed order.
for _, osType := range s.getPossibleOsTypes() {
osInfo := osTypeToInfo[osType]
if osInfo == nil {
continue
}
var osPropertySet android.BpPropertySet
var archOsPrefix string
if len(osTypeToInfo) == 1 {
// There is only one os type present in the variants sp don't bother
// with adding target specific properties.
// Create a structure that looks like:
// module_type {
// name: "...",
// ...
// <common properties>
// ...
// <single os type specific properties>
//
// arch: {
// <arch specific sections>
// }
//
osPropertySet = bpModule
// Arch specific properties need to be added to an arch specific section
// within arch.
archOsPrefix = ""
} else {
// Create a structure that looks like:
// module_type {
// name: "...",
// ...
// <common properties>
// ...
// target: {
// <arch independent os specific sections, e.g. android>
// ...
// <arch and os specific sections, e.g. android_x86>
// }
//
osPropertySet = targetPropertySet.AddPropertySet(osType.Name)
// Arch specific properties need to be added to an os and arch specific
// section prefixed with <os>_.
archOsPrefix = osType.Name + "_"
}
osInfo.Properties.AddToPropertySet(sdkModuleContext, builder, osPropertySet)
if !osInfo.commonArch {
// Either add the arch specific sections into the target or arch sections
// depending on whether they will also be os specific.
var archPropertySet android.BpPropertySet
if archOsPrefix == "" {
archPropertySet = osPropertySet.AddPropertySet("arch")
} else {
archPropertySet = targetPropertySet
}
// Add arch (and possibly os) specific sections for each set of
// arch (and possibly os) specific properties.
for _, av := range osInfo.archTypes {
archTypePropertySet := archPropertySet.AddPropertySet(av.archType.Name)
archTypePropertySet := archPropertySet.AddPropertySet(archOsPrefix + av.archType.Name)
av.Properties.AddToPropertySet(sdkModuleContext, builder, archTypePropertySet)
}
}
}
memberType.FinalizeModule(sdkModuleContext, builder, member, bpModule)
}
// Compute the list of possible os types that this sdk could support.
func (s *sdk) getPossibleOsTypes() []android.OsType {
var osTypes []android.OsType
for _, osType := range android.OsTypeList {
if s.DeviceSupported() {
if osType.Class == android.Device && osType != android.Fuchsia {
osTypes = append(osTypes, osType)
}
}
if s.HostSupported() {
if osType.Class == android.Host || osType.Class == android.HostCross {
osTypes = append(osTypes, osType)
}
}
}
sort.SliceStable(osTypes, func(i, j int) bool { return osTypes[i].Name < osTypes[j].Name })
return osTypes
}
// Extract common properties from a slice of property structures of the same type.
//
// All the property structures must be of the same type.
@ -766,6 +894,11 @@ func extractCommonProperties(commonProperties interface{}, inputPropertiesSlice
var commonValue *reflect.Value
sliceValue := reflect.ValueOf(inputPropertiesSlice)
field := propertiesStructType.Field(f)
if field.Name == "SdkMemberPropertiesBase" {
continue
}
for i := 0; i < sliceValue.Len(); i++ {
structValue := sliceValue.Index(i).Elem().Elem()
fieldValue := structValue.Field(f)