From 8471cdaced6a8c240ec9908916c567ef9c0dac5b Mon Sep 17 00:00:00 2001 From: Inseob Kim Date: Fri, 15 Nov 2019 09:59:12 +0900 Subject: [PATCH] Implement vendor snapshot Vendor snapshot can be captured with "m dist vendor-snapshot". With vendor snapshot and vndk snapshot, older version of /vendor and newer version of /system will be able to be built together by setting BOARD_VNDK_VERSION to past vendor's version. Only vendor modules under AOSP are to be captured. In detail, modules under following directories are ignored: - device/ - vendor/ - hardware/, except for interfaces/, libhardware/, libhardware_legacy/, and ril/ Test modules (cc_test, etc.) and sanitized modules are also ignored. Bug: 65377115 Test: m dist vendor-snapshot Change-Id: If7a2f6de7f36deee936930c0ccf7c47c4a0cebf6 --- Android.bp | 2 + android/module.go | 15 ++ cc/cc.go | 15 +- cc/cc_test.go | 106 ++++++++++-- cc/sanitize.go | 8 +- cc/snapshot_utils.go | 104 ++++++++++++ cc/testing.go | 1 + cc/vendor_snapshot.go | 369 ++++++++++++++++++++++++++++++++++++++++++ cc/vndk.go | 194 ++++++---------------- 9 files changed, 649 insertions(+), 165 deletions(-) create mode 100644 cc/snapshot_utils.go create mode 100644 cc/vendor_snapshot.go diff --git a/Android.bp b/Android.bp index 9b55c8c77..cb2f773a5 100644 --- a/Android.bp +++ b/Android.bp @@ -186,11 +186,13 @@ bootstrap_go_package { "cc/rs.go", "cc/sanitize.go", "cc/sabi.go", + "cc/snapshot_utils.go", "cc/stl.go", "cc/strip.go", "cc/sysprop.go", "cc/tidy.go", "cc/util.go", + "cc/vendor_snapshot.go", "cc/vndk.go", "cc/vndk_prebuilt.go", "cc/xom.go", diff --git a/android/module.go b/android/module.go index 96c2e1e90..0c732fb65 100644 --- a/android/module.go +++ b/android/module.go @@ -215,6 +215,8 @@ type Module interface { InstallBypassMake() bool SkipInstall() ExportedToMake() bool + InitRc() Paths + VintfFragments() Paths NoticeFile() OptionalPath AddProperties(props ...interface{}) @@ -662,6 +664,9 @@ type ModuleBase struct { ruleParams map[blueprint.Rule]blueprint.RuleParams variables map[string]string + initRcPaths Paths + vintfFragmentsPaths Paths + prefer32 func(ctx BaseModuleContext, base *ModuleBase, class OsClass) bool } @@ -932,6 +937,14 @@ func (m *ModuleBase) TargetRequiredModuleNames() []string { return m.base().commonProperties.Target_required } +func (m *ModuleBase) InitRc() Paths { + return append(Paths{}, m.initRcPaths...) +} + +func (m *ModuleBase) VintfFragments() Paths { + return append(Paths{}, m.vintfFragmentsPaths...) +} + func (m *ModuleBase) generateModuleTarget(ctx ModuleContext) { allInstalledFiles := Paths{} allCheckbuildFiles := Paths{} @@ -1148,6 +1161,8 @@ func (m *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) m.installFiles = append(m.installFiles, ctx.installFiles...) m.checkbuildFiles = append(m.checkbuildFiles, ctx.checkbuildFiles...) + m.initRcPaths = PathsForModuleSrc(ctx, m.commonProperties.Init_rc) + m.vintfFragmentsPaths = PathsForModuleSrc(ctx, m.commonProperties.Vintf_fragments) } else if ctx.Config().AllowMissingDependencies() { // If the module is not enabled it will not create any build rules, nothing will call // ctx.GetMissingDependencies(), and blueprint will consider the missing dependencies to be unhandled diff --git a/cc/cc.go b/cc/cc.go index 3b2af38ca..bef922a1f 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -242,6 +242,10 @@ type BaseProperties struct { // Even if DeviceConfig().VndkUseCoreVariant() is set, this module must use vendor variant. // see soong/cc/config/vndk.go MustUseVendorVariant bool `blueprint:"mutated"` + + // Used by vendor snapshot to record dependencies from snapshot modules. + SnapshotSharedLibs []string `blueprint:"mutated"` + SnapshotRuntimeLibs []string `blueprint:"mutated"` } type VendorProperties struct { @@ -453,8 +457,6 @@ type Module struct { pgo *pgo xom *xom - androidMkSharedLibDeps []string - outputFile android.OptionalPath cachedToolchain config.Toolchain @@ -930,6 +932,11 @@ func (c *Module) nativeCoverage() bool { return c.linker != nil && c.linker.nativeCoverage() } +func (c *Module) isSnapshotPrebuilt() bool { + _, ok := c.linker.(*vndkPrebuiltLibraryDecorator) + return ok +} + func (c *Module) ExportedIncludeDirs() android.Paths { if flagsProducer, ok := c.linker.(exportedFlagsProducer); ok { return flagsProducer.exportedDirs() @@ -2340,6 +2347,8 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { // they merely serve as Make dependencies and do not affect this lib itself. c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, makeLibName(depName)) + // Record depName as-is for snapshots. + c.Properties.SnapshotSharedLibs = append(c.Properties.SnapshotSharedLibs, depName) case ndkStubDepTag, ndkLateStubDepTag: c.Properties.AndroidMkSharedLibs = append( c.Properties.AndroidMkSharedLibs, @@ -2350,6 +2359,8 @@ func (c *Module) depsToPaths(ctx android.ModuleContext) PathDeps { case runtimeDepTag: c.Properties.AndroidMkRuntimeLibs = append( c.Properties.AndroidMkRuntimeLibs, makeLibName(depName)) + // Record depName as-is for snapshots. + c.Properties.SnapshotRuntimeLibs = append(c.Properties.SnapshotRuntimeLibs, depName) case wholeStaticDepTag: c.Properties.AndroidMkWholeStaticLibs = append( c.Properties.AndroidMkWholeStaticLibs, makeLibName(depName)) diff --git a/cc/cc_test.go b/cc/cc_test.go index 332cc45e7..216624978 100644 --- a/cc/cc_test.go +++ b/cc/cc_test.go @@ -258,8 +258,8 @@ func checkVndkModule(t *testing.T, ctx *android.TestContext, name, subDir string } } -func checkVndkSnapshot(t *testing.T, ctx *android.TestContext, moduleName, snapshotFilename, subDir, variant string) { - vndkSnapshot := ctx.SingletonForTests("vndk-snapshot") +func checkSnapshot(t *testing.T, ctx *android.TestContext, singletonName, moduleName, snapshotFilename, subDir, variant string) { + snapshotSingleton := ctx.SingletonForTests(singletonName) mod, ok := ctx.ModuleForTests(moduleName, variant).Module().(android.OutputFileProducer) if !ok { @@ -273,9 +273,9 @@ func checkVndkSnapshot(t *testing.T, ctx *android.TestContext, moduleName, snaps } snapshotPath := filepath.Join(subDir, snapshotFilename) - out := vndkSnapshot.Output(snapshotPath) + out := snapshotSingleton.Output(snapshotPath) if out.Input.String() != outputFiles[0].String() { - t.Errorf("The input of VNDK snapshot must be %q, but %q", out.Input.String(), outputFiles[0]) + t.Errorf("The input of snapshot %q must be %q, but %q", moduleName, out.Input.String(), outputFiles[0]) } } @@ -398,16 +398,16 @@ func TestVndk(t *testing.T) { variant := "android_vendor.VER_arm64_armv8-a_shared" variant2nd := "android_vendor.VER_arm_armv7-a-neon_shared" - checkVndkSnapshot(t, ctx, "libvndk", "libvndk.so", vndkCoreLibPath, variant) - checkVndkSnapshot(t, ctx, "libvndk", "libvndk.so", vndkCoreLib2ndPath, variant2nd) - checkVndkSnapshot(t, ctx, "libvndk_sp", "libvndk_sp-x.so", vndkSpLibPath, variant) - checkVndkSnapshot(t, ctx, "libvndk_sp", "libvndk_sp-x.so", vndkSpLib2ndPath, variant2nd) + checkSnapshot(t, ctx, "vndk-snapshot", "libvndk", "libvndk.so", vndkCoreLibPath, variant) + checkSnapshot(t, ctx, "vndk-snapshot", "libvndk", "libvndk.so", vndkCoreLib2ndPath, variant2nd) + checkSnapshot(t, ctx, "vndk-snapshot", "libvndk_sp", "libvndk_sp-x.so", vndkSpLibPath, variant) + checkSnapshot(t, ctx, "vndk-snapshot", "libvndk_sp", "libvndk_sp-x.so", vndkSpLib2ndPath, variant2nd) snapshotConfigsPath := filepath.Join(snapshotVariantPath, "configs") - checkVndkSnapshot(t, ctx, "llndk.libraries.txt", "llndk.libraries.txt", snapshotConfigsPath, "") - checkVndkSnapshot(t, ctx, "vndkcore.libraries.txt", "vndkcore.libraries.txt", snapshotConfigsPath, "") - checkVndkSnapshot(t, ctx, "vndksp.libraries.txt", "vndksp.libraries.txt", snapshotConfigsPath, "") - checkVndkSnapshot(t, ctx, "vndkprivate.libraries.txt", "vndkprivate.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, "vndk-snapshot", "llndk.libraries.txt", "llndk.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, "vndk-snapshot", "vndkcore.libraries.txt", "vndkcore.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, "vndk-snapshot", "vndksp.libraries.txt", "vndksp.libraries.txt", snapshotConfigsPath, "") + checkSnapshot(t, ctx, "vndk-snapshot", "vndkprivate.libraries.txt", "vndkprivate.libraries.txt", snapshotConfigsPath, "") checkVndkOutput(t, ctx, "vndk/vndk.libraries.txt", []string{ "LLNDK: libc.so", @@ -799,6 +799,88 @@ func TestDoubleLoadbleDep(t *testing.T) { `) } +func TestVendorSnapshot(t *testing.T) { + bp := ` + cc_library { + name: "libvndk", + vendor_available: true, + vndk: { + enabled: true, + }, + nocrt: true, + } + + cc_library { + name: "libvendor", + vendor: true, + nocrt: true, + } + + cc_library { + name: "libvendor_available", + vendor_available: true, + nocrt: true, + } + + cc_library_headers { + name: "libvendor_headers", + vendor_available: true, + nocrt: true, + } + + cc_binary { + name: "vendor_bin", + vendor: true, + nocrt: true, + } + + cc_binary { + name: "vendor_available_bin", + vendor_available: true, + nocrt: true, + } +` + config := TestConfig(buildDir, android.Android, nil, bp, nil) + config.TestProductVariables.DeviceVndkVersion = StringPtr("current") + config.TestProductVariables.Platform_vndk_version = StringPtr("VER") + ctx := testCcWithConfig(t, config) + + // Check Vendor snapshot output. + + snapshotDir := "vendor-snapshot" + snapshotVariantPath := filepath.Join(buildDir, snapshotDir, "arm64") + + for _, arch := range [][]string{ + []string{"arm64", "armv8-a"}, + []string{"arm", "armv7-a-neon"}, + } { + archType := arch[0] + archVariant := arch[1] + archDir := fmt.Sprintf("arch-%s-%s", archType, archVariant) + + // For shared libraries, only non-VNDK vendor_available modules are captured + sharedVariant := fmt.Sprintf("android_vendor.VER_%s_%s_shared", archType, archVariant) + sharedDir := filepath.Join(snapshotVariantPath, archDir, "shared") + checkSnapshot(t, ctx, "vendor-snapshot", "libvendor", "libvendor.so", sharedDir, sharedVariant) + checkSnapshot(t, ctx, "vendor-snapshot", "libvendor_available", "libvendor_available.so", sharedDir, sharedVariant) + + // For static libraries, all vendor:true and vendor_available modules (including VNDK) are captured. + staticVariant := fmt.Sprintf("android_vendor.VER_%s_%s_static", archType, archVariant) + staticDir := filepath.Join(snapshotVariantPath, archDir, "static") + checkSnapshot(t, ctx, "vendor-snapshot", "libvndk", "libvndk.a", staticDir, staticVariant) + checkSnapshot(t, ctx, "vendor-snapshot", "libvendor", "libvendor.a", staticDir, staticVariant) + checkSnapshot(t, ctx, "vendor-snapshot", "libvendor_available", "libvendor_available.a", staticDir, staticVariant) + + // For binary libraries, all vendor:true and vendor_available modules are captured. + if archType == "arm64" { + binaryVariant := fmt.Sprintf("android_vendor.VER_%s_%s", archType, archVariant) + binaryDir := filepath.Join(snapshotVariantPath, archDir, "binary") + checkSnapshot(t, ctx, "vendor-snapshot", "vendor_bin", "vendor_bin", binaryDir, binaryVariant) + checkSnapshot(t, ctx, "vendor-snapshot", "vendor_available_bin", "vendor_available_bin", binaryDir, binaryVariant) + } + } +} + func TestDoubleLoadableDepError(t *testing.T) { // Check whether an error is emitted when a LLNDK depends on a non-double_loadable VNDK lib. testCcError(t, "module \".*\" variant \".*\": link.* \".*\" which is not LL-NDK, VNDK-SP, .*double_loadable", ` diff --git a/cc/sanitize.go b/cc/sanitize.go index 93c4b410b..6f9dbef63 100644 --- a/cc/sanitize.go +++ b/cc/sanitize.go @@ -744,8 +744,7 @@ func sanitizerRuntimeDepsMutator(mctx android.TopDownMutatorContext) { // If a static dependency is built with the minimal runtime, // make sure we include the ubsan minimal runtime. c.sanitize.Properties.MinimalRuntimeDep = true - } else if Bool(d.sanitize.Properties.Sanitize.Diag.Integer_overflow) || - len(d.sanitize.Properties.Sanitize.Diag.Misc_undefined) > 0 { + } else if enableUbsanRuntime(d.sanitize) { // If a static dependency runs with full ubsan diagnostics, // make sure we include the ubsan runtime. c.sanitize.Properties.UbsanRuntimeDep = true @@ -1052,6 +1051,11 @@ func enableMinimalRuntime(sanitize *sanitize) bool { return false } +func enableUbsanRuntime(sanitize *sanitize) bool { + return Bool(sanitize.Properties.Sanitize.Diag.Integer_overflow) || + len(sanitize.Properties.Sanitize.Diag.Misc_undefined) > 0 +} + func cfiMakeVarsProvider(ctx android.MakeVarsContext) { cfiStaticLibs := cfiStaticLibs(ctx.Config()) sort.Strings(*cfiStaticLibs) diff --git a/cc/snapshot_utils.go b/cc/snapshot_utils.go new file mode 100644 index 000000000..1c872c2c9 --- /dev/null +++ b/cc/snapshot_utils.go @@ -0,0 +1,104 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cc + +import ( + "strings" + + "android/soong/android" +) + +var ( + headerExts = []string{".h", ".hh", ".hpp", ".hxx", ".h++", ".inl", ".inc", ".ipp", ".h.generic"} +) + +type snapshotLibraryInterface interface { + exportedFlagsProducer + libraryInterface +} + +var _ snapshotLibraryInterface = (*prebuiltLibraryLinker)(nil) +var _ snapshotLibraryInterface = (*libraryDecorator)(nil) + +func exportedHeaders(ctx android.SingletonContext, l exportedFlagsProducer) android.Paths { + var ret android.Paths + + // Headers in the source tree should be globbed. On the contrast, generated headers + // can't be globbed, and they should be manually collected. + // So, we first filter out intermediate directories (which contains generated headers) + // from exported directories, and then glob headers under remaining directories. + for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) { + dir := path.String() + // Skip if dir is for generated headers + if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) { + continue + } + exts := headerExts + // Glob all files under this special directory, because of C++ headers. + if strings.HasPrefix(dir, "external/libcxx/include") { + exts = []string{""} + } + for _, ext := range exts { + glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil) + if err != nil { + ctx.Errorf("%#v\n", err) + return nil + } + for _, header := range glob { + if strings.HasSuffix(header, "/") { + continue + } + ret = append(ret, android.PathForSource(ctx, header)) + } + } + } + + // Collect generated headers + for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) { + // TODO(b/148123511): remove exportedDeps after cleaning up genrule + if strings.HasSuffix(header.Base(), "-phony") { + continue + } + ret = append(ret, header) + } + + return ret +} + +func copyFile(ctx android.SingletonContext, path android.Path, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.Cp, + Input: path, + Output: outPath, + Description: "Cp " + out, + Args: map[string]string{ + "cpFlags": "-f -L", + }, + }) + return outPath +} + +func writeStringToFile(ctx android.SingletonContext, content, out string) android.OutputPath { + outPath := android.PathForOutput(ctx, out) + ctx.Build(pctx, android.BuildParams{ + Rule: android.WriteFile, + Output: outPath, + Description: "WriteFile " + out, + Args: map[string]string{ + "content": content, + }, + }) + return outPath +} diff --git a/cc/testing.go b/cc/testing.go index ba8ed95dd..60e5cf596 100644 --- a/cc/testing.go +++ b/cc/testing.go @@ -320,6 +320,7 @@ func CreateTestContext() *android.TestContext { RegisterRequiredBuildComponentsForTest(ctx) ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators) ctx.RegisterSingletonType("vndk-snapshot", VndkSnapshotSingleton) + ctx.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) return ctx } diff --git a/cc/vendor_snapshot.go b/cc/vendor_snapshot.go new file mode 100644 index 000000000..d952a4cb1 --- /dev/null +++ b/cc/vendor_snapshot.go @@ -0,0 +1,369 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cc + +import ( + "encoding/json" + "path/filepath" + "sort" + "strings" + + "github.com/google/blueprint/proptools" + + "android/soong/android" +) + +func init() { + android.RegisterSingletonType("vendor-snapshot", VendorSnapshotSingleton) +} + +func VendorSnapshotSingleton() android.Singleton { + return &vendorSnapshotSingleton{} +} + +type vendorSnapshotSingleton struct { + vendorSnapshotZipFile android.OptionalPath +} + +var ( + // Modules under following directories are ignored. They are OEM's and vendor's + // proprietary modules(device/, vendor/, and hardware/). + // TODO(b/65377115): Clean up these with more maintainable way + vendorProprietaryDirs = []string{ + "device", + "vendor", + "hardware", + } + + // Modules under following directories are included as they are in AOSP, + // although hardware/ is normally for vendor's own. + // TODO(b/65377115): Clean up these with more maintainable way + aospDirsUnderProprietary = []string{ + "hardware/interfaces", + "hardware/libhardware", + "hardware/libhardware_legacy", + "hardware/ril", + } +) + +// Determine if a dir under source tree is an SoC-owned proprietary directory, such as +// device/, vendor/, etc. +func isVendorProprietaryPath(dir string) bool { + for _, p := range vendorProprietaryDirs { + if strings.HasPrefix(dir, p) { + // filter out AOSP defined directories, e.g. hardware/interfaces/ + aosp := false + for _, p := range aospDirsUnderProprietary { + if strings.HasPrefix(dir, p) { + aosp = true + break + } + } + if !aosp { + return true + } + } + } + return false +} + +// Determine if a module is going to be included in vendor snapshot or not. +// +// Targets of vendor snapshot are "vendor: true" or "vendor_available: true" modules in +// AOSP. They are not guaranteed to be compatible with older vendor images. (e.g. might +// depend on newer VNDK) So they are captured as vendor snapshot To build older vendor +// image and newer system image altogether. +func isVendorSnapshotModule(ctx android.SingletonContext, m *Module) bool { + if !m.Enabled() { + return false + } + // skip proprietary modules, but include all VNDK (static) + if isVendorProprietaryPath(ctx.ModuleDir(m)) && !m.IsVndk() { + return false + } + if m.Target().Os.Class != android.Device { + return false + } + if m.Target().NativeBridge == android.NativeBridgeEnabled { + return false + } + // the module must be installed in /vendor + if !m.installable() || m.isSnapshotPrebuilt() || !m.inVendor() { + return false + } + // exclude test modules + if _, ok := m.linker.(interface{ gtest() bool }); ok { + return false + } + // TODO(b/65377115): add full support for sanitizer + if m.sanitize != nil && !m.sanitize.isUnsanitizedVariant() { + return false + } + + // Libraries + if l, ok := m.linker.(snapshotLibraryInterface); ok { + if l.static() { + return proptools.BoolDefault(m.VendorProperties.Vendor_available, true) + } + if l.shared() { + return !m.IsVndk() + } + return true + } + + // Binaries + _, ok := m.linker.(*binaryDecorator) + if !ok { + if _, ok := m.linker.(*prebuiltBinaryLinker); !ok { + return false + } + } + return proptools.BoolDefault(m.VendorProperties.Vendor_available, true) +} + +func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) { + // BOARD_VNDK_VERSION must be set to 'current' in order to generate a vendor snapshot. + if ctx.DeviceConfig().VndkVersion() != "current" { + return + } + + var snapshotOutputs android.Paths + + /* + Vendor snapshot zipped artifacts directory structure: + {SNAPSHOT_ARCH}/ + arch-{TARGET_ARCH}-{TARGET_ARCH_VARIANT}/ + shared/ + (.so shared libraries) + static/ + (.a static libraries) + header/ + (header only libraries) + binary/ + (executable binaries) + arch-{TARGET_2ND_ARCH}-{TARGET_2ND_ARCH_VARIANT}/ + shared/ + (.so shared libraries) + static/ + (.a static libraries) + header/ + (header only libraries) + binary/ + (executable binaries) + NOTICE_FILES/ + (notice files, e.g. libbase.txt) + configs/ + (config files, e.g. init.rc files, vintf_fragments.xml files, etc.) + include/ + (header files of same directory structure with source tree) + */ + + snapshotDir := "vendor-snapshot" + snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch()) + + includeDir := filepath.Join(snapshotArchDir, "include") + configsDir := filepath.Join(snapshotArchDir, "configs") + noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") + + installedNotices := make(map[string]bool) + installedConfigs := make(map[string]bool) + + var headers android.Paths + + type vendorSnapshotLibraryInterface interface { + exportedFlagsProducer + libraryInterface + } + + var _ vendorSnapshotLibraryInterface = (*prebuiltLibraryLinker)(nil) + var _ vendorSnapshotLibraryInterface = (*libraryDecorator)(nil) + + installSnapshot := func(m *Module) android.Paths { + targetArch := "arch-" + m.Target().Arch.ArchType.String() + if m.Target().Arch.ArchVariant != "" { + targetArch += "-" + m.Target().Arch.ArchVariant + } + + var ret android.Paths + + prop := struct { + ModuleName string `json:",omitempty"` + RelativeInstallPath string `json:",omitempty"` + + // library flags + ExportedDirs []string `json:",omitempty"` + ExportedSystemDirs []string `json:",omitempty"` + ExportedFlags []string `json:",omitempty"` + SanitizeMinimalDep bool `json:",omitempty"` + SanitizeUbsanDep bool `json:",omitempty"` + + // binary flags + Symlinks []string `json:",omitempty"` + + // dependencies + SharedLibs []string `json:",omitempty"` + RuntimeLibs []string `json:",omitempty"` + Required []string `json:",omitempty"` + + // extra config files + InitRc []string `json:",omitempty"` + VintfFragments []string `json:",omitempty"` + }{} + + // Common properties among snapshots. + prop.ModuleName = ctx.ModuleName(m) + prop.RelativeInstallPath = m.RelativeInstallPath() + prop.RuntimeLibs = m.Properties.SnapshotRuntimeLibs + prop.Required = m.RequiredModuleNames() + for _, path := range m.InitRc() { + prop.InitRc = append(prop.InitRc, filepath.Join("configs", path.Base())) + } + for _, path := range m.VintfFragments() { + prop.VintfFragments = append(prop.VintfFragments, filepath.Join("configs", path.Base())) + } + + // install config files. ignores any duplicates. + for _, path := range append(m.InitRc(), m.VintfFragments()...) { + out := filepath.Join(configsDir, path.Base()) + if !installedConfigs[out] { + installedConfigs[out] = true + ret = append(ret, copyFile(ctx, path, out)) + } + } + + var propOut string + + if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok { + // library flags + prop.ExportedFlags = l.exportedFlags() + for _, dir := range l.exportedDirs() { + prop.ExportedDirs = append(prop.ExportedDirs, filepath.Join("include", dir.String())) + } + for _, dir := range l.exportedSystemDirs() { + prop.ExportedSystemDirs = append(prop.ExportedSystemDirs, filepath.Join("include", dir.String())) + } + // shared libs dependencies aren't meaningful on static or header libs + if l.shared() { + prop.SharedLibs = m.Properties.SnapshotSharedLibs + } + if l.static() && m.sanitize != nil { + prop.SanitizeMinimalDep = m.sanitize.Properties.MinimalRuntimeDep || enableMinimalRuntime(m.sanitize) + prop.SanitizeUbsanDep = m.sanitize.Properties.UbsanRuntimeDep || enableUbsanRuntime(m.sanitize) + } + + var libType string + if l.static() { + libType = "static" + } else if l.shared() { + libType = "shared" + } else { + libType = "header" + } + + var stem string + + // install .a or .so + if libType != "header" { + libPath := m.outputFile.Path() + stem = libPath.Base() + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, libType, stem) + ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + } else { + stem = ctx.ModuleName(m) + } + + propOut = filepath.Join(snapshotArchDir, targetArch, libType, stem+".json") + } else { + // binary flags + prop.Symlinks = m.Symlinks() + prop.SharedLibs = m.Properties.SnapshotSharedLibs + + // install bin + binPath := m.outputFile.Path() + snapshotBinOut := filepath.Join(snapshotArchDir, targetArch, "binary", binPath.Base()) + ret = append(ret, copyFile(ctx, binPath, snapshotBinOut)) + propOut = snapshotBinOut + ".json" + } + + j, err := json.Marshal(prop) + if err != nil { + ctx.Errorf("json marshal to %q failed: %#v", propOut, err) + return nil + } + ret = append(ret, writeStringToFile(ctx, string(j), propOut)) + + return ret + } + + ctx.VisitAllModules(func(module android.Module) { + m, ok := module.(*Module) + if !ok || !isVendorSnapshotModule(ctx, m) { + return + } + + snapshotOutputs = append(snapshotOutputs, installSnapshot(m)...) + if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok { + headers = append(headers, exportedHeaders(ctx, l)...) + } + + if m.NoticeFile().Valid() { + noticeName := ctx.ModuleName(m) + ".txt" + noticeOut := filepath.Join(noticeDir, noticeName) + // skip already copied notice file + if !installedNotices[noticeOut] { + installedNotices[noticeOut] = true + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.NoticeFile().Path(), noticeOut)) + } + } + }) + + // install all headers after removing duplicates + for _, header := range android.FirstUniquePaths(headers) { + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, header, filepath.Join(includeDir, header.String()))) + } + + // All artifacts are ready. Sort them to normalize ninja and then zip. + sort.Slice(snapshotOutputs, func(i, j int) bool { + return snapshotOutputs[i].String() < snapshotOutputs[j].String() + }) + + zipPath := android.PathForOutput(ctx, snapshotDir, "vendor-"+ctx.Config().DeviceName()+".zip") + zipRule := android.NewRuleBuilder() + + // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with tr + snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "vendor-"+ctx.Config().DeviceName()+"_list") + zipRule.Command(). + Text("tr"). + FlagWithArg("-d ", "\\'"). + FlagWithRspFileInputList("< ", snapshotOutputs). + FlagWithOutput("> ", snapshotOutputList) + + zipRule.Temporary(snapshotOutputList) + + zipRule.Command(). + BuiltTool(ctx, "soong_zip"). + FlagWithOutput("-o ", zipPath). + FlagWithArg("-C ", android.PathForOutput(ctx, snapshotDir).String()). + FlagWithInput("-l ", snapshotOutputList) + + zipRule.Build(pctx, ctx, zipPath.String(), "vendor snapshot "+zipPath.String()) + zipRule.DeleteTemporaryFiles() + c.vendorSnapshotZipFile = android.OptionalPathForPath(zipPath) +} + +func (c *vendorSnapshotSingleton) MakeVars(ctx android.MakeVarsContext) { + ctx.Strict("SOONG_VENDOR_SNAPSHOT_ZIP", c.vendorSnapshotZipFile.String()) +} diff --git a/cc/vndk.go b/cc/vndk.go index ab730355c..4578a7d13 100644 --- a/cc/vndk.go +++ b/cc/vndk.go @@ -229,8 +229,6 @@ var ( vndkUsingCoreVariantLibrariesKey = android.NewOnceKey("vndkUsingCoreVariantLibraries") vndkMustUseVendorVariantListKey = android.NewOnceKey("vndkMustUseVendorVariantListKey") vndkLibrariesLock sync.Mutex - - headerExts = []string{".h", ".hh", ".hpp", ".hxx", ".h++", ".inl", ".inc", ".ipp", ".h.generic"} ) func vndkCoreLibraries(config android.Config) map[string]string { @@ -548,29 +546,10 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex snapshotDir := "vndk-snapshot" snapshotArchDir := filepath.Join(snapshotDir, ctx.DeviceConfig().DeviceArch()) - targetArchDirMap := make(map[android.ArchType]string) - for _, target := range ctx.Config().Targets[android.Android] { - dir := snapshotArchDir - if ctx.DeviceConfig().BinderBitness() == "32" { - dir = filepath.Join(dir, "binder32") - } - arch := "arch-" + target.Arch.ArchType.String() - if target.Arch.ArchVariant != "" { - arch += "-" + target.Arch.ArchVariant - } - dir = filepath.Join(dir, arch) - targetArchDirMap[target.Arch.ArchType] = dir - } configsDir := filepath.Join(snapshotArchDir, "configs") noticeDir := filepath.Join(snapshotArchDir, "NOTICE_FILES") includeDir := filepath.Join(snapshotArchDir, "include") - // set of include paths exported by VNDK libraries - exportedIncludes := make(map[string]bool) - - // generated header files among exported headers. - var generatedHeaders android.Paths - // set of notice files copied. noticeBuilt := make(map[string]bool) @@ -581,67 +560,20 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex // e.g. moduleNames["libprotobuf-cpp-full-3.9.1.so"] = "libprotobuf-cpp-full" moduleNames := make(map[string]string) - installSnapshotFileFromPath := func(path android.Path, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.Cp, - Input: path, - Output: outPath, - Description: "vndk snapshot " + out, - Args: map[string]string{ - "cpFlags": "-f -L", - }, - }) - return outPath - } - - installSnapshotFileFromContent := func(content, out string) android.OutputPath { - outPath := android.PathForOutput(ctx, out) - ctx.Build(pctx, android.BuildParams{ - Rule: android.WriteFile, - Output: outPath, - Description: "vndk snapshot " + out, - Args: map[string]string{ - "content": content, - }, - }) - return outPath - } - - type vndkSnapshotLibraryInterface interface { - exportedFlagsProducer - libraryInterface - } - - var _ vndkSnapshotLibraryInterface = (*prebuiltLibraryLinker)(nil) - var _ vndkSnapshotLibraryInterface = (*libraryDecorator)(nil) - - installVndkSnapshotLib := func(m *Module, l vndkSnapshotLibraryInterface, vndkType string) (android.Paths, bool) { - targetArchDir, ok := targetArchDirMap[m.Target().Arch.ArchType] - if !ok { - return nil, false - } + var headers android.Paths + installVndkSnapshotLib := func(m *Module, l snapshotLibraryInterface, vndkType string) (android.Paths, bool) { var ret android.Paths - libPath := m.outputFile.Path() - stem := libPath.Base() - snapshotLibOut := filepath.Join(targetArchDir, "shared", vndkType, stem) - ret = append(ret, installSnapshotFileFromPath(libPath, snapshotLibOut)) - - moduleNames[stem] = ctx.ModuleName(m) - modulePaths[stem] = ctx.ModuleDir(m) - - if m.NoticeFile().Valid() { - noticeName := stem + ".txt" - // skip already copied notice file - if _, ok := noticeBuilt[noticeName]; !ok { - noticeBuilt[noticeName] = true - ret = append(ret, installSnapshotFileFromPath( - m.NoticeFile().Path(), filepath.Join(noticeDir, noticeName))) - } + targetArch := "arch-" + m.Target().Arch.ArchType.String() + if m.Target().Arch.ArchVariant != "" { + targetArch += "-" + m.Target().Arch.ArchVariant } + libPath := m.outputFile.Path() + snapshotLibOut := filepath.Join(snapshotArchDir, targetArch, "shared", vndkType, libPath.Base()) + ret = append(ret, copyFile(ctx, libPath, snapshotLibOut)) + if ctx.Config().VndkSnapshotBuildArtifacts() { prop := struct { ExportedDirs []string `json:",omitempty"` @@ -661,19 +593,19 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex ctx.Errorf("json marshal to %q failed: %#v", propOut, err) return nil, false } - ret = append(ret, installSnapshotFileFromContent(string(j), propOut)) + ret = append(ret, writeStringToFile(ctx, string(j), propOut)) } return ret, true } - isVndkSnapshotLibrary := func(m *Module) (i vndkSnapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) { + isVndkSnapshotLibrary := func(m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) { if m.Target().NativeBridge == android.NativeBridgeEnabled { return nil, "", false } - if !m.UseVndk() || !m.installable() || !m.inVendor() { + if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() { return nil, "", false } - l, ok := m.linker.(vndkSnapshotLibraryInterface) + l, ok := m.linker.(snapshotLibraryInterface) if !ok || !l.shared() { return nil, "", false } @@ -699,75 +631,38 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex return } + // install .so files for appropriate modules. + // Also install .json files if VNDK_SNAPSHOT_BUILD_ARTIFACTS libs, ok := installVndkSnapshotLib(m, l, vndkType) if !ok { return } - snapshotOutputs = append(snapshotOutputs, libs...) - // We glob headers from include directories inside source tree. So we first gather - // all include directories inside our source tree. On the contrast, we manually - // collect generated headers from dependencies as they can't globbed. - generatedHeaders = append(generatedHeaders, l.exportedGeneratedHeaders()...) - for _, dir := range append(l.exportedDirs(), l.exportedSystemDirs()...) { - exportedIncludes[dir.String()] = true + // These are for generating module_names.txt and module_paths.txt + stem := m.outputFile.Path().Base() + moduleNames[stem] = ctx.ModuleName(m) + modulePaths[stem] = ctx.ModuleDir(m) + + if m.NoticeFile().Valid() { + noticeName := stem + ".txt" + // skip already copied notice file + if _, ok := noticeBuilt[noticeName]; !ok { + noticeBuilt[noticeName] = true + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.NoticeFile().Path(), filepath.Join(noticeDir, noticeName))) + } + } + + if ctx.Config().VndkSnapshotBuildArtifacts() { + headers = append(headers, exportedHeaders(ctx, l)...) } }) - if ctx.Config().VndkSnapshotBuildArtifacts() { - globbedHeaders := make(map[string]bool) - - for _, dir := range android.SortedStringKeys(exportedIncludes) { - // Skip if dir is for generated headers - if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) { - continue - } - exts := headerExts - // Glob all files under this special directory, because of C++ headers. - if strings.HasPrefix(dir, "external/libcxx/include") { - exts = []string{""} - } - for _, ext := range exts { - glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil) - if err != nil { - ctx.Errorf("%#v\n", err) - return - } - for _, header := range glob { - if strings.HasSuffix(header, "/") { - continue - } - globbedHeaders[header] = true - } - } - } - - for _, header := range android.SortedStringKeys(globbedHeaders) { - snapshotOutputs = append(snapshotOutputs, installSnapshotFileFromPath( - android.PathForSource(ctx, header), filepath.Join(includeDir, header))) - } - - isHeader := func(path string) bool { - for _, ext := range headerExts { - if strings.HasSuffix(path, ext) { - return true - } - } - return false - } - - // For generated headers, manually install one by one, rather than glob - for _, path := range android.PathsToDirectorySortedPaths(android.FirstUniquePaths(generatedHeaders)) { - header := path.String() - - if !isHeader(header) { - continue - } - - snapshotOutputs = append(snapshotOutputs, installSnapshotFileFromPath( - path, filepath.Join(includeDir, header))) - } + // install all headers after removing duplicates + for _, header := range android.FirstUniquePaths(headers) { + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, header, filepath.Join(includeDir, header.String()))) } // install *.libraries.txt except vndkcorevariant.libraries.txt @@ -776,7 +671,8 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex if !ok || !m.Enabled() || m.Name() == vndkUsingCoreVariantLibrariesTxt { return } - snapshotOutputs = append(snapshotOutputs, installSnapshotFileFromPath(m.OutputFile(), filepath.Join(configsDir, m.Name()))) + snapshotOutputs = append(snapshotOutputs, copyFile( + ctx, m.OutputFile(), filepath.Join(configsDir, m.Name()))) }) /* @@ -796,7 +692,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex txtBuilder.WriteString(" ") txtBuilder.WriteString(m[k]) } - return installSnapshotFileFromContent(txtBuilder.String(), path) + return writeStringToFile(ctx, txtBuilder.String(), path) } /* @@ -827,14 +723,13 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex zipPath := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+".zip") zipRule := android.NewRuleBuilder() - // If output files are too many, soong_zip command can exceed ARG_MAX. - // So first dump file lists into a single list file, and then feed it to Soong + // filenames in rspfile from FlagWithRspFileInputList might be single-quoted. Remove it with xargs snapshotOutputList := android.PathForOutput(ctx, snapshotDir, "android-vndk-"+ctx.DeviceConfig().DeviceArch()+"_list") zipRule.Command(). - Text("( xargs"). - FlagWithRspFileInputList("-n1 echo < ", snapshotOutputs). - FlagWithOutput("| tr -d \\' > ", snapshotOutputList). - Text(")") + Text("tr"). + FlagWithArg("-d ", "\\'"). + FlagWithRspFileInputList("< ", snapshotOutputs). + FlagWithOutput("> ", snapshotOutputList) zipRule.Temporary(snapshotOutputList) @@ -845,6 +740,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex FlagWithInput("-l ", snapshotOutputList) zipRule.Build(pctx, ctx, zipPath.String(), "vndk snapshot "+zipPath.String()) + zipRule.DeleteTemporaryFiles() c.vndkSnapshotZipFile = android.OptionalPathForPath(zipPath) }