Capture snapshot headers in parallel

VNDK and vendor snapshot singleton work in a single thread, so globbing
in singleton results in ridiculus running time. Moving codes to
GenerateAndroidBuildActions to reduce running time.

Bug: 150406226
Test: VNDK_SNAPSHOT_BUILD_ARTIFACTS=true m dist vndk vendor-snapshot
Test: vendorSnapshotSingleton build time became 0.56s (from 10s)
Test: build.ninja building time became 1m11s (from 1m21s)
Change-Id: I4a081eef5847c62ca00280ca426f5b4e10f87b59
Merged-In: I4a081eef5847c62ca00280ca426f5b4e10f87b59
(cherry picked from commit eda2e9c728)

cc/cc.go

@@ -1465,6 +1465,13 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
 			c.Properties.HideFromMake = false // unhide
 			// Note: this is still non-installable
 		}
+
+		// glob exported headers for snapshot, if BOARD_VNDK_VERSION is current.
+		if i, ok := c.linker.(snapshotLibraryInterface); ok && ctx.DeviceConfig().VndkVersion() == "current" {
+			if isSnapshotAware(ctx, c) {
+				i.collectHeadersForSnapshot(ctx)
+			}
+		}
 	}
 
 	if c.installable() {

cc/library.go

@@ -390,6 +390,68 @@ type libraryDecorator struct {
 	*baseCompiler
 	*baseLinker
 	*baseInstaller
+
+	collectedSnapshotHeaders android.Paths
+}
+
+// collectHeadersForSnapshot collects all exported headers from library.
+// It globs header files in the source tree for exported include directories,
+// and tracks generated header files separately.
+//
+// This is to be called from GenerateAndroidBuildActions, and then collected
+// header files can be retrieved by snapshotHeaders().
+func (l *libraryDecorator) collectHeadersForSnapshot(ctx android.ModuleContext) {
+	ret := android.Paths{}
+
+	// Headers in the source tree should be globbed. On the contrast, generated headers
+	// can't be globbed, and they should be manually collected.
+	// So, we first filter out intermediate directories (which contains generated headers)
+	// from exported directories, and then glob headers under remaining directories.
+	for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) {
+		dir := path.String()
+		// Skip if dir is for generated headers
+		if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) {
+			continue
+		}
+		exts := headerExts
+		// Glob all files under this special directory, because of C++ headers.
+		if strings.HasPrefix(dir, "external/libcxx/include") {
+			exts = []string{""}
+		}
+		for _, ext := range exts {
+			glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil)
+			if err != nil {
+				ctx.ModuleErrorf("glob failed: %#v", err)
+				return
+			}
+			for _, header := range glob {
+				if strings.HasSuffix(header, "/") {
+					continue
+				}
+				ret = append(ret, android.PathForSource(ctx, header))
+			}
+		}
+	}
+
+	// Collect generated headers
+	for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) {
+		// TODO(b/148123511): remove exportedDeps after cleaning up genrule
+		if strings.HasSuffix(header.Base(), "-phony") {
+			continue
+		}
+		ret = append(ret, header)
+	}
+
+	l.collectedSnapshotHeaders = ret
+}
+
+// This returns all exported header files, both generated ones and headers from source tree.
+// collectHeadersForSnapshot() must be called before calling this.
+func (l *libraryDecorator) snapshotHeaders() android.Paths {
+	if l.collectedSnapshotHeaders == nil {
+		panic("snapshotHeaders() must be called after collectHeadersForSnapshot()")
+	}
+	return l.collectedSnapshotHeaders
 }
 
 func (library *libraryDecorator) linkerProps() []interface{} {

cc/snapshot_utils.go

@@ -14,8 +14,6 @@
 package cc
 
 import (
-	"strings"
-
 	"android/soong/android"
 )
 
@@ -26,6 +24,8 @@ var (
 type snapshotLibraryInterface interface {
 	exportedFlagsProducer
 	libraryInterface
+	collectHeadersForSnapshot(ctx android.ModuleContext)
+	snapshotHeaders() android.Paths
 }
 
 var _ snapshotLibraryInterface = (*prebuiltLibraryLinker)(nil)
@@ -58,49 +58,13 @@ func (s *snapshotMap) get(name string, arch android.ArchType) (snapshot string,
 	return snapshot, found
 }
 
-func exportedHeaders(ctx android.SingletonContext, l exportedFlagsProducer) android.Paths {
+func isSnapshotAware(ctx android.ModuleContext, m *Module) bool {
-	var ret android.Paths
+	if _, _, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m); ok {
-
+		return ctx.Config().VndkSnapshotBuildArtifacts()
-	// Headers in the source tree should be globbed. On the contrast, generated headers
+	} else if isVendorSnapshotModule(m, ctx.ModuleDir()) {
-	// can't be globbed, and they should be manually collected.
+		return true
-	// So, we first filter out intermediate directories (which contains generated headers)
-	// from exported directories, and then glob headers under remaining directories.
-	for _, path := range append(l.exportedDirs(), l.exportedSystemDirs()...) {
-		dir := path.String()
-		// Skip if dir is for generated headers
-		if strings.HasPrefix(dir, android.PathForOutput(ctx).String()) {
-			continue
-		}
-		exts := headerExts
-		// Glob all files under this special directory, because of C++ headers.
-		if strings.HasPrefix(dir, "external/libcxx/include") {
-			exts = []string{""}
-		}
-		for _, ext := range exts {
-			glob, err := ctx.GlobWithDeps(dir+"/**/*"+ext, nil)
-			if err != nil {
-				ctx.Errorf("%#v\n", err)
-				return nil
-			}
-			for _, header := range glob {
-				if strings.HasSuffix(header, "/") {
-					continue
-				}
-				ret = append(ret, android.PathForSource(ctx, header))
-			}
-		}
 	}
-
+	return false
-	// Collect generated headers
-	for _, header := range append(l.exportedGeneratedHeaders(), l.exportedDeps()...) {
-		// TODO(b/148123511): remove exportedDeps after cleaning up genrule
-		if strings.HasSuffix(header.Base(), "-phony") {
-			continue
-		}
-		ret = append(ret, header)
-	}
-
-	return ret
 }
 
 func copyFile(ctx android.SingletonContext, path android.Path, out string) android.OutputPath {

cc/vendor_snapshot.go

@@ -428,12 +428,12 @@ func isVendorProprietaryPath(dir string) bool {
 // AOSP. They are not guaranteed to be compatible with older vendor images. (e.g. might
 // depend on newer VNDK) So they are captured as vendor snapshot To build older vendor
 // image and newer system image altogether.
-func isVendorSnapshotModule(ctx android.SingletonContext, m *Module) bool {
+func isVendorSnapshotModule(m *Module, moduleDir string) bool {
 	if !m.Enabled() {
 		return false
 	}
 	// skip proprietary modules, but include all VNDK (static)
-	if isVendorProprietaryPath(ctx.ModuleDir(m)) && !m.IsVndk() {
+	if isVendorProprietaryPath(moduleDir) && !m.IsVndk() {
 		return false
 	}
 	if m.Target().Os.Class != android.Device {
@@ -525,14 +525,6 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont
 
 	var headers android.Paths
 
-	type vendorSnapshotLibraryInterface interface {
-		exportedFlagsProducer
-		libraryInterface
-	}
-
-	var _ vendorSnapshotLibraryInterface = (*prebuiltLibraryLinker)(nil)
-	var _ vendorSnapshotLibraryInterface = (*libraryDecorator)(nil)
-
 	installSnapshot := func(m *Module) android.Paths {
 		targetArch := "arch-" + m.Target().Arch.ArchType.String()
 		if m.Target().Arch.ArchVariant != "" {
@@ -588,7 +580,7 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont
 
 		var propOut string
 
-		if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok {
+		if l, ok := m.linker.(snapshotLibraryInterface); ok {
 			// library flags
 			prop.ExportedFlags = l.exportedFlags()
 			for _, dir := range l.exportedDirs() {
@@ -652,13 +644,18 @@ func (c *vendorSnapshotSingleton) GenerateBuildActions(ctx android.SingletonCont
 
 	ctx.VisitAllModules(func(module android.Module) {
 		m, ok := module.(*Module)
-		if !ok || !isVendorSnapshotModule(ctx, m) {
+		if !ok {
+			return
+		}
+
+		moduleDir := ctx.ModuleDir(module)
+		if !isVendorSnapshotModule(m, moduleDir) {
 			return
 		}
 
 		snapshotOutputs = append(snapshotOutputs, installSnapshot(m)...)
-		if l, ok := m.linker.(vendorSnapshotLibraryInterface); ok {
+		if l, ok := m.linker.(snapshotLibraryInterface); ok {
-			headers = append(headers, exportedHeaders(ctx, l)...)
+			headers = append(headers, l.snapshotHeaders()...)
 		}
 
 		if m.NoticeFile().Valid() {

cc/vndk.go

@@ -496,6 +496,28 @@ type vndkSnapshotSingleton struct {
 	vndkSnapshotZipFile android.OptionalPath
 }
 
+func isVndkSnapshotLibrary(config android.DeviceConfig, m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) {
+	if m.Target().NativeBridge == android.NativeBridgeEnabled {
+		return nil, "", false
+	}
+	if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() {
+		return nil, "", false
+	}
+	l, ok := m.linker.(snapshotLibraryInterface)
+	if !ok || !l.shared() {
+		return nil, "", false
+	}
+	if m.VndkVersion() == config.PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() {
+		if m.isVndkSp() {
+			return l, "vndk-sp", true
+		} else {
+			return l, "vndk-core", true
+		}
+	}
+
+	return nil, "", false
+}
+
 func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContext) {
 	// build these files even if PlatformVndkVersion or BoardVndkVersion is not set
 	c.buildVndkLibrariesTxtFiles(ctx)
@@ -598,35 +620,13 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 		return ret, true
 	}
 
-	isVndkSnapshotLibrary := func(m *Module) (i snapshotLibraryInterface, vndkType string, isVndkSnapshotLib bool) {
-		if m.Target().NativeBridge == android.NativeBridgeEnabled {
-			return nil, "", false
-		}
-		if !m.inVendor() || !m.installable() || m.isSnapshotPrebuilt() {
-			return nil, "", false
-		}
-		l, ok := m.linker.(snapshotLibraryInterface)
-		if !ok || !l.shared() {
-			return nil, "", false
-		}
-		if m.VndkVersion() == ctx.DeviceConfig().PlatformVndkVersion() && m.IsVndk() && !m.isVndkExt() {
-			if m.isVndkSp() {
-				return l, "vndk-sp", true
-			} else {
-				return l, "vndk-core", true
-			}
-		}
-
-		return nil, "", false
-	}
-
 	ctx.VisitAllModules(func(module android.Module) {
 		m, ok := module.(*Module)
 		if !ok || !m.Enabled() {
 			return
 		}
 
-		l, vndkType, ok := isVndkSnapshotLibrary(m)
+		l, vndkType, ok := isVndkSnapshotLibrary(ctx.DeviceConfig(), m)
 		if !ok {
 			return
 		}
@@ -655,7 +655,7 @@ func (c *vndkSnapshotSingleton) GenerateBuildActions(ctx android.SingletonContex
 		}
 
 		if ctx.Config().VndkSnapshotBuildArtifacts() {
-			headers = append(headers, exportedHeaders(ctx, l)...)
+			headers = append(headers, l.snapshotHeaders()...)
 		}
 	})