Shard aidl compiles into srcjars

Group aidl files into groups of 50 and compile them together into a
srcjar.

Bug: 124333557
Test: m checkbuild
Change-Id: I18e0858eab434071d3ff0039bb21e07c7311b601

java/droiddoc.go

@@ -461,14 +461,14 @@ func (j *Javadoc) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 	flags droiddocBuilderFlags) android.Paths {
 
 	outSrcFiles := make(android.Paths, 0, len(srcFiles))
+	var aidlSrcs android.Paths
 
 	aidlIncludeFlags := genAidlIncludeFlags(srcFiles)
 
 	for _, srcFile := range srcFiles {
 		switch srcFile.Ext() {
 		case ".aidl":
-			javaFile := genAidl(ctx, srcFile, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps)
+			aidlSrcs = append(aidlSrcs, srcFile)
-			outSrcFiles = append(outSrcFiles, javaFile)
 		case ".logtags":
 			javaFile := genLogtags(ctx, srcFile)
 			outSrcFiles = append(outSrcFiles, javaFile)
@@ -477,6 +477,12 @@ func (j *Javadoc) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 		}
 	}
 
+	// Process all aidl files together to support sharding them into one or more rules that produce srcjars.
+	if len(aidlSrcs) > 0 {
+		srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps)
+		outSrcFiles = append(outSrcFiles, srcJarFiles...)
+	}
+
 	return outSrcFiles
 }
 

java/gen.go

@@ -15,9 +15,11 @@
 package java
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/google/blueprint"
+	"github.com/google/blueprint/pathtools"
 
 	"android/soong/android"
 )
@@ -29,13 +31,6 @@ func init() {
 }
 
 var (
-	aidl = pctx.AndroidStaticRule("aidl",
-		blueprint.RuleParams{
-			Command:     "${config.AidlCmd} -d$depFile $aidlFlags $in $out",
-			CommandDeps: []string{"${config.AidlCmd}"},
-		},
-		"depFile", "aidlFlags")
-
 	logtags = pctx.AndroidStaticRule("logtags",
 		blueprint.RuleParams{
 			Command:     "$logtagsCmd -o $out $in",
@@ -49,23 +44,64 @@ var (
 		})
 )
 
-func genAidl(ctx android.ModuleContext, aidlFile android.Path, aidlFlags string, deps android.Paths) android.Path {
+func genAidl(ctx android.ModuleContext, aidlFiles android.Paths, aidlFlags string, deps android.Paths) android.Paths {
-	javaFile := android.GenPathWithExt(ctx, "aidl", aidlFile, "java")
+	// Shard aidl files into groups of 50 to avoid having to recompile all of them if one changes and to avoid
-	depFile := javaFile.String() + ".d"
+	// hitting command line length limits.
+	shards := android.ShardPaths(aidlFiles, 50)
 
-	ctx.Build(pctx, android.BuildParams{
+	srcJarFiles := make(android.Paths, 0, len(shards))
-		Rule:        aidl,
-		Description: "aidl " + aidlFile.Rel(),
-		Output:      javaFile,
-		Input:       aidlFile,
-		Implicits:   deps,
-		Args: map[string]string{
-			"depFile":   depFile,
-			"aidlFlags": aidlFlags,
-		},
-	})
 
-	return javaFile
+	for i, shard := range shards {
+		srcJarFile := android.PathForModuleGen(ctx, "aidl", "aidl"+strconv.Itoa(i)+".srcjar")
+		srcJarFiles = append(srcJarFiles, srcJarFile)
+
+		outDir := srcJarFile.ReplaceExtension(ctx, "tmp")
+
+		rule := android.NewRuleBuilder()
+
+		rule.Command().Text("rm -rf").Flag(outDir.String())
+		rule.Command().Text("mkdir -p").Flag(outDir.String())
+		rule.Command().Text("FLAGS=' " + aidlFlags + "'")
+
+		for _, aidlFile := range shard {
+			depFile := srcJarFile.InSameDir(ctx, aidlFile.String()+".d")
+			javaFile := outDir.Join(ctx, pathtools.ReplaceExtension(aidlFile.String(), "java"))
+			rule.Command().
+				Tool(ctx.Config().HostToolPath(ctx, "aidl")).
+				FlagWithDepFile("-d", depFile).
+				Flag("$FLAGS").
+				Input(aidlFile).
+				Output(javaFile).
+				Implicits(deps)
+			rule.Temporary(javaFile)
+		}
+
+		rule.Command().
+			Tool(ctx.Config().HostToolPath(ctx, "soong_zip")).
+			// TODO(b/124333557): this can't use -srcjar for now, aidl on parcelables generates java files
+			//  without a package statement, which causes -srcjar to put them in the top level of the zip file.
+			//  Once aidl skips parcelables we can use -srcjar.
+			//Flag("-srcjar").
+			Flag("-write_if_changed").
+			FlagWithOutput("-o ", srcJarFile).
+			FlagWithArg("-C ", outDir.String()).
+			FlagWithArg("-D ", outDir.String())
+
+		rule.Command().Text("rm -rf").Flag(outDir.String())
+
+		rule.Restat()
+
+		ruleName := "aidl"
+		ruleDesc := "aidl"
+		if len(shards) > 1 {
+			ruleName += "_" + strconv.Itoa(i)
+			ruleDesc += " " + strconv.Itoa(i)
+		}
+
+		rule.Build(pctx, ctx, ruleName, ruleDesc)
+	}
+
+	return srcJarFiles
 }
 
 func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Path {
@@ -99,14 +135,14 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 
 	outSrcFiles := make(android.Paths, 0, len(srcFiles))
 	var protoSrcs android.Paths
+	var aidlSrcs android.Paths
 
 	aidlIncludeFlags := genAidlIncludeFlags(srcFiles)
 
 	for _, srcFile := range srcFiles {
 		switch srcFile.Ext() {
 		case ".aidl":
-			javaFile := genAidl(ctx, srcFile, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps)
+			aidlSrcs = append(aidlSrcs, srcFile)
-			outSrcFiles = append(outSrcFiles, javaFile)
 		case ".logtags":
 			j.logtagsSrcs = append(j.logtagsSrcs, srcFile)
 			javaFile := genLogtags(ctx, srcFile)
@@ -124,6 +160,12 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 		outSrcFiles = append(outSrcFiles, srcJarFiles...)
 	}
 
+	// Process all aidl files together to support sharding them into one or more rules that produce srcjars.
+	if len(aidlSrcs) > 0 {
+		srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps)
+		outSrcFiles = append(outSrcFiles, srcJarFiles...)
+	}
+
 	return outSrcFiles
 }
 

java/java_test.go

@@ -18,6 +18,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
 	"testing"
@@ -811,19 +812,22 @@ func TestDroiddoc(t *testing.T) {
 		}
 		`)
 
-	inputs := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc").Inputs
+	barDoc := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc")
 	var javaSrcs []string
-	for _, i := range inputs {
+	for _, i := range barDoc.Inputs {
 		javaSrcs = append(javaSrcs, i.Base())
 	}
-	if len(javaSrcs) != 3 || javaSrcs[0] != "a.java" || javaSrcs[1] != "IFoo.java" || javaSrcs[2] != "IBar.java" {
+	if len(javaSrcs) != 1 || javaSrcs[0] != "a.java" {
-		t.Errorf("inputs of bar-doc must be []string{\"a.java\", \"IFoo.java\", \"IBar.java\", but was %#v.", javaSrcs)
+		t.Errorf("inputs of bar-doc must be []string{\"a.java\"}, but was %#v.", javaSrcs)
 	}
 
-	aidlRule := ctx.ModuleForTests("bar-doc", "android_common").Output(inputs[2].String())
+	aidl := ctx.ModuleForTests("bar-doc", "android_common").Rule("aidl")
-	aidlFlags := aidlRule.Args["aidlFlags"]
+	if g, w := barDoc.Implicits.Strings(), aidl.Output.String(); !inList(w, g) {
-	if !strings.Contains(aidlFlags, "-Ibar-doc") {
+		t.Errorf("implicits of bar-doc must contain %q, but was %q.", w, g)
-		t.Errorf("aidl flags for IBar.aidl should contain \"-Ibar-doc\", but was %q", aidlFlags)
+	}
+
+	if g, w := aidl.Implicits.Strings(), []string{"bar-doc/IBar.aidl", "bar-doc/IFoo.aidl"}; !reflect.DeepEqual(w, g) {
+		t.Errorf("aidl inputs must be %q, but was %q", w, g)
 	}
 }
 

java/sdk_test.go

@@ -250,7 +250,10 @@ func TestClasspath(t *testing.T) {
 			}
 
 			checkClasspath := func(t *testing.T, ctx *android.TestContext) {
-				javac := ctx.ModuleForTests("foo", variant).Rule("javac")
+				foo := ctx.ModuleForTests("foo", variant)
+				javac := foo.Rule("javac")
+
+				aidl := foo.MaybeRule("aidl")
 
 				got := javac.Args["bootClasspath"]
 				if got != bc {
@@ -263,6 +266,9 @@ func TestClasspath(t *testing.T) {
 				}
 
 				var deps []string
+				if aidl.Rule != nil {
+					deps = append(deps, aidl.Output.String())
+				}
 				if len(bootclasspath) > 0 && bootclasspath[0] != `""` {
 					deps = append(deps, bootclasspath...)
 				}
@@ -290,12 +296,8 @@ func TestClasspath(t *testing.T) {
 				if testcase.host != android.Host {
 					aidl := ctx.ModuleForTests("foo", variant).Rule("aidl")
 
-					aidlFlags := aidl.Args["aidlFlags"]
+					if g, w := aidl.RuleParams.Command, testcase.aidl+" -I."; !strings.Contains(g, w) {
-					// Trim trailing "-I." to avoid having to specify it in every test
+						t.Errorf("want aidl command to contain %q, got %q", w, g)
-					aidlFlags = strings.TrimSpace(strings.TrimSuffix(aidlFlags, "-I."))
-
-					if g, w := aidlFlags, testcase.aidl; g != w {
-						t.Errorf("want aidl flags %q, got %q", w, g)
 					}
 				}
 			})