From c080617c8b0910fda257151aa8141676a1d6a328 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 14 Jun 2019 18:51:47 -0700 Subject: [PATCH] Shard aidl compiles into srcjars Group aidl files into groups of 50 and compile them together into a srcjar. Bug: 124333557 Test: m checkbuild Change-Id: I18e0858eab434071d3ff0039bb21e07c7311b601 --- java/droiddoc.go | 10 ++++-- java/gen.go | 90 ++++++++++++++++++++++++++++++++++------------- java/java_test.go | 20 ++++++----- java/sdk_test.go | 16 +++++---- 4 files changed, 95 insertions(+), 41 deletions(-) diff --git a/java/droiddoc.go b/java/droiddoc.go index 1d5331e64..aab61c5a7 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -461,14 +461,14 @@ func (j *Javadoc) genSources(ctx android.ModuleContext, srcFiles android.Paths, flags droiddocBuilderFlags) android.Paths { outSrcFiles := make(android.Paths, 0, len(srcFiles)) + var aidlSrcs android.Paths aidlIncludeFlags := genAidlIncludeFlags(srcFiles) for _, srcFile := range srcFiles { switch srcFile.Ext() { case ".aidl": - javaFile := genAidl(ctx, srcFile, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) - outSrcFiles = append(outSrcFiles, javaFile) + aidlSrcs = append(aidlSrcs, srcFile) case ".logtags": javaFile := genLogtags(ctx, srcFile) outSrcFiles = append(outSrcFiles, javaFile) @@ -477,6 +477,12 @@ func (j *Javadoc) genSources(ctx android.ModuleContext, srcFiles android.Paths, } } + // Process all aidl files together to support sharding them into one or more rules that produce srcjars. + if len(aidlSrcs) > 0 { + srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) + outSrcFiles = append(outSrcFiles, srcJarFiles...) + } + return outSrcFiles } diff --git a/java/gen.go b/java/gen.go index 8a90246be..d50a6653e 100644 --- a/java/gen.go +++ b/java/gen.go @@ -15,9 +15,11 @@ package java import ( + "strconv" "strings" "github.com/google/blueprint" + "github.com/google/blueprint/pathtools" "android/soong/android" ) @@ -29,13 +31,6 @@ func init() { } var ( - aidl = pctx.AndroidStaticRule("aidl", - blueprint.RuleParams{ - Command: "${config.AidlCmd} -d$depFile $aidlFlags $in $out", - CommandDeps: []string{"${config.AidlCmd}"}, - }, - "depFile", "aidlFlags") - logtags = pctx.AndroidStaticRule("logtags", blueprint.RuleParams{ Command: "$logtagsCmd -o $out $in", @@ -49,23 +44,64 @@ var ( }) ) -func genAidl(ctx android.ModuleContext, aidlFile android.Path, aidlFlags string, deps android.Paths) android.Path { - javaFile := android.GenPathWithExt(ctx, "aidl", aidlFile, "java") - depFile := javaFile.String() + ".d" +func genAidl(ctx android.ModuleContext, aidlFiles android.Paths, aidlFlags string, deps android.Paths) android.Paths { + // Shard aidl files into groups of 50 to avoid having to recompile all of them if one changes and to avoid + // hitting command line length limits. + shards := android.ShardPaths(aidlFiles, 50) - ctx.Build(pctx, android.BuildParams{ - Rule: aidl, - Description: "aidl " + aidlFile.Rel(), - Output: javaFile, - Input: aidlFile, - Implicits: deps, - Args: map[string]string{ - "depFile": depFile, - "aidlFlags": aidlFlags, - }, - }) + srcJarFiles := make(android.Paths, 0, len(shards)) - return javaFile + for i, shard := range shards { + srcJarFile := android.PathForModuleGen(ctx, "aidl", "aidl"+strconv.Itoa(i)+".srcjar") + srcJarFiles = append(srcJarFiles, srcJarFile) + + outDir := srcJarFile.ReplaceExtension(ctx, "tmp") + + rule := android.NewRuleBuilder() + + rule.Command().Text("rm -rf").Flag(outDir.String()) + rule.Command().Text("mkdir -p").Flag(outDir.String()) + rule.Command().Text("FLAGS=' " + aidlFlags + "'") + + for _, aidlFile := range shard { + depFile := srcJarFile.InSameDir(ctx, aidlFile.String()+".d") + javaFile := outDir.Join(ctx, pathtools.ReplaceExtension(aidlFile.String(), "java")) + rule.Command(). + Tool(ctx.Config().HostToolPath(ctx, "aidl")). + FlagWithDepFile("-d", depFile). + Flag("$FLAGS"). + Input(aidlFile). + Output(javaFile). + Implicits(deps) + rule.Temporary(javaFile) + } + + rule.Command(). + Tool(ctx.Config().HostToolPath(ctx, "soong_zip")). + // TODO(b/124333557): this can't use -srcjar for now, aidl on parcelables generates java files + // without a package statement, which causes -srcjar to put them in the top level of the zip file. + // Once aidl skips parcelables we can use -srcjar. + //Flag("-srcjar"). + Flag("-write_if_changed"). + FlagWithOutput("-o ", srcJarFile). + FlagWithArg("-C ", outDir.String()). + FlagWithArg("-D ", outDir.String()) + + rule.Command().Text("rm -rf").Flag(outDir.String()) + + rule.Restat() + + ruleName := "aidl" + ruleDesc := "aidl" + if len(shards) > 1 { + ruleName += "_" + strconv.Itoa(i) + ruleDesc += " " + strconv.Itoa(i) + } + + rule.Build(pctx, ctx, ruleName, ruleDesc) + } + + return srcJarFiles } func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Path { @@ -99,14 +135,14 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths, outSrcFiles := make(android.Paths, 0, len(srcFiles)) var protoSrcs android.Paths + var aidlSrcs android.Paths aidlIncludeFlags := genAidlIncludeFlags(srcFiles) for _, srcFile := range srcFiles { switch srcFile.Ext() { case ".aidl": - javaFile := genAidl(ctx, srcFile, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) - outSrcFiles = append(outSrcFiles, javaFile) + aidlSrcs = append(aidlSrcs, srcFile) case ".logtags": j.logtagsSrcs = append(j.logtagsSrcs, srcFile) javaFile := genLogtags(ctx, srcFile) @@ -124,6 +160,12 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths, outSrcFiles = append(outSrcFiles, srcJarFiles...) } + // Process all aidl files together to support sharding them into one or more rules that produce srcjars. + if len(aidlSrcs) > 0 { + srcJarFiles := genAidl(ctx, aidlSrcs, flags.aidlFlags+aidlIncludeFlags, flags.aidlDeps) + outSrcFiles = append(outSrcFiles, srcJarFiles...) + } + return outSrcFiles } diff --git a/java/java_test.go b/java/java_test.go index a9323199d..f0cb6f8bd 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -811,19 +812,22 @@ func TestDroiddoc(t *testing.T) { } `) - inputs := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc").Inputs + barDoc := ctx.ModuleForTests("bar-doc", "android_common").Rule("javadoc") var javaSrcs []string - for _, i := range inputs { + for _, i := range barDoc.Inputs { javaSrcs = append(javaSrcs, i.Base()) } - if len(javaSrcs) != 3 || javaSrcs[0] != "a.java" || javaSrcs[1] != "IFoo.java" || javaSrcs[2] != "IBar.java" { - t.Errorf("inputs of bar-doc must be []string{\"a.java\", \"IFoo.java\", \"IBar.java\", but was %#v.", javaSrcs) + if len(javaSrcs) != 1 || javaSrcs[0] != "a.java" { + t.Errorf("inputs of bar-doc must be []string{\"a.java\"}, but was %#v.", javaSrcs) } - aidlRule := ctx.ModuleForTests("bar-doc", "android_common").Output(inputs[2].String()) - aidlFlags := aidlRule.Args["aidlFlags"] - if !strings.Contains(aidlFlags, "-Ibar-doc") { - t.Errorf("aidl flags for IBar.aidl should contain \"-Ibar-doc\", but was %q", aidlFlags) + aidl := ctx.ModuleForTests("bar-doc", "android_common").Rule("aidl") + if g, w := barDoc.Implicits.Strings(), aidl.Output.String(); !inList(w, g) { + t.Errorf("implicits of bar-doc must contain %q, but was %q.", w, g) + } + + if g, w := aidl.Implicits.Strings(), []string{"bar-doc/IBar.aidl", "bar-doc/IFoo.aidl"}; !reflect.DeepEqual(w, g) { + t.Errorf("aidl inputs must be %q, but was %q", w, g) } } diff --git a/java/sdk_test.go b/java/sdk_test.go index 88e21d73d..5001b477b 100644 --- a/java/sdk_test.go +++ b/java/sdk_test.go @@ -250,7 +250,10 @@ func TestClasspath(t *testing.T) { } checkClasspath := func(t *testing.T, ctx *android.TestContext) { - javac := ctx.ModuleForTests("foo", variant).Rule("javac") + foo := ctx.ModuleForTests("foo", variant) + javac := foo.Rule("javac") + + aidl := foo.MaybeRule("aidl") got := javac.Args["bootClasspath"] if got != bc { @@ -263,6 +266,9 @@ func TestClasspath(t *testing.T) { } var deps []string + if aidl.Rule != nil { + deps = append(deps, aidl.Output.String()) + } if len(bootclasspath) > 0 && bootclasspath[0] != `""` { deps = append(deps, bootclasspath...) } @@ -290,12 +296,8 @@ func TestClasspath(t *testing.T) { if testcase.host != android.Host { aidl := ctx.ModuleForTests("foo", variant).Rule("aidl") - aidlFlags := aidl.Args["aidlFlags"] - // Trim trailing "-I." to avoid having to specify it in every test - aidlFlags = strings.TrimSpace(strings.TrimSuffix(aidlFlags, "-I.")) - - if g, w := aidlFlags, testcase.aidl; g != w { - t.Errorf("want aidl flags %q, got %q", w, g) + if g, w := aidl.RuleParams.Command, testcase.aidl+" -I."; !strings.Contains(g, w) { + t.Errorf("want aidl command to contain %q, got %q", w, g) } } })