From 61eaedbddf6e5d3ac5afa85118d8f082e7f592d6 Mon Sep 17 00:00:00 2001 From: Nan Zhang Date: Thu, 2 Nov 2017 13:28:15 -0700 Subject: [PATCH] Support Javac sharding in Soong. Test: m clean && m -j java and java_test.go Change-Id: I110a0ff029448d3319aed2788d25d90a6c1a7fa0 --- README.md | 6 +++-- java/builder.go | 35 ++++++++++++++++++------- java/java.go | 54 +++++++++++++++++++++++++++++++++----- java/java_test.go | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bc019ae63..3d24e75ad 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Variables and properties are strongly typed, variables dynamically based on the first assignment, and properties statically by the module type. The supported types are: * Bool (`true` or `false`) +* Integers (`int`) * Strings (`"string"`) * Lists of strings (`["string1", "string2"]`) * Maps (`{key1: "value1", key2: ["value2"]}`) @@ -71,8 +72,9 @@ trailing commas after the last value. ### Operators Strings, lists of strings, and maps can be appended using the `+` operator. -Appending a map produces the union of keys in both maps, appending the values -of any keys that are present in both maps. +Integers can be summed up using the `+` operator. Appending a map produces the +union of keys in both maps, appending the values of any keys that are present +in both maps. ### Defaults modules diff --git a/java/builder.go b/java/builder.go index 8b6eb9fa1..45e59a426 100644 --- a/java/builder.go +++ b/java/builder.go @@ -19,6 +19,8 @@ package java // functions. import ( + "path/filepath" + "strconv" "strings" "github.com/google/blueprint" @@ -211,12 +213,16 @@ func TransformKotlinToClasses(ctx android.ModuleContext, outputFile android.Writ }) } -func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, - srcFiles, srcJars android.Paths, - flags javaBuilderFlags, deps android.Paths) { +func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int, + srcFiles, srcJars android.Paths, flags javaBuilderFlags, deps android.Paths) { - transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, deps, - "javac", "javac", javac) + // Compile java sources into .class files + desc := "javac" + if shardIdx >= 0 { + desc += strconv.Itoa(shardIdx) + } + + transformJavaToClasses(ctx, outputFile, shardIdx, srcFiles, srcJars, flags, deps, "javac", desc, javac) } func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, @@ -226,7 +232,7 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?") } - transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, nil, + transformJavaToClasses(ctx, outputFile, -1, srcFiles, srcJars, flags, nil, "errorprone", "errorprone", errorprone) } @@ -275,7 +281,7 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. // suffix will be appended to various intermediate files and directories to avoid collisions when // this function is called twice in the same module directory. func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, - srcFiles, srcJars android.Paths, + shardIdx int, srcFiles, srcJars android.Paths, flags javaBuilderFlags, deps android.Paths, intermediatesDir, desc string, rule blueprint.Rule) { @@ -298,6 +304,15 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab deps = append(deps, flags.classpath...) + srcJarDir := "srcjars" + outDir := "classes" + annoDir := "anno" + if shardIdx >= 0 { + shardDir := "shard" + strconv.Itoa(shardIdx) + srcJarDir = filepath.Join(shardDir, srcJarDir) + outDir = filepath.Join(shardDir, outDir) + annoDir = filepath.Join(shardDir, annoDir) + } ctx.Build(pctx, android.BuildParams{ Rule: rule, Description: desc, @@ -309,9 +324,9 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab "bootClasspath": bootClasspath, "classpath": flags.classpath.FormJavaClassPath("-classpath"), "srcJars": strings.Join(srcJars.Strings(), " "), - "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars").String(), - "outDir": android.PathForModuleOut(ctx, intermediatesDir, "classes").String(), - "annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(), + "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, srcJarDir).String(), + "outDir": android.PathForModuleOut(ctx, intermediatesDir, outDir).String(), + "annoDir": android.PathForModuleOut(ctx, intermediatesDir, annoDir).String(), "javaVersion": flags.javaVersion, }, }) diff --git a/java/java.go b/java/java.go index f9a4c04d2..bb6e55667 100644 --- a/java/java.go +++ b/java/java.go @@ -117,6 +117,9 @@ type CompilerProperties struct { // List of classes to pass to javac to use as annotation processors Annotation_processor_classes []string + // The number of Java source entries each Javac instance can process + Javac_shard_size *int64 + Openjdk9 struct { // List of source files that should only be used when passing -source 1.9 Srcs []string @@ -355,6 +358,18 @@ func hasSrcExt(srcs []string, ext string) bool { return false } +func shardPaths(paths android.Paths, shardSize int) []android.Paths { + ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize) + for len(paths) > shardSize { + ret = append(ret, paths[0:shardSize]) + paths = paths[shardSize:] + } + if len(paths) > 0 { + ret = append(ret, paths) + } + return ret +} + func (j *Module) hasSrcExt(ext string) bool { return hasSrcExt(j.properties.Srcs, ext) } @@ -571,7 +586,17 @@ func (j *Module) compile(ctx android.ModuleContext) { } } + enable_sharding := false if ctx.Device() && !ctx.AConfig().IsEnvFalse("TURBINE_ENABLED") { + if j.properties.Javac_shard_size != nil && *(j.properties.Javac_shard_size) > 0 { + enable_sharding = true + if len(j.properties.Annotation_processors) != 0 || + len(j.properties.Annotation_processor_classes) != 0 { + ctx.PropertyErrorf("javac_shard_size", + "%q cannot be set when annotation processors are enabled.", + j.properties.Javac_shard_size) + } + } // If sdk jar is java module, then directly return classesJar as header.jar if j.Name() != "android_stubs_current" && j.Name() != "android_system_stubs_current" && j.Name() != "android_test_stubs_current" { @@ -590,18 +615,35 @@ func (j *Module) compile(ctx android.ModuleContext) { // TODO(ccross): Once we always compile with javac9 we may be able to conditionally // enable error-prone without affecting the output class files. errorprone := android.PathForModuleOut(ctx, "errorprone", jarName) - RunErrorProne(ctx, errorprone, javaSrcFiles, srcJars, flags) + RunErrorProne(ctx, errorprone, uniqueSrcFiles, srcJars, flags) extraJarDeps = append(extraJarDeps, errorprone) } - // Compile java sources into .class files - classes := android.PathForModuleOut(ctx, "javac", jarName) - TransformJavaToClasses(ctx, classes, javaSrcFiles, srcJars, flags, extraJarDeps) + if enable_sharding { + flags.classpath.AddPaths([]android.Path{j.headerJarFile}) + shardSize := int(*(j.properties.Javac_shard_size)) + var shardSrcs []android.Paths + if len(uniqueSrcFiles) > 0 { + shardSrcs = shardPaths(uniqueSrcFiles, shardSize) + for idx, shardSrc := range shardSrcs { + classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx)) + TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps) + jars = append(jars, classes) + } + } + if len(srcJars) > 0 { + classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(len(shardSrcs))) + TransformJavaToClasses(ctx, classes, len(shardSrcs), nil, srcJars, flags, extraJarDeps) + jars = append(jars, classes) + } + } else { + classes := android.PathForModuleOut(ctx, "javac", jarName) + TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps) + jars = append(jars, classes) + } if ctx.Failed() { return } - - jars = append(jars, classes) } dirArgs, dirDeps := ResourceDirsToJarArgs(ctx, j.properties.Java_resource_dirs, j.properties.Exclude_java_resource_dirs) diff --git a/java/java_test.go b/java/java_test.go index b819447c3..82eff1e93 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "testing" ) @@ -669,6 +670,71 @@ func TestKotlin(t *testing.T) { } } +func TestTurbine(t *testing.T) { + ctx := testJava(t, ` + java_library { + name: "foo", + srcs: ["a.java"], + } + + java_library { + name: "bar", + srcs: ["b.java"], + static_libs: ["foo"], + } + + java_library { + name: "baz", + srcs: ["c.java"], + libs: ["bar"], + sdk_version: "14", + } + `) + + fooTurbine := ctx.ModuleForTests("foo", "android_common").Rule("turbine") + barTurbine := ctx.ModuleForTests("bar", "android_common").Rule("turbine") + barJavac := ctx.ModuleForTests("bar", "android_common").Rule("javac") + barTurbineCombined := ctx.ModuleForTests("bar", "android_common").Description("for turbine") + bazJavac := ctx.ModuleForTests("baz", "android_common").Rule("javac") + + if len(fooTurbine.Inputs) != 1 || fooTurbine.Inputs[0].String() != "a.java" { + t.Errorf(`foo inputs %v != ["a.java"]`, fooTurbine.Inputs) + } + + fooHeaderJar := filepath.Join(buildDir, ".intermediates", "foo", "android_common", "turbine-combined", "foo.jar") + if !strings.Contains(barTurbine.Args["classpath"], fooHeaderJar) { + t.Errorf("bar turbine classpath %v does not contain %q", barTurbine.Args["classpath"], fooHeaderJar) + } + if !strings.Contains(barJavac.Args["classpath"], fooHeaderJar) { + t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], fooHeaderJar) + } + if len(barTurbineCombined.Inputs) != 2 || barTurbineCombined.Inputs[1].String() != fooHeaderJar { + t.Errorf("bar turbine combineJar inputs %v does not contain %q", barTurbineCombined.Inputs, fooHeaderJar) + } + if !strings.Contains(bazJavac.Args["classpath"], "prebuilts/sdk/14/android.jar") { + t.Errorf("baz javac classpath %v does not contain %q", bazJavac.Args["classpath"], + "prebuilts/sdk/14/android.jar") + } +} + +func TestSharding(t *testing.T) { + ctx := testJava(t, ` + java_library { + name: "bar", + srcs: ["a.java","b.java","c.java"], + javac_shard_size: 1 + } + `) + + barHeaderJar := filepath.Join(buildDir, ".intermediates", "bar", "android_common", "turbine-combined", "bar.jar") + for i := 0; i < 3; i++ { + barJavac := ctx.ModuleForTests("bar", "android_common").Description("javac" + strconv.Itoa(i)) + if !strings.Contains(barJavac.Args["classpath"], barHeaderJar) { + t.Errorf("bar javac classpath %v does not contain %q", barJavac.Args["classpath"], barHeaderJar) + } + } +} + func fail(t *testing.T, errs []error) { if len(errs) > 0 { for _, err := range errs {