From e9a275b440f2e23217081e16bdbe74f981f00239 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 16 Oct 2017 17:09:48 -0700 Subject: [PATCH 1/2] Pass output file names into java.Transform* functions Pass the output file name into the java.Transform* functions. This consistently puts control of the filename into java.go, which is often necessary to avoid collisions when the same rule is used multiple times in a single module. It also has the side-effect of removing the poorly named "stem" parameters. Test: java_test.go Change-Id: I7bc1d1f3bfae6f9d2c92870e6df381817817aab4 --- java/builder.go | 74 +++++++++++++++++-------------------------------- java/java.go | 36 ++++++++++++++++++------ 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/java/builder.go b/java/builder.go index 8992f681a..304cd2764 100644 --- a/java/builder.go +++ b/java/builder.go @@ -135,23 +135,24 @@ type javaBuilderFlags struct { protoOutFlag string } -func TransformJavaToClasses(ctx android.ModuleContext, srcFiles, srcFileLists android.Paths, - flags javaBuilderFlags, deps android.Paths) android.ModuleOutPath { +func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, + srcFiles, srcFileLists android.Paths, + flags javaBuilderFlags, deps android.Paths) { - return transformJavaToClasses(ctx, srcFiles, srcFileLists, flags, deps, - "classes-compiled.jar", "", "javac", javac) + transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, deps, + "", "javac", javac) } -func RunErrorProne(ctx android.ModuleContext, srcFiles, srcFileLists android.Paths, - flags javaBuilderFlags) android.Path { +func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, + srcFiles, srcFileLists android.Paths, + flags javaBuilderFlags) { if config.ErrorProneJar == "" { ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?") - return nil } - return transformJavaToClasses(ctx, srcFiles, srcFileLists, flags, nil, - "classes-errorprone.list", "-errorprone", "errorprone", errorprone) + transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, nil, + "-errorprone", "errorprone", errorprone) } // transformJavaToClasses takes source files and converts them to a jar containing .class files. @@ -166,11 +167,10 @@ func RunErrorProne(ctx android.ModuleContext, srcFiles, srcFileLists android.Pat // be printed at build time. The stem argument provides the file name of the output jar, and // suffix will be appended to various intermediate files and directories to avoid collisions when // this function is called twice in the same module directory. -func transformJavaToClasses(ctx android.ModuleContext, srcFiles, srcFileLists android.Paths, - flags javaBuilderFlags, deps android.Paths, stem, suffix, desc string, - rule blueprint.Rule) android.ModuleOutPath { - - outputFile := android.PathForModuleOut(ctx, stem) +func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, + srcFiles, srcFileLists android.Paths, + flags javaBuilderFlags, deps android.Paths, + intermediatesSuffix, desc string, rule blueprint.Rule) { javacFlags := flags.javacFlags if len(srcFileLists) > 0 { @@ -200,19 +200,15 @@ func transformJavaToClasses(ctx android.ModuleContext, srcFiles, srcFileLists an "javacFlags": javacFlags, "bootClasspath": bootClasspath, "classpath": flags.classpath.JavaClasspath(), - "outDir": android.PathForModuleOut(ctx, "classes"+suffix).String(), - "annoDir": android.PathForModuleOut(ctx, "anno"+suffix).String(), + "outDir": android.PathForModuleOut(ctx, "classes"+intermediatesSuffix).String(), + "annoDir": android.PathForModuleOut(ctx, "anno"+intermediatesSuffix).String(), "javaVersion": flags.javaVersion, }, }) - - return outputFile } -func TransformResourcesToJar(ctx android.ModuleContext, jarArgs []string, - deps android.Paths) android.Path { - - outputFile := android.PathForModuleOut(ctx, "res.jar") +func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath, + jarArgs []string, deps android.Paths) { ctx.ModuleBuild(pctx, android.ModuleBuildParams{ Rule: jar, @@ -223,18 +219,10 @@ func TransformResourcesToJar(ctx android.ModuleContext, jarArgs []string, "jarArgs": strings.Join(jarArgs, " "), }, }) - - return outputFile } -func TransformJarsToJar(ctx android.ModuleContext, stem string, jars android.Paths, - manifest android.OptionalPath, stripDirs bool) android.Path { - - outputFile := android.PathForModuleOut(ctx, stem) - - if len(jars) == 1 && !manifest.Valid() { - return jars[0] - } +func TransformJarsToJar(ctx android.ModuleContext, outputFile android.WritablePath, + jars android.Paths, manifest android.OptionalPath, stripDirs bool) { var deps android.Paths @@ -258,14 +246,11 @@ func TransformJarsToJar(ctx android.ModuleContext, stem string, jars android.Pat "jarArgs": strings.Join(jarArgs, " "), }, }) - - return outputFile } -func TransformDesugar(ctx android.ModuleContext, classesJar android.Path, - flags javaBuilderFlags) android.Path { +func TransformDesugar(ctx android.ModuleContext, outputFile android.WritablePath, + classesJar android.Path, flags javaBuilderFlags) { - outputFile := android.PathForModuleOut(ctx, "classes-desugar.jar") dumpDir := android.PathForModuleOut(ctx, "desugar_dumped_classes") javaFlags := "" @@ -294,17 +279,14 @@ func TransformDesugar(ctx android.ModuleContext, classesJar android.Path, "desugarFlags": flags.desugarFlags, }, }) - - return outputFile } // Converts a classes.jar file to classes*.dex, then combines the dex files with any resources // in the classes.jar file into a dex jar. -func TransformClassesJarToDexJar(ctx android.ModuleContext, stem string, classesJar android.Path, - flags javaBuilderFlags) android.Path { +func TransformClassesJarToDexJar(ctx android.ModuleContext, outputFile android.WritablePath, + classesJar android.Path, flags javaBuilderFlags) { outDir := android.PathForModuleOut(ctx, "dex") - outputFile := android.PathForModuleOut(ctx, stem) ctx.ModuleBuild(pctx, android.ModuleBuildParams{ Rule: dx, @@ -316,12 +298,10 @@ func TransformClassesJarToDexJar(ctx android.ModuleContext, stem string, classes "outDir": outDir.String(), }, }) - - return outputFile } -func TransformJarJar(ctx android.ModuleContext, classesJar android.Path, rulesFile android.Path) android.ModuleOutPath { - outputFile := android.PathForModuleOut(ctx, "classes-jarjar.jar") +func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath, + classesJar android.Path, rulesFile android.Path) { ctx.ModuleBuild(pctx, android.ModuleBuildParams{ Rule: jarjar, Description: "jarjar", @@ -332,8 +312,6 @@ func TransformJarJar(ctx android.ModuleContext, classesJar android.Path, rulesFi "rulesFile": rulesFile.String(), }, }) - - return outputFile } type classpath []android.Path diff --git a/java/java.go b/java/java.go index c3b7557fd..6485f0654 100644 --- a/java/java.go +++ b/java/java.go @@ -501,12 +501,14 @@ func (j *Module) compile(ctx android.ModuleContext) { // a rebuild when error-prone is turned off). // TODO(ccross): Once we always compile with javac9 we may be able to conditionally // enable error-prone without affecting the output class files. - errorprone := RunErrorProne(ctx, srcFiles, srcFileLists, flags) + errorprone := android.PathForModuleOut(ctx, "classes-errorprone.list") + RunErrorProne(ctx, errorprone, srcFiles, srcFileLists, flags) extraJarDeps = append(extraJarDeps, errorprone) } // Compile java sources into .class files - classes := TransformJavaToClasses(ctx, srcFiles, srcFileLists, flags, extraJarDeps) + classes := android.PathForModuleOut(ctx, "classes-compiled.jar") + TransformJavaToClasses(ctx, classes, srcFiles, srcFileLists, flags, extraJarDeps) if ctx.Failed() { return } @@ -533,7 +535,8 @@ func (j *Module) compile(ctx android.ModuleContext) { } if len(resArgs) > 0 { - resourceJar := TransformResourcesToJar(ctx, resArgs, resDeps) + resourceJar := android.PathForModuleOut(ctx, "res.jar") + TransformResourcesToJar(ctx, resourceJar, resArgs, resDeps) if ctx.Failed() { return } @@ -548,12 +551,23 @@ func (j *Module) compile(ctx android.ModuleContext) { // Combine the classes built from sources, any manifests, and any static libraries into // classes.jar. If there is only one input jar this step will be skipped. - outputFile := TransformJarsToJar(ctx, "classes.jar", jars, manifest, false) + var outputFile android.Path + + if len(jars) == 1 && !manifest.Valid() { + // Optimization: skip the combine step if there is nothing to do + outputFile = jars[0] + } else { + combinedJar := android.PathForModuleOut(ctx, "classes.jar") + TransformJarsToJar(ctx, combinedJar, jars, manifest, false) + outputFile = combinedJar + } if j.properties.Jarjar_rules != nil { jarjar_rules := android.PathForModuleSrc(ctx, *j.properties.Jarjar_rules) // Transform classes.jar into classes-jarjar.jar - outputFile = TransformJarJar(ctx, outputFile, jarjar_rules) + jarjarFile := android.PathForModuleOut(ctx, "classes-jarjar.jar") + TransformJarJar(ctx, jarjarFile, outputFile, jarjar_rules) + outputFile = jarjarFile if ctx.Failed() { return } @@ -609,13 +623,17 @@ func (j *Module) compile(ctx android.ModuleContext) { flags.desugarFlags = strings.Join(desugarFlags, " ") - desugarJar := TransformDesugar(ctx, outputFile, flags) + desugarJar := android.PathForModuleOut(ctx, "classes-desugar.jar") + TransformDesugar(ctx, desugarJar, outputFile, flags) + outputFile = desugarJar if ctx.Failed() { return } // Compile classes.jar into classes.dex and then javalib.jar - outputFile = TransformClassesJarToDexJar(ctx, "javalib.jar", desugarJar, flags) + javalibJar := android.PathForModuleOut(ctx, "javalib.jar") + TransformClassesJarToDexJar(ctx, javalibJar, desugarJar, flags) + outputFile = javalibJar if ctx.Failed() { return } @@ -790,7 +808,9 @@ func (j *Import) DepsMutator(ctx android.BottomUpMutatorContext) { func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { j.classpathFiles = android.PathsForModuleSrc(ctx, j.properties.Jars) - j.combinedClasspathFile = TransformJarsToJar(ctx, "classes.jar", j.classpathFiles, android.OptionalPath{}, false) + outputFile := android.PathForModuleOut(ctx, "classes.jar") + TransformJarsToJar(ctx, outputFile, j.classpathFiles, android.OptionalPath{}, false) + j.combinedClasspathFile = outputFile } var _ Dependency = (*Import)(nil) From 59149b6df59b8dcc7b4eddd2ebc95352608291a8 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 16 Oct 2017 18:07:29 -0700 Subject: [PATCH 2/2] Use jars containg sources for java generators srcFileLists was an ill-fated attempt to deal with generators that produce a set of java sources that is not known ahead of time. For example, the list of files produced by protoc depends on the package statement in the .proto file. srcFileLists put the list of generated files into a file, which was then passed to javac using the @file syntax. This worked, but it was too easy to cause missing dependencies, and will not work well in a future distributed build environment. Switch to putting generated sources into a jar, and then pass them jar to javac using -sourcepath. Test: m checkbuild Change-Id: Iaab7a588a6c1239f7bf46e4f1b102b3ef517619b --- java/app.go | 2 +- java/builder.go | 45 ++++++++++++++++++++++++--------------------- java/gen.go | 11 ++++++----- java/java.go | 21 ++++++++++----------- java/proto.go | 17 ++++++++--------- 5 files changed, 49 insertions(+), 47 deletions(-) diff --git a/java/app.go b/java/app.go index 490a03d9a..42ae23663 100644 --- a/java/app.go +++ b/java/app.go @@ -89,7 +89,7 @@ func (a *AndroidApp) GenerateAndroidBuildActions(ctx android.ModuleContext) { publicResourcesFile, proguardOptionsFile, aaptJavaFileList := CreateResourceJavaFiles(ctx, aaptRJavaFlags, aaptDeps) a.aaptJavaFileList = aaptJavaFileList - a.ExtraSrcLists = append(a.ExtraSrcLists, aaptJavaFileList) + // TODO(ccross): export aapt generated java files as a src jar if a.appProperties.Export_package_resources { aaptPackageFlags := append([]string(nil), aaptFlags...) diff --git a/java/builder.go b/java/builder.go index 304cd2764..efe625665 100644 --- a/java/builder.go +++ b/java/builder.go @@ -40,7 +40,7 @@ var ( blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" && mkdir -p "$outDir" "$annoDir" && ` + `${config.JavacWrapper}${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + - `$javacFlags $bootClasspath $classpath ` + + `$javacFlags $sourcepath $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + `-d $outDir -s $annoDir @$out.rsp && ` + `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`, @@ -48,13 +48,13 @@ var ( Rspfile: "$out.rsp", RspfileContent: "$in", }, - "javacFlags", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion") + "javacFlags", "sourcepath", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion") errorprone = pctx.AndroidStaticRule("errorprone", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" && mkdir -p "$outDir" "$annoDir" && ` + `${config.ErrorProneCmd} ` + - `$javacFlags $bootClasspath $classpath ` + + `$javacFlags $sourcepath $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + `-d $outDir -s $annoDir @$out.rsp && ` + `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`, @@ -67,7 +67,7 @@ var ( Rspfile: "$out.rsp", RspfileContent: "$in", }, - "javacFlags", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion") + "javacFlags", "sourcepath", "bootClasspath", "classpath", "outDir", "annoDir", "javaVersion") jar = pctx.AndroidStaticRule("jar", blueprint.RuleParams{ @@ -136,31 +136,28 @@ type javaBuilderFlags struct { } func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, - srcFiles, srcFileLists android.Paths, + srcFiles android.Paths, srcJars classpath, flags javaBuilderFlags, deps android.Paths) { - transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, deps, + transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, deps, "", "javac", javac) } func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, - srcFiles, srcFileLists android.Paths, + srcFiles android.Paths, srcJars classpath, flags javaBuilderFlags) { if config.ErrorProneJar == "" { ctx.ModuleErrorf("cannot build with Error Prone, missing external/error_prone?") } - transformJavaToClasses(ctx, outputFile, srcFiles, srcFileLists, flags, nil, + transformJavaToClasses(ctx, outputFile, srcFiles, srcJars, flags, nil, "-errorprone", "errorprone", errorprone) } // transformJavaToClasses takes source files and converts them to a jar containing .class files. -// srcFiles is a list of paths to sources, srcFileLists is a list of paths to files that contain -// paths to sources. There is no dependency on the sources passed through srcFileLists, those -// must be added through the deps argument, which contains a list of paths that should be added -// as implicit dependencies. flags contains various command line flags to be passed to the -// compiler. +// srcFiles is a list of paths to sources, srcJars is a list of paths to jar files that contain +// sources. flags contains various command line flags to be passed to the compiler. // // This method may be used for different compilers, including javac and Error Prone. The rule // argument specifies which command line to use and desc sets the description of the rule that will @@ -168,16 +165,11 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, // suffix will be appended to various intermediate files and directories to avoid collisions when // this function is called twice in the same module directory. func transformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, - srcFiles, srcFileLists android.Paths, + srcFiles android.Paths, srcJars classpath, flags javaBuilderFlags, deps android.Paths, intermediatesSuffix, desc string, rule blueprint.Rule) { - javacFlags := flags.javacFlags - if len(srcFileLists) > 0 { - javacFlags += " " + android.JoinWithPrefix(srcFileLists.Strings(), "@") - } - - deps = append(deps, srcFileLists...) + deps = append(deps, srcJars...) var bootClasspath string if flags.javaVersion == "1.9" { @@ -197,8 +189,9 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab Inputs: srcFiles, Implicits: deps, Args: map[string]string{ - "javacFlags": javacFlags, + "javacFlags": flags.javacFlags, "bootClasspath": bootClasspath, + "sourcepath": srcJars.JavaSourcepath(), "classpath": flags.classpath.JavaClasspath(), "outDir": android.PathForModuleOut(ctx, "classes"+intermediatesSuffix).String(), "annoDir": android.PathForModuleOut(ctx, "anno"+intermediatesSuffix).String(), @@ -316,6 +309,16 @@ func TransformJarJar(ctx android.ModuleContext, outputFile android.WritablePath, type classpath []android.Path +// Returns a -sourcepath argument in the form javac expects. If the list is empty returns +// -sourcepath "" to ensure javac does not fall back to searching the classpath for sources. +func (x *classpath) JavaSourcepath() string { + if len(*x) > 0 { + return "-sourcepath " + strings.Join(x.Strings(), ":") + } else { + return `-sourcepath ""` + } +} + // Returns a -classpath argument in the form java or javac expects func (x *classpath) JavaClasspath() string { if len(*x) > 0 { diff --git a/java/gen.go b/java/gen.go index e55be9167..e12a71c4e 100644 --- a/java/gen.go +++ b/java/gen.go @@ -85,7 +85,7 @@ func genLogtags(ctx android.ModuleContext, logtagsFile android.Path) android.Pat } func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths, - flags javaBuilderFlags) (android.Paths, android.Paths) { + flags javaBuilderFlags) (android.Paths, classpath) { var protoFiles android.Paths outSrcFiles := make(android.Paths, 0, len(srcFiles)) @@ -106,16 +106,17 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths, } } - var outSrcFileLists android.Paths + var outSrcJars classpath if len(protoFiles) > 0 { - protoFileList := genProto(ctx, protoFiles, + protoSrcJar := android.PathForModuleGen(ctx, "proto.src.jar") + genProto(ctx, protoSrcJar, protoFiles, flags.protoFlags, flags.protoOutFlag, "") - outSrcFileLists = append(outSrcFileLists, protoFileList) + outSrcJars = append(outSrcJars, protoSrcJar) } - return outSrcFiles, outSrcFileLists + return outSrcFiles, outSrcJars } func LogtagsSingleton() blueprint.Singleton { diff --git a/java/java.go b/java/java.go index 6485f0654..770c9c1fd 100644 --- a/java/java.go +++ b/java/java.go @@ -169,9 +169,9 @@ type Module struct { logtagsSrcs android.Paths - // filelists of extra source files that should be included in the javac command line, + // jars containing source files that should be included in the javac command line, // for example R.java generated by aapt for android apps - ExtraSrcLists android.Paths + ExtraSrcJars android.Paths // installed file for binary dependency installFile android.Path @@ -370,7 +370,7 @@ type deps struct { staticJars android.Paths staticJarResources android.Paths aidlIncludeDirs android.Paths - srcFileLists android.Paths + srcJars android.Paths systemModules android.Path aidlPreprocess android.OptionalPath } @@ -422,7 +422,7 @@ func (j *Module) collectDeps(ctx android.ModuleContext) deps { if ctx.ModuleName() == "framework" { // framework.jar has a one-off dependency on the R.java and Manifest.java files // generated by framework-res.apk - deps.srcFileLists = append(deps.srcFileLists, module.(*AndroidApp).aaptJavaFileList) + // TODO(ccross): aapt java files should go in a src jar } default: panic(fmt.Errorf("unknown dependency %q for %q", otherName, ctx.ModuleName())) @@ -483,13 +483,12 @@ func (j *Module) compile(ctx android.ModuleContext) { flags = protoFlags(ctx, &j.protoProperties, flags) } - var srcFileLists android.Paths + var srcJars classpath + srcFiles, srcJars = j.genSources(ctx, srcFiles, flags) - srcFiles, srcFileLists = j.genSources(ctx, srcFiles, flags) + srcJars = append(srcJars, deps.srcJars...) - srcFileLists = append(srcFileLists, deps.srcFileLists...) - - srcFileLists = append(srcFileLists, j.ExtraSrcLists...) + srcJars = append(srcJars, j.ExtraSrcJars...) var jars android.Paths @@ -502,13 +501,13 @@ func (j *Module) compile(ctx android.ModuleContext) { // TODO(ccross): Once we always compile with javac9 we may be able to conditionally // enable error-prone without affecting the output class files. errorprone := android.PathForModuleOut(ctx, "classes-errorprone.list") - RunErrorProne(ctx, errorprone, srcFiles, srcFileLists, flags) + RunErrorProne(ctx, errorprone, srcFiles, srcJars, flags) extraJarDeps = append(extraJarDeps, errorprone) } // Compile java sources into .class files classes := android.PathForModuleOut(ctx, "classes-compiled.jar") - TransformJavaToClasses(ctx, classes, srcFiles, srcFileLists, flags, extraJarDeps) + TransformJavaToClasses(ctx, classes, srcFiles, srcJars, flags, extraJarDeps) if ctx.Failed() { return } diff --git a/java/proto.go b/java/proto.go index dd8cabd97..fc259a5d2 100644 --- a/java/proto.go +++ b/java/proto.go @@ -30,20 +30,21 @@ var ( blueprint.RuleParams{ Command: `rm -rf $outDir && mkdir -p $outDir && ` + `$protocCmd $protoOut=$protoOutFlags:$outDir $protoFlags $in && ` + - `find $outDir -name "*.java" > $out`, - CommandDeps: []string{"$protocCmd"}, + `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir`, + CommandDeps: []string{ + "$protocCmd", + "${config.SoongZipCmd}", + }, }, "protoFlags", "protoOut", "protoOutFlags", "outDir") ) -func genProto(ctx android.ModuleContext, protoFiles android.Paths, - protoFlags string, protoOut, protoOutFlags string) android.WritablePath { - - protoFileList := android.PathForModuleGen(ctx, "proto.filelist") +func genProto(ctx android.ModuleContext, outputSrcJar android.WritablePath, + protoFiles android.Paths, protoFlags string, protoOut, protoOutFlags string) { ctx.ModuleBuild(pctx, android.ModuleBuildParams{ Rule: proto, Description: "protoc " + protoFiles[0].Rel(), - Output: protoFileList, + Output: outputSrcJar, Inputs: protoFiles, Args: map[string]string{ "outDir": android.ProtoDir(ctx).String(), @@ -52,8 +53,6 @@ func genProto(ctx android.ModuleContext, protoFiles android.Paths, "protoFlags": protoFlags, }, }) - - return protoFileList } func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) {