Add support for remote-execution / caching of jar/zip actions
Test: Ran a build with `RBE_JAR="true" RBE_JAR_EXEC_STRATEGY="remote" RBE_ZIP="true" RBE_ZIP_EXEC_STRATEGY="remote" ... use_rbe m` and that succeeded. Bug: b/156765207 Change-Id: Ia17b532fbb282be414adf879914870082dd0841b
This commit is contained in:
Kousik Kumar
parent
8f37ec82d9
commit
366afc5011
|
|
@ -232,14 +232,20 @@ func TransformJniLibsToJar(ctx android.ModuleContext, outputFile android.Writabl
|
|||
"-f", j.path.String())
|
||||
}
|
||||
|
||||
rule := zip
|
||||
args := map[string]string{
|
||||
"jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),
|
||||
}
|
||||
if ctx.Config().IsEnvTrue("RBE_ZIP") {
|
||||
rule = zipRE
|
||||
args["implicits"] = strings.Join(deps.Strings(), ",")
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: zip,
|
||||
Rule: rule,
|
||||
Description: "zip jni libs",
|
||||
Output: outputFile,
|
||||
Implicits: deps,
|
||||
Args: map[string]string{
|
||||
"jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),
|
||||
},
|
||||
Args: args,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,17 +40,17 @@ var (
|
|||
// (if the rule produces .class files) or a .srcjar file (if the rule produces .java files).
|
||||
// .srcjar files are unzipped into a temporary directory when compiled with javac.
|
||||
// TODO(b/143658984): goma can't handle the --system argument to javac.
|
||||
javac, javacRE = remoteexec.StaticRules(pctx, "javac",
|
||||
javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac",
|
||||
blueprint.RuleParams{
|
||||
Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +
|
||||
`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +
|
||||
`(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` +
|
||||
`${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ` +
|
||||
`${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` +
|
||||
`${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` +
|
||||
`$processorpath $processor $javacFlags $bootClasspath $classpath ` +
|
||||
`-source $javaVersion -target $javaVersion ` +
|
||||
`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` +
|
||||
`${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +
|
||||
`$zipTemplate${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +
|
||||
`rm -rf "$srcJarDir"`,
|
||||
CommandDeps: []string{
|
||||
"${config.JavacCmd}",
|
||||
|
|
@ -60,10 +60,19 @@ var (
|
|||
CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
|
||||
Rspfile: "$out.rsp",
|
||||
RspfileContent: "$in",
|
||||
}, &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},
|
||||
ExecStrategy: "${config.REJavacExecStrategy}",
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, map[string]*remoteexec.REParams{
|
||||
"$javaTemplate": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},
|
||||
ExecStrategy: "${config.REJavacExecStrategy}",
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
"$zipTemplate": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "tool", "name": "soong_zip"},
|
||||
Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
|
||||
OutputFiles: []string{"$out"},
|
||||
ExecStrategy: "${config.REJavacExecStrategy}",
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
}, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
|
||||
"outDir", "annoDir", "javaVersion"}, nil)
|
||||
|
||||
|
|
@ -144,23 +153,35 @@ var (
|
|||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"})
|
||||
|
||||
jar = pctx.AndroidStaticRule("jar",
|
||||
jar, jarRE = remoteexec.StaticRules(pctx, "jar",
|
||||
blueprint.RuleParams{
|
||||
Command: `${config.SoongZipCmd} -jar -o $out @$out.rsp`,
|
||||
Command: `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`,
|
||||
CommandDeps: []string{"${config.SoongZipCmd}"},
|
||||
Rspfile: "$out.rsp",
|
||||
RspfileContent: "$jarArgs",
|
||||
},
|
||||
"jarArgs")
|
||||
&remoteexec.REParams{
|
||||
ExecStrategy: "${config.REJarExecStrategy}",
|
||||
Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp"},
|
||||
RSPFile: "${out}.rsp",
|
||||
OutputFiles: []string{"$out"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, []string{"jarArgs"}, nil)
|
||||
|
||||
zip = pctx.AndroidStaticRule("zip",
|
||||
zip, zipRE = remoteexec.StaticRules(pctx, "zip",
|
||||
blueprint.RuleParams{
|
||||
Command: `${config.SoongZipCmd} -o $out @$out.rsp`,
|
||||
CommandDeps: []string{"${config.SoongZipCmd}"},
|
||||
Rspfile: "$out.rsp",
|
||||
RspfileContent: "$jarArgs",
|
||||
},
|
||||
"jarArgs")
|
||||
&remoteexec.REParams{
|
||||
ExecStrategy: "${config.REZipExecStrategy}",
|
||||
Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"},
|
||||
RSPFile: "${out}.rsp",
|
||||
OutputFiles: []string{"$out"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, []string{"jarArgs"}, []string{"implicits"})
|
||||
|
||||
combineJar = pctx.AndroidStaticRule("combineJar",
|
||||
blueprint.RuleParams{
|
||||
|
|
@ -457,8 +478,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab
|
|||
func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath,
|
||||
jarArgs []string, deps android.Paths) {
|
||||
|
||||
rule := jar
|
||||
if ctx.Config().IsEnvTrue("RBE_JAR") {
|
||||
rule = jarRE
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: jar,
|
||||
Rule: rule,
|
||||
Description: "jar",
|
||||
Output: outputFile,
|
||||
Implicits: deps,
|
||||
|
|
|
|||
|
|
@ -150,6 +150,8 @@ func init() {
|
|||
pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
|
||||
pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
|
||||
pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
|
||||
pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
|
||||
pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
|
||||
|
||||
pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")
|
||||
|
||||
|
|
|
|||
54
java/dex.go
54
java/dex.go
|
|
@ -24,48 +24,66 @@ import (
|
|||
"android/soong/remoteexec"
|
||||
)
|
||||
|
||||
var d8, d8RE = remoteexec.StaticRules(pctx, "d8",
|
||||
var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",
|
||||
blueprint.RuleParams{
|
||||
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
|
||||
`$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +
|
||||
`${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
|
||||
`$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +
|
||||
`$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
|
||||
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
|
||||
CommandDeps: []string{
|
||||
"${config.D8Cmd}",
|
||||
"${config.SoongZipCmd}",
|
||||
"${config.MergeZipsCmd}",
|
||||
},
|
||||
}, &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "compiler": "d8"},
|
||||
Inputs: []string{"${config.D8Jar}"},
|
||||
ExecStrategy: "${config.RED8ExecStrategy}",
|
||||
ToolchainInputs: []string{"${config.JavaCmd}"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, map[string]*remoteexec.REParams{
|
||||
"$d8Template": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "compiler": "d8"},
|
||||
Inputs: []string{"${config.D8Jar}"},
|
||||
ExecStrategy: "${config.RED8ExecStrategy}",
|
||||
ToolchainInputs: []string{"${config.JavaCmd}"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
"$zipTemplate": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "tool", "name": "soong_zip"},
|
||||
Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
|
||||
OutputFiles: []string{"$outDir/classes.dex.jar"},
|
||||
ExecStrategy: "${config.RED8ExecStrategy}",
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
}, []string{"outDir", "d8Flags", "zipFlags"}, nil)
|
||||
|
||||
var r8, r8RE = remoteexec.StaticRules(pctx, "r8",
|
||||
var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8",
|
||||
blueprint.RuleParams{
|
||||
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
|
||||
`rm -f "$outDict" && ` +
|
||||
`$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +
|
||||
`$r8Template${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +
|
||||
`--force-proguard-compatibility ` +
|
||||
`--no-data-resources ` +
|
||||
`-printmapping $outDict ` +
|
||||
`$r8Flags && ` +
|
||||
`touch "$outDict" && ` +
|
||||
`${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
|
||||
`$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
|
||||
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
|
||||
CommandDeps: []string{
|
||||
"${config.R8Cmd}",
|
||||
"${config.SoongZipCmd}",
|
||||
"${config.MergeZipsCmd}",
|
||||
},
|
||||
}, &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "compiler": "r8"},
|
||||
Inputs: []string{"$implicits", "${config.R8Jar}"},
|
||||
ExecStrategy: "${config.RER8ExecStrategy}",
|
||||
ToolchainInputs: []string{"${config.JavaCmd}"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
}, map[string]*remoteexec.REParams{
|
||||
"$r8Template": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "compile", "compiler": "r8"},
|
||||
Inputs: []string{"$implicits", "${config.R8Jar}"},
|
||||
ExecStrategy: "${config.RER8ExecStrategy}",
|
||||
ToolchainInputs: []string{"${config.JavaCmd}"},
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
"$zipTemplate": &remoteexec.REParams{
|
||||
Labels: map[string]string{"type": "tool", "name": "soong_zip"},
|
||||
Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
|
||||
OutputFiles: []string{"$outDir/classes.dex.jar"},
|
||||
ExecStrategy: "${config.RER8ExecStrategy}",
|
||||
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
|
||||
},
|
||||
}, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"})
|
||||
|
||||
func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string {
|
||||
|
|
|
|||
14
java/java.go
14
java/java.go
|
|
@ -1456,13 +1456,19 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) {
|
|||
serviceFile := file.String()
|
||||
zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile)
|
||||
}
|
||||
rule := zip
|
||||
args := map[string]string{
|
||||
"jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),
|
||||
}
|
||||
if ctx.Config().IsEnvTrue("RBE_ZIP") {
|
||||
rule = zipRE
|
||||
args["implicits"] = strings.Join(services.Strings(), ",")
|
||||
}
|
||||
ctx.Build(pctx, android.BuildParams{
|
||||
Rule: zip,
|
||||
Rule: rule,
|
||||
Output: servicesJar,
|
||||
Implicits: services,
|
||||
Args: map[string]string{
|
||||
"jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),
|
||||
},
|
||||
Args: args,
|
||||
})
|
||||
jars = append(jars, servicesJar)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,6 +178,22 @@ func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.R
|
|||
ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
|
||||
}
|
||||
|
||||
// MultiCommandStaticRules returns a pair of rules based on the given RuleParams, where the first
|
||||
// rule is a locally executable rule and the second rule is a remotely executable rule. This
|
||||
// function supports multiple remote execution wrappers placed in the template when commands are
|
||||
// chained together with &&. commonArgs are args used for both the local and remotely executable
|
||||
// rules. reArgs are args used only for remote execution.
|
||||
func MultiCommandStaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams map[string]*REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) {
|
||||
ruleParamsRE := ruleParams
|
||||
for k, v := range reParams {
|
||||
ruleParams.Command = strings.ReplaceAll(ruleParams.Command, k, "")
|
||||
ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, k, v.Template())
|
||||
}
|
||||
|
||||
return ctx.AndroidStaticRule(name, ruleParams, commonArgs...),
|
||||
ctx.AndroidRemoteStaticRule(name+"RE", android.RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
|
||||
}
|
||||
|
||||
// EnvOverrideFunc retrieves a variable func that evaluates to the value of the given environment
|
||||
// variable if set, otherwise the given default.
|
||||
func EnvOverrideFunc(envVar, defaultVal string) func(ctx android.PackageVarContext) string {
|
||||
|
|
|
|||
Loading…
Reference in New Issue