From 1372c1b430d441aa28aa87926f49e2e2cc00c0b6 Mon Sep 17 00:00:00 2001 From: Kousik Kumar Date: Wed, 20 May 2020 07:55:56 -0700 Subject: [PATCH] Add support for remote-execution / caching of turbine actions Test: Ran a sample turbine action with: RBE_TURBINE_EXEC_STRATEGY="remote" RBE_TURBINE="true" ... use_rbe m out/soong/.intermediates/external/protobuf/libprotobuf-java-micro/android_common/turbine/libprotobuf-java-micro.jar and checked in reproxy_log.txt that remote-metadata had status SUCCESS. Bug: b/156765475 Change-Id: I85c334e6998c6dd2a68460b3580d50fffd900e6f --- java/builder.go | 38 ++++++++++++++++++++++++++------------ java/config/config.go | 1 + remoteexec/remoteexec.go | 11 ++++++++++- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/java/builder.go b/java/builder.go index b8340295b..b6cb2aeeb 100644 --- a/java/builder.go +++ b/java/builder.go @@ -116,10 +116,10 @@ var ( }, "abis", "allow-prereleased", "screen-densities", "sdk-version", "stem") - turbine = pctx.AndroidStaticRule("turbine", + turbine, turbineRE = remoteexec.StaticRules(pctx, "turbine", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + - `${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} --output $out.tmp ` + + `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} --output $out.tmp ` + `--temp_dir "$outDir" --sources @$out.rsp --source_jars $srcJars ` + `--javacopts ${config.CommonJdkFlags} ` + `$javacFlags -source $javaVersion -target $javaVersion -- $bootClasspath $classpath && ` + @@ -134,7 +134,15 @@ var ( RspfileContent: "$in", Restat: true, }, - "javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion") + &remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"}, + ExecStrategy: "${config.RETurbineExecStrategy}", + Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"}, + RSPFile: "${out}.rsp", + OutputFiles: []string{"$out.tmp"}, + OutputDirectories: []string{"$outDir"}, + ToolchainInputs: []string{"${config.JavaCmd}"}, + Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, + }, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"}) jar = pctx.AndroidStaticRule("jar", blueprint.RuleParams{ @@ -346,20 +354,26 @@ func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android. deps = append(deps, classpath...) deps = append(deps, flags.processorPath...) + rule := turbine + args := map[string]string{ + "javacFlags": flags.javacFlags, + "bootClasspath": bootClasspath, + "srcJars": strings.Join(srcJars.Strings(), " "), + "classpath": classpath.FormTurbineClassPath("--classpath "), + "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(), + "javaVersion": flags.javaVersion.String(), + } + if ctx.Config().IsEnvTrue("RBE_TURBINE") { + rule = turbineRE + args["implicits"] = strings.Join(deps.Strings(), ",") + } ctx.Build(pctx, android.BuildParams{ - Rule: turbine, + Rule: rule, Description: "turbine", Output: outputFile, Inputs: srcFiles, Implicits: deps, - Args: map[string]string{ - "javacFlags": flags.javacFlags, - "bootClasspath": bootClasspath, - "srcJars": strings.Join(srcJars.Strings(), " "), - "classpath": classpath.FormTurbineClassPath("--classpath "), - "outDir": android.PathForModuleOut(ctx, "turbine", "classes").String(), - "javaVersion": flags.javaVersion.String(), - }, + Args: args, }) } diff --git a/java/config/config.go b/java/config/config.go index 1344ceb22..fa19afbf5 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -148,6 +148,7 @@ func init() { pctx.VariableFunc("REJavacExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAVAC_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("RED8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_D8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) + pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy)) pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar") diff --git a/remoteexec/remoteexec.go b/remoteexec/remoteexec.go index 99e29dc0f..2b513b244 100644 --- a/remoteexec/remoteexec.go +++ b/remoteexec/remoteexec.go @@ -75,6 +75,9 @@ type REParams struct { // OutputFiles is a list of output file paths or ninja variables as placeholders for rule // outputs. OutputFiles []string + // OutputDirectories is a list of output directory paths or ninja variables as placeholders + // for rule outputs. + OutputDirectories []string // ToolchainInputs is a list of paths or ninja variables pointing to the location of // toolchain binaries used by the rule. ToolchainInputs []string @@ -151,6 +154,10 @@ func (r *REParams) wrapperArgs() string { args += " --output_files=" + strings.Join(r.OutputFiles, ",") } + if len(r.OutputDirectories) > 0 { + args += " --output_directories=" + strings.Join(r.OutputDirectories, ",") + } + if len(r.ToolchainInputs) > 0 { args += " --toolchain_inputs=" + strings.Join(r.ToolchainInputs, ",") } @@ -159,7 +166,9 @@ func (r *REParams) wrapperArgs() string { } // StaticRules returns a pair of rules based on the given RuleParams, where the first rule is a -// locally executable rule and the second rule is a remotely executable rule. +// locally executable rule and the second rule is a remotely executable rule. commonArgs are args +// used for both the local and remotely executable rules. reArgs are used only for remote +// execution. func StaticRules(ctx android.PackageContext, name string, ruleParams blueprint.RuleParams, reParams *REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) { ruleParamsRE := ruleParams ruleParams.Command = strings.ReplaceAll(ruleParams.Command, "$reTemplate", "")