openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge changes Ie8721dcd,If1827b9d,Iec250a2d,I3f46f611,Idf136919, ... 

* changes:
  Replace ANDROID_SDK_HOME with ANDROID_PREFS_ROOT for metalava
  Simplify lint rules using improved RuleBuilder rsp support
  Support multiple rsp files in RuleBuilder
  Pass rsp files into sbox and rewrapper
  Add test for sbox input sandboxing
  Support multiple rsp files in REParams
  Move response file handling to a separate package
This commit is contained in:
Colin Cross 2021-03-25 20:55:56 +00:00 committed by Gerrit Code Review
parent d411f069ff 7ee54ffd70
commit ba5fff85a7
23 changed files with 748 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
android/Android.bp
View File

@ -14,6 +14,7 @@ bootstrap_go_package {
"soong-bazel",
"soong-cquery",
"soong-remoteexec",
"soong-response",
"soong-shared",
"soong-ui-metrics_proto",
],

157
android/rule_builder.go
View File

@ -28,6 +28,7 @@ import (
"android/soong/cmd/sbox/sbox_proto"
"android/soong/remoteexec"
"android/soong/response"
"android/soong/shared"
)
@ -408,30 +409,21 @@ func (r *RuleBuilder) Tools() Paths {
func (r *RuleBuilder) RspFileInputs() Paths {
var rspFileInputs Paths
for _, c := range r.commands {
if c.rspFileInputs != nil {
if rspFileInputs != nil {
panic("Multiple commands in a rule may not have rsp file inputs")
}
rspFileInputs = c.rspFileInputs
for _, rspFile := range c.rspFiles {
rspFileInputs = append(rspFileInputs, rspFile.paths...)
}
}
return rspFileInputs
}
// RspFile returns the path to the rspfile that was passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
func (r *RuleBuilder) RspFile() WritablePath {
var rspFile WritablePath
func (r *RuleBuilder) rspFiles() []rspFileAndPaths {
var rspFiles []rspFileAndPaths
for _, c := range r.commands {
if c.rspFile != nil {
if rspFile != nil {
panic("Multiple commands in a rule may not have rsp file inputs")
}
rspFile = c.rspFile
}
rspFiles = append(rspFiles, c.rspFiles...)
}
return rspFile
return rspFiles
}
// Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
@ -459,26 +451,6 @@ func (r *RuleBuilder) depFileMergerCmd(depFiles WritablePaths) *RuleBuilderComma
Inputs(depFiles.Paths())
}
// composeRspFileContent returns a string that will serve as the contents of the rsp file to pass
// the listed input files to the command running in the sandbox.
func (r *RuleBuilder) composeRspFileContent(rspFileInputs Paths) string {
if r.sboxInputs {
if len(rspFileInputs) > 0 {
// When SandboxInputs is used the paths need to be rewritten to be relative to the sandbox
// directory so that they are valid after sbox chdirs into the sandbox directory.
return proptools.NinjaEscape(strings.Join(r.sboxPathsForInputsRel(rspFileInputs), " "))
} else {
// If the list of inputs is empty fall back to "$in" so that the rspfilecontent Ninja
// variable is set to something non-empty, otherwise ninja will complain. The inputs
// will be empty (all the non-rspfile inputs are implicits), so $in will evaluate to
// an empty string.
return "$in"
}
} else {
return "$in"
}
}
// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
// Outputs.
func (r *RuleBuilder) Build(name string, desc string) {
@ -520,8 +492,7 @@ func (r *RuleBuilder) Build(name string, desc string) {
commands := r.Commands()
outputs := r.Outputs()
inputs := r.Inputs()
rspFileInputs := r.RspFileInputs()
rspFilePath := r.RspFile()
rspFiles := r.rspFiles()
if len(commands) == 0 {
return
@ -575,11 +546,22 @@ func (r *RuleBuilder) Build(name string, desc string) {
})
}
// If using an rsp file copy it into the sbox directory.
if rspFilePath != nil {
command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
From: proto.String(rspFilePath.String()),
To: proto.String(r.sboxPathForInputRel(rspFilePath)),
// If using rsp files copy them and their contents into the sbox directory with
// the appropriate path mappings.
for _, rspFile := range rspFiles {
command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
File: proto.String(rspFile.file.String()),
// These have to match the logic in sboxPathForInputRel
PathMappings: []*sbox_proto.PathMapping{
{
From: proto.String(r.outDir.String()),
To: proto.String(sboxOutSubDir),
},
{
From: proto.String(PathForOutput(r.ctx).String()),
To: proto.String(sboxOutSubDir),
},
},
})
}
@ -641,20 +623,30 @@ func (r *RuleBuilder) Build(name string, desc string) {
inputs = append(inputs, sboxCmd.inputs...)
if r.rbeParams != nil {
var remoteInputs []string
remoteInputs = append(remoteInputs, inputs.Strings()...)
remoteInputs = append(remoteInputs, tools.Strings()...)
remoteInputs = append(remoteInputs, rspFileInputs.Strings()...)
if rspFilePath != nil {
remoteInputs = append(remoteInputs, rspFilePath.String())
// RBE needs a list of input files to copy to the remote builder. For inputs already
// listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
// create a new rsp file to pass to rewrapper.
var remoteRspFiles Paths
var remoteInputs Paths
remoteInputs = append(remoteInputs, inputs...)
remoteInputs = append(remoteInputs, tools...)
for _, rspFile := range rspFiles {
remoteInputs = append(remoteInputs, rspFile.file)
remoteRspFiles = append(remoteRspFiles, rspFile.file)
}
if len(remoteInputs) > 0 {
inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
remoteRspFiles = append(remoteRspFiles, inputsListFile)
// Add the new rsp file as an extra input to the rule.
inputs = append(inputs, inputsListFile)
}
inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
inputsListContents := rspFileForInputs(remoteInputs)
WriteFileRule(r.ctx, inputsListFile, inputsListContents)
inputs = append(inputs, inputsListFile)
r.rbeParams.OutputFiles = outputs.Strings()
r.rbeParams.RSPFile = inputsListFile.String()
r.rbeParams.RSPFiles = remoteRspFiles.Strings()
rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
}
@ -672,9 +664,24 @@ func (r *RuleBuilder) Build(name string, desc string) {
implicitOutputs := outputs[1:]
var rspFile, rspFileContent string
if rspFilePath != nil {
rspFile = rspFilePath.String()
rspFileContent = r.composeRspFileContent(rspFileInputs)
var rspFileInputs Paths
if len(rspFiles) > 0 {
// The first rsp files uses Ninja's rsp file support for the rule
rspFile = rspFiles[0].file.String()
// Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
// list and in the contents of the rsp file. Inputs to the rule that are not in the
// rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
rspFileContent = "$in"
rspFileInputs = append(rspFileInputs, rspFiles[0].paths...)
for _, rspFile := range rspFiles[1:] {
// Any additional rsp files need an extra rule to write the file.
writeRspFileRule(r.ctx, rspFile.file, rspFile.paths)
// The main rule needs to depend on the inputs listed in the extra rsp file.
inputs = append(inputs, rspFile.paths...)
// The main rule needs to depend on the extra rsp file.
inputs = append(inputs, rspFile.file)
}
}
var pool blueprint.Pool
@ -725,8 +732,12 @@ type RuleBuilderCommand struct {
depFiles WritablePaths
tools Paths
packagedTools []PackagingSpec
rspFileInputs Paths
rspFile WritablePath
rspFiles []rspFileAndPaths
}
type rspFileAndPaths struct {
file WritablePath
paths Paths
}
func (c *RuleBuilderCommand) addInput(path Path) string {
@ -1170,22 +1181,19 @@ func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *Ru
return c.Text(flag + c.PathForOutput(path))
}
// FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with no separator
// between them. The paths will be written to the rspfile. If sbox is enabled, the rspfile must
// be outside the sbox directory.
// FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with
// no separator between them. The paths will be written to the rspfile. If sbox is enabled, the
// rspfile must be outside the sbox directory. The first use of FlagWithRspFileInputList in any
// RuleBuilderCommand of a RuleBuilder will use Ninja's rsp file support for the rule, additional
// uses will result in an auxiliary rules to write the rspFile contents.
func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile WritablePath, paths Paths) *RuleBuilderCommand {
if c.rspFileInputs != nil {
panic("FlagWithRspFileInputList cannot be called if rsp file inputs have already been provided")
}
// Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
// generated.
if paths == nil {
paths = Paths{}
}
c.rspFileInputs = paths
c.rspFile = rspFile
c.rspFiles = append(c.rspFiles, rspFileAndPaths{rspFile, paths})
if c.rule.sbox {
if _, isRel, _ := maybeRelErr(c.rule.outDir.String(), rspFile.String()); isRel {
@ -1264,13 +1272,12 @@ func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams,
}
func (builderContextForTests) Build(PackageContext, BuildParams) {}
func rspFileForInputs(paths []string) string {
s := strings.Builder{}
for i, path := range paths {
if i != 0 {
s.WriteByte(' ')
}
s.WriteString(proptools.ShellEscape(path))
func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
buf := &strings.Builder{}
err := response.WriteRspFile(buf, paths.Strings())
if err != nil {
// There should never be I/O errors writing to a bytes.Buffer.
panic(err)
}
return s.String()
WriteFileRule(ctx, rspFile, buf.String())
}

139
android/rule_builder_test.go
View File

@ -376,8 +376,6 @@ func TestRuleBuilder(t *testing.T) {
wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer " +
"out_local/module/DepFile out_local/module/depfile out_local/module/ImplicitDepFile out_local/module/depfile2"
wantRspFileContent := "$in"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
@ -389,8 +387,6 @@ func TestRuleBuilder(t *testing.T) {
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
t.Run("sbox", func(t *testing.T) {
@ -409,8 +405,6 @@ func TestRuleBuilder(t *testing.T) {
wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
wantRspFileContent := "$in"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
@ -422,8 +416,6 @@ func TestRuleBuilder(t *testing.T) {
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
t.Run("sbox tools", func(t *testing.T) {
@ -442,7 +434,34 @@ func TestRuleBuilder(t *testing.T) {
wantDepMergerCommand := "__SBOX_SANDBOX_DIR__/tools/out/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
wantRspFileContent := "$in"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
AssertDeepEquals(t, "rule.RspfileInputs()", wantRspFileInputs, rule.RspFileInputs())
AssertDeepEquals(t, "rule.Outputs()", wantOutputs, rule.Outputs())
AssertDeepEquals(t, "rule.SymlinkOutputs()", wantSymlinkOutputs, rule.SymlinkOutputs())
AssertDeepEquals(t, "rule.DepFiles()", wantDepFiles, rule.DepFiles())
AssertDeepEquals(t, "rule.Tools()", wantTools, rule.Tools())
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
})
t.Run("sbox inputs", func(t *testing.T) {
rule := NewRuleBuilder(pctx, ctx).Sbox(PathForOutput(ctx, "module"),
PathForOutput(ctx, "sbox.textproto")).SandboxInputs()
addCommands(rule)
wantCommands := []string{
"__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " +
"FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " +
"FlagWithRspFileInputList=__SBOX_SANDBOX_DIR__/out/rsp Input __SBOX_SANDBOX_DIR__/out/Output " +
"__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text __SBOX_SANDBOX_DIR__/tools/src/Tool after command2 old cmd",
"command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/tools/src/tool2",
"command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2",
}
wantDepMergerCommand := "__SBOX_SANDBOX_DIR__/tools/out/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
@ -455,8 +474,6 @@ func TestRuleBuilder(t *testing.T) {
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
}
@ -472,8 +489,9 @@ type testRuleBuilderModule struct {
properties struct {
Srcs []string
Restat bool
Sbox bool
Restat bool
Sbox bool
Sbox_inputs bool
}
}
@ -482,9 +500,15 @@ func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) {
out := PathForModuleOut(ctx, "gen", ctx.ModuleName())
outDep := PathForModuleOut(ctx, "gen", ctx.ModuleName()+".d")
outDir := PathForModuleOut(ctx, "gen")
rspFile := PathForModuleOut(ctx, "rsp")
rspFile2 := PathForModuleOut(ctx, "rsp2")
rspFileContents := PathsForSource(ctx, []string{"rsp_in"})
rspFileContents2 := PathsForSource(ctx, []string{"rsp_in2"})
manifestPath := PathForModuleOut(ctx, "sbox.textproto")
testRuleBuilder_Build(ctx, in, out, outDep, outDir, manifestPath, t.properties.Restat, t.properties.Sbox)
testRuleBuilder_Build(ctx, in, out, outDep, outDir, manifestPath, t.properties.Restat,
t.properties.Sbox, t.properties.Sbox_inputs, rspFile, rspFileContents,
rspFile2, rspFileContents2)
}
type testRuleBuilderSingleton struct{}
@ -498,18 +522,35 @@ func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) {
out := PathForOutput(ctx, "singleton/gen/baz")
outDep := PathForOutput(ctx, "singleton/gen/baz.d")
outDir := PathForOutput(ctx, "singleton/gen")
rspFile := PathForOutput(ctx, "singleton/rsp")
rspFile2 := PathForOutput(ctx, "singleton/rsp2")
rspFileContents := PathsForSource(ctx, []string{"rsp_in"})
rspFileContents2 := PathsForSource(ctx, []string{"rsp_in2"})
manifestPath := PathForOutput(ctx, "singleton/sbox.textproto")
testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, manifestPath, true, false)
testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, manifestPath, true, false, false,
rspFile, rspFileContents, rspFile2, rspFileContents2)
}
func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir, manifestPath WritablePath, restat, sbox bool) {
func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir, manifestPath WritablePath,
restat, sbox, sboxInputs bool,
rspFile WritablePath, rspFileContents Paths, rspFile2 WritablePath, rspFileContents2 Paths) {
rule := NewRuleBuilder(pctx, ctx)
if sbox {
rule.Sbox(outDir, manifestPath)
if sboxInputs {
rule.SandboxInputs()
}
}
rule.Command().Tool(PathForSource(ctx, "cp")).Inputs(in).Output(out).ImplicitDepFile(outDep)
rule.Command().
Tool(PathForSource(ctx, "cp")).
Inputs(in).
Output(out).
ImplicitDepFile(outDep).
FlagWithRspFileInputList("@", rspFile, rspFileContents).
FlagWithRspFileInputList("@", rspFile2, rspFileContents2)
if restat {
rule.Restat()
@ -540,6 +581,12 @@ func TestRuleBuilder_Build(t *testing.T) {
srcs: ["bar"],
sbox: true,
}
rule_builder_test {
name: "foo_sbox_inputs",
srcs: ["bar"],
sbox: true,
sbox_inputs: true,
}
`
result := GroupFixturePreparers(
@ -548,7 +595,10 @@ func TestRuleBuilder_Build(t *testing.T) {
fs.AddToFixture(),
).RunTest(t)
check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraImplicits, extraCmdDeps []string) {
check := func(t *testing.T, params TestingBuildParams, rspFile2Params TestingBuildParams,
wantCommand, wantOutput, wantDepfile, wantRspFile, wantRspFile2 string,
wantRestat bool, extraImplicits, extraCmdDeps []string) {
t.Helper()
command := params.RuleParams.Command
re := regexp.MustCompile(" # hash of input list: [a-z0-9]*$")
@ -561,9 +611,19 @@ func TestRuleBuilder_Build(t *testing.T) {
AssertBoolEquals(t, "RuleParams.Restat", wantRestat, params.RuleParams.Restat)
wantInputs := []string{"rsp_in"}
AssertArrayString(t, "Inputs", wantInputs, params.Inputs.Strings())
wantImplicits := append([]string{"bar"}, extraImplicits...)
// The second rsp file and the files listed in it should be in implicits
wantImplicits = append(wantImplicits, "rsp_in2", wantRspFile2)
AssertPathsRelativeToTopEquals(t, "Implicits", wantImplicits, params.Implicits)
wantRspFileContent := "$in"
AssertStringEquals(t, "RspfileContent", wantRspFileContent, params.RuleParams.RspfileContent)
AssertStringEquals(t, "Rspfile", wantRspFile, params.RuleParams.Rspfile)
AssertPathRelativeToTopEquals(t, "Output", wantOutput, params.Output)
if len(params.ImplicitOutputs) != 0 {
@ -575,18 +635,42 @@ func TestRuleBuilder_Build(t *testing.T) {
if params.Deps != blueprint.DepsGCC {
t.Errorf("want Deps = %q, got %q", blueprint.DepsGCC, params.Deps)
}
rspFile2Content := ContentFromFileRuleForTests(t, rspFile2Params)
AssertStringEquals(t, "rspFile2 content", "rsp_in2\n", rspFile2Content)
}
t.Run("module", func(t *testing.T) {
outFile := "out/soong/.intermediates/foo/gen/foo"
check(t, result.ModuleForTests("foo", "").Rule("rule").RelativeToTop(),
"cp bar "+outFile,
outFile, outFile+".d", true, nil, nil)
rspFile := "out/soong/.intermediates/foo/rsp"
rspFile2 := "out/soong/.intermediates/foo/rsp2"
module := result.ModuleForTests("foo", "")
check(t, module.Rule("rule").RelativeToTop(), module.Output(rspFile2).RelativeToTop(),
"cp bar "+outFile+" @"+rspFile+" @"+rspFile2,
outFile, outFile+".d", rspFile, rspFile2, true, nil, nil)
})
t.Run("sbox", func(t *testing.T) {
outDir := "out/soong/.intermediates/foo_sbox"
outFile := filepath.Join(outDir, "gen/foo_sbox")
depFile := filepath.Join(outDir, "gen/foo_sbox.d")
rspFile := filepath.Join(outDir, "rsp")
rspFile2 := filepath.Join(outDir, "rsp2")
manifest := filepath.Join(outDir, "sbox.textproto")
sbox := filepath.Join("out", "soong", "host", result.Config.PrebuiltOS(), "bin/sbox")
sandboxPath := shared.TempDirForOutDir("out/soong")
cmd := `rm -rf ` + outDir + `/gen && ` +
sbox + ` --sandbox-path ` + sandboxPath + ` --manifest ` + manifest
module := result.ModuleForTests("foo_sbox", "")
check(t, module.Output("gen/foo_sbox").RelativeToTop(), module.Output(rspFile2).RelativeToTop(),
cmd, outFile, depFile, rspFile, rspFile2, false, []string{manifest}, []string{sbox})
})
t.Run("sbox_inputs", func(t *testing.T) {
outDir := "out/soong/.intermediates/foo_sbox_inputs"
outFile := filepath.Join(outDir, "gen/foo_sbox_inputs")
depFile := filepath.Join(outDir, "gen/foo_sbox_inputs.d")
rspFile := filepath.Join(outDir, "rsp")
rspFile2 := filepath.Join(outDir, "rsp2")
manifest := filepath.Join(outDir, "sbox.textproto")
sbox := filepath.Join("out", "soong", "host", result.Config.PrebuiltOS(), "bin/sbox")
sandboxPath := shared.TempDirForOutDir("out/soong")
@ -594,13 +678,18 @@ func TestRuleBuilder_Build(t *testing.T) {
cmd := `rm -rf ` + outDir + `/gen && ` +
sbox + ` --sandbox-path ` + sandboxPath + ` --manifest ` + manifest
check(t, result.ModuleForTests("foo_sbox", "").Output("gen/foo_sbox").RelativeToTop(),
cmd, outFile, depFile, false, []string{manifest}, []string{sbox})
module := result.ModuleForTests("foo_sbox_inputs", "")
check(t, module.Output("gen/foo_sbox_inputs").RelativeToTop(), module.Output(rspFile2).RelativeToTop(),
cmd, outFile, depFile, rspFile, rspFile2, false, []string{manifest}, []string{sbox})
})
t.Run("singleton", func(t *testing.T) {
outFile := filepath.Join("out/soong/singleton/gen/baz")
check(t, result.SingletonForTests("rule_builder_test").Rule("rule").RelativeToTop(),
"cp bar "+outFile, outFile, outFile+".d", true, nil, nil)
rspFile := filepath.Join("out/soong/singleton/rsp")
rspFile2 := filepath.Join("out/soong/singleton/rsp2")
singleton := result.SingletonForTests("rule_builder_test")
check(t, singleton.Rule("rule").RelativeToTop(), singleton.Output(rspFile2).RelativeToTop(),
"cp bar "+outFile+" @"+rspFile+" @"+rspFile2,
outFile, outFile+".d", rspFile, rspFile2, true, nil, nil)
})
}

4
cc/builder.go
View File

@ -73,7 +73,7 @@ var (
Labels: map[string]string{"type": "link", "tool": "clang"},
ExecStrategy: "${config.RECXXLinksExecStrategy}",
Inputs: []string{"${out}.rsp", "$implicitInputs"},
RSPFile: "${out}.rsp",
RSPFiles: []string{"${out}.rsp"},
OutputFiles: []string{"${out}", "$implicitOutputs"},
ToolchainInputs: []string{"$ldCmd"},
Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXLinksPool}"},
@ -256,7 +256,7 @@ var (
Labels: map[string]string{"type": "tool", "name": "abi-linker"},
ExecStrategy: "${config.REAbiLinkerExecStrategy}",
Inputs: []string{"$sAbiLinkerLibs", "${out}.rsp", "$implicitInputs"},
RSPFile: "${out}.rsp",
RSPFiles: []string{"${out}.rsp"},
OutputFiles: []string{"$out"},
ToolchainInputs: []string{"$sAbiLinker"},
Platform: map[string]string{remoteexec.PoolKey: "${config.RECXXPool}"},

2
cmd/merge_zips/Android.bp
View File

@ -22,7 +22,7 @@ blueprint_go_binary {
"android-archive-zip",
"blueprint-pathtools",
"soong-jar",
"soong-zip",
"soong-response",
],
srcs: [
"merge_zips.go",

19
cmd/merge_zips/merge_zips.go
View File

@ -25,12 +25,14 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"android/soong/response"
"github.com/google/blueprint/pathtools"
"android/soong/jar"
"android/soong/third_party/zip"
soongZip "android/soong/zip"
)
// Input zip: we can open it, close it, and obtain an array of entries
@ -690,15 +692,20 @@ func main() {
inputs := make([]string, 0)
for _, input := range args[1:] {
if input[0] == '@' {
bytes, err := ioutil.ReadFile(input[1:])
f, err := os.Open(strings.TrimPrefix(input[1:], "@"))
if err != nil {
log.Fatal(err)
}
inputs = append(inputs, soongZip.ReadRespFile(bytes)...)
continue
rspInputs, err := response.ReadRspFile(f)
f.Close()
if err != nil {
log.Fatal(err)
}
inputs = append(inputs, rspInputs...)
} else {
inputs = append(inputs, input)
}
inputs = append(inputs, input)
continue
}
log.SetFlags(log.Lshortfile)

1
cmd/sbox/Android.bp
View File

@ -21,6 +21,7 @@ blueprint_go_binary {
deps: [
"sbox_proto",
"soong-makedeps",
"soong-response",
],
srcs: [
"sbox.go",

90
cmd/sbox/sbox.go
View File

@ -32,6 +32,7 @@ import (
"android/soong/cmd/sbox/sbox_proto"
"android/soong/makedeps"
"android/soong/response"
"github.com/golang/protobuf/proto"
)
@ -218,6 +219,11 @@ func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, er
return "", fmt.Errorf("command is required")
}
pathToTempDirInSbox := tempDir
if command.GetChdir() {
pathToTempDirInSbox = "."
}
err = os.MkdirAll(tempDir, 0777)
if err != nil {
return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
@ -228,10 +234,9 @@ func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, er
if err != nil {
return "", err
}
pathToTempDirInSbox := tempDir
if command.GetChdir() {
pathToTempDirInSbox = "."
err = copyRspFiles(command.RspFiles, tempDir, pathToTempDirInSbox)
if err != nil {
return "", err
}
if strings.Contains(rawCommand, depFilePlaceholder) {
@ -409,6 +414,83 @@ func copyOneFile(from string, to string, executable bool) error {
return nil
}
// copyRspFiles copies rsp files into the sandbox with path mappings, and also copies the files
// listed into the sandbox.
func copyRspFiles(rspFiles []*sbox_proto.RspFile, toDir, toDirInSandbox string) error {
for _, rspFile := range rspFiles {
err := copyOneRspFile(rspFile, toDir, toDirInSandbox)
if err != nil {
return err
}
}
return nil
}
// copyOneRspFiles copies an rsp file into the sandbox with path mappings, and also copies the files
// listed into the sandbox.
func copyOneRspFile(rspFile *sbox_proto.RspFile, toDir, toDirInSandbox string) error {
in, err := os.Open(rspFile.GetFile())
if err != nil {
return err
}
defer in.Close()
files, err := response.ReadRspFile(in)
if err != nil {
return err
}
for i, from := range files {
// Convert the real path of the input file into the path inside the sandbox using the
// path mappings.
to := applyPathMappings(rspFile.PathMappings, from)
// Copy the file into the sandbox.
err := copyOneFile(from, joinPath(toDir, to), false)
if err != nil {
return err
}
// Rewrite the name in the list of files to be relative to the sandbox directory.
files[i] = joinPath(toDirInSandbox, to)
}
// Convert the real path of the rsp file into the path inside the sandbox using the path
// mappings.
outRspFile := joinPath(toDir, applyPathMappings(rspFile.PathMappings, rspFile.GetFile()))
err = os.MkdirAll(filepath.Dir(outRspFile), 0777)
if err != nil {
return err
}
out, err := os.Create(outRspFile)
if err != nil {
return err
}
defer out.Close()
// Write the rsp file with converted paths into the sandbox.
err = response.WriteRspFile(out, files)
if err != nil {
return err
}
return nil
}
// applyPathMappings takes a list of path mappings and a path, and returns the path with the first
// matching path mapping applied. If the path does not match any of the path mappings then it is
// returned unmodified.
func applyPathMappings(pathMappings []*sbox_proto.PathMapping, path string) string {
for _, mapping := range pathMappings {
if strings.HasPrefix(path, mapping.GetFrom()+"/") {
return joinPath(mapping.GetTo()+"/", strings.TrimPrefix(path, mapping.GetFrom()+"/"))
}
}
return path
}
// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
// to moving files where the source and destination are in the same filesystem. This is OK for
// sbox because the temporary directory is inside the out directory. It updates the timestamp

159
cmd/sbox/sbox_proto/sbox.pb.go
View File

@ -86,10 +86,13 @@ type Command struct {
CopyAfter []*Copy `protobuf:"bytes,4,rep,name=copy_after,json=copyAfter" json:"copy_after,omitempty"`
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
// A list of files that will be copied before the sandboxed command, and whose contents should be
// copied as if they were listed in copy_before.
RspFiles []*RspFile `protobuf:"bytes,6,rep,name=rsp_files,json=rspFiles" json:"rsp_files,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Command) Reset() { *m = Command{} }
@ -152,6 +155,13 @@ func (m *Command) GetInputHash() string {
return ""
}
func (m *Command) GetRspFiles() []*RspFile {
if m != nil {
return m.RspFiles
}
return nil
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
// are relative to is specific to the context the Copy is used in and will be different for
// from and to.
@ -211,10 +221,110 @@ func (m *Copy) GetExecutable() bool {
return false
}
// RspFile describes an rspfile that should be copied into the sandbox directory.
type RspFile struct {
// The path to the rsp file.
File *string `protobuf:"bytes,1,req,name=file" json:"file,omitempty"`
// A list of path mappings that should be applied to each file listed in the rsp file.
PathMappings []*PathMapping `protobuf:"bytes,2,rep,name=path_mappings,json=pathMappings" json:"path_mappings,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RspFile) Reset() { *m = RspFile{} }
func (m *RspFile) String() string { return proto.CompactTextString(m) }
func (*RspFile) ProtoMessage() {}
func (*RspFile) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{3}
}
func (m *RspFile) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RspFile.Unmarshal(m, b)
}
func (m *RspFile) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RspFile.Marshal(b, m, deterministic)
}
func (m *RspFile) XXX_Merge(src proto.Message) {
xxx_messageInfo_RspFile.Merge(m, src)
}
func (m *RspFile) XXX_Size() int {
return xxx_messageInfo_RspFile.Size(m)
}
func (m *RspFile) XXX_DiscardUnknown() {
xxx_messageInfo_RspFile.DiscardUnknown(m)
}
var xxx_messageInfo_RspFile proto.InternalMessageInfo
func (m *RspFile) GetFile() string {
if m != nil && m.File != nil {
return *m.File
}
return ""
}
func (m *RspFile) GetPathMappings() []*PathMapping {
if m != nil {
return m.PathMappings
}
return nil
}
// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
type PathMapping struct {
From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"`
To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *PathMapping) Reset() { *m = PathMapping{} }
func (m *PathMapping) String() string { return proto.CompactTextString(m) }
func (*PathMapping) ProtoMessage() {}
func (*PathMapping) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{4}
}
func (m *PathMapping) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_PathMapping.Unmarshal(m, b)
}
func (m *PathMapping) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_PathMapping.Marshal(b, m, deterministic)
}
func (m *PathMapping) XXX_Merge(src proto.Message) {
xxx_messageInfo_PathMapping.Merge(m, src)
}
func (m *PathMapping) XXX_Size() int {
return xxx_messageInfo_PathMapping.Size(m)
}
func (m *PathMapping) XXX_DiscardUnknown() {
xxx_messageInfo_PathMapping.DiscardUnknown(m)
}
var xxx_messageInfo_PathMapping proto.InternalMessageInfo
func (m *PathMapping) GetFrom() string {
if m != nil && m.From != nil {
return *m.From
}
return ""
}
func (m *PathMapping) GetTo() string {
if m != nil && m.To != nil {
return *m.To
}
return ""
}
func init() {
proto.RegisterType((*Manifest)(nil), "sbox.Manifest")
proto.RegisterType((*Command)(nil), "sbox.Command")
proto.RegisterType((*Copy)(nil), "sbox.Copy")
proto.RegisterType((*RspFile)(nil), "sbox.RspFile")
proto.RegisterType((*PathMapping)(nil), "sbox.PathMapping")
}
func init() {
@ -222,22 +332,27 @@ func init() {
}
var fileDescriptor_9d0425bf0de86ed1 = []byte{
// 268 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x4f, 0x4b, 0xc3, 0x40,
0x10, 0xc5, 0xc9, 0x9f, 0xd2, 0x64, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x09, 0x01, 0x21, 0x45,
0xe8, 0xc1, 0x6f, 0x60, 0xf5, 0x20, 0x82, 0x97, 0x1c, 0x45, 0x08, 0x9b, 0x64, 0x43, 0x02, 0x4d,
0x26, 0xec, 0x6e, 0xa0, 0xfd, 0x56, 0x7e, 0x44, 0xd9, 0x49, 0x2a, 0x82, 0xb7, 0x99, 0xdf, 0xe3,
0xcd, 0x7b, 0x0c, 0x80, 0x29, 0xe9, 0x7c, 0x18, 0x35, 0x59, 0xc2, 0xd0, 0xcd, 0xe9, 0x17, 0x44,
0x1f, 0x72, 0xe8, 0x1a, 0x65, 0x2c, 0xee, 0x21, 0xaa, 0xa8, 0xef, 0xe5, 0x50, 0x1b, 0xe1, 0x25,
0x41, 0xb6, 0x79, 0xda, 0x1e, 0xd8, 0xf0, 0x32, 0xd3, 0xfc, 0x57, 0xc6, 0x07, 0xd8, 0xd1, 0x64,
0xc7, 0xc9, 0x16, 0xb5, 0x1a, 0x9b, 0xee, 0xa4, 0x84, 0x9f, 0x78, 0x59, 0x9c, 0x6f, 0x67, 0xfa,
0x3a, 0xc3, 0xf4, 0xdb, 0x83, 0xf5, 0x62, 0xc6, 0x47, 0xd8, 0x54, 0x34, 0x5e, 0x8a, 0x52, 0x35,
0xa4, 0xd5, 0x12, 0x00, 0xd7, 0x80, 0xf1, 0x92, 0x83, 0x93, 0x8f, 0xac, 0xe2, 0x2d, 0xac, 0xaa,
0xb6, 0xee, 0x34, 0x9f, 0x8d, 0xf2, 0x79, 0x41, 0x01, 0xeb, 0xa5, 0x81, 0x08, 0x12, 0x3f, 0x8b,
0xf3, 0xeb, 0x8a, 0x7b, 0x60, 0x77, 0x21, 0x1b, 0xab, 0xb4, 0x08, 0xff, 0xdd, 0x8e, 0x9d, 0xfa,
0xec, 0x44, 0xbc, 0x03, 0xe8, 0x06, 0xd7, 0xbc, 0x95, 0xa6, 0x15, 0x2b, 0xae, 0x1d, 0x33, 0x79,
0x93, 0xa6, 0x4d, 0xdf, 0x21, 0x74, 0x0e, 0x44, 0x08, 0x1b, 0x4d, 0xbd, 0xf0, 0x38, 0x88, 0x67,
0xdc, 0x81, 0x6f, 0x49, 0xf8, 0x4c, 0x7c, 0x4b, 0x78, 0x0f, 0xa0, 0xce, 0xaa, 0x9a, 0xac, 0x2c,
0x4f, 0x4a, 0x04, 0x5c, 0xf5, 0x0f, 0x39, 0xde, 0x7c, 0xf2, 0xc3, 0x0b, 0x7e, 0xf8, 0x4f, 0x00,
0x00, 0x00, 0xff, 0xff, 0x78, 0x37, 0x3e, 0x6a, 0x7d, 0x01, 0x00, 0x00,
// 342 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x5f, 0x4b, 0xc3, 0x30,
0x14, 0xc5, 0x69, 0xd7, 0xb9, 0xf6, 0xee, 0x0f, 0x18, 0x7c, 0xc8, 0x8b, 0x32, 0x0a, 0xc2, 0xa6,
0x30, 0xd0, 0x07, 0xdf, 0x9d, 0x22, 0x22, 0x0c, 0x24, 0xe0, 0x8b, 0x08, 0x25, 0xeb, 0x52, 0x5b,
0x58, 0x9b, 0x90, 0x64, 0xb0, 0x7d, 0x57, 0x3f, 0x8c, 0xe4, 0xa6, 0xd3, 0x82, 0x2f, 0xbe, 0xdd,
0x7b, 0x0e, 0xf7, 0xdc, 0x5f, 0xc2, 0x05, 0x30, 0x6b, 0xb9, 0x5f, 0x28, 0x2d, 0xad, 0x24, 0x91,
0xab, 0xd3, 0x0f, 0x88, 0x57, 0xbc, 0xa9, 0x0a, 0x61, 0x2c, 0x99, 0x43, 0x9c, 0xcb, 0xba, 0xe6,
0xcd, 0xc6, 0xd0, 0x60, 0xda, 0x9b, 0x0d, 0x6f, 0xc7, 0x0b, 0x1c, 0x78, 0xf0, 0x2a, 0xfb, 0xb1,
0xc9, 0x25, 0x4c, 0xe4, 0xce, 0xaa, 0x9d, 0xcd, 0x36, 0x42, 0x15, 0xd5, 0x56, 0xd0, 0x70, 0x1a,
0xcc, 0x12, 0x36, 0xf6, 0xea, 0xa3, 0x17, 0xd3, 0xaf, 0x00, 0x06, 0xed, 0x30, 0xb9, 0x86, 0x61,
0x2e, 0xd5, 0x21, 0x5b, 0x8b, 0x42, 0x6a, 0xd1, 0x2e, 0x80, 0xe3, 0x02, 0x75, 0x60, 0xe0, 0xec,
0x25, 0xba, 0xe4, 0x0c, 0xfa, 0x79, 0xb9, 0xa9, 0x34, 0xc6, 0xc6, 0xcc, 0x37, 0x84, 0xc2, 0xa0,
0x25, 0xa0, 0xbd, 0x69, 0x38, 0x4b, 0xd8, 0xb1, 0x25, 0x73, 0xc0, 0xe9, 0x8c, 0x17, 0x56, 0x68,
0x1a, 0xfd, 0xc9, 0x4e, 0x9c, 0x7b, 0xef, 0x4c, 0x72, 0x0e, 0x50, 0x35, 0x8e, 0xbc, 0xe4, 0xa6,
0xa4, 0x7d, 0xc4, 0x4e, 0x50, 0x79, 0xe6, 0xa6, 0x24, 0x57, 0x90, 0x68, 0xa3, 0x32, 0x87, 0x6f,
0xe8, 0x49, 0xf7, 0x17, 0x98, 0x51, 0x4f, 0xd5, 0x56, 0xb0, 0x58, 0xfb, 0xc2, 0xa4, 0x2f, 0x10,
0xb9, 0x74, 0x42, 0x20, 0x2a, 0xb4, 0xac, 0x69, 0x80, 0x50, 0x58, 0x93, 0x09, 0x84, 0x56, 0xd2,
0x10, 0x95, 0xd0, 0x4a, 0x72, 0x01, 0x20, 0xf6, 0x22, 0xdf, 0x59, 0xbe, 0xde, 0x0a, 0xda, 0xc3,
0x67, 0x75, 0x94, 0xf4, 0x0d, 0x06, 0xed, 0x02, 0x8c, 0x73, 0x5f, 0x7a, 0x8c, 0x73, 0xda, 0x1d,
0x8c, 0x15, 0xb7, 0x65, 0x56, 0x73, 0xa5, 0xaa, 0xe6, 0xd3, 0xd0, 0x10, 0xd1, 0x4e, 0x3d, 0xda,
0x2b, 0xb7, 0xe5, 0xca, 0x3b, 0x6c, 0xa4, 0x7e, 0x1b, 0x93, 0xde, 0xc0, 0xb0, 0x63, 0xfe, 0x87,
0x74, 0x39, 0x7a, 0xc7, 0x33, 0xc9, 0xf0, 0x4c, 0xbe, 0x03, 0x00, 0x00, 0xff, 0xff, 0x83, 0x82,
0xb0, 0xc3, 0x33, 0x02, 0x00, 0x00,
}

21
cmd/sbox/sbox_proto/sbox.proto
View File

@ -47,6 +47,10 @@ message Command {
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
optional string input_hash = 5;
// A list of files that will be copied before the sandboxed command, and whose contents should be
// copied as if they were listed in copy_before.
repeated RspFile rsp_files = 6;
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
@ -58,4 +62,19 @@ message Copy {
// If true, make the file executable after copying it.
optional bool executable = 3;
}
}
// RspFile describes an rspfile that should be copied into the sandbox directory.
message RspFile {
// The path to the rsp file.
required string file = 1;
// A list of path mappings that should be applied to each file listed in the rsp file.
repeated PathMapping path_mappings = 2;
}
// PathMapping describes a mapping from a path outside the sandbox to the path inside the sandbox.
message PathMapping {
required string from = 1;
required string to = 2;
}

6
java/builder.go
View File

@ -150,7 +150,7 @@ var (
&remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"},
ExecStrategy: "${config.RETurbineExecStrategy}",
Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"},
RSPFile: "${out}.rsp",
RSPFiles: []string{"${out}.rsp"},
OutputFiles: []string{"$out.tmp"},
OutputDirectories: []string{"$outDir"},
ToolchainInputs: []string{"${config.JavaCmd}"},
@ -167,7 +167,7 @@ var (
&remoteexec.REParams{
ExecStrategy: "${config.REJarExecStrategy}",
Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp"},
RSPFile: "${out}.rsp",
RSPFiles: []string{"${out}.rsp"},
OutputFiles: []string{"$out"},
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
}, []string{"jarArgs"}, nil)
@ -182,7 +182,7 @@ var (
&remoteexec.REParams{
ExecStrategy: "${config.REZipExecStrategy}",
Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"},
RSPFile: "${out}.rsp",
RSPFiles: []string{"${out}.rsp"},
OutputFiles: []string{"$out"},
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
}, []string{"jarArgs"}, []string{"implicits"})

6
java/droiddoc.go
View File

@ -1209,7 +1209,7 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi
rule.Command().Text("mkdir -p").Flag(homeDir.String())
cmd := rule.Command()
cmd.FlagWithArg("ANDROID_SDK_HOME=", homeDir.String())
cmd.FlagWithArg("ANDROID_PREFS_ROOT=", homeDir.String())
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_METALAVA") {
rule.Remoteable(android.RemoteRuleSupports{RBE: true})
@ -1231,10 +1231,10 @@ func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersi
Labels: labels,
ExecStrategy: execStrategy,
Inputs: inputs,
RSPFile: implicitsRsp.String(),
RSPFiles: []string{implicitsRsp.String()},
ToolchainInputs: []string{config.JavaCmd(ctx).String()},
Platform: map[string]string{remoteexec.PoolKey: pool},
EnvironmentVariables: []string{"ANDROID_SDK_HOME"},
EnvironmentVariables: []string{"ANDROID_PREFS_ROOT"},
}).NoVarTemplate(ctx.Config().RBEWrapper()))
}

2
java/java_test.go
View File

@ -628,7 +628,7 @@ prebuilt_stubs_sources {
}
t.Run("empty/missing directory", func(t *testing.T) {
test(t, "empty-directory", []string{})
test(t, "empty-directory", nil)
})
t.Run("non-empty set of sources", func(t *testing.T) {

85
java/lint.go
View File

@ -182,11 +182,6 @@ type lintPaths struct {
cacheDir android.WritablePath
homeDir android.WritablePath
srcjarDir android.WritablePath
deps android.Paths
remoteInputs android.Paths
remoteRSPInputs android.Paths
}
func lintRBEExecStrategy(ctx android.ModuleContext) string {
@ -194,39 +189,6 @@ func lintRBEExecStrategy(ctx android.ModuleContext) string {
}
func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.RuleBuilder) lintPaths {
var deps android.Paths
var remoteInputs android.Paths
var remoteRSPInputs android.Paths
// Paths passed to trackInputDependency will be added as dependencies of the rule that runs
// lint and passed as inputs to the remote execution proxy.
trackInputDependency := func(paths ...android.Path) {
deps = append(deps, paths...)
remoteInputs = append(remoteInputs, paths...)
}
// Paths passed to trackRSPDependency will be added as dependencies of the rule that runs
// lint, but the RSP file will be used by the remote execution proxy to find the files so that
// it doesn't overflow command line limits.
trackRSPDependency := func(paths android.Paths, rsp android.Path) {
deps = append(deps, paths...)
remoteRSPInputs = append(remoteRSPInputs, rsp)
}
var resourcesList android.WritablePath
if len(l.resources) > 0 {
// The list of resources may be too long to put on the command line, but
// we can't use the rsp file because it is already being used for srcs.
// Insert a second rule to write out the list of resources to a file.
resourcesList = android.PathForModuleOut(ctx, "resources.list")
resListRule := android.NewRuleBuilder(pctx, ctx)
resListRule.Command().Text("cp").
FlagWithRspFileInputList("", resourcesList.ReplaceExtension(ctx, "rsp"), l.resources).
Output(resourcesList)
resListRule.Build("lint_resources_list", "lint resources list")
trackRSPDependency(l.resources, resourcesList)
}
projectXMLPath := android.PathForModuleOut(ctx, "lint", "project.xml")
// Lint looks for a lint.xml file next to the project.xml file, give it one.
configXMLPath := android.PathForModuleOut(ctx, "lint", "lint.xml")
@ -235,20 +197,6 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
srcJarDir := android.PathForModuleOut(ctx, "lint", "srcjars")
srcJarList := zipSyncCmd(ctx, rule, srcJarDir, l.srcJars)
// TODO(ccross): this is a little fishy. The files extracted from the srcjars are referenced
// by the project.xml and used by the later lint rule, but the lint rule depends on the srcjars,
// not the extracted files.
trackRSPDependency(l.srcJars, srcJarList)
// TODO(ccross): some of the files in l.srcs are generated sources and should be passed to
// lint separately.
srcsList := android.PathForModuleOut(ctx, "lint", "srcs.list")
srcsListRsp := android.PathForModuleOut(ctx, "lint-srcs.list.rsp")
rule.Command().Text("cp").
FlagWithRspFileInputList("", srcsListRsp, l.srcs).
Output(srcsList)
trackRSPDependency(l.srcs, srcsList)
rule.Temporary(srcsList)
cmd := rule.Command().
BuiltTool("lint-project-xml").
@ -263,32 +211,31 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
cmd.Flag("--test")
}
if l.manifest != nil {
cmd.FlagWithArg("--manifest ", cmd.PathForInput(l.manifest))
trackInputDependency(l.manifest)
cmd.FlagWithInput("--manifest ", l.manifest)
}
if l.mergedManifest != nil {
cmd.FlagWithArg("--merged_manifest ", cmd.PathForInput(l.mergedManifest))
trackInputDependency(l.mergedManifest)
cmd.FlagWithInput("--merged_manifest ", l.mergedManifest)
}
cmd.FlagWithInput("--srcs ", srcsList)
// TODO(ccross): some of the files in l.srcs are generated sources and should be passed to
// lint separately.
srcsList := android.PathForModuleOut(ctx, "lint-srcs.list")
cmd.FlagWithRspFileInputList("--srcs ", srcsList, l.srcs)
cmd.FlagWithInput("--generated_srcs ", srcJarList)
if resourcesList != nil {
cmd.FlagWithInput("--resources ", resourcesList)
if len(l.resources) > 0 {
resourcesList := android.PathForModuleOut(ctx, "lint-resources.list")
cmd.FlagWithRspFileInputList("--resources ", resourcesList, l.resources)
}
if l.classes != nil {
cmd.FlagWithArg("--classes ", cmd.PathForInput(l.classes))
trackInputDependency(l.classes)
cmd.FlagWithInput("--classes ", l.classes)
}
cmd.FlagForEachArg("--classpath ", cmd.PathsForInputs(l.classpath))
trackInputDependency(l.classpath...)
cmd.FlagForEachInput("--classpath ", l.classpath)
cmd.FlagForEachArg("--extra_checks_jar ", cmd.PathsForInputs(l.extraLintCheckJars))
trackInputDependency(l.extraLintCheckJars...)
cmd.FlagForEachInput("--extra_checks_jar ", l.extraLintCheckJars)
cmd.FlagWithArg("--root_dir ", "$PWD")
@ -309,11 +256,6 @@ func (l *linter) writeLintProjectXML(ctx android.ModuleContext, rule *android.Ru
configXML: configXMLPath,
cacheDir: cacheDir,
homeDir: homeDir,
deps: deps,
remoteInputs: remoteInputs,
remoteRSPInputs: remoteRSPInputs,
}
}
@ -424,8 +366,7 @@ func (l *linter) lint(ctx android.ModuleContext) {
Flag("--exitcode").
Flags(l.properties.Lint.Flags).
Implicit(annotationsZipPath).
Implicit(apiVersionsXMLPath).
Implicits(lintPaths.deps)
Implicit(apiVersionsXMLPath)
rule.Temporary(lintPaths.projectXML)
rule.Temporary(lintPaths.configXML)

9
remoteexec/remoteexec.go
View File

@ -64,9 +64,8 @@ type REParams struct {
ExecStrategy string
// Inputs is a list of input paths or ninja variables.
Inputs []string
// RSPFile is the name of the ninja variable used by the rule as a placeholder for an rsp
// input.
RSPFile string
// RSPFiles is the name of the files used by the rule as a placeholder for an rsp input.
RSPFiles []string
// OutputFiles is a list of output file paths or ninja variables as placeholders for rule
// outputs.
OutputFiles []string
@ -134,8 +133,8 @@ func (r *REParams) wrapperArgs() string {
args += " --inputs=" + strings.Join(r.Inputs, ",")
}
if r.RSPFile != "" {
args += " --input_list_paths=" + r.RSPFile
if len(r.RSPFiles) > 0 {
args += " --input_list_paths=" + strings.Join(r.RSPFiles, ",")
}
if len(r.OutputFiles) > 0 {

4
remoteexec/remoteexec_test.go
View File

@ -45,14 +45,14 @@ func TestTemplate(t *testing.T) {
Inputs: []string{"$in"},
OutputFiles: []string{"$out"},
ExecStrategy: "remote",
RSPFile: "$out.rsp",
RSPFiles: []string{"$out.rsp", "out2.rsp"},
ToolchainInputs: []string{"clang++"},
Platform: map[string]string{
ContainerImageKey: DefaultImage,
PoolKey: "default",
},
},
want: fmt.Sprintf("${android.RBEWrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=remote --inputs=$in --input_list_paths=$out.rsp --output_files=$out --toolchain_inputs=clang++ -- ", DefaultImage),
want: fmt.Sprintf("${android.RBEWrapper} --labels=compiler=clang,lang=cpp,type=compile --platform=\"Pool=default,container-image=%s\" --exec_strategy=remote --inputs=$in --input_list_paths=$out.rsp,out2.rsp --output_files=$out --toolchain_inputs=clang++ -- ", DefaultImage),
},
}
for _, test := range tests {

16
response/Android.bp Normal file
View File

@ -0,0 +1,16 @@
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
bootstrap_go_package {
name: "soong-response",
pkgPath: "android/soong/response",
deps: [
],
srcs: [
"response.go",
],
testSrcs: [
"response_test.go",
],
}

112
response/response.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package response
import (
"io"
"io/ioutil"
"strings"
"unicode"
)
const noQuote = '\x00'
// ReadRspFile reads a file in Ninja's response file format and returns its contents.
func ReadRspFile(r io.Reader) ([]string, error) {
var files []string
var file []byte
buf, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
isEscaping := false
quotingStart := byte(noQuote)
for _, c := range buf {
switch {
case isEscaping:
if quotingStart == '"' {
if !(c == '"' || c == '\\') {
// '\"' or '\\' will be escaped under double quoting.
file = append(file, '\\')
}
}
file = append(file, c)
isEscaping = false
case c == '\\' && quotingStart != '\'':
isEscaping = true
case quotingStart == noQuote && (c == '\'' || c == '"'):
quotingStart = c
case quotingStart != noQuote && c == quotingStart:
quotingStart = noQuote
case quotingStart == noQuote && unicode.IsSpace(rune(c)):
// Current character is a space outside quotes
if len(file) != 0 {
files = append(files, string(file))
}
file = file[:0]
default:
file = append(file, c)
}
}
if len(file) != 0 {
files = append(files, string(file))
}
return files, nil
}
func rspUnsafeChar(r rune) bool {
switch {
case 'A' <= r && r <= 'Z',
'a' <= r && r <= 'z',
'0' <= r && r <= '9',
r == '_',
r == '+',
r == '-',
r == '.',
r == '/':
return false
default:
return true
}
}
var rspEscaper = strings.NewReplacer(`'`, `'\''`)
// WriteRspFile writes a list of files to a file in Ninja's response file format.
func WriteRspFile(w io.Writer, files []string) error {
for i, f := range files {
if i != 0 {
_, err := io.WriteString(w, " ")
if err != nil {
return err
}
}
if strings.IndexFunc(f, rspUnsafeChar) != -1 {
f = `'` + rspEscaper.Replace(f) + `'`
}
_, err := io.WriteString(w, f)
if err != nil {
return err
}
}
return nil
}

123
response/response_test.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package response
import (
"bytes"
"reflect"
"testing"
)
func TestReadRspFile(t *testing.T) {
testCases := []struct {
name, in string
out []string
}{
{
name: "single quoting test case 1",
in: `./cmd '"'-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "single quoting test case 2",
in: `./cmd '-C`,
out: []string{"./cmd", `-C`},
},
{
name: "single quoting test case 3",
in: `./cmd '\"'-C`,
out: []string{"./cmd", `\"-C`},
},
{
name: "single quoting test case 4",
in: `./cmd '\\'-C`,
out: []string{"./cmd", `\\-C`},
},
{
name: "none quoting test case 1",
in: `./cmd \'-C`,
out: []string{"./cmd", `'-C`},
},
{
name: "none quoting test case 2",
in: `./cmd \\-C`,
out: []string{"./cmd", `\-C`},
},
{
name: "none quoting test case 3",
in: `./cmd \"-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "double quoting test case 1",
in: `./cmd "'"-C`,
out: []string{"./cmd", `'-C`},
},
{
name: "double quoting test case 2",
in: `./cmd "\\"-C`,
out: []string{"./cmd", `\-C`},
},
{
name: "double quoting test case 3",
in: `./cmd "\""-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "ninja rsp file",
in: "'a'\nb\n'@'\n'foo'\\''bar'\n'foo\"bar'",
out: []string{"a", "b", "@", "foo'bar", `foo"bar`},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
got, err := ReadRspFile(bytes.NewBuffer([]byte(testCase.in)))
if err != nil {
t.Errorf("unexpected error: %q", err)
}
if !reflect.DeepEqual(got, testCase.out) {
t.Errorf("expected %q got %q", testCase.out, got)
}
})
}
}
func TestWriteRspFile(t *testing.T) {
testCases := []struct {
name string
in []string
out string
}{
{
name: "ninja rsp file",
in: []string{"a", "b", "@", "foo'bar", `foo"bar`},
out: "a b '@' 'foo'\\''bar' 'foo\"bar'",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
buf := &bytes.Buffer{}
err := WriteRspFile(buf, testCase.in)
if err != nil {
t.Errorf("unexpected error: %q", err)
}
if buf.String() != testCase.out {
t.Errorf("expected %q got %q", testCase.out, buf.String())
}
})
}
}

1
zip/Android.bp
View File

@ -25,6 +25,7 @@ bootstrap_go_package {
"android-archive-zip",
"blueprint-pathtools",
"soong-jar",
"soong-response",
],
srcs: [
"zip.go",

12
zip/cmd/main.go
View File

@ -24,7 +24,6 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"os"
"runtime"
"runtime/pprof"
@ -32,6 +31,7 @@ import (
"strconv"
"strings"
"android/soong/response"
"android/soong/zip"
)
@ -125,12 +125,18 @@ func main() {
var expandedArgs []string
for _, arg := range os.Args {
if strings.HasPrefix(arg, "@") {
bytes, err := ioutil.ReadFile(strings.TrimPrefix(arg, "@"))
f, err := os.Open(strings.TrimPrefix(arg, "@"))
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
respArgs, err := response.ReadRspFile(f)
f.Close()
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
respArgs := zip.ReadRespFile(bytes)
expandedArgs = append(expandedArgs, respArgs...)
} else {
expandedArgs = append(expandedArgs, arg)

52
zip/zip.go
View File

@ -29,7 +29,8 @@ import (
"sync"
"syscall"
"time"
"unicode"
"android/soong/response"
"github.com/google/blueprint/pathtools"
@ -164,14 +165,12 @@ func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder {
}
defer f.Close()
list, err := ioutil.ReadAll(f)
arg := b.state
arg.SourceFiles, err = response.ReadRspFile(f)
if err != nil {
b.err = err
return b
}
arg := b.state
arg.SourceFiles = ReadRespFile(list)
for i := range arg.SourceFiles {
arg.SourceFiles[i] = pathtools.MatchEscape(arg.SourceFiles[i])
}
@ -253,49 +252,6 @@ type ZipArgs struct {
Filesystem pathtools.FileSystem
}
const NOQUOTE = '\x00'
func ReadRespFile(bytes []byte) []string {
var args []string
var arg []rune
isEscaping := false
quotingStart := NOQUOTE
for _, c := range string(bytes) {
switch {
case isEscaping:
if quotingStart == '"' {
if !(c == '"' || c == '\\') {
// '\"' or '\\' will be escaped under double quoting.
arg = append(arg, '\\')
}
}
arg = append(arg, c)
isEscaping = false
case c == '\\' && quotingStart != '\'':
isEscaping = true
case quotingStart == NOQUOTE && (c == '\'' || c == '"'):
quotingStart = c
case quotingStart != NOQUOTE && c == quotingStart:
quotingStart = NOQUOTE
case quotingStart == NOQUOTE && unicode.IsSpace(c):
// Current character is a space outside quotes
if len(arg) != 0 {
args = append(args, string(arg))
}
arg = arg[:0]
default:
arg = append(arg, c)
}
}
if len(arg) != 0 {
args = append(args, string(arg))
}
return args
}
func zipTo(args ZipArgs, w io.Writer) error {
if args.EmulateJar {
args.AddDirectoryEntriesToZip = true

72
zip/zip_test.go
View File

@ -535,78 +535,6 @@ func TestZip(t *testing.T) {
}
}
func TestReadRespFile(t *testing.T) {
testCases := []struct {
name, in string
out []string
}{
{
name: "single quoting test case 1",
in: `./cmd '"'-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "single quoting test case 2",
in: `./cmd '-C`,
out: []string{"./cmd", `-C`},
},
{
name: "single quoting test case 3",
in: `./cmd '\"'-C`,
out: []string{"./cmd", `\"-C`},
},
{
name: "single quoting test case 4",
in: `./cmd '\\'-C`,
out: []string{"./cmd", `\\-C`},
},
{
name: "none quoting test case 1",
in: `./cmd \'-C`,
out: []string{"./cmd", `'-C`},
},
{
name: "none quoting test case 2",
in: `./cmd \\-C`,
out: []string{"./cmd", `\-C`},
},
{
name: "none quoting test case 3",
in: `./cmd \"-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "double quoting test case 1",
in: `./cmd "'"-C`,
out: []string{"./cmd", `'-C`},
},
{
name: "double quoting test case 2",
in: `./cmd "\\"-C`,
out: []string{"./cmd", `\-C`},
},
{
name: "double quoting test case 3",
in: `./cmd "\""-C`,
out: []string{"./cmd", `"-C`},
},
{
name: "ninja rsp file",
in: "'a'\nb\n'@'\n'foo'\\''bar'\n'foo\"bar'",
out: []string{"a", "b", "@", "foo'bar", `foo"bar`},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
got := ReadRespFile([]byte(testCase.in))
if !reflect.DeepEqual(got, testCase.out) {
t.Errorf("expected %q got %q", testCase.out, got)
}
})
}
}
func TestSrcJar(t *testing.T) {
mockFs := pathtools.MockFs(map[string][]byte{
"wrong_package.java": []byte("package foo;"),