openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge changes I8af00c8c,Ia526ae89 

* changes:
  Reland: Rewrite sbox to use a textproto manifest
  Fix gensrcs subdirs
This commit is contained in:
Colin Cross 2020-11-24 19:27:35 +00:00 committed by Gerrit Code Review
parent e5218b6be6 e16ce36818
commit 38214f5ac5
12 changed files with 803 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
android/Android.bp
View File

@ -4,6 +4,7 @@ bootstrap_go_package {
deps: [
"blueprint",
"blueprint-bootstrap",
"sbox_proto",
"soong",
"soong-android-soongconfig",
"soong-bazel",

119
android/rule_builder.go
View File

@ -20,27 +20,33 @@ import (
"path/filepath"
"sort"
"strings"
"testing"
"github.com/golang/protobuf/proto"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
"android/soong/cmd/sbox/sbox_proto"
"android/soong/shared"
)
const sboxOutDir = "__SBOX_OUT_DIR__"
const sboxSandboxBaseDir = "__SBOX_SANDBOX_DIR__"
const sboxOutSubDir = "out"
const sboxOutDir = sboxSandboxBaseDir + "/" + sboxOutSubDir
// RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
// graph.
type RuleBuilder struct {
commands []*RuleBuilderCommand
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
sbox bool
highmem bool
remoteable RemoteRuleSupports
sboxOutDir WritablePath
missingDeps []string
commands []*RuleBuilderCommand
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
sbox bool
highmem bool
remoteable RemoteRuleSupports
sboxOutDir WritablePath
sboxManifestPath WritablePath
missingDeps []string
}
// NewRuleBuilder returns a newly created RuleBuilder.
@ -106,12 +112,14 @@ func (r *RuleBuilder) Remoteable(supports RemoteRuleSupports) *RuleBuilder {
return r
}
// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output
// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure
// that all outputs have been written, and will discard any output files that were not specified.
// Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
// directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
// point to a location where sbox's manifest will be written and must be outside outputDir. sbox
// will ensure that all outputs have been written, and will discard any output files that were not
// specified.
//
// Sbox is not compatible with Restat()
func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
if r.sbox {
panic("Sbox() may not be called more than once")
}
@ -123,6 +131,7 @@ func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
}
r.sbox = true
r.sboxOutDir = outputDir
r.sboxManifestPath = manifestPath
return r
}
@ -420,7 +429,8 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
r.depFileMergerCmd(ctx, depFiles)
if r.sbox {
// Check for Rel() errors, as all depfiles should be in the output dir
// Check for Rel() errors, as all depfiles should be in the output dir. Errors
// will be reported to the ctx.
for _, path := range depFiles[1:] {
Rel(ctx, r.sboxOutDir.String(), path.String())
}
@ -443,34 +453,60 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
commandString := strings.Join(commands, " && ")
if r.sbox {
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
sboxOutputs[i] = filepath.Join(sboxOutDir, Rel(ctx, r.sboxOutDir.String(), output.String()))
}
commandString = proptools.ShellEscape(commandString)
if !strings.HasPrefix(commandString, `'`) {
commandString = `'` + commandString + `'`
}
sboxCmd := &RuleBuilderCommand{}
sboxCmd.BuiltTool(ctx, "sbox").
Flag("-c").Text(commandString).
Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
Flag("--output-root").Text(r.sboxOutDir.String())
// If running the command inside sbox, write the rule data out to an sbox
// manifest.textproto.
manifest := sbox_proto.Manifest{}
command := sbox_proto.Command{}
manifest.Commands = append(manifest.Commands, &command)
command.Command = proto.String(commandString)
if depFile != nil {
sboxCmd.Flag("--depfile-out").Text(depFile.String())
manifest.OutputDepfile = proto.String(depFile.String())
}
// Add a hash of the list of input files to the xbox command line so that ninja reruns
// it when the list of input files changes.
sboxCmd.FlagWithArg("--input-hash ", hashSrcFiles(inputs))
// Add copy rules to the manifest to copy each output file from the sbox directory.
// to the output directory.
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
rel := Rel(ctx, r.sboxOutDir.String(), output.String())
sboxOutputs[i] = filepath.Join(sboxOutDir, rel)
command.CopyAfter = append(command.CopyAfter, &sbox_proto.Copy{
From: proto.String(filepath.Join(sboxOutSubDir, rel)),
To: proto.String(output.String()),
})
}
sboxCmd.Flags(sboxOutputs)
// Add a hash of the list of input files to the manifest so that the textproto file
// changes when the list of input files changes and causes the sbox rule that
// depends on it to rerun.
command.InputHash = proto.String(hashSrcFiles(inputs))
// Verify that the manifest textproto is not inside the sbox output directory, otherwise
// it will get deleted when the sbox rule clears its output directory.
_, manifestInOutDir := MaybeRel(ctx, r.sboxOutDir.String(), r.sboxManifestPath.String())
if manifestInOutDir {
ReportPathErrorf(ctx, "sbox rule %q manifestPath %q must not be in outputDir %q",
name, r.sboxManifestPath.String(), r.sboxOutDir.String())
}
// Create a rule to write the manifest as a the textproto.
WriteFileRule(ctx, r.sboxManifestPath, proto.MarshalTextString(&manifest))
// Generate a new string to use as the command line of the sbox rule. This uses
// a RuleBuilderCommand as a convenience method of building the command line, then
// converts it to a string to replace commandString.
sboxCmd := &RuleBuilderCommand{}
sboxCmd.Text("rm -rf").Output(r.sboxOutDir)
sboxCmd.Text("&&")
sboxCmd.BuiltTool(ctx, "sbox").
Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
Flag("--manifest").Input(r.sboxManifestPath)
// Replace the command string, and add the sbox tool and manifest textproto to the
// dependencies of the final sbox rule.
commandString = sboxCmd.buf.String()
tools = append(tools, sboxCmd.tools...)
inputs = append(inputs, sboxCmd.inputs...)
} else {
// If not using sbox the rule will run the command directly, put the hash of the
// list of input files in a comment at the end of the command line to ensure ninja
@ -890,6 +926,19 @@ func (c *RuleBuilderCommand) NinjaEscapedString() string {
return ninjaEscapeExceptForSpans(c.String(), c.unescapedSpans)
}
// RuleBuilderSboxProtoForTests takes the BuildParams for the manifest passed to RuleBuilder.Sbox()
// and returns sbox testproto generated by the RuleBuilder.
func RuleBuilderSboxProtoForTests(t *testing.T, params TestingBuildParams) *sbox_proto.Manifest {
t.Helper()
content := ContentFromFileRuleForTests(t, params)
manifest := sbox_proto.Manifest{}
err := proto.UnmarshalText(content, &manifest)
if err != nil {
t.Fatalf("failed to unmarshal manifest: %s", err.Error())
}
return &manifest
}
func ninjaEscapeExceptForSpans(s string, spans [][2]int) string {
if len(spans) == 0 {
return proptools.NinjaEscape(s)

70
android/rule_builder_test.go
View File

@ -395,16 +395,17 @@ func TestRuleBuilder(t *testing.T) {
})
t.Run("sbox", func(t *testing.T) {
rule := NewRuleBuilder().Sbox(PathForOutput(ctx))
rule := NewRuleBuilder().Sbox(PathForOutput(ctx, ""),
PathForOutput(ctx, "sbox.textproto"))
addCommands(rule)
wantCommands := []string{
"__SBOX_OUT_DIR__/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_OUT_DIR__/depfile FlagWithInput=input FlagWithOutput=__SBOX_OUT_DIR__/output Input __SBOX_OUT_DIR__/Output __SBOX_OUT_DIR__/SymlinkOutput Text Tool after command2 old cmd",
"command2 __SBOX_OUT_DIR__/depfile2 input2 __SBOX_OUT_DIR__/output2 tool2",
"command3 input3 __SBOX_OUT_DIR__/output2 __SBOX_OUT_DIR__/output3",
"__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output Input __SBOX_SANDBOX_DIR__/out/Output __SBOX_SANDBOX_DIR__/out/SymlinkOutput Text Tool after command2 old cmd",
"command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 tool2",
"command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3",
}
wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_OUT_DIR__/DepFile __SBOX_OUT_DIR__/depfile __SBOX_OUT_DIR__/ImplicitDepFile __SBOX_OUT_DIR__/depfile2"
wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) {
t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g)
@ -451,11 +452,12 @@ type testRuleBuilderModule struct {
func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) {
in := PathsForSource(ctx, t.properties.Srcs)
out := PathForModuleOut(ctx, ctx.ModuleName())
outDep := PathForModuleOut(ctx, ctx.ModuleName()+".d")
outDir := PathForModuleOut(ctx)
out := PathForModuleOut(ctx, "gen", ctx.ModuleName())
outDep := PathForModuleOut(ctx, "gen", ctx.ModuleName()+".d")
outDir := PathForModuleOut(ctx, "gen")
manifestPath := PathForModuleOut(ctx, "sbox.textproto")
testRuleBuilder_Build(ctx, in, out, outDep, outDir, t.properties.Restat, t.properties.Sbox)
testRuleBuilder_Build(ctx, in, out, outDep, outDir, manifestPath, t.properties.Restat, t.properties.Sbox)
}
type testRuleBuilderSingleton struct{}
@ -466,17 +468,18 @@ func testRuleBuilderSingletonFactory() Singleton {
func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) {
in := PathForSource(ctx, "bar")
out := PathForOutput(ctx, "baz")
outDep := PathForOutput(ctx, "baz.d")
outDir := PathForOutput(ctx)
testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, true, false)
out := PathForOutput(ctx, "singleton/gen/baz")
outDep := PathForOutput(ctx, "singleton/gen/baz.d")
outDir := PathForOutput(ctx, "singleton/gen")
manifestPath := PathForOutput(ctx, "singleton/sbox.textproto")
testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, manifestPath, true, false)
}
func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir WritablePath, restat, sbox bool) {
func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir, manifestPath WritablePath, restat, sbox bool) {
rule := NewRuleBuilder()
if sbox {
rule.Sbox(outDir)
rule.Sbox(outDir, manifestPath)
}
rule.Command().Tool(PathForSource(ctx, "cp")).Inputs(in).Output(out).ImplicitDepFile(outDep)
@ -518,10 +521,10 @@ func TestRuleBuilder_Build(t *testing.T) {
_, errs = ctx.PrepareBuildActions(config)
FailIfErrored(t, errs)
check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraCmdDeps []string) {
check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraImplicits, extraCmdDeps []string) {
t.Helper()
command := params.RuleParams.Command
re := regexp.MustCompile(" (# hash of input list:|--input-hash) [a-z0-9]*")
re := regexp.MustCompile(" # hash of input list: [a-z0-9]*$")
command = re.ReplaceAllLiteralString(command, "")
if command != wantCommand {
t.Errorf("\nwant RuleParams.Command = %q\n got %q", wantCommand, params.RuleParams.Command)
@ -536,7 +539,8 @@ func TestRuleBuilder_Build(t *testing.T) {
t.Errorf("want RuleParams.Restat = %v, got %v", wantRestat, params.RuleParams.Restat)
}
if len(params.Implicits) != 1 || params.Implicits[0].String() != "bar" {
wantImplicits := append([]string{"bar"}, extraImplicits...)
if !reflect.DeepEqual(params.Implicits.Strings(), wantImplicits) {
t.Errorf("want Implicits = [%q], got %q", "bar", params.Implicits.Strings())
}
@ -558,27 +562,29 @@ func TestRuleBuilder_Build(t *testing.T) {
}
t.Run("module", func(t *testing.T) {
outFile := filepath.Join(buildDir, ".intermediates", "foo", "foo")
outFile := filepath.Join(buildDir, ".intermediates", "foo", "gen", "foo")
check(t, ctx.ModuleForTests("foo", "").Rule("rule"),
"cp bar "+outFile,
outFile, outFile+".d", true, nil)
outFile, outFile+".d", true, nil, nil)
})
t.Run("sbox", func(t *testing.T) {
outDir := filepath.Join(buildDir, ".intermediates", "foo_sbox")
outFile := filepath.Join(outDir, "foo_sbox")
depFile := filepath.Join(outDir, "foo_sbox.d")
outFile := filepath.Join(outDir, "gen/foo_sbox")
depFile := filepath.Join(outDir, "gen/foo_sbox.d")
manifest := filepath.Join(outDir, "sbox.textproto")
sbox := filepath.Join(buildDir, "host", config.PrebuiltOS(), "bin/sbox")
sandboxPath := shared.TempDirForOutDir(buildDir)
cmd := sbox + ` -c 'cp bar __SBOX_OUT_DIR__/foo_sbox' --sandbox-path ` + sandboxPath + " --output-root " + outDir + " --depfile-out " + depFile + " __SBOX_OUT_DIR__/foo_sbox"
cmd := `rm -rf ` + outDir + `/gen && ` +
sbox + ` --sandbox-path ` + sandboxPath + ` --manifest ` + manifest
check(t, ctx.ModuleForTests("foo_sbox", "").Rule("rule"),
cmd, outFile, depFile, false, []string{sbox})
check(t, ctx.ModuleForTests("foo_sbox", "").Output("gen/foo_sbox"),
cmd, outFile, depFile, false, []string{manifest}, []string{sbox})
})
t.Run("singleton", func(t *testing.T) {
outFile := filepath.Join(buildDir, "baz")
outFile := filepath.Join(buildDir, "singleton/gen/baz")
check(t, ctx.SingletonForTests("rule_builder_test").Rule("rule"),
"cp bar "+outFile, outFile, outFile+".d", true, nil)
"cp bar "+outFile, outFile, outFile+".d", true, nil, nil)
})
}
@ -715,14 +721,16 @@ func TestRuleBuilderHashInputs(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Run("sbox", func(t *testing.T) {
gen := ctx.ModuleForTests(test.name+"_sbox", "")
command := gen.Output(test.name + "_sbox").RuleParams.Command
if g, w := command, " --input-hash "+test.expectedHash; !strings.Contains(g, w) {
t.Errorf("Expected command line to end with %q, got %q", w, g)
manifest := RuleBuilderSboxProtoForTests(t, gen.Output("sbox.textproto"))
hash := manifest.Commands[0].GetInputHash()
if g, w := hash, test.expectedHash; g != w {
t.Errorf("Expected has %q, got %q", w, g)
}
})
t.Run("", func(t *testing.T) {
gen := ctx.ModuleForTests(test.name+"", "")
command := gen.Output(test.name).RuleParams.Command
command := gen.Output("gen/" + test.name).RuleParams.Command
if g, w := command, " # hash of input list: "+test.expectedHash; !strings.HasSuffix(g, w) {
t.Errorf("Expected command line to end with %q, got %q", w, g)
}

6
cc/gen.go
View File

@ -232,7 +232,8 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths,
var yaccRule_ *android.RuleBuilder
yaccRule := func() *android.RuleBuilder {
if yaccRule_ == nil {
yaccRule_ = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "yacc"))
yaccRule_ = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "yacc"),
android.PathForModuleGen(ctx, "yacc.sbox.textproto"))
}
return yaccRule_
}
@ -261,7 +262,8 @@ func genSources(ctx android.ModuleContext, srcFiles android.Paths,
deps = append(deps, headerFile)
case ".aidl":
if aidlRule == nil {
aidlRule = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "aidl"))
aidlRule = android.NewRuleBuilder().Sbox(android.PathForModuleGen(ctx, "aidl"),
android.PathForModuleGen(ctx, "aidl.sbox.textproto"))
}
cppFile := android.GenPathWithExt(ctx, "aidl", srcFile, "cpp")
depFile := android.GenPathWithExt(ctx, "aidl", srcFile, "cpp.d")

5
cc/gen_test.go
View File

@ -18,6 +18,8 @@ import (
"path/filepath"
"strings"
"testing"
"android/soong/android"
)
func TestGen(t *testing.T) {
@ -56,13 +58,14 @@ func TestGen(t *testing.T) {
}`)
aidl := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Rule("aidl")
aidlManifest := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Output("aidl.sbox.textproto")
libfoo := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_shared").Module().(*Module)
if !inList("-I"+filepath.Dir(aidl.Output.String()), libfoo.flags.Local.CommonFlags) {
t.Errorf("missing aidl includes in global flags")
}
aidlCommand := aidl.RuleParams.Command
aidlCommand := android.RuleBuilderSboxProtoForTests(t, aidlManifest).Commands[0].GetCommand()
if !strings.Contains(aidlCommand, "-Isub") {
t.Errorf("aidl command for c.aidl should contain \"-Isub\", but was %q", aidlCommand)
}

14
cmd/sbox/Android.bp
View File

@ -14,8 +14,20 @@
blueprint_go_binary {
name: "sbox",
deps: ["soong-makedeps"],
deps: [
"sbox_proto",
"soong-makedeps",
],
srcs: [
"sbox.go",
],
}
bootstrap_go_package {
name: "sbox_proto",
pkgPath: "android/soong/cmd/sbox/sbox_proto",
deps: ["golang-protobuf-proto"],
srcs: [
"sbox_proto/sbox.pb.go",
],
}

428
cmd/sbox/sbox.go
View File

@ -19,41 +19,39 @@ import (
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"android/soong/cmd/sbox/sbox_proto"
"android/soong/makedeps"
"github.com/golang/protobuf/proto"
)
var (
sandboxesRoot string
rawCommand string
outputRoot string
manifestFile string
keepOutDir bool
depfileOut string
inputHash string
)
const (
depFilePlaceholder = "__SBOX_DEPFILE__"
sandboxDirPlaceholder = "__SBOX_SANDBOX_DIR__"
)
func init() {
flag.StringVar(&sandboxesRoot, "sandbox-path", "",
"root of temp directory to put the sandbox into")
flag.StringVar(&rawCommand, "c", "",
"command to run")
flag.StringVar(&outputRoot, "output-root", "",
"root of directory to copy outputs into")
flag.StringVar(&manifestFile, "manifest", "",
"textproto manifest describing the sandboxed command(s)")
flag.BoolVar(&keepOutDir, "keep-out-dir", false,
"whether to keep the sandbox directory when done")
flag.StringVar(&depfileOut, "depfile-out", "",
"file path of the depfile to generate. This value will replace '__SBOX_DEPFILE__' in the command and will be treated as an output but won't be added to __SBOX_OUT_FILES__")
flag.StringVar(&inputHash, "input-hash", "",
"This option is ignored. Typical usage is to supply a hash of the list of input names so that the module will be rebuilt if the list (and thus the hash) changes.")
}
func usageViolation(violation string) {
@ -62,11 +60,7 @@ func usageViolation(violation string) {
}
fmt.Fprintf(os.Stderr,
"Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> [--depfile-out depFile] [--input-hash hash] <outputFile> [<outputFile>...]\n"+
"\n"+
"Deletes <outputRoot>,"+
"runs <commandToRun>,"+
"and moves each <outputFile> out of <sandboxPath> and into <outputRoot>\n")
"Usage: sbox --manifest <manifest> --sandbox-path <sandboxPath>\n")
flag.PrintDefaults()
@ -103,8 +97,8 @@ func findAllFilesUnder(root string) (paths []string) {
}
func run() error {
if rawCommand == "" {
usageViolation("-c <commandToRun> is required and must be non-empty")
if manifestFile == "" {
usageViolation("--manifest <manifest> is required and must be non-empty")
}
if sandboxesRoot == "" {
// In practice, the value of sandboxesRoot will mostly likely be at a fixed location relative to OUT_DIR,
@ -114,61 +108,28 @@ func run() error {
// and by passing it as a parameter we don't need to duplicate its value
usageViolation("--sandbox-path <sandboxPath> is required and must be non-empty")
}
if len(outputRoot) == 0 {
usageViolation("--output-root <outputRoot> is required and must be non-empty")
manifest, err := readManifest(manifestFile)
if len(manifest.Commands) == 0 {
return fmt.Errorf("at least one commands entry is required in %q", manifestFile)
}
// the contents of the __SBOX_OUT_FILES__ variable
outputsVarEntries := flag.Args()
if len(outputsVarEntries) == 0 {
usageViolation("at least one output file must be given")
}
// all outputs
var allOutputs []string
// setup directories
err := os.MkdirAll(sandboxesRoot, 0777)
// setup sandbox directory
err = os.MkdirAll(sandboxesRoot, 0777)
if err != nil {
return err
}
err = os.RemoveAll(outputRoot)
if err != nil {
return err
}
err = os.MkdirAll(outputRoot, 0777)
if err != nil {
return err
return fmt.Errorf("failed to create %q: %w", sandboxesRoot, err)
}
tempDir, err := ioutil.TempDir(sandboxesRoot, "sbox")
for i, filePath := range outputsVarEntries {
if !strings.HasPrefix(filePath, "__SBOX_OUT_DIR__/") {
return fmt.Errorf("output files must start with `__SBOX_OUT_DIR__/`")
}
outputsVarEntries[i] = strings.TrimPrefix(filePath, "__SBOX_OUT_DIR__/")
}
allOutputs = append([]string(nil), outputsVarEntries...)
if depfileOut != "" {
sandboxedDepfile, err := filepath.Rel(outputRoot, depfileOut)
if err != nil {
return err
}
allOutputs = append(allOutputs, sandboxedDepfile)
rawCommand = strings.Replace(rawCommand, "__SBOX_DEPFILE__", filepath.Join(tempDir, sandboxedDepfile), -1)
}
if err != nil {
return fmt.Errorf("Failed to create temp dir: %s", err)
return fmt.Errorf("failed to create temporary dir in %q: %w", sandboxesRoot, err)
}
// In the common case, the following line of code is what removes the sandbox
// If a fatal error occurs (such as if our Go process is killed unexpectedly),
// then at the beginning of the next build, Soong will retry the cleanup
// then at the beginning of the next build, Soong will wipe the temporary
// directory.
defer func() {
// in some cases we decline to remove the temp dir, to facilitate debugging
if !keepOutDir {
@ -176,27 +137,95 @@ func run() error {
}
}()
if strings.Contains(rawCommand, "__SBOX_OUT_DIR__") {
rawCommand = strings.Replace(rawCommand, "__SBOX_OUT_DIR__", tempDir, -1)
}
// If there is more than one command in the manifest use a separate directory for each one.
useSubDir := len(manifest.Commands) > 1
var commandDepFiles []string
if strings.Contains(rawCommand, "__SBOX_OUT_FILES__") {
// expands into a space-separated list of output files to be generated into the sandbox directory
tempOutPaths := []string{}
for _, outputPath := range outputsVarEntries {
tempOutPath := path.Join(tempDir, outputPath)
tempOutPaths = append(tempOutPaths, tempOutPath)
for i, command := range manifest.Commands {
localTempDir := tempDir
if useSubDir {
localTempDir = filepath.Join(localTempDir, strconv.Itoa(i))
}
pathsText := strings.Join(tempOutPaths, " ")
rawCommand = strings.Replace(rawCommand, "__SBOX_OUT_FILES__", pathsText, -1)
}
for _, filePath := range allOutputs {
dir := path.Join(tempDir, filepath.Dir(filePath))
err = os.MkdirAll(dir, 0777)
depFile, err := runCommand(command, localTempDir)
if err != nil {
// Running the command failed, keep the temporary output directory around in
// case a user wants to inspect it for debugging purposes. Soong will delete
// it at the beginning of the next build anyway.
keepOutDir = true
return err
}
if depFile != "" {
commandDepFiles = append(commandDepFiles, depFile)
}
}
outputDepFile := manifest.GetOutputDepfile()
if len(commandDepFiles) > 0 && outputDepFile == "" {
return fmt.Errorf("Sandboxed commands used %s but output depfile is not set in manifest file",
depFilePlaceholder)
}
if outputDepFile != "" {
// Merge the depfiles from each command in the manifest to a single output depfile.
err = rewriteDepFiles(commandDepFiles, outputDepFile)
if err != nil {
return fmt.Errorf("failed merging depfiles: %w", err)
}
}
return nil
}
// readManifest reads an sbox manifest from a textproto file.
func readManifest(file string) (*sbox_proto.Manifest, error) {
manifestData, err := ioutil.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("error reading manifest %q: %w", file, err)
}
manifest := sbox_proto.Manifest{}
err = proto.UnmarshalText(string(manifestData), &manifest)
if err != nil {
return nil, fmt.Errorf("error parsing manifest %q: %w", file, err)
}
return &manifest, nil
}
// runCommand runs a single command from a manifest. If the command references the
// __SBOX_DEPFILE__ placeholder it returns the name of the depfile that was used.
func runCommand(command *sbox_proto.Command, tempDir string) (depFile string, err error) {
rawCommand := command.GetCommand()
if rawCommand == "" {
return "", fmt.Errorf("command is required")
}
err = os.MkdirAll(tempDir, 0777)
if err != nil {
return "", fmt.Errorf("failed to create %q: %w", tempDir, err)
}
// Copy in any files specified by the manifest.
err = linkOrCopyFiles(command.CopyBefore, "", tempDir)
if err != nil {
return "", err
}
if strings.Contains(rawCommand, depFilePlaceholder) {
depFile = filepath.Join(tempDir, "deps.d")
rawCommand = strings.Replace(rawCommand, depFilePlaceholder, depFile, -1)
}
if strings.Contains(rawCommand, sandboxDirPlaceholder) {
rawCommand = strings.Replace(rawCommand, sandboxDirPlaceholder, tempDir, -1)
}
// Emulate ninja's behavior of creating the directories for any output files before
// running the command.
err = makeOutputDirs(command.CopyAfter, tempDir)
if err != nil {
return "", err
}
commandDescription := rawCommand
@ -205,27 +234,20 @@ func run() error {
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if command.GetChdir() {
cmd.Dir = tempDir
}
err = cmd.Run()
if exit, ok := err.(*exec.ExitError); ok && !exit.Success() {
return fmt.Errorf("sbox command (%s) failed with err %#v\n", commandDescription, err.Error())
return "", fmt.Errorf("sbox command failed with err:\n%s\n%w\n", commandDescription, err)
} else if err != nil {
return err
return "", err
}
// validate that all files are created properly
var missingOutputErrors []string
for _, filePath := range allOutputs {
tempPath := filepath.Join(tempDir, filePath)
fileInfo, err := os.Stat(tempPath)
if err != nil {
missingOutputErrors = append(missingOutputErrors, fmt.Sprintf("%s: does not exist", filePath))
continue
}
if fileInfo.IsDir() {
missingOutputErrors = append(missingOutputErrors, fmt.Sprintf("%s: not a file", filePath))
}
}
missingOutputErrors := validateOutputFiles(command.CopyAfter, tempDir)
if len(missingOutputErrors) > 0 {
// find all created files for making a more informative error message
createdFiles := findAllFilesUnder(tempDir)
@ -236,7 +258,7 @@ func run() error {
errorMessage += "in sandbox " + tempDir + ",\n"
errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
for _, missingOutputError := range missingOutputErrors {
errorMessage += " " + missingOutputError + "\n"
errorMessage += " " + missingOutputError.Error() + "\n"
}
if len(createdFiles) < 1 {
errorMessage += "created 0 files."
@ -253,19 +275,137 @@ func run() error {
}
}
// Keep the temporary output directory around in case a user wants to inspect it for debugging purposes.
// Soong will delete it later anyway.
keepOutDir = true
return errors.New(errorMessage)
return "", errors.New(errorMessage)
}
// the created files match the declared files; now move them
for _, filePath := range allOutputs {
tempPath := filepath.Join(tempDir, filePath)
destPath := filePath
if len(outputRoot) != 0 {
destPath = filepath.Join(outputRoot, filePath)
err = moveFiles(command.CopyAfter, tempDir, "")
return depFile, nil
}
// makeOutputDirs creates directories in the sandbox dir for every file that has a rule to be copied
// out of the sandbox. This emulate's Ninja's behavior of creating directories for output files
// so that the tools don't have to.
func makeOutputDirs(copies []*sbox_proto.Copy, sandboxDir string) error {
for _, copyPair := range copies {
dir := joinPath(sandboxDir, filepath.Dir(copyPair.GetFrom()))
err := os.MkdirAll(dir, 0777)
if err != nil {
return err
}
err := os.MkdirAll(filepath.Dir(destPath), 0777)
}
return nil
}
// validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
// were created by the command.
func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir string) []error {
var missingOutputErrors []error
for _, copyPair := range copies {
fromPath := joinPath(sandboxDir, copyPair.GetFrom())
fileInfo, err := os.Stat(fromPath)
if err != nil {
missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: does not exist", fromPath))
continue
}
if fileInfo.IsDir() {
missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
}
}
return missingOutputErrors
}
// linkOrCopyFiles hardlinks or copies files in or out of the sandbox.
func linkOrCopyFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
for _, copyPair := range copies {
fromPath := joinPath(fromDir, copyPair.GetFrom())
toPath := joinPath(toDir, copyPair.GetTo())
err := linkOrCopyOneFile(fromPath, toPath)
if err != nil {
return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
}
}
return nil
}
// linkOrCopyOneFile first attempts to hardlink a file to a destination, and falls back to making
// a copy if the hardlink fails.
func linkOrCopyOneFile(from string, to string) error {
err := os.MkdirAll(filepath.Dir(to), 0777)
if err != nil {
return err
}
// First try hardlinking
err = os.Link(from, to)
if err != nil {
// Retry with copying in case the source an destination are on different filesystems.
// TODO: check for specific hardlink error?
err = copyOneFile(from, to)
if err != nil {
return err
}
}
return nil
}
// copyOneFile copies a file.
func copyOneFile(from string, to string) error {
stat, err := os.Stat(from)
if err != nil {
return err
}
perm := stat.Mode()
in, err := os.Open(from)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(to)
if err != nil {
return err
}
defer func() {
out.Close()
if err != nil {
os.Remove(to)
}
}()
_, err = io.Copy(out, in)
if err != nil {
return err
}
if err = out.Close(); err != nil {
return err
}
if err = os.Chmod(to, perm); err != nil {
return err
}
return nil
}
// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
// to moving files where the source and destination are in the same filesystem. This is OK for
// sbox because the temporary directory is inside the out directory. It updates the timestamp
// of the new file.
func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
for _, copyPair := range copies {
fromPath := joinPath(fromDir, copyPair.GetFrom())
toPath := joinPath(toDir, copyPair.GetTo())
err := os.MkdirAll(filepath.Dir(toPath), 0777)
if err != nil {
return err
}
err = os.Rename(fromPath, toPath)
if err != nil {
return err
}
@ -273,37 +413,53 @@ func run() error {
// Update the timestamp of the output file in case the tool wrote an old timestamp (for example, tar can extract
// files with old timestamps).
now := time.Now()
err = os.Chtimes(tempPath, now, now)
if err != nil {
return err
}
err = os.Rename(tempPath, destPath)
err = os.Chtimes(toPath, now, now)
if err != nil {
return err
}
}
// Rewrite the depfile so that it doesn't include the (randomized) sandbox directory
if depfileOut != "" {
in, err := ioutil.ReadFile(depfileOut)
if err != nil {
return err
}
deps, err := makedeps.Parse(depfileOut, bytes.NewBuffer(in))
if err != nil {
return err
}
deps.Output = "outputfile"
err = ioutil.WriteFile(depfileOut, deps.Print(), 0666)
if err != nil {
return err
}
}
// TODO(jeffrygaston) if a process creates more output files than it declares, should there be a warning?
return nil
}
// Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
// to an output file.
func rewriteDepFiles(ins []string, out string) error {
var mergedDeps []string
for _, in := range ins {
data, err := ioutil.ReadFile(in)
if err != nil {
return err
}
deps, err := makedeps.Parse(in, bytes.NewBuffer(data))
if err != nil {
return err
}
mergedDeps = append(mergedDeps, deps.Inputs...)
}
deps := makedeps.Deps{
// Ninja doesn't care what the output file is, so we can use any string here.
Output: "outputfile",
Inputs: mergedDeps,
}
// Make the directory for the output depfile in case it is in a different directory
// than any of the output files.
outDir := filepath.Dir(out)
err := os.MkdirAll(outDir, 0777)
if err != nil {
return fmt.Errorf("failed to create %q: %w", outDir, err)
}
return ioutil.WriteFile(out, deps.Print(), 0666)
}
// joinPath wraps filepath.Join but returns file without appending to dir if file is
// absolute.
func joinPath(dir, file string) string {
if filepath.IsAbs(file) {
return file
}
return filepath.Join(dir, file)
}

233
cmd/sbox/sbox_proto/sbox.pb.go Normal file
View File

@ -0,0 +1,233 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: sbox.proto
package sbox_proto
import (
fmt "fmt"
proto "github.com/golang/protobuf/proto"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
// A set of commands to run in a sandbox.
type Manifest struct {
// A list of commands to run in the sandbox.
Commands []*Command `protobuf:"bytes,1,rep,name=commands" json:"commands,omitempty"`
// If set, GCC-style dependency files from any command that references __SBOX_DEPFILE__ will be
// merged into the given output file relative to the $PWD when sbox was started.
OutputDepfile *string `protobuf:"bytes,2,opt,name=output_depfile,json=outputDepfile" json:"output_depfile,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Manifest) Reset() { *m = Manifest{} }
func (m *Manifest) String() string { return proto.CompactTextString(m) }
func (*Manifest) ProtoMessage() {}
func (*Manifest) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{0}
}
func (m *Manifest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Manifest.Unmarshal(m, b)
}
func (m *Manifest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Manifest.Marshal(b, m, deterministic)
}
func (m *Manifest) XXX_Merge(src proto.Message) {
xxx_messageInfo_Manifest.Merge(m, src)
}
func (m *Manifest) XXX_Size() int {
return xxx_messageInfo_Manifest.Size(m)
}
func (m *Manifest) XXX_DiscardUnknown() {
xxx_messageInfo_Manifest.DiscardUnknown(m)
}
var xxx_messageInfo_Manifest proto.InternalMessageInfo
func (m *Manifest) GetCommands() []*Command {
if m != nil {
return m.Commands
}
return nil
}
func (m *Manifest) GetOutputDepfile() string {
if m != nil && m.OutputDepfile != nil {
return *m.OutputDepfile
}
return ""
}
// SandboxManifest describes a command to run in the sandbox.
type Command struct {
// A list of copy rules to run before the sandboxed command. The from field is relative to the
// $PWD when sbox was run, the to field is relative to the top of the temporary sandbox directory.
CopyBefore []*Copy `protobuf:"bytes,1,rep,name=copy_before,json=copyBefore" json:"copy_before,omitempty"`
// If true, change the working directory to the top of the temporary sandbox directory before
// running the command. If false, leave the working directory where it was when sbox was started.
Chdir *bool `protobuf:"varint,2,opt,name=chdir" json:"chdir,omitempty"`
// The command to run.
Command *string `protobuf:"bytes,3,req,name=command" json:"command,omitempty"`
// A list of copy rules to run after the sandboxed command. The from field is relative to the
// top of the temporary sandbox directory, the to field is relative to the $PWD when sbox was run.
CopyAfter []*Copy `protobuf:"bytes,4,rep,name=copy_after,json=copyAfter" json:"copy_after,omitempty"`
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
InputHash *string `protobuf:"bytes,5,opt,name=input_hash,json=inputHash" json:"input_hash,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Command) Reset() { *m = Command{} }
func (m *Command) String() string { return proto.CompactTextString(m) }
func (*Command) ProtoMessage() {}
func (*Command) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{1}
}
func (m *Command) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Command.Unmarshal(m, b)
}
func (m *Command) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Command.Marshal(b, m, deterministic)
}
func (m *Command) XXX_Merge(src proto.Message) {
xxx_messageInfo_Command.Merge(m, src)
}
func (m *Command) XXX_Size() int {
return xxx_messageInfo_Command.Size(m)
}
func (m *Command) XXX_DiscardUnknown() {
xxx_messageInfo_Command.DiscardUnknown(m)
}
var xxx_messageInfo_Command proto.InternalMessageInfo
func (m *Command) GetCopyBefore() []*Copy {
if m != nil {
return m.CopyBefore
}
return nil
}
func (m *Command) GetChdir() bool {
if m != nil && m.Chdir != nil {
return *m.Chdir
}
return false
}
func (m *Command) GetCommand() string {
if m != nil && m.Command != nil {
return *m.Command
}
return ""
}
func (m *Command) GetCopyAfter() []*Copy {
if m != nil {
return m.CopyAfter
}
return nil
}
func (m *Command) GetInputHash() string {
if m != nil && m.InputHash != nil {
return *m.InputHash
}
return ""
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
// are relative to is specific to the context the Copy is used in and will be different for
// from and to.
type Copy struct {
From *string `protobuf:"bytes,1,req,name=from" json:"from,omitempty"`
To *string `protobuf:"bytes,2,req,name=to" json:"to,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Copy) Reset() { *m = Copy{} }
func (m *Copy) String() string { return proto.CompactTextString(m) }
func (*Copy) ProtoMessage() {}
func (*Copy) Descriptor() ([]byte, []int) {
return fileDescriptor_9d0425bf0de86ed1, []int{2}
}
func (m *Copy) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Copy.Unmarshal(m, b)
}
func (m *Copy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Copy.Marshal(b, m, deterministic)
}
func (m *Copy) XXX_Merge(src proto.Message) {
xxx_messageInfo_Copy.Merge(m, src)
}
func (m *Copy) XXX_Size() int {
return xxx_messageInfo_Copy.Size(m)
}
func (m *Copy) XXX_DiscardUnknown() {
xxx_messageInfo_Copy.DiscardUnknown(m)
}
var xxx_messageInfo_Copy proto.InternalMessageInfo
func (m *Copy) GetFrom() string {
if m != nil && m.From != nil {
return *m.From
}
return ""
}
func (m *Copy) GetTo() string {
if m != nil && m.To != nil {
return *m.To
}
return ""
}
func init() {
proto.RegisterType((*Manifest)(nil), "sbox.Manifest")
proto.RegisterType((*Command)(nil), "sbox.Command")
proto.RegisterType((*Copy)(nil), "sbox.Copy")
}
func init() {
proto.RegisterFile("sbox.proto", fileDescriptor_9d0425bf0de86ed1)
}
var fileDescriptor_9d0425bf0de86ed1 = []byte{
// 252 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x41, 0x4b, 0xc3, 0x40,
0x10, 0x85, 0x49, 0x9a, 0xd2, 0x66, 0x6a, 0x7b, 0x18, 0x3c, 0xec, 0x45, 0x08, 0x01, 0x21, 0x55,
0xe8, 0xc1, 0x7f, 0x60, 0xf5, 0xe0, 0xc5, 0xcb, 0x1e, 0x45, 0x08, 0xdb, 0x64, 0x97, 0x04, 0x4c,
0x66, 0xd9, 0xdd, 0x82, 0xfd, 0x57, 0xfe, 0x44, 0xd9, 0x49, 0xea, 0xc5, 0xdb, 0xcc, 0xfb, 0x78,
0xf3, 0x1e, 0x03, 0xe0, 0x4f, 0xf4, 0x7d, 0xb0, 0x8e, 0x02, 0x61, 0x16, 0xe7, 0xf2, 0x13, 0xd6,
0xef, 0x6a, 0xec, 0x8d, 0xf6, 0x01, 0xf7, 0xb0, 0x6e, 0x68, 0x18, 0xd4, 0xd8, 0x7a, 0x91, 0x14,
0x8b, 0x6a, 0xf3, 0xb4, 0x3d, 0xb0, 0xe1, 0x65, 0x52, 0xe5, 0x1f, 0xc6, 0x7b, 0xd8, 0xd1, 0x39,
0xd8, 0x73, 0xa8, 0x5b, 0x6d, 0x4d, 0xff, 0xa5, 0x45, 0x5a, 0x24, 0x55, 0x2e, 0xb7, 0x93, 0xfa,
0x3a, 0x89, 0xe5, 0x4f, 0x02, 0xab, 0xd9, 0x8c, 0x8f, 0xb0, 0x69, 0xc8, 0x5e, 0xea, 0x93, 0x36,
0xe4, 0xf4, 0x1c, 0x00, 0xd7, 0x00, 0x7b, 0x91, 0x10, 0xf1, 0x91, 0x29, 0xde, 0xc2, 0xb2, 0xe9,
0xda, 0xde, 0xf1, 0xd9, 0xb5, 0x9c, 0x16, 0x14, 0xb0, 0x9a, 0x1b, 0x88, 0x45, 0x91, 0x56, 0xb9,
0xbc, 0xae, 0xb8, 0x07, 0x76, 0xd7, 0xca, 0x04, 0xed, 0x44, 0xf6, 0xef, 0x76, 0x1e, 0xe9, 0x73,
0x84, 0x78, 0x07, 0xd0, 0x8f, 0xb1, 0x79, 0xa7, 0x7c, 0x27, 0x96, 0x5c, 0x3b, 0x67, 0xe5, 0x4d,
0xf9, 0xae, 0x7c, 0x80, 0x2c, 0x3a, 0x10, 0x21, 0x33, 0x8e, 0x06, 0x91, 0x70, 0x10, 0xcf, 0xb8,
0x83, 0x34, 0x90, 0x48, 0x59, 0x49, 0x03, 0x1d, 0x6f, 0x3e, 0xf8, 0xa1, 0x35, 0x3f, 0xf4, 0x37,
0x00, 0x00, 0xff, 0xff, 0x95, 0x4d, 0xee, 0x7d, 0x5d, 0x01, 0x00, 0x00,
}

58
cmd/sbox/sbox_proto/sbox.proto Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package sbox;
option go_package = "sbox_proto";
// A set of commands to run in a sandbox.
message Manifest {
// A list of commands to run in the sandbox.
repeated Command commands = 1;
// If set, GCC-style dependency files from any command that references __SBOX_DEPFILE__ will be
// merged into the given output file relative to the $PWD when sbox was started.
optional string output_depfile = 2;
}
// SandboxManifest describes a command to run in the sandbox.
message Command {
// A list of copy rules to run before the sandboxed command. The from field is relative to the
// $PWD when sbox was run, the to field is relative to the top of the temporary sandbox directory.
repeated Copy copy_before = 1;
// If true, change the working directory to the top of the temporary sandbox directory before
// running the command. If false, leave the working directory where it was when sbox was started.
optional bool chdir = 2;
// The command to run.
required string command = 3;
// A list of copy rules to run after the sandboxed command. The from field is relative to the
// top of the temporary sandbox directory, the to field is relative to the $PWD when sbox was run.
repeated Copy copy_after = 4;
// An optional hash of the input files to ensure the textproto files and the sbox rule reruns
// when the lists of inputs changes, even if the inputs are not on the command line.
optional string input_hash = 5;
}
// Copy describes a from-to pair of files to copy. The paths may be relative, the root that they
// are relative to is specific to the context the Copy is used in and will be different for
// from and to.
message Copy {
required string from = 1;
required string to = 2;
}

1
genrule/Android.bp
View File

@ -4,6 +4,7 @@ bootstrap_go_package {
deps: [
"blueprint",
"blueprint-pathtools",
"sbox_proto",
"soong",
"soong-android",
"soong-bazel",

61
genrule/genrule.go
View File

@ -200,6 +200,7 @@ func (c *Module) generateBazelBuildActions(ctx android.ModuleContext, label stri
}
return ok
}
func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
g.subName = ctx.ModuleSubDir()
@ -411,18 +412,23 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
}
g.rawCommands = append(g.rawCommands, rawCommand)
// Pick a unique rule name and the user-visible description.
// Pick a unique path outside the task.genDir for the sbox manifest textproto,
// a unique rule name, and the user-visible description.
manifestName := "genrule.sbox.textproto"
desc := "generate"
name := "generator"
if task.shards > 0 {
manifestName = "genrule_" + strconv.Itoa(task.shard) + ".sbox.textproto"
desc += " " + strconv.Itoa(task.shard)
name += strconv.Itoa(task.shard)
} else if len(task.out) == 1 {
desc += " " + task.out[0].Base()
}
manifestPath := android.PathForModuleOut(ctx, manifestName)
// Use a RuleBuilder to create a rule that runs the command inside an sbox sandbox.
rule := android.NewRuleBuilder().Sbox(task.genDir)
rule := android.NewRuleBuilder().Sbox(task.genDir, manifestPath)
cmd := rule.Command()
cmd.Text(rawCommand)
cmd.ImplicitOutputs(task.out)
@ -455,9 +461,10 @@ func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
// Create a rule that zips all the per-shard directories into a single zip and then
// uses zipsync to unzip it into the final directory.
ctx.Build(pctx, android.BuildParams{
Rule: gensrcsMerge,
Implicits: copyFrom,
Outputs: outputFiles,
Rule: gensrcsMerge,
Implicits: copyFrom,
Outputs: outputFiles,
Description: "merge shards",
Args: map[string]string{
"zipArgs": zipArgs.String(),
"tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
@ -558,13 +565,19 @@ func (x noopImageInterface) SetImageVariation(ctx android.BaseModuleContext, var
func NewGenSrcs() *Module {
properties := &genSrcsProperties{}
// finalSubDir is the name of the subdirectory that output files will be generated into.
// It is used so that per-shard directories can be placed alongside it an then finally
// merged into it.
const finalSubDir = "gensrcs"
taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
genDir := android.PathForModuleGen(ctx, "gensrcs")
shardSize := defaultShardSize
if s := properties.Shard_size; s != nil {
shardSize = int(*s)
}
// gensrcs rules can easily hit command line limits by repeating the command for
// every input file. Shard the input files into groups.
shards := android.ShardPaths(srcFiles, shardSize)
var generateTasks []generateTask
@ -572,35 +585,45 @@ func NewGenSrcs() *Module {
var commands []string
var outFiles android.WritablePaths
var copyTo android.WritablePaths
var shardDir android.WritablePath
var depFile android.WritablePath
// When sharding is enabled (i.e. len(shards) > 1), the sbox rules for each
// shard will be write to their own directories and then be merged together
// into finalSubDir. If sharding is not enabled (i.e. len(shards) == 1),
// the sbox rule will write directly to finalSubDir.
genSubDir := finalSubDir
if len(shards) > 1 {
shardDir = android.PathForModuleGen(ctx, strconv.Itoa(i))
} else {
shardDir = genDir
genSubDir = strconv.Itoa(i)
}
for j, in := range shard {
outFile := android.GenPathWithExt(ctx, "gensrcs", in, String(properties.Output_extension))
if j == 0 {
depFile = outFile.ReplaceExtension(ctx, "d")
}
genDir := android.PathForModuleGen(ctx, genSubDir)
for j, in := range shard {
outFile := android.GenPathWithExt(ctx, finalSubDir, in, String(properties.Output_extension))
// If sharding is enabled, then outFile is the path to the output file in
// the shard directory, and copyTo is the path to the output file in the
// final directory.
if len(shards) > 1 {
shardFile := android.GenPathWithExt(ctx, strconv.Itoa(i), in, String(properties.Output_extension))
shardFile := android.GenPathWithExt(ctx, genSubDir, in, String(properties.Output_extension))
copyTo = append(copyTo, outFile)
outFile = shardFile
}
if j == 0 {
depFile = outFile.ReplaceExtension(ctx, "d")
}
outFiles = append(outFiles, outFile)
// pre-expand the command line to replace $in and $out with references to
// a single input and output file.
command, err := android.Expand(rawCommand, func(name string) (string, error) {
switch name {
case "in":
return in.String(), nil
case "out":
return android.SboxPathForOutput(outFile, shardDir), nil
return android.SboxPathForOutput(outFile, genDir), nil
default:
return "$(" + name + ")", nil
}
@ -620,7 +643,7 @@ func NewGenSrcs() *Module {
out: outFiles,
depFile: depFile,
copyTo: copyTo,
genDir: shardDir,
genDir: genDir,
cmd: fullCommand,
shard: i,
shards: len(shards),
@ -631,7 +654,7 @@ func NewGenSrcs() *Module {
}
g := generatorFactory(taskGenerator, properties)
g.subDir = "gensrcs"
g.subDir = finalSubDir
return g
}

73
genrule/genrule_test.go
View File

@ -141,7 +141,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location) > $(out)",
`,
expect: "out/tool > __SBOX_OUT_DIR__/out",
expect: "out/tool > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "empty location tool2",
@ -150,7 +150,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location) > $(out)",
`,
expect: "out/tool > __SBOX_OUT_DIR__/out",
expect: "out/tool > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "empty location tool file",
@ -159,7 +159,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location) > $(out)",
`,
expect: "tool_file1 > __SBOX_OUT_DIR__/out",
expect: "tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "empty location tool file fg",
@ -168,7 +168,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location) > $(out)",
`,
expect: "tool_file1 > __SBOX_OUT_DIR__/out",
expect: "tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "empty location tool and tool file",
@ -178,7 +178,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location) > $(out)",
`,
expect: "out/tool > __SBOX_OUT_DIR__/out",
expect: "out/tool > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool",
@ -187,7 +187,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location tool) > $(out)",
`,
expect: "out/tool > __SBOX_OUT_DIR__/out",
expect: "out/tool > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool2",
@ -196,7 +196,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location :tool) > $(out)",
`,
expect: "out/tool > __SBOX_OUT_DIR__/out",
expect: "out/tool > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool file",
@ -205,7 +205,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location tool_file1) > $(out)",
`,
expect: "tool_file1 > __SBOX_OUT_DIR__/out",
expect: "tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool file fg",
@ -214,7 +214,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(location :1tool_file) > $(out)",
`,
expect: "tool_file1 > __SBOX_OUT_DIR__/out",
expect: "tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool files",
@ -223,7 +223,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "$(locations :tool_files) > $(out)",
`,
expect: "tool_file1 tool_file2 > __SBOX_OUT_DIR__/out",
expect: "tool_file1 tool_file2 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "in1",
@ -232,7 +232,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(in) > $(out)",
`,
expect: "cat in1 > __SBOX_OUT_DIR__/out",
expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "in1 fg",
@ -241,7 +241,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(in) > $(out)",
`,
expect: "cat in1 > __SBOX_OUT_DIR__/out",
expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "ins",
@ -250,7 +250,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(in) > $(out)",
`,
expect: "cat in1 in2 > __SBOX_OUT_DIR__/out",
expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "ins fg",
@ -259,7 +259,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(in) > $(out)",
`,
expect: "cat in1 in2 > __SBOX_OUT_DIR__/out",
expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "location in1",
@ -268,7 +268,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(location in1) > $(out)",
`,
expect: "cat in1 > __SBOX_OUT_DIR__/out",
expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "location in1 fg",
@ -277,7 +277,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(location :1in) > $(out)",
`,
expect: "cat in1 > __SBOX_OUT_DIR__/out",
expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "location ins",
@ -286,7 +286,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(location in1) > $(out)",
`,
expect: "cat in1 > __SBOX_OUT_DIR__/out",
expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "location ins fg",
@ -295,7 +295,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "cat $(locations :ins) > $(out)",
`,
expect: "cat in1 in2 > __SBOX_OUT_DIR__/out",
expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "outs",
@ -303,7 +303,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out", "out2"],
cmd: "echo foo > $(out)",
`,
expect: "echo foo > __SBOX_OUT_DIR__/out __SBOX_OUT_DIR__/out2",
expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out __SBOX_SANDBOX_DIR__/out/out2",
},
{
name: "location out",
@ -311,7 +311,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out", "out2"],
cmd: "echo foo > $(location out2)",
`,
expect: "echo foo > __SBOX_OUT_DIR__/out2",
expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out2",
},
{
name: "depfile",
@ -320,7 +320,7 @@ func TestGenruleCmd(t *testing.T) {
depfile: true,
cmd: "echo foo > $(out) && touch $(depfile)",
`,
expect: "echo foo > __SBOX_OUT_DIR__/out && touch __SBOX_DEPFILE__",
expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out && touch __SBOX_DEPFILE__",
},
{
name: "gendir",
@ -328,7 +328,7 @@ func TestGenruleCmd(t *testing.T) {
out: ["out"],
cmd: "echo foo > $(genDir)/foo && cp $(genDir)/foo $(out)",
`,
expect: "echo foo > __SBOX_OUT_DIR__/foo && cp __SBOX_OUT_DIR__/foo __SBOX_OUT_DIR__/out",
expect: "echo foo > __SBOX_SANDBOX_DIR__/out/foo && cp __SBOX_SANDBOX_DIR__/out/foo __SBOX_SANDBOX_DIR__/out/out",
},
{
@ -443,7 +443,7 @@ func TestGenruleCmd(t *testing.T) {
allowMissingDependencies: true,
expect: "cat ***missing srcs :missing*** > __SBOX_OUT_DIR__/out",
expect: "cat ***missing srcs :missing*** > __SBOX_SANDBOX_DIR__/out/out",
},
{
name: "tool allow missing dependencies",
@ -455,7 +455,7 @@ func TestGenruleCmd(t *testing.T) {
allowMissingDependencies: true,
expect: "***missing tool :missing*** > __SBOX_OUT_DIR__/out",
expect: "***missing tool :missing*** > __SBOX_SANDBOX_DIR__/out/out",
},
}
@ -570,20 +570,11 @@ func TestGenruleHashInputs(t *testing.T) {
for _, test := range testcases {
t.Run(test.name, func(t *testing.T) {
gen := ctx.ModuleForTests(test.name, "")
command := gen.Rule("generator").RuleParams.Command
manifest := android.RuleBuilderSboxProtoForTests(t, gen.Output("genrule.sbox.textproto"))
hash := manifest.Commands[0].GetInputHash()
if len(test.expectedHash) > 0 {
// We add spaces before and after to make sure that
// this option doesn't abutt another sbox option.
expectedInputHashOption := " --input-hash " + test.expectedHash
if !strings.Contains(command, expectedInputHashOption) {
t.Errorf("Expected command \"%s\" to contain \"%s\"", command, expectedInputHashOption)
}
} else {
if strings.Contains(command, "--input-hash") {
t.Errorf("Unexpected \"--input-hash\" found in command: \"%s\"", command)
}
if g, w := hash, test.expectedHash; g != w {
t.Errorf("Expected has %q, got %q", w, g)
}
})
}
@ -609,7 +600,7 @@ func TestGenSrcs(t *testing.T) {
cmd: "$(location) $(in) > $(out)",
`,
cmds: []string{
"bash -c 'out/tool in1.txt > __SBOX_OUT_DIR__/in1.h' && bash -c 'out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'",
"bash -c 'out/tool in1.txt > __SBOX_SANDBOX_DIR__/out/in1.h' && bash -c 'out/tool in2.txt > __SBOX_SANDBOX_DIR__/out/in2.h'",
},
deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"},
files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"},
@ -623,8 +614,8 @@ func TestGenSrcs(t *testing.T) {
shard_size: 2,
`,
cmds: []string{
"bash -c 'out/tool in1.txt > __SBOX_OUT_DIR__/in1.h' && bash -c 'out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'",
"bash -c 'out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'",
"bash -c 'out/tool in1.txt > __SBOX_SANDBOX_DIR__/out/in1.h' && bash -c 'out/tool in2.txt > __SBOX_SANDBOX_DIR__/out/in2.h'",
"bash -c 'out/tool in3.txt > __SBOX_SANDBOX_DIR__/out/in3.h'",
},
deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"},
files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"},
@ -710,7 +701,7 @@ func TestGenruleDefaults(t *testing.T) {
}
gen := ctx.ModuleForTests("gen", "").Module().(*Module)
expectedCmd := "cp in1 __SBOX_OUT_DIR__/out"
expectedCmd := "cp in1 __SBOX_SANDBOX_DIR__/out/out"
if gen.rawCommands[0] != expectedCmd {
t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommands[0])
}