From e348076296f29c877f4eab4fd3e563aa3961e61e Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Tue, 7 Nov 2017 11:23:27 -0800 Subject: [PATCH] Change storage behavior of multiproduct_kati Instead of deleting artifacts/logs from successful build (unless -keep is set), and keeping unsuccessful artifacts, keep all logs and remove all artifacts (unless -keep is set, then we'll compress the artifacts). If -dist is passed in, we'll put an archive of the logs into the DIST_DIR. Even compressed, the rest of the artifacts are still a bit too large to dist (~5.6GB on AOSP). Test: build/soong/build_test.bash Test: build/soong/build_test.bash -keep Test: build/soong/build_test.bash -dist Change-Id: I87f55978c18c8ff2e517b8271554ba383003742f --- cmd/multiproduct_kati/Android.bp | 1 + cmd/multiproduct_kati/main.go | 70 +++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/cmd/multiproduct_kati/Android.bp b/cmd/multiproduct_kati/Android.bp index b264c3544..04a58023b 100644 --- a/cmd/multiproduct_kati/Android.bp +++ b/cmd/multiproduct_kati/Android.bp @@ -18,6 +18,7 @@ blueprint_go_binary { "soong-ui-build", "soong-ui-logger", "soong-ui-tracer", + "soong-zip", ], srcs: [ "main.go", diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 1c853d63e..e4a05fc08 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -30,6 +30,7 @@ import ( "android/soong/ui/build" "android/soong/ui/logger" "android/soong/ui/tracer" + "android/soong/zip" ) // We default to number of cpus / 4, which seems to be the sweet spot for my @@ -45,7 +46,7 @@ func detectNumJobs() int { var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") -var keep = flag.Bool("keep", false, "keep successful output files") +var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts") var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)") var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or /dist when empty)") @@ -200,24 +201,19 @@ func main() { if err := os.MkdirAll(*outDir, 0777); err != nil { log.Fatalf("Failed to create tempdir: %v", err) } - - if !*keep { - defer func() { - if status.Finished() == 0 { - os.RemoveAll(*outDir) - } - }() - } } config.Environment().Set("OUT_DIR", *outDir) log.Println("Output directory:", *outDir) + logsDir := filepath.Join(config.OutDir(), "logs") + os.MkdirAll(logsDir, 0777) + build.SetupOutDir(buildCtx, config) if *alternateResultDir { - logsDir := filepath.Join(config.DistDir(), "logs") - os.MkdirAll(logsDir, 0777) - log.SetOutput(filepath.Join(logsDir, "soong.log")) - trace.SetOutput(filepath.Join(logsDir, "build.trace")) + distLogsDir := filepath.Join(config.DistDir(), "logs") + os.MkdirAll(distLogsDir, 0777) + log.SetOutput(filepath.Join(distLogsDir, "soong.log")) + trace.SetOutput(filepath.Join(distLogsDir, "build.trace")) } else { log.SetOutput(filepath.Join(config.OutDir(), "soong.log")) trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) @@ -269,17 +265,14 @@ func main() { }) productOutDir := filepath.Join(config.OutDir(), product) - productLogDir := productOutDir - if *alternateResultDir { - productLogDir = filepath.Join(config.DistDir(), product) - if err := os.MkdirAll(productLogDir, 0777); err != nil { - log.Fatalf("Error creating log directory: %v", err) - } - } + productLogDir := filepath.Join(logsDir, product) if err := os.MkdirAll(productOutDir, 0777); err != nil { log.Fatalf("Error creating out directory: %v", err) } + if err := os.MkdirAll(productLogDir, 0777); err != nil { + log.Fatalf("Error creating log directory: %v", err) + } stdLog = filepath.Join(productLogDir, "std.log") f, err := os.Create(stdLog) @@ -324,6 +317,26 @@ func main() { status.Fail(product.config.TargetProduct(), err, product.logFile) }) + defer func() { + if *keepArtifacts { + args := zip.ZipArgs{ + FileArgs: []zip.FileArg{ + { + GlobDir: product.config.OutDir(), + SourcePrefixToStrip: product.config.OutDir(), + }, + }, + OutputFilePath: filepath.Join(config.OutDir(), product.config.TargetProduct()+".zip"), + NumParallelJobs: runtime.NumCPU(), + CompressionLevel: 5, + } + if err := zip.Run(args); err != nil { + log.Fatalf("Error zipping artifacts: %v", err) + } + } + os.RemoveAll(product.config.OutDir()) + }() + buildWhat := 0 if !*onlyConfig { buildWhat |= build.BuildSoong @@ -332,9 +345,6 @@ func main() { } } build.Build(product.ctx, product.config, buildWhat) - if !*keep { - os.RemoveAll(product.config.OutDir()) - } status.Finish(product.config.TargetProduct()) }() } @@ -342,6 +352,20 @@ func main() { } wg2.Wait() + if *alternateResultDir { + args := zip.ZipArgs{ + FileArgs: []zip.FileArg{ + {GlobDir: logsDir, SourcePrefixToStrip: logsDir}, + }, + OutputFilePath: filepath.Join(config.DistDir(), "logs.zip"), + NumParallelJobs: runtime.NumCPU(), + CompressionLevel: 5, + } + if err := zip.Run(args); err != nil { + log.Fatalf("Error zipping logs: %v", err) + } + } + if count := status.Finished(); count > 0 { log.Fatalln(count, "products failed") }