From 1bdbdec5a0bd0501665867d7d7c395b34038241e Mon Sep 17 00:00:00 2001 From: Dan Willemsen Date: Fri, 27 Dec 2019 09:54:11 -0800 Subject: [PATCH] Take into account RAM usage for multiproduct_kati Apparently we have some instances of Soong taking 16GB of RAM. While that's a problem to solve on it's own, make multiproduct_kati's auto-parallelism detection smarter by taking into account the total RAM on the machine instead of just the number of CPUs. Bug: 146925549 Test: check soong.log for autodetected parallelism values Change-Id: Ice34c2cf73ea4f8f235521cbefc8654a79a04eef --- cmd/multiproduct_kati/main.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/cmd/multiproduct_kati/main.go b/cmd/multiproduct_kati/main.go index 47712062f..7329aee88 100644 --- a/cmd/multiproduct_kati/main.go +++ b/cmd/multiproduct_kati/main.go @@ -37,18 +37,7 @@ import ( "android/soong/zip" ) -// We default to number of cpus / 4, which seems to be the sweet spot for my -// system. I suspect this is mostly due to memory or disk bandwidth though, and -// may depend on the size ofthe source tree, so this probably isn't a great -// default. -func detectNumJobs() int { - if runtime.NumCPU() < 4 { - return 1 - } - return runtime.NumCPU() / 4 -} - -var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") +var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]") var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts") var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)") @@ -233,6 +222,21 @@ func main() { trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) } + var jobs = *numJobs + if jobs < 1 { + jobs = runtime.NumCPU() / 4 + + ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024) + if ramJobs := ramGb / 20; ramGb > 0 && jobs > ramJobs { + jobs = ramJobs + } + + if jobs < 1 { + jobs = 1 + } + } + log.Verbosef("Using %d parallel jobs", jobs) + setMaxFiles(log) finder := build.NewSourceFinder(buildCtx, config) @@ -318,7 +322,7 @@ func main() { }() var wg sync.WaitGroup - for i := 0; i < *numJobs; i++ { + for i := 0; i < jobs; i++ { wg.Add(1) go func() { defer wg.Done()