Shard java proto files into groups of 100

Group protoc calls to generate java files into groups of 100
to avoid having a srcjar for every proto file.

Test: m checkbuild
Change-Id: I72a5efaabad24c975282df321d00ec2d7900645e

java/gen.go

@@ -98,6 +98,7 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 	flags javaBuilderFlags) android.Paths {
 
 	outSrcFiles := make(android.Paths, 0, len(srcFiles))
+	var protoSrcs android.Paths
 
 	aidlIncludeFlags := genAidlIncludeFlags(srcFiles)
 
@@ -111,13 +112,18 @@ func (j *Module) genSources(ctx android.ModuleContext, srcFiles android.Paths,
 			javaFile := genLogtags(ctx, srcFile)
 			outSrcFiles = append(outSrcFiles, javaFile)
 		case ".proto":
-			srcJarFile := genProto(ctx, srcFile, flags.proto)
+			protoSrcs = append(protoSrcs, srcFile)
-			outSrcFiles = append(outSrcFiles, srcJarFile)
 		default:
 			outSrcFiles = append(outSrcFiles, srcFile)
 		}
 	}
 
+	// Process all proto files together to support sharding them into one or more rules that produce srcjars.
+	if len(protoSrcs) > 0 {
+		srcJarFiles := genProto(ctx, protoSrcs, flags.proto)
+		outSrcFiles = append(outSrcFiles, srcJarFiles...)
+	}
+
 	return outSrcFiles
 }
 

java/proto.go

@@ -15,36 +15,61 @@
 package java
 
 import (
+	"path/filepath"
+	"strconv"
+
 	"android/soong/android"
 )
 
-func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.ProtoFlags) android.Path {
+func genProto(ctx android.ModuleContext, protoFiles android.Paths, flags android.ProtoFlags) android.Paths {
-	srcJarFile := android.GenPathWithExt(ctx, "proto", protoFile, "srcjar")
+	// Shard proto files into groups of 100 to avoid having to recompile all of them if one changes and to avoid
+	// hitting command line length limits.
+	shards := android.ShardPaths(protoFiles, 100)
 
-	outDir := srcJarFile.ReplaceExtension(ctx, "tmp")
+	srcJarFiles := make(android.Paths, 0, len(shards))
-	depFile := srcJarFile.ReplaceExtension(ctx, "srcjar.d")
 
-	rule := android.NewRuleBuilder()
+	for i, shard := range shards {
+		srcJarFile := android.PathForModuleGen(ctx, "proto", "proto"+strconv.Itoa(i)+".srcjar")
+		srcJarFiles = append(srcJarFiles, srcJarFile)
 
-	rule.Command().Text("rm -rf").Flag(outDir.String())
+		outDir := srcJarFile.ReplaceExtension(ctx, "tmp")
-	rule.Command().Text("mkdir -p").Flag(outDir.String())
 
-	android.ProtoRule(ctx, rule, protoFile, flags, flags.Deps, outDir, depFile, nil)
+		rule := android.NewRuleBuilder()
 
-	// Proto generated java files have an unknown package name in the path, so package the entire output directory
+		rule.Command().Text("rm -rf").Flag(outDir.String())
-	// into a srcjar.
+		rule.Command().Text("mkdir -p").Flag(outDir.String())
-	rule.Command().
-		BuiltTool(ctx, "soong_zip").
-		Flag("-jar").
-		FlagWithOutput("-o ", srcJarFile).
-		FlagWithArg("-C ", outDir.String()).
-		FlagWithArg("-D ", outDir.String())
 
-	rule.Command().Text("rm -rf").Flag(outDir.String())
+		for _, protoFile := range shard {
+			depFile := srcJarFile.InSameDir(ctx, protoFile.String()+".d")
+			rule.Command().Text("mkdir -p").Flag(filepath.Dir(depFile.String()))
+			android.ProtoRule(ctx, rule, protoFile, flags, flags.Deps, outDir, depFile, nil)
+		}
 
-	rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
+		// Proto generated java files have an unknown package name in the path, so package the entire output directory
+		// into a srcjar.
+		rule.Command().
+			BuiltTool(ctx, "soong_zip").
+			Flag("-jar").
+			Flag("-write_if_changed").
+			FlagWithOutput("-o ", srcJarFile).
+			FlagWithArg("-C ", outDir.String()).
+			FlagWithArg("-D ", outDir.String())
 
-	return srcJarFile
+		rule.Command().Text("rm -rf").Flag(outDir.String())
+
+		rule.Restat()
+
+		ruleName := "protoc"
+		ruleDesc := "protoc"
+		if len(shards) > 1 {
+			ruleName += "_" + strconv.Itoa(i)
+			ruleDesc += " " + strconv.Itoa(i)
+		}
+
+		rule.Build(pctx, ctx, ruleName, ruleDesc)
+	}
+
+	return srcJarFiles
 }
 
 func protoDeps(ctx android.BottomUpMutatorContext, p *android.ProtoProperties) {