From d3f2bd79e8351547da4e868780da5a56020ac8d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hans=20M=C3=A5nsson?= <hans.mansson@sony.com>
Date: Fri, 27 Nov 2020 12:37:28 +0100
Subject: [PATCH] WriteFileRule: Chunk long content and merge them to result

sbox.textproto files are created when handling genrule. The command
for the genrule and output files are written to this file in the
following format:

commands <
  command: "...."
  copy_after: <
    from: "out/asm-generic/auxvec.h"
    to: "out/soong/.intermediates/kernel/msm-4.19/qti_generate_kernel_headers_arm/gen/asm-generic/auxvec.h"
  >
  copy_after: <
    from: "out/asm-generic/bitsperlong.h"
     to:
     "out/soong/.intermediates/kernel/msm-4.19/qti_generate_kernel_headers_arm/gen/asm-generic/bitsperlong.h"
  >
  ....
>

This file grow by one copy_after entry for each output file.

When generating kenrnel headers where the number of output files are
~1000 we run into problems as the contents of sbox.textproto files
are written to disk by generating a shell script using the following
template: /bin/bash -c 'echo -e "$$0" > $out' $content
If $content is very long as in the case of generating kernel headers we
run into the issue where the command line is so long that the shell
script return E2BIG.

Fix this issue by chuking contents into smaller files and then merge
them as a final step.

Test: Build
Issue: 174444967
Change-Id: I1a74023b4222d19672e4df7edb19810a9cf2136f
---
 android/defs.go  | 32 ++++++++++++++++++++++++++++----
 android/paths.go | 10 ++++++++++
 android/util.go  | 17 +++++++++++++++++
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/android/defs.go b/android/defs.go
index 631dfe8f8..f5bd362cd 100644
--- a/android/defs.go
+++ b/android/defs.go
@@ -15,6 +15,7 @@
 package android
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -97,7 +98,7 @@ var (
 	// content to file.
 	writeFile = pctx.AndroidStaticRule("writeFile",
 		blueprint.RuleParams{
-			Command:     `/bin/bash -c 'echo -e "$$0" > $out' $content`,
+			Command:     `/bin/bash -c 'echo -e -n "$$0" > $out' $content`,
 			Description: "writing file $out",
 		},
 		"content")
@@ -133,9 +134,7 @@ var (
 	shellUnescaper = strings.NewReplacer(`'\''`, `'`)
 )
 
-// WriteFileRule creates a ninja rule to write contents to a file.  The contents will be escaped
-// so that the file contains exactly the contents passed to the function, plus a trailing newline.
-func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
+func buildWriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
 	content = echoEscaper.Replace(content)
 	content = proptools.ShellEscape(content)
 	if content == "" {
@@ -151,6 +150,31 @@ func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string)
 	})
 }
 
+// WriteFileRule creates a ninja rule to write contents to a file.  The contents will be escaped
+// so that the file contains exactly the contents passed to the function, plus a trailing newline.
+func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
+	// This is MAX_ARG_STRLEN subtracted with some safety to account for shell escapes
+	const SHARD_SIZE = 131072 - 10000
+
+	content += "\n"
+	if len(content) > SHARD_SIZE {
+		var chunks WritablePaths
+		for i, c := range ShardString(content, SHARD_SIZE) {
+			tempPath := outputFile.ReplaceExtension(ctx, fmt.Sprintf("%s.%d", outputFile.Ext(), i))
+			buildWriteFileRule(ctx, tempPath, c)
+			chunks = append(chunks, tempPath)
+		}
+		ctx.Build(pctx, BuildParams{
+			Rule:        Cat,
+			Inputs:      chunks.Paths(),
+			Output:      outputFile,
+			Description: "Merging to " + outputFile.Base(),
+		})
+		return
+	}
+	buildWriteFileRule(ctx, outputFile, content)
+}
+
 // shellUnescape reverses proptools.ShellEscape
 func shellUnescape(s string) string {
 	// Remove leading and trailing quotes if present
diff --git a/android/paths.go b/android/paths.go
index b7117a376..5a41cf16f 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -138,6 +138,8 @@ type WritablePath interface {
 	// the writablePath method doesn't directly do anything,
 	// but it allows a struct to distinguish between whether or not it implements the WritablePath interface
 	writablePath()
+
+	ReplaceExtension(ctx PathContext, ext string) OutputPath
 }
 
 type genPathProvider interface {
@@ -1249,6 +1251,10 @@ func (p InstallPath) buildDir() string {
 	return p.config.buildDir
 }
 
+func (p InstallPath) ReplaceExtension(ctx PathContext, ext string) OutputPath {
+	panic("Not implemented")
+}
+
 var _ Path = InstallPath{}
 var _ WritablePath = InstallPath{}
 
@@ -1511,6 +1517,10 @@ func (p PhonyPath) buildDir() string {
 	return p.config.buildDir
 }
 
+func (p PhonyPath) ReplaceExtension(ctx PathContext, ext string) OutputPath {
+	panic("Not implemented")
+}
+
 var _ Path = PhonyPath{}
 var _ WritablePath = PhonyPath{}
 
diff --git a/android/util.go b/android/util.go
index 8e4c0f476..0f940fa40 100644
--- a/android/util.go
+++ b/android/util.go
@@ -401,6 +401,23 @@ func ShardPaths(paths Paths, shardSize int) []Paths {
 	return ret
 }
 
+// ShardString takes a string and returns a slice of strings where the length of each one is
+// at most shardSize.
+func ShardString(s string, shardSize int) []string {
+	if len(s) == 0 {
+		return nil
+	}
+	ret := make([]string, 0, (len(s)+shardSize-1)/shardSize)
+	for len(s) > shardSize {
+		ret = append(ret, s[0:shardSize])
+		s = s[shardSize:]
+	}
+	if len(s) > 0 {
+		ret = append(ret, s)
+	}
+	return ret
+}
+
 // ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize
 // elements.
 func ShardStrings(s []string, shardSize int) [][]string {