soong_zip: add --ignore_missing_files flag

soong_zip builds a list of files to zip early and then starts
zipping them all.  If a directory being zipped is concurrently
modified, a file that existed when soong_zip started may not
still exist.  Add a flag that continues when an expected file
does not exist.  Print a warning, since this should be rare
in normal usages but is a sign of a problem if it happens
regularly.

Test: zip_test.go
Test: m checkbuild
Test: m platform
Change-Id: I78426fe66fded8528ddd436c0f71a7442183cfeb

zip/cmd/main.go

@@ -136,7 +136,7 @@ func main() {
 	compLevel := flags.Int("L", 5, "deflate compression level (0-9)")
 	emulateJar := flags.Bool("jar", false, "modify the resultant .zip to emulate the output of 'jar'")
 	writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
-
+	ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
 	symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
 
 	parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
@@ -200,6 +200,7 @@ func main() {
 		NonDeflatedFiles:         nonDeflatedFiles,
 		WriteIfChanged:           *writeIfChanged,
 		StoreSymlinks:            *symlinks,
+		IgnoreMissingFiles:       *ignoreMissingFiles,
 	})
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "error:", err.Error())

zip/zip.go

@@ -188,9 +188,11 @@ type ZipWriter struct {
 	compressorPool sync.Pool
 	compLevel      int
 
-	followSymlinks pathtools.ShouldFollowSymlinks
+	followSymlinks     pathtools.ShouldFollowSymlinks
+	ignoreMissingFiles bool
 
-	fs pathtools.FileSystem
+	stderr io.Writer
+	fs     pathtools.FileSystem
 }
 
 type zipEntry struct {
@@ -215,7 +217,9 @@ type ZipArgs struct {
 	NonDeflatedFiles         map[string]bool
 	WriteIfChanged           bool
 	StoreSymlinks            bool
+	IgnoreMissingFiles       bool
 
+	Stderr     io.Writer
 	Filesystem pathtools.FileSystem
 }
 
@@ -271,19 +275,25 @@ func ZipTo(args ZipArgs, w io.Writer) error {
 	followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
 
 	z := &ZipWriter{
-		time:           jar.DefaultTime,
-		createdDirs:    make(map[string]string),
-		createdFiles:   make(map[string]string),
-		directories:    args.AddDirectoryEntriesToZip,
-		compLevel:      args.CompressionLevel,
-		followSymlinks: followSymlinks,
-		fs:             args.Filesystem,
+		time:               jar.DefaultTime,
+		createdDirs:        make(map[string]string),
+		createdFiles:       make(map[string]string),
+		directories:        args.AddDirectoryEntriesToZip,
+		compLevel:          args.CompressionLevel,
+		followSymlinks:     followSymlinks,
+		ignoreMissingFiles: args.IgnoreMissingFiles,
+		stderr:             args.Stderr,
+		fs:                 args.Filesystem,
 	}
 
 	if z.fs == nil {
 		z.fs = pathtools.OsFs
 	}
 
+	if z.stderr == nil {
+		z.stderr = os.Stderr
+	}
+
 	pathMappings := []pathMapping{}
 
 	noCompression := args.CompressionLevel == 0
@@ -301,29 +311,44 @@ func ZipTo(args ZipArgs, w io.Writer) error {
 				return err
 			}
 			if len(globbed) == 0 {
-				return &os.PathError{
-					Op:   "stat",
+				err := &os.PathError{
+					Op:   "lstat",
 					Path: s,
 					Err:  os.ErrNotExist,
 				}
+				if args.IgnoreMissingFiles {
+					fmt.Fprintln(args.Stderr, "warning:", err)
+				} else {
+					return err
+				}
 			}
 			srcs = append(srcs, globbed...)
 		}
 		if fa.GlobDir != "" {
 			if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
 				return err
-			} else if !exists {
-				return &os.PathError{
-					Op:   "stat",
+			} else if !exists && !args.IgnoreMissingFiles {
+				err := &os.PathError{
+					Op:   "lstat",
 					Path: fa.GlobDir,
 					Err:  os.ErrNotExist,
 				}
-			} else if !isDir {
-				return &os.PathError{
-					Op:   "stat",
+				if args.IgnoreMissingFiles {
+					fmt.Fprintln(args.Stderr, "warning:", err)
+				} else {
+					return err
+				}
+			} else if !isDir && !args.IgnoreMissingFiles {
+				err := &os.PathError{
+					Op:   "lstat",
 					Path: fa.GlobDir,
 					Err:  syscall.ENOTDIR,
 				}
+				if args.IgnoreMissingFiles {
+					fmt.Fprintln(args.Stderr, "warning:", err)
+				} else {
+					return err
+				}
 			}
 			globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
 			if err != nil {
@@ -576,6 +601,10 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
 	}
 
 	if err != nil {
+		if os.IsNotExist(err) && z.ignoreMissingFiles {
+			fmt.Fprintln(z.stderr, "warning:", err)
+			return nil
+		}
 		return err
 	} else if s.IsDir() {
 		if z.directories {

zip/zip_test.go

@@ -98,14 +98,15 @@ func fileArgsBuilder() *FileArgsBuilder {
 
 func TestZip(t *testing.T) {
 	testCases := []struct {
-		name             string
-		args             *FileArgsBuilder
-		compressionLevel int
-		emulateJar       bool
-		nonDeflatedFiles map[string]bool
-		dirEntries       bool
-		manifest         string
-		storeSymlinks    bool
+		name               string
+		args               *FileArgsBuilder
+		compressionLevel   int
+		emulateJar         bool
+		nonDeflatedFiles   map[string]bool
+		dirEntries         bool
+		manifest           string
+		storeSymlinks      bool
+		ignoreMissingFiles bool
 
 		files []zip.FileHeader
 		err   error
@@ -338,6 +339,20 @@ func TestZip(t *testing.T) {
 				fh("a/a/b", fileB, zip.Deflate),
 			},
 		},
+		{
+			name: "ignore missing files",
+			args: fileArgsBuilder().
+				File("a/a/a").
+				File("a/a/b").
+				File("missing"),
+			compressionLevel:   9,
+			ignoreMissingFiles: true,
+
+			files: []zip.FileHeader{
+				fh("a/a/a", fileA, zip.Deflate),
+				fh("a/a/b", fileB, zip.Deflate),
+			},
+		},
 
 		// errors
 		{
@@ -381,7 +396,9 @@ func TestZip(t *testing.T) {
 			args.NonDeflatedFiles = test.nonDeflatedFiles
 			args.ManifestSourcePath = test.manifest
 			args.StoreSymlinks = test.storeSymlinks
+			args.IgnoreMissingFiles = test.ignoreMissingFiles
 			args.Filesystem = mockFs
+			args.Stderr = &bytes.Buffer{}
 
 			buf := &bytes.Buffer{}
 			err := ZipTo(args, buf)