// Copyright 2015 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package zip import ( "bytes" "compress/flate" "errors" "fmt" "hash/crc32" "io" "io/ioutil" "os" "path/filepath" "sort" "strings" "sync" "syscall" "time" "unicode" "github.com/google/blueprint/pathtools" "android/soong/jar" "android/soong/third_party/zip" ) // Block size used during parallel compression of a single file. const parallelBlockSize = 1 * 1024 * 1024 // 1MB // Minimum file size to use parallel compression. It requires more // flate.Writer allocations, since we can't change the dictionary // during Reset const minParallelFileSize = parallelBlockSize * 6 // Size of the ZIP compression window (32KB) const windowSize = 32 * 1024 type nopCloser struct { io.Writer } func (nopCloser) Close() error { return nil } type byteReaderCloser struct { *bytes.Reader io.Closer } type pathMapping struct { dest, src string zipMethod uint16 } type FileArg struct { PathPrefixInZip, SourcePrefixToStrip string SourceFiles []string JunkPaths bool GlobDir string } type FileArgsBuilder struct { state FileArg err error fs pathtools.FileSystem fileArgs []FileArg } func NewFileArgsBuilder() *FileArgsBuilder { return &FileArgsBuilder{ fs: pathtools.OsFs, } } func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder { b.state.JunkPaths = v b.state.SourcePrefixToStrip = "" return b } func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder { b.state.JunkPaths = false b.state.SourcePrefixToStrip = prefixToStrip return b } func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder { b.state.PathPrefixInZip = rootPrefix return b } func (b *FileArgsBuilder) File(name string) *FileArgsBuilder { if b.err != nil { return b } arg := b.state arg.SourceFiles = []string{name} b.fileArgs = append(b.fileArgs, arg) return b } func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder { if b.err != nil { return b } arg := b.state arg.GlobDir = name b.fileArgs = append(b.fileArgs, arg) return b } func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { if b.err != nil { return b } f, err := b.fs.Open(name) if err != nil { b.err = err return b } defer f.Close() list, err := ioutil.ReadAll(f) if err != nil { b.err = err return b } arg := b.state arg.SourceFiles = strings.Fields(string(list)) b.fileArgs = append(b.fileArgs, arg) return b } func (b *FileArgsBuilder) RspFile(name string) *FileArgsBuilder { if b.err != nil { return b } f, err := b.fs.Open(name) if err != nil { b.err = err return b } defer f.Close() list, err := ioutil.ReadAll(f) if err != nil { b.err = err return b } arg := b.state arg.SourceFiles = ReadRespFile(list) b.fileArgs = append(b.fileArgs, arg) return b } func (b *FileArgsBuilder) Error() error { if b == nil { return nil } return b.err } func (b *FileArgsBuilder) FileArgs() []FileArg { if b == nil { return nil } return b.fileArgs } type IncorrectRelativeRootError struct { RelativeRoot string Path string } func (x IncorrectRelativeRootError) Error() string { return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot) } type ZipWriter struct { time time.Time createdFiles map[string]string createdDirs map[string]string directories bool errors chan error writeOps chan chan *zipEntry cpuRateLimiter *CPURateLimiter memoryRateLimiter *MemoryRateLimiter compressorPool sync.Pool compLevel int followSymlinks pathtools.ShouldFollowSymlinks ignoreMissingFiles bool stderr io.Writer fs pathtools.FileSystem } type zipEntry struct { fh *zip.FileHeader // List of delayed io.Reader futureReaders chan chan io.Reader // Only used for passing into the MemoryRateLimiter to ensure we // release as much memory as much as we request allocatedSize int64 } type ZipArgs struct { FileArgs []FileArg OutputFilePath string EmulateJar bool SrcJar bool AddDirectoryEntriesToZip bool CompressionLevel int ManifestSourcePath string NumParallelJobs int NonDeflatedFiles map[string]bool WriteIfChanged bool StoreSymlinks bool IgnoreMissingFiles bool Stderr io.Writer Filesystem pathtools.FileSystem } const NOQUOTE = '\x00' func ReadRespFile(bytes []byte) []string { var args []string var arg []rune isEscaping := false quotingStart := NOQUOTE for _, c := range string(bytes) { switch { case isEscaping: if quotingStart == '"' { if !(c == '"' || c == '\\') { // '\"' or '\\' will be escaped under double quoting. arg = append(arg, '\\') } } arg = append(arg, c) isEscaping = false case c == '\\' && quotingStart != '\'': isEscaping = true case quotingStart == NOQUOTE && (c == '\'' || c == '"'): quotingStart = c case quotingStart != NOQUOTE && c == quotingStart: quotingStart = NOQUOTE case quotingStart == NOQUOTE && unicode.IsSpace(c): // Current character is a space outside quotes if len(arg) != 0 { args = append(args, string(arg)) } arg = arg[:0] default: arg = append(arg, c) } } if len(arg) != 0 { args = append(args, string(arg)) } return args } func ZipTo(args ZipArgs, w io.Writer) error { if args.EmulateJar { args.AddDirectoryEntriesToZip = true } // Have Glob follow symlinks if they are not being stored as symlinks in the zip file. followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks) z := &ZipWriter{ time: jar.DefaultTime, createdDirs: make(map[string]string), createdFiles: make(map[string]string), directories: args.AddDirectoryEntriesToZip, compLevel: args.CompressionLevel, followSymlinks: followSymlinks, ignoreMissingFiles: args.IgnoreMissingFiles, stderr: args.Stderr, fs: args.Filesystem, } if z.fs == nil { z.fs = pathtools.OsFs } if z.stderr == nil { z.stderr = os.Stderr } pathMappings := []pathMapping{} noCompression := args.CompressionLevel == 0 for _, fa := range args.FileArgs { var srcs []string for _, s := range fa.SourceFiles { s = strings.TrimSpace(s) if s == "" { continue } globbed, _, err := z.fs.Glob(s, nil, followSymlinks) if err != nil { return err } if len(globbed) == 0 { err := &os.PathError{ Op: "lstat", Path: s, Err: os.ErrNotExist, } if args.IgnoreMissingFiles { fmt.Fprintln(z.stderr, "warning:", err) } else { return err } } srcs = append(srcs, globbed...) } if fa.GlobDir != "" { if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil { return err } else if !exists && !args.IgnoreMissingFiles { err := &os.PathError{ Op: "lstat", Path: fa.GlobDir, Err: os.ErrNotExist, } if args.IgnoreMissingFiles { fmt.Fprintln(z.stderr, "warning:", err) } else { return err } } else if !isDir && !args.IgnoreMissingFiles { err := &os.PathError{ Op: "lstat", Path: fa.GlobDir, Err: syscall.ENOTDIR, } if args.IgnoreMissingFiles { fmt.Fprintln(z.stderr, "warning:", err) } else { return err } } globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks) if err != nil { return err } srcs = append(srcs, globbed...) } for _, src := range srcs { err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression) if err != nil { return err } } } return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs) } func Zip(args ZipArgs) error { if args.OutputFilePath == "" { return fmt.Errorf("output file path must be nonempty") } buf := &bytes.Buffer{} var out io.Writer = buf if !args.WriteIfChanged { f, err := os.Create(args.OutputFilePath) if err != nil { return err } defer f.Close() defer func() { if err != nil { os.Remove(args.OutputFilePath) } }() out = f } err := ZipTo(args, out) if err != nil { return err } if args.WriteIfChanged { err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666) if err != nil { return err } } return nil } func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, nonDeflatedFiles map[string]bool, noCompression bool) error { var dest string if fa.JunkPaths { dest = filepath.Base(src) } else { var err error dest, err = filepath.Rel(fa.SourcePrefixToStrip, src) if err != nil { return err } if strings.HasPrefix(dest, "../") { return IncorrectRelativeRootError{ Path: src, RelativeRoot: fa.SourcePrefixToStrip, } } } dest = filepath.Join(fa.PathPrefixInZip, dest) zipMethod := zip.Deflate if _, found := nonDeflatedFiles[dest]; found || noCompression { zipMethod = zip.Store } *pathMappings = append(*pathMappings, pathMapping{dest: dest, src: src, zipMethod: zipMethod}) return nil } func jarSort(mappings []pathMapping) { less := func(i int, j int) (smaller bool) { return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest) } sort.SliceStable(mappings, less) } func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool, parallelJobs int) error { z.errors = make(chan error) defer close(z.errors) // This channel size can be essentially unlimited -- it's used as a fifo // queue decouple the CPU and IO loads. Directories don't require any // compression time, but still cost some IO. Similar with small files that // can be very fast to compress. Some files that are more difficult to // compress won't take a corresponding longer time writing out. // // The optimum size here depends on your CPU and IO characteristics, and // the the layout of your zip file. 1000 was chosen mostly at random as // something that worked reasonably well for a test file. // // The RateLimit object will put the upper bounds on the number of // parallel compressions and outstanding buffers. z.writeOps = make(chan chan *zipEntry, 1000) z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs)) z.memoryRateLimiter = NewMemoryRateLimiter(0) defer func() { z.cpuRateLimiter.Stop() z.memoryRateLimiter.Stop() }() if manifest != "" && !emulateJar { return errors.New("must specify --jar when specifying a manifest via -m") } if emulateJar { // manifest may be empty, in which case addManifest will fill in a default pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate}) jarSort(pathMappings) } go func() { var err error defer close(z.writeOps) for _, ele := range pathMappings { if emulateJar && ele.dest == jar.ManifestFile { err = z.addManifest(ele.dest, ele.src, ele.zipMethod) } else { err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar) } if err != nil { z.errors <- err return } } }() zipw := zip.NewWriter(f) var currentWriteOpChan chan *zipEntry var currentWriter io.WriteCloser var currentReaders chan chan io.Reader var currentReader chan io.Reader var done bool for !done { var writeOpsChan chan chan *zipEntry var writeOpChan chan *zipEntry var readersChan chan chan io.Reader if currentReader != nil { // Only read and process errors } else if currentReaders != nil { readersChan = currentReaders } else if currentWriteOpChan != nil { writeOpChan = currentWriteOpChan } else { writeOpsChan = z.writeOps } select { case writeOp, ok := <-writeOpsChan: if !ok { done = true } currentWriteOpChan = writeOp case op := <-writeOpChan: currentWriteOpChan = nil var err error if op.fh.Method == zip.Deflate { currentWriter, err = zipw.CreateCompressedHeader(op.fh) } else { var zw io.Writer op.fh.CompressedSize64 = op.fh.UncompressedSize64 zw, err = zipw.CreateHeaderAndroid(op.fh) currentWriter = nopCloser{zw} } if err != nil { return err } currentReaders = op.futureReaders if op.futureReaders == nil { currentWriter.Close() currentWriter = nil } z.memoryRateLimiter.Finish(op.allocatedSize) case futureReader, ok := <-readersChan: if !ok { // Done with reading currentWriter.Close() currentWriter = nil currentReaders = nil } currentReader = futureReader case reader := <-currentReader: _, err := io.Copy(currentWriter, reader) if err != nil { return err } currentReader = nil case err := <-z.errors: return err } } // One last chance to catch an error select { case err := <-z.errors: return err default: zipw.Close() return nil } } // imports (possibly with compression) into the zip at sub-path func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error { var fileSize int64 var executable bool var s os.FileInfo var err error if z.followSymlinks { s, err = z.fs.Stat(src) } else { s, err = z.fs.Lstat(src) } if err != nil { if os.IsNotExist(err) && z.ignoreMissingFiles { fmt.Fprintln(z.stderr, "warning:", err) return nil } return err } createParentDirs := func(dest, src string) error { if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil { return err } if prev, exists := z.createdDirs[dest]; exists { return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) } if prev, exists := z.createdFiles[dest]; exists { return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) } z.createdFiles[dest] = src return nil } if s.IsDir() { if z.directories { return z.writeDirectory(dest, src, emulateJar) } return nil } else if s.Mode()&os.ModeSymlink != 0 { err = createParentDirs(dest, src) if err != nil { return err } return z.writeSymlink(dest, src) } else if s.Mode().IsRegular() { r, err := z.fs.Open(src) if err != nil { return err } if srcJar && filepath.Ext(src) == ".java" { // rewrite the destination using the package path if it can be determined pkg, err := jar.JavaPackage(r, src) if err != nil { // ignore errors for now, leaving the file at in its original location in the zip } else { dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src)) } _, err = r.Seek(0, io.SeekStart) if err != nil { return err } } fileSize = s.Size() executable = s.Mode()&0100 != 0 header := &zip.FileHeader{ Name: dest, Method: method, UncompressedSize64: uint64(fileSize), } if executable { header.SetMode(0700) } err = createParentDirs(dest, src) if err != nil { return err } return z.writeFileContents(header, r) } else { return fmt.Errorf("%s is not a file, directory, or symlink", src) } } func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { if prev, exists := z.createdDirs[dest]; exists { return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) } if prev, exists := z.createdFiles[dest]; exists { return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) } if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil { return err } var contents []byte if src != "" { f, err := z.fs.Open(src) if err != nil { return err } contents, err = ioutil.ReadAll(f) f.Close() if err != nil { return err } } fh, buf, err := jar.ManifestFileContents(contents) if err != nil { return err } reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)} return z.writeFileContents(fh, reader) } func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) { header.SetModTime(z.time) compressChan := make(chan *zipEntry, 1) z.writeOps <- compressChan // Pre-fill a zipEntry, it will be sent in the compressChan once // we're sure about the Method and CRC. ze := &zipEntry{ fh: header, } ze.allocatedSize = int64(header.UncompressedSize64) z.cpuRateLimiter.Request() z.memoryRateLimiter.Request(ze.allocatedSize) fileSize := int64(header.UncompressedSize64) if fileSize == 0 { fileSize = int64(header.UncompressedSize) } if header.Method == zip.Deflate && fileSize >= minParallelFileSize { wg := new(sync.WaitGroup) // Allocate enough buffer to hold all readers. We'll limit // this based on actual buffer sizes in RateLimit. ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1) // Calculate the CRC in the background, since reading the entire // file could take a while. // // We could split this up into chunks as well, but it's faster // than the compression. Due to the Go Zip API, we also need to // know the result before we can begin writing the compressed // data out to the zipfile. wg.Add(1) go z.crcFile(r, ze, compressChan, wg) for start := int64(0); start < fileSize; start += parallelBlockSize { sr := io.NewSectionReader(r, start, parallelBlockSize) resultChan := make(chan io.Reader, 1) ze.futureReaders <- resultChan z.cpuRateLimiter.Request() last := !(start+parallelBlockSize < fileSize) var dict []byte if start >= windowSize { dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize)) if err != nil { return err } } wg.Add(1) go z.compressPartialFile(sr, dict, last, resultChan, wg) } close(ze.futureReaders) // Close the file handle after all readers are done go func(wg *sync.WaitGroup, closer io.Closer) { wg.Wait() closer.Close() }(wg, r) } else { go func() { z.compressWholeFile(ze, r, compressChan) r.Close() }() } return nil } func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) { defer wg.Done() defer z.cpuRateLimiter.Finish() crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { z.errors <- err return } ze.fh.CRC32 = crc.Sum32() resultChan <- ze close(resultChan) } func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) { defer wg.Done() result, err := z.compressBlock(r, dict, last) if err != nil { z.errors <- err return } z.cpuRateLimiter.Finish() resultChan <- result } func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) { buf := new(bytes.Buffer) var fw *flate.Writer var err error if len(dict) > 0 { // There's no way to Reset a Writer with a new dictionary, so // don't use the Pool fw, err = flate.NewWriterDict(buf, z.compLevel, dict) } else { var ok bool if fw, ok = z.compressorPool.Get().(*flate.Writer); ok { fw.Reset(buf) } else { fw, err = flate.NewWriter(buf, z.compLevel) } defer z.compressorPool.Put(fw) } if err != nil { return nil, err } _, err = io.Copy(fw, r) if err != nil { return nil, err } if last { fw.Close() } else { fw.Flush() } return buf, nil } func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) { crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { z.errors <- err return } ze.fh.CRC32 = crc.Sum32() _, err = r.Seek(0, 0) if err != nil { z.errors <- err return } readFile := func(reader io.ReadSeeker) ([]byte, error) { _, err := reader.Seek(0, 0) if err != nil { return nil, err } buf, err := ioutil.ReadAll(reader) if err != nil { return nil, err } return buf, nil } ze.futureReaders = make(chan chan io.Reader, 1) futureReader := make(chan io.Reader, 1) ze.futureReaders <- futureReader close(ze.futureReaders) if ze.fh.Method == zip.Deflate { compressed, err := z.compressBlock(r, nil, true) if err != nil { z.errors <- err return } if uint64(compressed.Len()) < ze.fh.UncompressedSize64 { futureReader <- compressed } else { buf, err := readFile(r) if err != nil { z.errors <- err return } ze.fh.Method = zip.Store futureReader <- bytes.NewReader(buf) } } else { buf, err := readFile(r) if err != nil { z.errors <- err return } ze.fh.Method = zip.Store futureReader <- bytes.NewReader(buf) } z.cpuRateLimiter.Finish() close(futureReader) compressChan <- ze close(compressChan) } // writeDirectory annotates that dir is a directory created for the src file or directory, and adds // the directory entry to the zip file if directories are enabled. func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error { // clean the input dir = filepath.Clean(dir) // discover any uncreated directories in the path zipDirs := []string{} for dir != "" && dir != "." { if _, exists := z.createdDirs[dir]; exists { break } if prev, exists := z.createdFiles[dir]; exists { return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev) } z.createdDirs[dir] = src // parent directories precede their children zipDirs = append([]string{dir}, zipDirs...) dir = filepath.Dir(dir) } if z.directories { // make a directory entry for each uncreated directory for _, cleanDir := range zipDirs { var dirHeader *zip.FileHeader if emulateJar && cleanDir+"/" == jar.MetaDir { dirHeader = jar.MetaDirFileHeader() } else { dirHeader = &zip.FileHeader{ Name: cleanDir + "/", } dirHeader.SetMode(0700 | os.ModeDir) } dirHeader.SetModTime(z.time) ze := make(chan *zipEntry, 1) ze <- &zipEntry{ fh: dirHeader, } close(ze) z.writeOps <- ze } } return nil } func (z *ZipWriter) writeSymlink(rel, file string) error { fileHeader := &zip.FileHeader{ Name: rel, } fileHeader.SetModTime(z.time) fileHeader.SetMode(0777 | os.ModeSymlink) dest, err := z.fs.Readlink(file) if err != nil { return err } fileHeader.UncompressedSize64 = uint64(len(dest)) fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest)) ze := make(chan *zipEntry, 1) futureReaders := make(chan chan io.Reader, 1) futureReader := make(chan io.Reader, 1) futureReaders <- futureReader close(futureReaders) futureReader <- bytes.NewBufferString(dest) close(futureReader) ze <- &zipEntry{ fh: fileHeader, futureReaders: futureReaders, } close(ze) z.writeOps <- ze return nil }