openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

zip2zip: Support sorting globbed arguments, '**' 

When '-s' is passed, any globbed arguments will have their results
sorted. When there are multiple arguments, the files will still be
inserted in argument order.

A bare '**' is now special cased to mean every file in the input zip.

This allows zip2zip to sort entire zip files efficiently by using
`zip2zip -s -i <> -o <> '**'`. This can be useful if your original zip
program used filesystem ordering which was not reproducible.

Test: m -j blueprint_tools (new tests pass)
Change-Id: Ic3512c5fe14c94c6f3e134296905121d2ff8b58a
This commit is contained in:
Dan Willemsen 2017-06-19 16:35:00 -07:00
parent 83cf1cee58
commit 82218f285c
3 changed files with 274 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/zip2zip/Android.bp
View File

@ -18,5 +18,6 @@ blueprint_go_binary {
srcs: [
"zip2zip.go",
],
testSrcs: ["zip2zip_test.go"],
}

129
cmd/zip2zip/zip2zip.go
View File

@ -17,49 +17,59 @@ package main
import (
"flag"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"time"
"android/soong/third_party/zip"
)
var (
input = flag.String("i", "", "zip file to read from")
output = flag.String("o", "", "output file")
input = flag.String("i", "", "zip file to read from")
output = flag.String("o", "", "output file")
sortGlobs = flag.Bool("s", false, "sort matches from each glob (defaults to the order from the input zip file)")
setTime = flag.Bool("t", false, "set timestamps to 2009-01-01 00:00:00")
staticTime = time.Date(2009, 1, 1, 0, 0, 0, 0, time.UTC)
)
func usage() {
fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [filespec]...")
flag.PrintDefaults()
fmt.Fprintln(os.Stderr, " filespec:")
fmt.Fprintln(os.Stderr, " <name>")
fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
os.Exit(2)
}
func main() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "usage: zip2zip -i zipfile -o zipfile [-s] [-t] [filespec]...")
flag.PrintDefaults()
fmt.Fprintln(os.Stderr, " filespec:")
fmt.Fprintln(os.Stderr, " <name>")
fmt.Fprintln(os.Stderr, " <in_name>:<out_name>")
fmt.Fprintln(os.Stderr, " <glob>:<out_dir>/")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "<glob> uses the rules at https://golang.org/pkg/path/filepath/#Match")
fmt.Fprintln(os.Stderr, "As a special exception, '**' is supported to specify all files in the input zip")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Files will be copied with their existing compression from the input zipfile to")
fmt.Fprintln(os.Stderr, "the output zipfile, in the order of filespec arguments")
}
flag.Parse()
if flag.NArg() == 0 || *input == "" || *output == "" {
usage()
flag.Usage()
os.Exit(1)
}
log.SetFlags(log.Lshortfile)
reader, err := zip.OpenReader(*input)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(3)
log.Fatal(err)
}
defer reader.Close()
output, err := os.Create(*output)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(4)
log.Fatal(err)
}
defer output.Close()
@ -67,20 +77,24 @@ func main() {
defer func() {
err := writer.Close()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(5)
log.Fatal(err)
}
}()
for _, arg := range flag.Args() {
if err := zip2zip(&reader.Reader, writer, *sortGlobs, *setTime, flag.Args()); err != nil {
log.Fatal(err)
}
}
func zip2zip(reader *zip.Reader, writer *zip.Writer, sortGlobs, setTime bool, args []string) error {
for _, arg := range args {
var input string
var output string
// Reserve escaping for future implementation, so make sure no
// one is using \ and expecting a certain behavior.
if strings.Contains(arg, "\\") {
fmt.Fprintln(os.Stderr, "\\ characters are not currently supported")
os.Exit(6)
return fmt.Errorf("\\ characters are not currently supported")
}
args := strings.SplitN(arg, ":", 2)
@ -89,25 +103,45 @@ func main() {
output = args[1]
}
type pair struct {
*zip.File
newName string
}
matches := []pair{}
if strings.IndexAny(input, "*?[") >= 0 {
matchAll := input == "**"
if !matchAll && strings.Contains(input, "**") {
return fmt.Errorf("** is only supported on its own, not with other characters")
}
for _, file := range reader.File {
if match, err := filepath.Match(input, file.Name); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(7)
} else if match {
var newFileName string
if output == "" {
newFileName = file.Name
} else {
_, name := filepath.Split(file.Name)
newFileName = filepath.Join(output, name)
}
err = writer.CopyFrom(file, newFileName)
match := matchAll
if !match {
var err error
match, err = filepath.Match(input, file.Name)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(8)
return err
}
}
if match {
var newName string
if output == "" {
newName = file.Name
} else {
_, name := filepath.Split(file.Name)
newName = filepath.Join(output, name)
}
matches = append(matches, pair{file, newName})
}
}
if sortGlobs {
sort.SliceStable(matches, func(i, j int) bool {
return matches[i].newName < matches[j].newName
})
}
} else {
if output == "" {
@ -115,14 +149,21 @@ func main() {
}
for _, file := range reader.File {
if input == file.Name {
err = writer.CopyFrom(file, output)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(8)
}
matches = append(matches, pair{file, output})
break
}
}
}
for _, match := range matches {
if setTime {
match.File.SetModTime(staticTime)
}
if err := writer.CopyFrom(match.File, match.newName); err != nil {
return err
}
}
}
return nil
}

188
cmd/zip2zip/zip2zip_test.go Normal file
View File

@ -0,0 +1,188 @@
// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"fmt"
"reflect"
"testing"
"android/soong/third_party/zip"
)
var testCases = []struct {
name string
inputFiles []string
sortGlobs bool
args []string
outputFiles []string
err error
}{
{
name: "unsupported \\",
args: []string{"a\\b:b"},
err: fmt.Errorf("\\ characters are not currently supported"),
},
{
name: "unsupported **",
args: []string{"a/**:b"},
err: fmt.Errorf("** is only supported on its own, not with other characters"),
},
{ // This is modelled after the update package build rules in build/make/core/Makefile
name: "filter globs",
inputFiles: []string{
"RADIO/a",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
"OTA/android-info.txt",
"OTA/b",
},
args: []string{"OTA/android-info.txt:android-info.txt", "IMAGES/*.img:."},
outputFiles: []string{
"android-info.txt",
"system.img",
"recovery.img",
"vendor.img",
},
},
{
name: "sorted filter globs",
inputFiles: []string{
"RADIO/a",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
"OTA/android-info.txt",
"OTA/b",
},
sortGlobs: true,
args: []string{"IMAGES/*.img:.", "OTA/android-info.txt:android-info.txt"},
outputFiles: []string{
"recovery.img",
"system.img",
"vendor.img",
"android-info.txt",
},
},
{
name: "sort all",
inputFiles: []string{
"RADIO/a",
"IMAGES/system.img",
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/vendor.img",
"OTA/b",
"OTA/android-info.txt",
},
sortGlobs: true,
args: []string{"**"},
outputFiles: []string{
"IMAGES/b.txt",
"IMAGES/recovery.img",
"IMAGES/system.img",
"IMAGES/vendor.img",
"OTA/android-info.txt",
"OTA/b",
"RADIO/a",
},
},
{
name: "double input",
inputFiles: []string{
"b",
"a",
},
args: []string{"a:a2", "**"},
outputFiles: []string{
"a2",
"b",
"a",
},
},
}
func errorString(e error) string {
if e == nil {
return ""
}
return e.Error()
}
func TestZip2Zip(t *testing.T) {
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
inputBuf := &bytes.Buffer{}
outputBuf := &bytes.Buffer{}
inputWriter := zip.NewWriter(inputBuf)
for _, file := range testCase.inputFiles {
w, err := inputWriter.Create(file)
if err != nil {
t.Fatal(err)
}
fmt.Fprintln(w, "test")
}
inputWriter.Close()
inputBytes := inputBuf.Bytes()
inputReader, err := zip.NewReader(bytes.NewReader(inputBytes), int64(len(inputBytes)))
if err != nil {
t.Fatal(err)
}
outputWriter := zip.NewWriter(outputBuf)
err = zip2zip(inputReader, outputWriter, testCase.sortGlobs, false, testCase.args)
if errorString(testCase.err) != errorString(err) {
t.Fatalf("Unexpected error:\n got: %q\nwant: %q", errorString(err), errorString(testCase.err))
}
outputWriter.Close()
outputBytes := outputBuf.Bytes()
outputReader, err := zip.NewReader(bytes.NewReader(outputBytes), int64(len(outputBytes)))
if err != nil {
t.Fatal(err)
}
var outputFiles []string
if len(outputReader.File) > 0 {
outputFiles = make([]string, len(outputReader.File))
for i, file := range outputReader.File {
outputFiles[i] = file.Name
}
}
if !reflect.DeepEqual(testCase.outputFiles, outputFiles) {
t.Fatalf("Output file list does not match:\n got: %v\nwant: %v", outputFiles, testCase.outputFiles)
}
})
}
}