diff --git a/cmd/diff_target_files/compare.go b/cmd/diff_target_files/compare.go new file mode 100644 index 000000000..00cd9ca10 --- /dev/null +++ b/cmd/diff_target_files/compare.go @@ -0,0 +1,133 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" +) + +// compareTargetFiles takes two ZipArtifacts and compares the files they contain by examining +// the path, size, and CRC of each file. +func compareTargetFiles(priZip, refZip ZipArtifact, artifact string, whitelists []whitelist, filters []string) (zipDiff, error) { + priZipFiles, err := priZip.Files() + if err != nil { + return zipDiff{}, fmt.Errorf("error fetching target file lists from primary zip %v", err) + } + + refZipFiles, err := refZip.Files() + if err != nil { + return zipDiff{}, fmt.Errorf("error fetching target file lists from reference zip %v", err) + } + + priZipFiles, err = filterTargetZipFiles(priZipFiles, artifact, filters) + if err != nil { + return zipDiff{}, err + } + + refZipFiles, err = filterTargetZipFiles(refZipFiles, artifact, filters) + if err != nil { + return zipDiff{}, err + } + + // Compare the file lists from both builds + diff := diffTargetFilesLists(refZipFiles, priZipFiles) + + return applyWhitelists(diff, whitelists) +} + +// zipDiff contains the list of files that differ between two zip files. +type zipDiff struct { + modified [][2]*ZipArtifactFile + onlyInA, onlyInB []*ZipArtifactFile +} + +// String pretty-prints the list of files that differ between two zip files. +func (d *zipDiff) String() string { + buf := &bytes.Buffer{} + + must := func(n int, err error) { + if err != nil { + panic(err) + } + } + + var sizeChange int64 + + if len(d.modified) > 0 { + must(fmt.Fprintln(buf, "files modified:")) + for _, f := range d.modified { + must(fmt.Fprintf(buf, " %v (%v bytes -> %v bytes)\n", f[0].Name, f[0].UncompressedSize64, f[1].UncompressedSize64)) + sizeChange += int64(f[1].UncompressedSize64) - int64(f[0].UncompressedSize64) + } + } + + if len(d.onlyInA) > 0 { + must(fmt.Fprintln(buf, "files removed:")) + for _, f := range d.onlyInA { + must(fmt.Fprintf(buf, " - %v (%v bytes)\n", f.Name, f.UncompressedSize64)) + sizeChange -= int64(f.UncompressedSize64) + } + } + + if len(d.onlyInB) > 0 { + must(fmt.Fprintln(buf, "files added:")) + for _, f := range d.onlyInB { + must(fmt.Fprintf(buf, " + %v (%v bytes)\n", f.Name, f.UncompressedSize64)) + sizeChange += int64(f.UncompressedSize64) + } + } + + if len(d.modified) > 0 || len(d.onlyInA) > 0 || len(d.onlyInB) > 0 { + must(fmt.Fprintf(buf, "total size change: %v bytes\n", sizeChange)) + } + + return buf.String() +} + +func diffTargetFilesLists(a, b []*ZipArtifactFile) zipDiff { + i := 0 + j := 0 + + diff := zipDiff{} + + for i < len(a) && j < len(b) { + if a[i].Name == b[j].Name { + if a[i].UncompressedSize64 != b[j].UncompressedSize64 || a[i].CRC32 != b[j].CRC32 { + diff.modified = append(diff.modified, [2]*ZipArtifactFile{a[i], b[j]}) + } + i++ + j++ + } else if a[i].Name < b[j].Name { + // a[i] is not present in b + diff.onlyInA = append(diff.onlyInA, a[i]) + i++ + } else { + // b[j] is not present in a + diff.onlyInB = append(diff.onlyInB, b[j]) + j++ + } + } + for i < len(a) { + diff.onlyInA = append(diff.onlyInA, a[i]) + i++ + } + for j < len(b) { + diff.onlyInB = append(diff.onlyInB, b[j]) + j++ + } + + return diff +} diff --git a/cmd/diff_target_files/compare_test.go b/cmd/diff_target_files/compare_test.go new file mode 100644 index 000000000..9d3f8a547 --- /dev/null +++ b/cmd/diff_target_files/compare_test.go @@ -0,0 +1,131 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/zip" + "reflect" + "testing" +) + +func TestDiffTargetFilesLists(t *testing.T) { + zipArtifactFile := func(name string, crc32 uint32, size uint64) *ZipArtifactFile { + return &ZipArtifactFile{ + File: &zip.File{ + FileHeader: zip.FileHeader{ + Name: name, + CRC32: crc32, + UncompressedSize64: size, + }, + }, + } + } + x0 := zipArtifactFile("x", 0, 0) + x1 := zipArtifactFile("x", 1, 0) + x2 := zipArtifactFile("x", 0, 2) + y0 := zipArtifactFile("y", 0, 0) + //y1 := zipArtifactFile("y", 1, 0) + //y2 := zipArtifactFile("y", 1, 2) + z0 := zipArtifactFile("z", 0, 0) + z1 := zipArtifactFile("z", 1, 0) + //z2 := zipArtifactFile("z", 1, 2) + + testCases := []struct { + name string + a, b []*ZipArtifactFile + diff zipDiff + }{ + { + name: "same", + a: []*ZipArtifactFile{x0, y0, z0}, + b: []*ZipArtifactFile{x0, y0, z0}, + diff: zipDiff{nil, nil, nil}, + }, + { + name: "first only in a", + a: []*ZipArtifactFile{x0, y0, z0}, + b: []*ZipArtifactFile{y0, z0}, + diff: zipDiff{nil, []*ZipArtifactFile{x0}, nil}, + }, + { + name: "middle only in a", + a: []*ZipArtifactFile{x0, y0, z0}, + b: []*ZipArtifactFile{x0, z0}, + diff: zipDiff{nil, []*ZipArtifactFile{y0}, nil}, + }, + { + name: "last only in a", + a: []*ZipArtifactFile{x0, y0, z0}, + b: []*ZipArtifactFile{x0, y0}, + diff: zipDiff{nil, []*ZipArtifactFile{z0}, nil}, + }, + + { + name: "first only in b", + a: []*ZipArtifactFile{y0, z0}, + b: []*ZipArtifactFile{x0, y0, z0}, + diff: zipDiff{nil, nil, []*ZipArtifactFile{x0}}, + }, + { + name: "middle only in b", + a: []*ZipArtifactFile{x0, z0}, + b: []*ZipArtifactFile{x0, y0, z0}, + diff: zipDiff{nil, nil, []*ZipArtifactFile{y0}}, + }, + { + name: "last only in b", + a: []*ZipArtifactFile{x0, y0}, + b: []*ZipArtifactFile{x0, y0, z0}, + diff: zipDiff{nil, nil, []*ZipArtifactFile{z0}}, + }, + + { + name: "diff", + a: []*ZipArtifactFile{x0}, + b: []*ZipArtifactFile{x1}, + diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, nil, nil}, + }, + { + name: "diff plus unique last", + a: []*ZipArtifactFile{x0, y0}, + b: []*ZipArtifactFile{x1, z0}, + diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, []*ZipArtifactFile{y0}, []*ZipArtifactFile{z0}}, + }, + { + name: "diff plus unique first", + a: []*ZipArtifactFile{x0, z0}, + b: []*ZipArtifactFile{y0, z1}, + diff: zipDiff{[][2]*ZipArtifactFile{{z0, z1}}, []*ZipArtifactFile{x0}, []*ZipArtifactFile{y0}}, + }, + { + name: "diff size", + a: []*ZipArtifactFile{x0}, + b: []*ZipArtifactFile{x2}, + diff: zipDiff{[][2]*ZipArtifactFile{{x0, x2}}, nil, nil}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + diff := diffTargetFilesLists(test.a, test.b) + + if !reflect.DeepEqual(diff, test.diff) { + + t.Errorf("diffTargetFilesLists = %v, %v, %v", diff.modified, diff.onlyInA, diff.onlyInB) + t.Errorf(" want %v, %v, %v", test.diff.modified, test.diff.onlyInA, test.diff.onlyInB) + } + }) + } +} diff --git a/cmd/diff_target_files/glob.go b/cmd/diff_target_files/glob.go new file mode 100644 index 000000000..ed91af717 --- /dev/null +++ b/cmd/diff_target_files/glob.go @@ -0,0 +1,81 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "path/filepath" + "strings" +) + +// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting +// recursive globs (**). +func Match(pattern, name string) (bool, error) { + if filepath.Base(pattern) == "**" { + return false, errors.New("pattern has '**' as last path element") + } + + patternDir := pattern[len(pattern)-1] == '/' + nameDir := name[len(name)-1] == '/' + + if patternDir != nameDir { + return false, nil + } + + if nameDir { + name = name[:len(name)-1] + pattern = pattern[:len(pattern)-1] + } + + for { + var patternFile, nameFile string + pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern) + + if patternFile == "**" { + if strings.Contains(pattern, "**") { + return false, errors.New("pattern contains multiple '**'") + } + // Test if the any prefix of name matches the part of the pattern before ** + for { + if name == "." || name == "/" { + return name == pattern, nil + } + if match, err := filepath.Match(pattern, name); err != nil { + return false, err + } else if match { + return true, nil + } + name = filepath.Dir(name) + } + } else if strings.Contains(patternFile, "**") { + return false, errors.New("pattern contains other characters between '**' and path separator") + } + + name, nameFile = filepath.Dir(name), filepath.Base(name) + + if nameFile == "." && patternFile == "." { + return true, nil + } else if nameFile == "/" && patternFile == "/" { + return true, nil + } else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" { + return false, nil + } + + match, err := filepath.Match(patternFile, nameFile) + if err != nil || !match { + return match, err + } + } +} diff --git a/cmd/diff_target_files/glob_test.go b/cmd/diff_target_files/glob_test.go new file mode 100644 index 000000000..63df68d7b --- /dev/null +++ b/cmd/diff_target_files/glob_test.go @@ -0,0 +1,158 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" +) + +func TestMatch(t *testing.T) { + testCases := []struct { + pattern, name string + match bool + }{ + {"a/*", "b/", false}, + {"a/*", "b/a", false}, + {"a/*", "b/b/", false}, + {"a/*", "b/b/c", false}, + {"a/**/*", "b/", false}, + {"a/**/*", "b/a", false}, + {"a/**/*", "b/b/", false}, + {"a/**/*", "b/b/c", false}, + + {"a/*", "a/", false}, + {"a/*", "a/a", true}, + {"a/*", "a/b/", false}, + {"a/*", "a/b/c", false}, + + {"a/*/", "a/", false}, + {"a/*/", "a/a", false}, + {"a/*/", "a/b/", true}, + {"a/*/", "a/b/c", false}, + + {"a/**/*", "a/", false}, + {"a/**/*", "a/a", true}, + {"a/**/*", "a/b/", false}, + {"a/**/*", "a/b/c", true}, + + {"a/**/*/", "a/", false}, + {"a/**/*/", "a/a", false}, + {"a/**/*/", "a/b/", true}, + {"a/**/*/", "a/b/c", false}, + + {"**/*", "a/", false}, + {"**/*", "a/a", true}, + {"**/*", "a/b/", false}, + {"**/*", "a/b/c", true}, + + {"**/*/", "a/", true}, + {"**/*/", "a/a", false}, + {"**/*/", "a/b/", true}, + {"**/*/", "a/b/c", false}, + + {`a/\*\*/\*`, `a/**/*`, true}, + {`a/\*\*/\*`, `a/a/*`, false}, + {`a/\*\*/\*`, `a/**/a`, false}, + {`a/\*\*/\*`, `a/a/a`, false}, + + {`a/**/\*`, `a/**/*`, true}, + {`a/**/\*`, `a/a/*`, true}, + {`a/**/\*`, `a/**/a`, false}, + {`a/**/\*`, `a/a/a`, false}, + + {`a/\*\*/*`, `a/**/*`, true}, + {`a/\*\*/*`, `a/a/*`, false}, + {`a/\*\*/*`, `a/**/a`, true}, + {`a/\*\*/*`, `a/a/a`, false}, + + {`*/**/a`, `a/a/a`, true}, + {`*/**/a`, `*/a/a`, true}, + {`*/**/a`, `a/**/a`, true}, + {`*/**/a`, `*/**/a`, true}, + + {`\*/\*\*/a`, `a/a/a`, false}, + {`\*/\*\*/a`, `*/a/a`, false}, + {`\*/\*\*/a`, `a/**/a`, false}, + {`\*/\*\*/a`, `*/**/a`, true}, + + {`a/?`, `a/?`, true}, + {`a/?`, `a/a`, true}, + {`a/\?`, `a/?`, true}, + {`a/\?`, `a/a`, false}, + + {`a/?`, `a/?`, true}, + {`a/?`, `a/a`, true}, + {`a/\?`, `a/?`, true}, + {`a/\?`, `a/a`, false}, + + {`a/[a-c]`, `a/b`, true}, + {`a/[abc]`, `a/b`, true}, + + {`a/\[abc]`, `a/b`, false}, + {`a/\[abc]`, `a/[abc]`, true}, + + {`a/\[abc\]`, `a/b`, false}, + {`a/\[abc\]`, `a/[abc]`, true}, + + {`a/?`, `a/?`, true}, + {`a/?`, `a/a`, true}, + {`a/\?`, `a/?`, true}, + {`a/\?`, `a/a`, false}, + + {"/a/*", "/a/", false}, + {"/a/*", "/a/a", true}, + {"/a/*", "/a/b/", false}, + {"/a/*", "/a/b/c", false}, + + {"/a/*/", "/a/", false}, + {"/a/*/", "/a/a", false}, + {"/a/*/", "/a/b/", true}, + {"/a/*/", "/a/b/c", false}, + + {"/a/**/*", "/a/", false}, + {"/a/**/*", "/a/a", true}, + {"/a/**/*", "/a/b/", false}, + {"/a/**/*", "/a/b/c", true}, + + {"/**/*", "/a/", false}, + {"/**/*", "/a/a", true}, + {"/**/*", "/a/b/", false}, + {"/**/*", "/a/b/c", true}, + + {"/**/*/", "/a/", true}, + {"/**/*/", "/a/a", false}, + {"/**/*/", "/a/b/", true}, + {"/**/*/", "/a/b/c", false}, + + {`a`, `/a`, false}, + {`/a`, `a`, false}, + {`*`, `/a`, false}, + {`/*`, `a`, false}, + {`**/*`, `/a`, false}, + {`/**/*`, `a`, false}, + } + + for _, test := range testCases { + t.Run(test.pattern+","+test.name, func(t *testing.T) { + match, err := Match(test.pattern, test.name) + if err != nil { + t.Fatal(err) + } + if match != test.match { + t.Errorf("want: %v, got %v", test.match, match) + } + }) + } +} diff --git a/cmd/diff_target_files/props.whitelist b/cmd/diff_target_files/props.whitelist new file mode 100644 index 000000000..9245b8bdb --- /dev/null +++ b/cmd/diff_target_files/props.whitelist @@ -0,0 +1,18 @@ +[ + // Ignore date, version and hostname properties in build.prop and prop.default files. + { + "Paths": [ + "**/build.prop", + "**/prop.default" + ], + "IgnoreMatchingLines": [ + "ro\\..*build\\.date=.*", + "ro\\..*build\\.date\\.utc=.*", + "ro\\..*build\\.version\\.incremental=.*", + "ro\\..*build\\.fingerprint=.*", + "ro\\.build\\.display\\.id=.*", + "ro\\.build\\.description=.*", + "ro\\.build\\.host=.*" + ] + } +] \ No newline at end of file diff --git a/cmd/diff_target_files/target_files.go b/cmd/diff_target_files/target_files.go new file mode 100644 index 000000000..8705ca700 --- /dev/null +++ b/cmd/diff_target_files/target_files.go @@ -0,0 +1,86 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "strings" +) + +const targetFilesPattern = "*-target_files-*.zip" + +var targetZipPartitions = []string{ + "BOOT/RAMDISK/", + "BOOT/", + "DATA/", + "ODM/", + "OEM/", + "PRODUCT/", + "PRODUCT_SERVICES/", + "ROOT/", + "SYSTEM/", + "SYSTEM_OTHER/", + "VENDOR/", +} + +var targetZipFilter = []string{ + "IMAGES/", + "OTA/", + "META/", + "PREBUILT_IMAGES/", + "RADIO/", +} + +func filterTargetZipFiles(files []*ZipArtifactFile, artifact string, patterns []string) ([]*ZipArtifactFile, error) { + var ret []*ZipArtifactFile +outer: + for _, f := range files { + if f.FileInfo().IsDir() { + continue + } + + if artifact == targetFilesPattern { + found := false + for _, p := range targetZipPartitions { + if strings.HasPrefix(f.Name, p) { + f.Name = strings.ToLower(p) + strings.TrimPrefix(f.Name, p) + found = true + } + } + for _, filter := range targetZipFilter { + if strings.HasPrefix(f.Name, filter) { + continue outer + } + } + + if !found { + return nil, fmt.Errorf("unmatched prefix for %s", f.Name) + } + } + + if patterns != nil { + for _, pattern := range patterns { + match, _ := Match(pattern, f.Name) + if match { + ret = append(ret, f) + } + } + } else { + ret = append(ret, f) + } + } + + return ret, nil +} diff --git a/cmd/diff_target_files/whitelist.go b/cmd/diff_target_files/whitelist.go new file mode 100644 index 000000000..f00fc1ee2 --- /dev/null +++ b/cmd/diff_target_files/whitelist.go @@ -0,0 +1,251 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "os" + "regexp" + "strings" + "unicode" +) + +type jsonWhitelist struct { + Paths []string + IgnoreMatchingLines []string +} + +type whitelist struct { + path string + ignoreMatchingLines []string +} + +func parseWhitelists(whitelists []string, whitelistFiles []string) ([]whitelist, error) { + var ret []whitelist + + add := func(path string, ignoreMatchingLines []string) { + for _, x := range ret { + if x.path == path { + x.ignoreMatchingLines = append(x.ignoreMatchingLines, ignoreMatchingLines...) + return + } + } + + ret = append(ret, whitelist{ + path: path, + ignoreMatchingLines: ignoreMatchingLines, + }) + } + + for _, file := range whitelistFiles { + newWhitelists, err := parseWhitelistFile(file) + if err != nil { + return nil, err + } + + for _, w := range newWhitelists { + add(w.path, w.ignoreMatchingLines) + } + } + + for _, s := range whitelists { + colon := strings.IndexRune(s, ':') + var ignoreMatchingLines []string + if colon >= 0 { + ignoreMatchingLines = []string{s[colon+1:]} + } + add(s, ignoreMatchingLines) + } + + return ret, nil +} + +func parseWhitelistFile(file string) ([]whitelist, error) { + r, err := os.Open(file) + if err != nil { + return nil, err + } + defer r.Close() + + d := json.NewDecoder(newJSONCommentStripper(r)) + + var jsonWhitelists []jsonWhitelist + + err = d.Decode(&jsonWhitelists) + + var whitelists []whitelist + for _, w := range jsonWhitelists { + for _, p := range w.Paths { + whitelists = append(whitelists, whitelist{ + path: p, + ignoreMatchingLines: w.IgnoreMatchingLines, + }) + } + } + + return whitelists, err +} + +func filterModifiedPaths(l [][2]*ZipArtifactFile, whitelists []whitelist) ([][2]*ZipArtifactFile, error) { +outer: + for i := 0; i < len(l); i++ { + for _, w := range whitelists { + if match, err := Match(w.path, l[i][0].Name); err != nil { + return l, err + } else if match { + if match, err := diffIgnoringMatchingLines(l[i][0], l[i][1], w.ignoreMatchingLines); err != nil { + return l, err + } else if match || len(w.ignoreMatchingLines) == 0 { + l = append(l[:i], l[i+1:]...) + i-- + } + continue outer + } + } + } + + if len(l) == 0 { + l = nil + } + + return l, nil +} + +func filterNewPaths(l []*ZipArtifactFile, whitelists []whitelist) ([]*ZipArtifactFile, error) { +outer: + for i := 0; i < len(l); i++ { + for _, w := range whitelists { + if match, err := Match(w.path, l[i].Name); err != nil { + return l, err + } else if match && len(w.ignoreMatchingLines) == 0 { + l = append(l[:i], l[i+1:]...) + i-- + } + continue outer + } + } + + if len(l) == 0 { + l = nil + } + + return l, nil +} + +func diffIgnoringMatchingLines(a *ZipArtifactFile, b *ZipArtifactFile, ignoreMatchingLines []string) (match bool, err error) { + lineMatchesIgnores := func(b []byte) (bool, error) { + for _, m := range ignoreMatchingLines { + if match, err := regexp.Match(m, b); err != nil { + return false, err + } else if match { + return match, nil + } + } + return false, nil + } + + filter := func(z *ZipArtifactFile) ([]byte, error) { + var ret []byte + + r, err := z.Open() + if err != nil { + return nil, err + } + s := bufio.NewScanner(r) + + for s.Scan() { + if match, err := lineMatchesIgnores(s.Bytes()); err != nil { + return nil, err + } else if !match { + ret = append(ret, "\n"...) + ret = append(ret, s.Bytes()...) + } + } + + return ret, nil + } + + bufA, err := filter(a) + if err != nil { + return false, err + } + bufB, err := filter(b) + if err != nil { + return false, err + } + + return bytes.Compare(bufA, bufB) == 0, nil +} + +func applyWhitelists(diff zipDiff, whitelists []whitelist) (zipDiff, error) { + var err error + + diff.modified, err = filterModifiedPaths(diff.modified, whitelists) + if err != nil { + return diff, err + } + diff.onlyInA, err = filterNewPaths(diff.onlyInA, whitelists) + if err != nil { + return diff, err + } + diff.onlyInB, err = filterNewPaths(diff.onlyInB, whitelists) + if err != nil { + return diff, err + } + + return diff, nil +} + +func newJSONCommentStripper(r io.Reader) *jsonCommentStripper { + return &jsonCommentStripper{ + r: bufio.NewReader(r), + } +} + +type jsonCommentStripper struct { + r *bufio.Reader + b []byte + err error +} + +func (j *jsonCommentStripper) Read(buf []byte) (int, error) { + for len(j.b) == 0 { + if j.err != nil { + return 0, j.err + } + + j.b, j.err = j.r.ReadBytes('\n') + + if isComment(j.b) { + j.b = nil + } + } + + n := copy(buf, j.b) + j.b = j.b[n:] + return n, nil +} + +var commentPrefix = []byte("//") + +func isComment(b []byte) bool { + for len(b) > 0 && unicode.IsSpace(rune(b[0])) { + b = b[1:] + } + return bytes.HasPrefix(b, commentPrefix) +} diff --git a/cmd/diff_target_files/whitelist_test.go b/cmd/diff_target_files/whitelist_test.go new file mode 100644 index 000000000..4b19fdd21 --- /dev/null +++ b/cmd/diff_target_files/whitelist_test.go @@ -0,0 +1,126 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/zip" + "bytes" + "reflect" + "testing" +) + +func bytesToZipArtifactFile(name string, data []byte) *ZipArtifactFile { + buf := &bytes.Buffer{} + w := zip.NewWriter(buf) + f, err := w.Create(name) + if err != nil { + panic(err) + } + _, err = f.Write(data) + if err != nil { + panic(err) + } + + w.Close() + + r, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + panic(err) + } + + return &ZipArtifactFile{r.File[0]} +} + +var f1a = bytesToZipArtifactFile("dir/f1", []byte(` +a +foo: bar +c +`)) + +var f1b = bytesToZipArtifactFile("dir/f1", []byte(` +a +foo: baz +c +`)) + +var f2 = bytesToZipArtifactFile("dir/f2", nil) + +func Test_applyWhitelists(t *testing.T) { + type args struct { + diff zipDiff + whitelists []whitelist + } + tests := []struct { + name string + args args + want zipDiff + wantErr bool + }{ + { + name: "simple", + args: args{ + diff: zipDiff{ + onlyInA: []*ZipArtifactFile{f1a, f2}, + }, + whitelists: []whitelist{{path: "dir/f1"}}, + }, + want: zipDiff{ + onlyInA: []*ZipArtifactFile{f2}, + }, + }, + { + name: "glob", + args: args{ + diff: zipDiff{ + onlyInA: []*ZipArtifactFile{f1a, f2}, + }, + whitelists: []whitelist{{path: "dir/*"}}, + }, + want: zipDiff{}, + }, + { + name: "modified", + args: args{ + diff: zipDiff{ + modified: [][2]*ZipArtifactFile{{f1a, f1b}}, + }, + whitelists: []whitelist{{path: "dir/*"}}, + }, + want: zipDiff{}, + }, + { + name: "matching lines", + args: args{ + diff: zipDiff{ + modified: [][2]*ZipArtifactFile{{f1a, f1b}}, + }, + whitelists: []whitelist{{path: "dir/*", ignoreMatchingLines: []string{"foo: .*"}}}, + }, + want: zipDiff{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := applyWhitelists(tt.args.diff, tt.args.whitelists) + if (err != nil) != tt.wantErr { + t.Errorf("applyWhitelists() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("applyWhitelists() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/diff_target_files/zip_artifact.go b/cmd/diff_target_files/zip_artifact.go new file mode 100644 index 000000000..08ce889a4 --- /dev/null +++ b/cmd/diff_target_files/zip_artifact.go @@ -0,0 +1,174 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/zip" + "context" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// ZipArtifact represents a zip file that may be local or remote. +type ZipArtifact interface { + // Files returns the list of files contained in the zip file. + Files() ([]*ZipArtifactFile, error) + + // Close closes the zip file artifact. + Close() +} + +// localZipArtifact is a handle to a local zip file artifact. +type localZipArtifact struct { + zr *zip.ReadCloser + files []*ZipArtifactFile +} + +// NewLocalZipArtifact returns a ZipArtifact for a local zip file.. +func NewLocalZipArtifact(name string) (ZipArtifact, error) { + zr, err := zip.OpenReader(name) + if err != nil { + return nil, err + } + + var files []*ZipArtifactFile + for _, zf := range zr.File { + files = append(files, &ZipArtifactFile{zf}) + } + + return &localZipArtifact{ + zr: zr, + files: files, + }, nil +} + +// Files returns the list of files contained in the local zip file artifact. +func (z *localZipArtifact) Files() ([]*ZipArtifactFile, error) { + return z.files, nil +} + +// Close closes the buffered reader of the local zip file artifact. +func (z *localZipArtifact) Close() { + z.zr.Close() +} + +// ZipArtifactFile contains a zip.File handle to the data inside the remote *-target_files-*.zip +// build artifact. +type ZipArtifactFile struct { + *zip.File +} + +// Extract begins extract a file from inside a ZipArtifact. It returns an +// ExtractedZipArtifactFile handle. +func (zf *ZipArtifactFile) Extract(ctx context.Context, dir string, + limiter chan bool) *ExtractedZipArtifactFile { + + d := &ExtractedZipArtifactFile{ + initCh: make(chan struct{}), + } + + go func() { + defer close(d.initCh) + limiter <- true + defer func() { <-limiter }() + + zr, err := zf.Open() + if err != nil { + d.err = err + return + } + defer zr.Close() + + crc := crc32.NewIEEE() + r := io.TeeReader(zr, crc) + + if filepath.Clean(zf.Name) != zf.Name { + d.err = fmt.Errorf("invalid filename %q", zf.Name) + return + } + path := filepath.Join(dir, zf.Name) + + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + d.err = err + return + } + + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + d.err = err + return + } + + if zf.Mode().IsRegular() { + w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, zf.Mode()) + if err != nil { + d.err = err + return + } + defer w.Close() + + _, err = io.Copy(w, r) + if err != nil { + d.err = err + return + } + } else if zf.Mode()&os.ModeSymlink != 0 { + target, err := ioutil.ReadAll(r) + if err != nil { + d.err = err + return + } + + err = os.Symlink(string(target), path) + if err != nil { + d.err = err + return + } + } else { + d.err = fmt.Errorf("unknown mode %q", zf.Mode()) + return + } + + if crc.Sum32() != zf.CRC32 { + d.err = fmt.Errorf("crc mismatch for %v", zf.Name) + return + } + + d.path = path + }() + + return d +} + +// ExtractedZipArtifactFile is a handle to a downloaded file from a remoteZipArtifact. The download +// may still be in progress, and will be complete with Path() returns. +type ExtractedZipArtifactFile struct { + initCh chan struct{} + err error + + path string +} + +// Path returns the path to the downloaded file and any errors that occurred during the download. +// It will block until the download is complete. +func (d *ExtractedZipArtifactFile) Path() (string, error) { + <-d.initCh + return d.path, d.err +}