openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Import files from compare_target_files for use in diff_target_files 

Copied from cl/240594925.

Bug: 121158314
Test: copied unit tests
Change-Id: I2e91126285dcd33171ff8b8dbfcfa5d48501f535
This commit is contained in:
Colin Cross 2019-03-26 15:50:03 -07:00
parent 65c95ff1fb
commit c45c3b5ec6
9 changed files with 1158 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
cmd/diff_target_files/compare.go Normal file
View File

@ -0,0 +1,133 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"fmt"
)
// compareTargetFiles takes two ZipArtifacts and compares the files they contain by examining
// the path, size, and CRC of each file.
func compareTargetFiles(priZip, refZip ZipArtifact, artifact string, whitelists []whitelist, filters []string) (zipDiff, error) {
priZipFiles, err := priZip.Files()
if err != nil {
return zipDiff{}, fmt.Errorf("error fetching target file lists from primary zip %v", err)
}
refZipFiles, err := refZip.Files()
if err != nil {
return zipDiff{}, fmt.Errorf("error fetching target file lists from reference zip %v", err)
}
priZipFiles, err = filterTargetZipFiles(priZipFiles, artifact, filters)
if err != nil {
return zipDiff{}, err
}
refZipFiles, err = filterTargetZipFiles(refZipFiles, artifact, filters)
if err != nil {
return zipDiff{}, err
}
// Compare the file lists from both builds
diff := diffTargetFilesLists(refZipFiles, priZipFiles)
return applyWhitelists(diff, whitelists)
}
// zipDiff contains the list of files that differ between two zip files.
type zipDiff struct {
modified [][2]*ZipArtifactFile
onlyInA, onlyInB []*ZipArtifactFile
}
// String pretty-prints the list of files that differ between two zip files.
func (d *zipDiff) String() string {
buf := &bytes.Buffer{}
must := func(n int, err error) {
if err != nil {
panic(err)
}
}
var sizeChange int64
if len(d.modified) > 0 {
must(fmt.Fprintln(buf, "files modified:"))
for _, f := range d.modified {
must(fmt.Fprintf(buf, " %v (%v bytes -> %v bytes)\n", f[0].Name, f[0].UncompressedSize64, f[1].UncompressedSize64))
sizeChange += int64(f[1].UncompressedSize64) - int64(f[0].UncompressedSize64)
}
}
if len(d.onlyInA) > 0 {
must(fmt.Fprintln(buf, "files removed:"))
for _, f := range d.onlyInA {
must(fmt.Fprintf(buf, " - %v (%v bytes)\n", f.Name, f.UncompressedSize64))
sizeChange -= int64(f.UncompressedSize64)
}
}
if len(d.onlyInB) > 0 {
must(fmt.Fprintln(buf, "files added:"))
for _, f := range d.onlyInB {
must(fmt.Fprintf(buf, " + %v (%v bytes)\n", f.Name, f.UncompressedSize64))
sizeChange += int64(f.UncompressedSize64)
}
}
if len(d.modified) > 0 || len(d.onlyInA) > 0 || len(d.onlyInB) > 0 {
must(fmt.Fprintf(buf, "total size change: %v bytes\n", sizeChange))
}
return buf.String()
}
func diffTargetFilesLists(a, b []*ZipArtifactFile) zipDiff {
i := 0
j := 0
diff := zipDiff{}
for i < len(a) && j < len(b) {
if a[i].Name == b[j].Name {
if a[i].UncompressedSize64 != b[j].UncompressedSize64 || a[i].CRC32 != b[j].CRC32 {
diff.modified = append(diff.modified, [2]*ZipArtifactFile{a[i], b[j]})
}
i++
j++
} else if a[i].Name < b[j].Name {
// a[i] is not present in b
diff.onlyInA = append(diff.onlyInA, a[i])
i++
} else {
// b[j] is not present in a
diff.onlyInB = append(diff.onlyInB, b[j])
j++
}
}
for i < len(a) {
diff.onlyInA = append(diff.onlyInA, a[i])
i++
}
for j < len(b) {
diff.onlyInB = append(diff.onlyInB, b[j])
j++
}
return diff
}

131
cmd/diff_target_files/compare_test.go Normal file
View File

@ -0,0 +1,131 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/zip"
"reflect"
"testing"
)
func TestDiffTargetFilesLists(t *testing.T) {
zipArtifactFile := func(name string, crc32 uint32, size uint64) *ZipArtifactFile {
return &ZipArtifactFile{
File: &zip.File{
FileHeader: zip.FileHeader{
Name: name,
CRC32: crc32,
UncompressedSize64: size,
},
},
}
}
x0 := zipArtifactFile("x", 0, 0)
x1 := zipArtifactFile("x", 1, 0)
x2 := zipArtifactFile("x", 0, 2)
y0 := zipArtifactFile("y", 0, 0)
//y1 := zipArtifactFile("y", 1, 0)
//y2 := zipArtifactFile("y", 1, 2)
z0 := zipArtifactFile("z", 0, 0)
z1 := zipArtifactFile("z", 1, 0)
//z2 := zipArtifactFile("z", 1, 2)
testCases := []struct {
name string
a, b []*ZipArtifactFile
diff zipDiff
}{
{
name: "same",
a: []*ZipArtifactFile{x0, y0, z0},
b: []*ZipArtifactFile{x0, y0, z0},
diff: zipDiff{nil, nil, nil},
},
{
name: "first only in a",
a: []*ZipArtifactFile{x0, y0, z0},
b: []*ZipArtifactFile{y0, z0},
diff: zipDiff{nil, []*ZipArtifactFile{x0}, nil},
},
{
name: "middle only in a",
a: []*ZipArtifactFile{x0, y0, z0},
b: []*ZipArtifactFile{x0, z0},
diff: zipDiff{nil, []*ZipArtifactFile{y0}, nil},
},
{
name: "last only in a",
a: []*ZipArtifactFile{x0, y0, z0},
b: []*ZipArtifactFile{x0, y0},
diff: zipDiff{nil, []*ZipArtifactFile{z0}, nil},
},
{
name: "first only in b",
a: []*ZipArtifactFile{y0, z0},
b: []*ZipArtifactFile{x0, y0, z0},
diff: zipDiff{nil, nil, []*ZipArtifactFile{x0}},
},
{
name: "middle only in b",
a: []*ZipArtifactFile{x0, z0},
b: []*ZipArtifactFile{x0, y0, z0},
diff: zipDiff{nil, nil, []*ZipArtifactFile{y0}},
},
{
name: "last only in b",
a: []*ZipArtifactFile{x0, y0},
b: []*ZipArtifactFile{x0, y0, z0},
diff: zipDiff{nil, nil, []*ZipArtifactFile{z0}},
},
{
name: "diff",
a: []*ZipArtifactFile{x0},
b: []*ZipArtifactFile{x1},
diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, nil, nil},
},
{
name: "diff plus unique last",
a: []*ZipArtifactFile{x0, y0},
b: []*ZipArtifactFile{x1, z0},
diff: zipDiff{[][2]*ZipArtifactFile{{x0, x1}}, []*ZipArtifactFile{y0}, []*ZipArtifactFile{z0}},
},
{
name: "diff plus unique first",
a: []*ZipArtifactFile{x0, z0},
b: []*ZipArtifactFile{y0, z1},
diff: zipDiff{[][2]*ZipArtifactFile{{z0, z1}}, []*ZipArtifactFile{x0}, []*ZipArtifactFile{y0}},
},
{
name: "diff size",
a: []*ZipArtifactFile{x0},
b: []*ZipArtifactFile{x2},
diff: zipDiff{[][2]*ZipArtifactFile{{x0, x2}}, nil, nil},
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
diff := diffTargetFilesLists(test.a, test.b)
if !reflect.DeepEqual(diff, test.diff) {
t.Errorf("diffTargetFilesLists = %v, %v, %v", diff.modified, diff.onlyInA, diff.onlyInB)
t.Errorf(" want %v, %v, %v", test.diff.modified, test.diff.onlyInA, test.diff.onlyInB)
}
})
}
}

81
cmd/diff_target_files/glob.go Normal file
View File

@ -0,0 +1,81 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"errors"
"path/filepath"
"strings"
)
// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
// recursive globs (**).
func Match(pattern, name string) (bool, error) {
if filepath.Base(pattern) == "**" {
return false, errors.New("pattern has '**' as last path element")
}
patternDir := pattern[len(pattern)-1] == '/'
nameDir := name[len(name)-1] == '/'
if patternDir != nameDir {
return false, nil
}
if nameDir {
name = name[:len(name)-1]
pattern = pattern[:len(pattern)-1]
}
for {
var patternFile, nameFile string
pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)
if patternFile == "**" {
if strings.Contains(pattern, "**") {
return false, errors.New("pattern contains multiple '**'")
}
// Test if the any prefix of name matches the part of the pattern before **
for {
if name == "." || name == "/" {
return name == pattern, nil
}
if match, err := filepath.Match(pattern, name); err != nil {
return false, err
} else if match {
return true, nil
}
name = filepath.Dir(name)
}
} else if strings.Contains(patternFile, "**") {
return false, errors.New("pattern contains other characters between '**' and path separator")
}
name, nameFile = filepath.Dir(name), filepath.Base(name)
if nameFile == "." && patternFile == "." {
return true, nil
} else if nameFile == "/" && patternFile == "/" {
return true, nil
} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
return false, nil
}
match, err := filepath.Match(patternFile, nameFile)
if err != nil || !match {
return match, err
}
}
}

158
cmd/diff_target_files/glob_test.go Normal file
View File

@ -0,0 +1,158 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"testing"
)
func TestMatch(t *testing.T) {
testCases := []struct {
pattern, name string
match bool
}{
{"a/*", "b/", false},
{"a/*", "b/a", false},
{"a/*", "b/b/", false},
{"a/*", "b/b/c", false},
{"a/**/*", "b/", false},
{"a/**/*", "b/a", false},
{"a/**/*", "b/b/", false},
{"a/**/*", "b/b/c", false},
{"a/*", "a/", false},
{"a/*", "a/a", true},
{"a/*", "a/b/", false},
{"a/*", "a/b/c", false},
{"a/*/", "a/", false},
{"a/*/", "a/a", false},
{"a/*/", "a/b/", true},
{"a/*/", "a/b/c", false},
{"a/**/*", "a/", false},
{"a/**/*", "a/a", true},
{"a/**/*", "a/b/", false},
{"a/**/*", "a/b/c", true},
{"a/**/*/", "a/", false},
{"a/**/*/", "a/a", false},
{"a/**/*/", "a/b/", true},
{"a/**/*/", "a/b/c", false},
{"**/*", "a/", false},
{"**/*", "a/a", true},
{"**/*", "a/b/", false},
{"**/*", "a/b/c", true},
{"**/*/", "a/", true},
{"**/*/", "a/a", false},
{"**/*/", "a/b/", true},
{"**/*/", "a/b/c", false},
{`a/\*\*/\*`, `a/**/*`, true},
{`a/\*\*/\*`, `a/a/*`, false},
{`a/\*\*/\*`, `a/**/a`, false},
{`a/\*\*/\*`, `a/a/a`, false},
{`a/**/\*`, `a/**/*`, true},
{`a/**/\*`, `a/a/*`, true},
{`a/**/\*`, `a/**/a`, false},
{`a/**/\*`, `a/a/a`, false},
{`a/\*\*/*`, `a/**/*`, true},
{`a/\*\*/*`, `a/a/*`, false},
{`a/\*\*/*`, `a/**/a`, true},
{`a/\*\*/*`, `a/a/a`, false},
{`*/**/a`, `a/a/a`, true},
{`*/**/a`, `*/a/a`, true},
{`*/**/a`, `a/**/a`, true},
{`*/**/a`, `*/**/a`, true},
{`\*/\*\*/a`, `a/a/a`, false},
{`\*/\*\*/a`, `*/a/a`, false},
{`\*/\*\*/a`, `a/**/a`, false},
{`\*/\*\*/a`, `*/**/a`, true},
{`a/?`, `a/?`, true},
{`a/?`, `a/a`, true},
{`a/\?`, `a/?`, true},
{`a/\?`, `a/a`, false},
{`a/?`, `a/?`, true},
{`a/?`, `a/a`, true},
{`a/\?`, `a/?`, true},
{`a/\?`, `a/a`, false},
{`a/[a-c]`, `a/b`, true},
{`a/[abc]`, `a/b`, true},
{`a/\[abc]`, `a/b`, false},
{`a/\[abc]`, `a/[abc]`, true},
{`a/\[abc\]`, `a/b`, false},
{`a/\[abc\]`, `a/[abc]`, true},
{`a/?`, `a/?`, true},
{`a/?`, `a/a`, true},
{`a/\?`, `a/?`, true},
{`a/\?`, `a/a`, false},
{"/a/*", "/a/", false},
{"/a/*", "/a/a", true},
{"/a/*", "/a/b/", false},
{"/a/*", "/a/b/c", false},
{"/a/*/", "/a/", false},
{"/a/*/", "/a/a", false},
{"/a/*/", "/a/b/", true},
{"/a/*/", "/a/b/c", false},
{"/a/**/*", "/a/", false},
{"/a/**/*", "/a/a", true},
{"/a/**/*", "/a/b/", false},
{"/a/**/*", "/a/b/c", true},
{"/**/*", "/a/", false},
{"/**/*", "/a/a", true},
{"/**/*", "/a/b/", false},
{"/**/*", "/a/b/c", true},
{"/**/*/", "/a/", true},
{"/**/*/", "/a/a", false},
{"/**/*/", "/a/b/", true},
{"/**/*/", "/a/b/c", false},
{`a`, `/a`, false},
{`/a`, `a`, false},
{`*`, `/a`, false},
{`/*`, `a`, false},
{`**/*`, `/a`, false},
{`/**/*`, `a`, false},
}
for _, test := range testCases {
t.Run(test.pattern+","+test.name, func(t *testing.T) {
match, err := Match(test.pattern, test.name)
if err != nil {
t.Fatal(err)
}
if match != test.match {
t.Errorf("want: %v, got %v", test.match, match)
}
})
}
}

18
cmd/diff_target_files/props.whitelist Normal file
View File

@ -0,0 +1,18 @@
[
// Ignore date, version and hostname properties in build.prop and prop.default files.
{
"Paths": [
"**/build.prop",
"**/prop.default"
],
"IgnoreMatchingLines": [
"ro\\..*build\\.date=.*",
"ro\\..*build\\.date\\.utc=.*",
"ro\\..*build\\.version\\.incremental=.*",
"ro\\..*build\\.fingerprint=.*",
"ro\\.build\\.display\\.id=.*",
"ro\\.build\\.description=.*",
"ro\\.build\\.host=.*"
]
}
]

86
cmd/diff_target_files/target_files.go Normal file
View File

@ -0,0 +1,86 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"strings"
)
const targetFilesPattern = "*-target_files-*.zip"
var targetZipPartitions = []string{
"BOOT/RAMDISK/",
"BOOT/",
"DATA/",
"ODM/",
"OEM/",
"PRODUCT/",
"PRODUCT_SERVICES/",
"ROOT/",
"SYSTEM/",
"SYSTEM_OTHER/",
"VENDOR/",
}
var targetZipFilter = []string{
"IMAGES/",
"OTA/",
"META/",
"PREBUILT_IMAGES/",
"RADIO/",
}
func filterTargetZipFiles(files []*ZipArtifactFile, artifact string, patterns []string) ([]*ZipArtifactFile, error) {
var ret []*ZipArtifactFile
outer:
for _, f := range files {
if f.FileInfo().IsDir() {
continue
}
if artifact == targetFilesPattern {
found := false
for _, p := range targetZipPartitions {
if strings.HasPrefix(f.Name, p) {
f.Name = strings.ToLower(p) + strings.TrimPrefix(f.Name, p)
found = true
}
}
for _, filter := range targetZipFilter {
if strings.HasPrefix(f.Name, filter) {
continue outer
}
}
if !found {
return nil, fmt.Errorf("unmatched prefix for %s", f.Name)
}
}
if patterns != nil {
for _, pattern := range patterns {
match, _ := Match(pattern, f.Name)
if match {
ret = append(ret, f)
}
}
} else {
ret = append(ret, f)
}
}
return ret, nil
}

251
cmd/diff_target_files/whitelist.go Normal file
View File

@ -0,0 +1,251 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bufio"
"bytes"
"encoding/json"
"io"
"os"
"regexp"
"strings"
"unicode"
)
type jsonWhitelist struct {
Paths []string
IgnoreMatchingLines []string
}
type whitelist struct {
path string
ignoreMatchingLines []string
}
func parseWhitelists(whitelists []string, whitelistFiles []string) ([]whitelist, error) {
var ret []whitelist
add := func(path string, ignoreMatchingLines []string) {
for _, x := range ret {
if x.path == path {
x.ignoreMatchingLines = append(x.ignoreMatchingLines, ignoreMatchingLines...)
return
}
}
ret = append(ret, whitelist{
path: path,
ignoreMatchingLines: ignoreMatchingLines,
})
}
for _, file := range whitelistFiles {
newWhitelists, err := parseWhitelistFile(file)
if err != nil {
return nil, err
}
for _, w := range newWhitelists {
add(w.path, w.ignoreMatchingLines)
}
}
for _, s := range whitelists {
colon := strings.IndexRune(s, ':')
var ignoreMatchingLines []string
if colon >= 0 {
ignoreMatchingLines = []string{s[colon+1:]}
}
add(s, ignoreMatchingLines)
}
return ret, nil
}
func parseWhitelistFile(file string) ([]whitelist, error) {
r, err := os.Open(file)
if err != nil {
return nil, err
}
defer r.Close()
d := json.NewDecoder(newJSONCommentStripper(r))
var jsonWhitelists []jsonWhitelist
err = d.Decode(&jsonWhitelists)
var whitelists []whitelist
for _, w := range jsonWhitelists {
for _, p := range w.Paths {
whitelists = append(whitelists, whitelist{
path: p,
ignoreMatchingLines: w.IgnoreMatchingLines,
})
}
}
return whitelists, err
}
func filterModifiedPaths(l [][2]*ZipArtifactFile, whitelists []whitelist) ([][2]*ZipArtifactFile, error) {
outer:
for i := 0; i < len(l); i++ {
for _, w := range whitelists {
if match, err := Match(w.path, l[i][0].Name); err != nil {
return l, err
} else if match {
if match, err := diffIgnoringMatchingLines(l[i][0], l[i][1], w.ignoreMatchingLines); err != nil {
return l, err
} else if match || len(w.ignoreMatchingLines) == 0 {
l = append(l[:i], l[i+1:]...)
i--
}
continue outer
}
}
}
if len(l) == 0 {
l = nil
}
return l, nil
}
func filterNewPaths(l []*ZipArtifactFile, whitelists []whitelist) ([]*ZipArtifactFile, error) {
outer:
for i := 0; i < len(l); i++ {
for _, w := range whitelists {
if match, err := Match(w.path, l[i].Name); err != nil {
return l, err
} else if match && len(w.ignoreMatchingLines) == 0 {
l = append(l[:i], l[i+1:]...)
i--
}
continue outer
}
}
if len(l) == 0 {
l = nil
}
return l, nil
}
func diffIgnoringMatchingLines(a *ZipArtifactFile, b *ZipArtifactFile, ignoreMatchingLines []string) (match bool, err error) {
lineMatchesIgnores := func(b []byte) (bool, error) {
for _, m := range ignoreMatchingLines {
if match, err := regexp.Match(m, b); err != nil {
return false, err
} else if match {
return match, nil
}
}
return false, nil
}
filter := func(z *ZipArtifactFile) ([]byte, error) {
var ret []byte
r, err := z.Open()
if err != nil {
return nil, err
}
s := bufio.NewScanner(r)
for s.Scan() {
if match, err := lineMatchesIgnores(s.Bytes()); err != nil {
return nil, err
} else if !match {
ret = append(ret, "\n"...)
ret = append(ret, s.Bytes()...)
}
}
return ret, nil
}
bufA, err := filter(a)
if err != nil {
return false, err
}
bufB, err := filter(b)
if err != nil {
return false, err
}
return bytes.Compare(bufA, bufB) == 0, nil
}
func applyWhitelists(diff zipDiff, whitelists []whitelist) (zipDiff, error) {
var err error
diff.modified, err = filterModifiedPaths(diff.modified, whitelists)
if err != nil {
return diff, err
}
diff.onlyInA, err = filterNewPaths(diff.onlyInA, whitelists)
if err != nil {
return diff, err
}
diff.onlyInB, err = filterNewPaths(diff.onlyInB, whitelists)
if err != nil {
return diff, err
}
return diff, nil
}
func newJSONCommentStripper(r io.Reader) *jsonCommentStripper {
return &jsonCommentStripper{
r: bufio.NewReader(r),
}
}
type jsonCommentStripper struct {
r *bufio.Reader
b []byte
err error
}
func (j *jsonCommentStripper) Read(buf []byte) (int, error) {
for len(j.b) == 0 {
if j.err != nil {
return 0, j.err
}
j.b, j.err = j.r.ReadBytes('\n')
if isComment(j.b) {
j.b = nil
}
}
n := copy(buf, j.b)
j.b = j.b[n:]
return n, nil
}
var commentPrefix = []byte("//")
func isComment(b []byte) bool {
for len(b) > 0 && unicode.IsSpace(rune(b[0])) {
b = b[1:]
}
return bytes.HasPrefix(b, commentPrefix)
}

126
cmd/diff_target_files/whitelist_test.go Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/zip"
"bytes"
"reflect"
"testing"
)
func bytesToZipArtifactFile(name string, data []byte) *ZipArtifactFile {
buf := &bytes.Buffer{}
w := zip.NewWriter(buf)
f, err := w.Create(name)
if err != nil {
panic(err)
}
_, err = f.Write(data)
if err != nil {
panic(err)
}
w.Close()
r, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
if err != nil {
panic(err)
}
return &ZipArtifactFile{r.File[0]}
}
var f1a = bytesToZipArtifactFile("dir/f1", []byte(`
a
foo: bar
c
`))
var f1b = bytesToZipArtifactFile("dir/f1", []byte(`
a
foo: baz
c
`))
var f2 = bytesToZipArtifactFile("dir/f2", nil)
func Test_applyWhitelists(t *testing.T) {
type args struct {
diff zipDiff
whitelists []whitelist
}
tests := []struct {
name string
args args
want zipDiff
wantErr bool
}{
{
name: "simple",
args: args{
diff: zipDiff{
onlyInA: []*ZipArtifactFile{f1a, f2},
},
whitelists: []whitelist{{path: "dir/f1"}},
},
want: zipDiff{
onlyInA: []*ZipArtifactFile{f2},
},
},
{
name: "glob",
args: args{
diff: zipDiff{
onlyInA: []*ZipArtifactFile{f1a, f2},
},
whitelists: []whitelist{{path: "dir/*"}},
},
want: zipDiff{},
},
{
name: "modified",
args: args{
diff: zipDiff{
modified: [][2]*ZipArtifactFile{{f1a, f1b}},
},
whitelists: []whitelist{{path: "dir/*"}},
},
want: zipDiff{},
},
{
name: "matching lines",
args: args{
diff: zipDiff{
modified: [][2]*ZipArtifactFile{{f1a, f1b}},
},
whitelists: []whitelist{{path: "dir/*", ignoreMatchingLines: []string{"foo: .*"}}},
},
want: zipDiff{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := applyWhitelists(tt.args.diff, tt.args.whitelists)
if (err != nil) != tt.wantErr {
t.Errorf("applyWhitelists() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("applyWhitelists() = %v, want %v", got, tt.want)
}
})
}
}

174
cmd/diff_target_files/zip_artifact.go Normal file
View File

@ -0,0 +1,174 @@
// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/zip"
"context"
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"os"
"path/filepath"
)
// ZipArtifact represents a zip file that may be local or remote.
type ZipArtifact interface {
// Files returns the list of files contained in the zip file.
Files() ([]*ZipArtifactFile, error)
// Close closes the zip file artifact.
Close()
}
// localZipArtifact is a handle to a local zip file artifact.
type localZipArtifact struct {
zr *zip.ReadCloser
files []*ZipArtifactFile
}
// NewLocalZipArtifact returns a ZipArtifact for a local zip file..
func NewLocalZipArtifact(name string) (ZipArtifact, error) {
zr, err := zip.OpenReader(name)
if err != nil {
return nil, err
}
var files []*ZipArtifactFile
for _, zf := range zr.File {
files = append(files, &ZipArtifactFile{zf})
}
return &localZipArtifact{
zr: zr,
files: files,
}, nil
}
// Files returns the list of files contained in the local zip file artifact.
func (z *localZipArtifact) Files() ([]*ZipArtifactFile, error) {
return z.files, nil
}
// Close closes the buffered reader of the local zip file artifact.
func (z *localZipArtifact) Close() {
z.zr.Close()
}
// ZipArtifactFile contains a zip.File handle to the data inside the remote *-target_files-*.zip
// build artifact.
type ZipArtifactFile struct {
*zip.File
}
// Extract begins extract a file from inside a ZipArtifact. It returns an
// ExtractedZipArtifactFile handle.
func (zf *ZipArtifactFile) Extract(ctx context.Context, dir string,
limiter chan bool) *ExtractedZipArtifactFile {
d := &ExtractedZipArtifactFile{
initCh: make(chan struct{}),
}
go func() {
defer close(d.initCh)
limiter <- true
defer func() { <-limiter }()
zr, err := zf.Open()
if err != nil {
d.err = err
return
}
defer zr.Close()
crc := crc32.NewIEEE()
r := io.TeeReader(zr, crc)
if filepath.Clean(zf.Name) != zf.Name {
d.err = fmt.Errorf("invalid filename %q", zf.Name)
return
}
path := filepath.Join(dir, zf.Name)
err = os.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
d.err = err
return
}
err = os.Remove(path)
if err != nil && !os.IsNotExist(err) {
d.err = err
return
}
if zf.Mode().IsRegular() {
w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, zf.Mode())
if err != nil {
d.err = err
return
}
defer w.Close()
_, err = io.Copy(w, r)
if err != nil {
d.err = err
return
}
} else if zf.Mode()&os.ModeSymlink != 0 {
target, err := ioutil.ReadAll(r)
if err != nil {
d.err = err
return
}
err = os.Symlink(string(target), path)
if err != nil {
d.err = err
return
}
} else {
d.err = fmt.Errorf("unknown mode %q", zf.Mode())
return
}
if crc.Sum32() != zf.CRC32 {
d.err = fmt.Errorf("crc mismatch for %v", zf.Name)
return
}
d.path = path
}()
return d
}
// ExtractedZipArtifactFile is a handle to a downloaded file from a remoteZipArtifact. The download
// may still be in progress, and will be complete with Path() returns.
type ExtractedZipArtifactFile struct {
initCh chan struct{}
err error
path string
}
// Path returns the path to the downloaded file and any errors that occurred during the download.
// It will block until the download is complete.
func (d *ExtractedZipArtifactFile) Path() (string, error) {
<-d.initCh
return d.path, d.err
}