openkylin
/
platform_build_soong
mirror of https://gitee.com/openkylin/platform_build_soong.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Support moving sources in srcjars in soong_zip 

Add a -srcjar argument to soong_zip that causes it to read the
package statement of each .java file and use that to place the
source file at a path that matches the package.

Test: jar_test.go, zip_test.go
Change-Id: I36017e42445ba3b0a82a10a8d81e8ac0cca096f2
This commit is contained in:
Colin Cross 2019-06-17 14:12:41 -07:00
parent 5d7409ba0f
commit 9cb51dbc0b
6 changed files with 445 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
jar/Android.bp
View File

@ -18,8 +18,10 @@ bootstrap_go_package {
srcs: [
"jar.go",
],
testSrcs: [
"jar_test.go",
],
deps: [
"android-archive-zip",
],
}

111
jar/jar.go
View File

@ -17,9 +17,12 @@ package jar
import (
"bytes"
"fmt"
"io"
"os"
"strings"
"text/scanner"
"time"
"unicode"
"android/soong/third_party/zip"
)
@ -112,3 +115,111 @@ func manifestContents(contents []byte) ([]byte, error) {
return finalBytes, nil
}
var javaIgnorableIdentifier = &unicode.RangeTable{
R16: []unicode.Range16{
{0x00, 0x08, 1},
{0x0e, 0x1b, 1},
{0x7f, 0x9f, 1},
},
LatinOffset: 3,
}
func javaIdentRune(ch rune, i int) bool {
if unicode.IsLetter(ch) {
return true
}
if unicode.IsDigit(ch) && i > 0 {
return true
}
if unicode.In(ch,
unicode.Nl, // letter number
unicode.Sc, // currency symbol
unicode.Pc, // connecting punctuation
) {
return true
}
if unicode.In(ch,
unicode.Cf, // format
unicode.Mc, // combining mark
unicode.Mn, // non-spacing mark
javaIgnorableIdentifier,
) && i > 0 {
return true
}
return false
}
// JavaPackage parses the package out of a java source file by looking for the package statement, or the first valid
// non-package statement, in which case it returns an empty string for the package.
func JavaPackage(r io.Reader, src string) (string, error) {
var s scanner.Scanner
var sErr error
s.Init(r)
s.Filename = src
s.Error = func(s *scanner.Scanner, msg string) {
sErr = fmt.Errorf("error parsing %q: %s", src, msg)
}
s.IsIdentRune = javaIdentRune
tok := s.Scan()
if sErr != nil {
return "", sErr
}
if tok == scanner.Ident {
switch s.TokenText() {
case "package":
// Nothing
case "import":
// File has no package statement, first keyword is an import
return "", nil
case "class", "enum", "interface":
// File has no package statement, first keyword is a type declaration
return "", nil
case "public", "protected", "private", "abstract", "static", "final", "strictfp":
// File has no package statement, first keyword is a modifier
return "", nil
case "module", "open":
// File has no package statement, first keyword is a module declaration
return "", nil
default:
return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
}
} else if tok == '@' {
// File has no package statement, first token is an annotation
return "", nil
} else if tok == scanner.EOF {
// File no package statement, it has no non-whitespace non-comment tokens
return "", nil
} else {
return "", fmt.Errorf(`expected first token of java file to be "package", got %q`, s.TokenText())
}
var pkg string
for {
tok = s.Scan()
if sErr != nil {
return "", sErr
}
if tok != scanner.Ident {
return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
}
pkg += s.TokenText()
tok = s.Scan()
if sErr != nil {
return "", sErr
}
if tok == ';' {
return pkg, nil
} else if tok == '.' {
pkg += "."
} else {
return "", fmt.Errorf(`expected "package <package>;", got "package %s%s"`, pkg, s.TokenText())
}
}
}

182
jar/jar_test.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package jar
import (
"bytes"
"io"
"testing"
)
func TestGetJavaPackage(t *testing.T) {
type args struct {
r io.Reader
src string
}
tests := []struct {
name string
in string
want string
wantErr bool
}{
{
name: "simple",
in: "package foo.bar;",
want: "foo.bar",
},
{
name: "comment",
in: "/* test */\npackage foo.bar;",
want: "foo.bar",
},
{
name: "no package",
in: "import foo.bar;",
want: "",
},
{
name: "missing semicolon error",
in: "package foo.bar",
wantErr: true,
},
{
name: "parser error",
in: "/*",
wantErr: true,
},
{
name: "parser ident error",
in: "package 0foo.bar;",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buf := bytes.NewBufferString(tt.in)
got, err := JavaPackage(buf, "<test>")
if (err != nil) != tt.wantErr {
t.Errorf("JavaPackage() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("JavaPackage() = %v, want %v", got, tt.want)
}
})
}
}
func Test_javaIdentRune(t *testing.T) {
// runes that should be valid anywhere in an identifier
validAnywhere := []rune{
// letters, $, _
'a',
'A',
'$',
'_',
// assorted unicode
'𐐀',
'𐐨',
'Dž',
'ῼ',
'ʰ',
'ﾟ',
'ƻ',
'㡢',
'￦',
'＿',
'Ⅰ',
'𐍊',
}
// runes that should be invalid as the first rune in an identifier, but valid anywhere else
validAfterFirst := []rune{
// digits
'0',
// assorted unicode
'᥍',
'𝟎',
'ྂ',
'𝆀',
// control characters
'\x00',
'\b',
'\u000e',
'\u001b',
'\u007f',
'\u009f',
'\u00ad',
0xE007F,
// zero width space
'\u200b',
}
// runes that should never be valid in an identifier
invalid := []rune{
';',
0x110000,
}
validFirst := validAnywhere
invalidFirst := append(validAfterFirst, invalid...)
validPart := append(validAnywhere, validAfterFirst...)
invalidPart := invalid
check := func(t *testing.T, ch rune, i int, want bool) {
t.Helper()
if got := javaIdentRune(ch, i); got != want {
t.Errorf("javaIdentRune() = %v, want %v", got, want)
}
}
t.Run("first", func(t *testing.T) {
t.Run("valid", func(t *testing.T) {
for _, ch := range validFirst {
t.Run(string(ch), func(t *testing.T) {
check(t, ch, 0, true)
})
}
})
t.Run("invalid", func(t *testing.T) {
for _, ch := range invalidFirst {
t.Run(string(ch), func(t *testing.T) {
check(t, ch, 0, false)
})
}
})
})
t.Run("part", func(t *testing.T) {
t.Run("valid", func(t *testing.T) {
for _, ch := range validPart {
t.Run(string(ch), func(t *testing.T) {
check(t, ch, 1, true)
})
}
})
t.Run("invalid", func(t *testing.T) {
for _, ch := range invalidPart {
t.Run(string(ch), func(t *testing.T) {
check(t, ch, 1, false)
})
}
})
})
}

2
zip/cmd/main.go
View File

@ -136,6 +136,7 @@ func main() {
writeIfChanged := flags.Bool("write_if_changed", false, "only update resultant .zip if it has changed")
ignoreMissingFiles := flags.Bool("ignore_missing_files", false, "continue if a requested file does not exist")
symlinks := flags.Bool("symlinks", true, "store symbolic links in zip instead of following them")
srcJar := flags.Bool("srcjar", false, "move .java files to locations that match their package statement")
parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file")
@ -191,6 +192,7 @@ func main() {
FileArgs: fileArgsBuilder.FileArgs(),
OutputFilePath: *out,
EmulateJar: *emulateJar,
SrcJar: *srcJar,
AddDirectoryEntriesToZip: *directories,
CompressionLevel: *compLevel,
ManifestSourcePath: *manifest,

88
zip/zip.go
View File

@ -210,6 +210,7 @@ type ZipArgs struct {
FileArgs []FileArg
OutputFilePath string
EmulateJar bool
SrcJar bool
AddDirectoryEntriesToZip bool
CompressionLevel int
ManifestSourcePath string
@ -364,7 +365,7 @@ func ZipTo(args ZipArgs, w io.Writer) error {
}
}
return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
}
func Zip(args ZipArgs) error {
@ -446,7 +447,9 @@ func jarSort(mappings []pathMapping) {
sort.SliceStable(mappings, less)
}
func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
parallelJobs int) error {
z.errors = make(chan error)
defer close(z.errors)
@ -489,7 +492,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
if emulateJar && ele.dest == jar.ManifestFile {
err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
} else {
err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
}
if err != nil {
z.errors <- err
@ -588,7 +591,7 @@ func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest stri
}
// imports (possibly with compression) <src> into the zip at sub-path <dest>
func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
var fileSize int64
var executable bool
@ -606,12 +609,9 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
return nil
}
return err
} else if s.IsDir() {
if z.directories {
return z.writeDirectory(dest, src, emulateJar)
}
return nil
} else {
}
createParentDirs := func(dest, src string) error {
if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
return err
}
@ -625,32 +625,64 @@ func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) er
z.createdFiles[dest] = src
if s.Mode()&os.ModeSymlink != 0 {
return z.writeSymlink(dest, src)
} else if !s.Mode().IsRegular() {
return fmt.Errorf("%s is not a file, directory, or symlink", src)
return nil
}
if s.IsDir() {
if z.directories {
return z.writeDirectory(dest, src, emulateJar)
}
return nil
} else if s.Mode()&os.ModeSymlink != 0 {
err = createParentDirs(dest, src)
if err != nil {
return err
}
return z.writeSymlink(dest, src)
} else if s.Mode().IsRegular() {
r, err := z.fs.Open(src)
if err != nil {
return err
}
if srcJar && filepath.Ext(src) == ".java" {
// rewrite the destination using the package path if it can be determined
pkg, err := jar.JavaPackage(r, src)
if err != nil {
// ignore errors for now, leaving the file at in its original location in the zip
} else {
dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
}
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return err
}
}
fileSize = s.Size()
executable = s.Mode()&0100 != 0
}
r, err := z.fs.Open(src)
if err != nil {
return err
}
header := &zip.FileHeader{
Name: dest,
Method: method,
UncompressedSize64: uint64(fileSize),
}
header := &zip.FileHeader{
Name: dest,
Method: method,
UncompressedSize64: uint64(fileSize),
}
if executable {
header.SetMode(0700)
}
if executable {
header.SetMode(0700)
}
err = createParentDirs(dest, src)
if err != nil {
return err
}
return z.writeFileContents(header, r)
return z.writeFileContents(header, r)
} else {
return fmt.Errorf("%s is not a file, directory, or symlink", src)
}
}
func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {

95
zip/zip_test.go
View File

@ -40,14 +40,15 @@ var (
)
var mockFs = pathtools.MockFs(map[string][]byte{
"a/a/a": fileA,
"a/a/b": fileB,
"a/a/c -> ../../c": nil,
"a/a/d -> b": nil,
"c": fileC,
"l": []byte("a/a/a\na/a/b\nc\n"),
"l2": []byte("missing\n"),
"manifest.txt": fileCustomManifest,
"a/a/a": fileA,
"a/a/b": fileB,
"a/a/c -> ../../c": nil,
"dangling -> missing": nil,
"a/a/d -> b": nil,
"c": fileC,
"l": []byte("a/a/a\na/a/b\nc\n"),
"l2": []byte("missing\n"),
"manifest.txt": fileCustomManifest,
})
func fh(name string, contents []byte, method uint16) zip.FileHeader {
@ -209,6 +210,17 @@ func TestZip(t *testing.T) {
fh("a/a/d", fileB, zip.Deflate),
},
},
{
name: "dangling symlinks",
args: fileArgsBuilder().
File("dangling"),
compressionLevel: 9,
storeSymlinks: true,
files: []zip.FileHeader{
fhLink("dangling", "missing"),
},
},
{
name: "list",
args: fileArgsBuilder().
@ -554,3 +566,70 @@ func TestReadRespFile(t *testing.T) {
})
}
}
func TestSrcJar(t *testing.T) {
mockFs := pathtools.MockFs(map[string][]byte{
"wrong_package.java": []byte("package foo;"),
"foo/correct_package.java": []byte("package foo;"),
"src/no_package.java": nil,
"src2/parse_error.java": []byte("error"),
})
want := []string{
"foo/",
"foo/wrong_package.java",
"foo/correct_package.java",
"no_package.java",
"src2/",
"src2/parse_error.java",
}
args := ZipArgs{}
args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs()
args.SrcJar = true
args.AddDirectoryEntriesToZip = true
args.Filesystem = mockFs
args.Stderr = &bytes.Buffer{}
buf := &bytes.Buffer{}
err := ZipTo(args, buf)
if err != nil {
t.Fatalf("got error %v", err)
}
br := bytes.NewReader(buf.Bytes())
zr, err := zip.NewReader(br, int64(br.Len()))
if err != nil {
t.Fatal(err)
}
var got []string
for _, f := range zr.File {
r, err := f.Open()
if err != nil {
t.Fatalf("error when opening %s: %s", f.Name, err)
}
crc := crc32.NewIEEE()
len, err := io.Copy(crc, r)
r.Close()
if err != nil {
t.Fatalf("error when reading %s: %s", f.Name, err)
}
if uint64(len) != f.UncompressedSize64 {
t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len)
}
if crc.Sum32() != f.CRC32 {
t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc)
}
got = append(got, f.Name)
}
if !reflect.DeepEqual(want, got) {
t.Errorf("want files %q, got %q", want, got)
}
}