From 436b76564d83586c68025985bc5d9fc7774109bf Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Thu, 15 Mar 2018 16:24:10 -0700 Subject: [PATCH] Replace extract-srcjars.sh with zipsync tool extract_srcjars.sh uses zipinfo and unzip, which fail with an error on an empty zip file. Instead of trying to hack around this (which is hard to make guarantees for since they are non-hermetic host tools), replace them with a go tool to unzip a set of zip files into a directory. Bug: 73885582 Test: m checkbuild Change-Id: I151fed347ed5196726e36866ffc27bc831799afb --- cmd/zipsync/Android.bp | 25 ++++++++ cmd/zipsync/zipsync.go | 124 +++++++++++++++++++++++++++++++++++++ java/builder.go | 12 ++-- java/config/config.go | 2 +- java/config/makevars.go | 2 +- java/droiddoc.go | 4 +- scripts/extract-srcjars.sh | 44 ------------- 7 files changed, 159 insertions(+), 54 deletions(-) create mode 100644 cmd/zipsync/Android.bp create mode 100644 cmd/zipsync/zipsync.go delete mode 100755 scripts/extract-srcjars.sh diff --git a/cmd/zipsync/Android.bp b/cmd/zipsync/Android.bp new file mode 100644 index 000000000..22feadc8f --- /dev/null +++ b/cmd/zipsync/Android.bp @@ -0,0 +1,25 @@ +// Copyright 2016 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +blueprint_go_binary { + name: "zipsync", + deps: [ + "android-archive-zip", + "blueprint-pathtools", + ], + srcs: [ + "zipsync.go", + ], +} + diff --git a/cmd/zipsync/zipsync.go b/cmd/zipsync/zipsync.go new file mode 100644 index 000000000..035a1455f --- /dev/null +++ b/cmd/zipsync/zipsync.go @@ -0,0 +1,124 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/zip" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +var ( + outputDir = flag.String("d", "", "output dir") + outputFile = flag.String("l", "", "output list file") + filter = flag.String("f", "", "optional filter pattern") +) + +func must(err error) { + if err != nil { + log.Fatal(err) + } +} + +func writeFile(filename string, in io.Reader, perm os.FileMode) error { + out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = io.Copy(out, in) + if err != nil { + out.Close() + return err + } + + return out.Close() +} + +func main() { + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: zipsync -d [-l ] [-f ] [zip]...") + flag.PrintDefaults() + } + + flag.Parse() + + if *outputDir == "" { + flag.Usage() + os.Exit(1) + } + + inputs := flag.Args() + + // For now, just wipe the output directory and replace its contents with the zip files + // Eventually this could only modify the directory contents as necessary to bring it up + // to date with the zip files. + must(os.RemoveAll(*outputDir)) + + must(os.MkdirAll(*outputDir, 0777)) + + var files []string + seen := make(map[string]string) + + for _, input := range inputs { + reader, err := zip.OpenReader(input) + if err != nil { + log.Fatal(err) + } + defer reader.Close() + + for _, f := range reader.File { + if *filter != "" { + if match, err := filepath.Match(*filter, filepath.Base(f.Name)); err != nil { + log.Fatal(err) + } else if !match { + continue + } + } + if filepath.IsAbs(f.Name) { + log.Fatal("%q in %q is an absolute path", f.Name, input) + } + + if prev, exists := seen[f.Name]; exists { + log.Fatal("%q found in both %q and %q", f.Name, prev, input) + } + seen[f.Name] = input + + filename := filepath.Join(*outputDir, f.Name) + if f.FileInfo().IsDir() { + must(os.MkdirAll(filename, f.FileInfo().Mode())) + } else { + must(os.MkdirAll(filepath.Dir(filename), 0777)) + in, err := f.Open() + if err != nil { + log.Fatal(err) + } + must(writeFile(filename, in, f.FileInfo().Mode())) + in.Close() + files = append(files, filename) + } + } + } + + if *outputFile != "" { + data := strings.Join(files, "\n") + "\n" + must(ioutil.WriteFile(*outputFile, []byte(data), 0666)) + } +} diff --git a/java/builder.go b/java/builder.go index ee0d8a830..12110f2f0 100644 --- a/java/builder.go +++ b/java/builder.go @@ -41,7 +41,7 @@ var ( javac = pctx.AndroidGomaStaticRule("javac", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + - `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` + + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `${config.SoongJavacWrapper} ${config.JavacWrapper}${config.JavacCmd} ${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + `$javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + @@ -50,7 +50,7 @@ var ( CommandDeps: []string{ "${config.JavacCmd}", "${config.SoongZipCmd}", - "${config.ExtractSrcJarsCmd}", + "${config.ZipSyncCmd}", }, CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, Rspfile: "$out.rsp", @@ -62,7 +62,7 @@ var ( kotlinc = pctx.AndroidGomaStaticRule("kotlinc", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$srcJarDir" && mkdir -p "$outDir" "$srcJarDir" && ` + - `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` + + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `${config.GenKotlinBuildFileCmd} $classpath $outDir $out.rsp $srcJarDir/list > $outDir/kotlinc-build.xml &&` + `${config.KotlincCmd} $kotlincFlags ` + `-jvm-target $kotlinJvmTarget -Xbuild-file=$outDir/kotlinc-build.xml && ` + @@ -72,7 +72,7 @@ var ( "${config.KotlinCompilerJar}", "${config.GenKotlinBuildFileCmd}", "${config.SoongZipCmd}", - "${config.ExtractSrcJarsCmd}", + "${config.ZipSyncCmd}", }, Rspfile: "$out.rsp", RspfileContent: `$in`, @@ -82,7 +82,7 @@ var ( errorprone = pctx.AndroidStaticRule("errorprone", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` + - `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` + + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `${config.SoongJavacWrapper} ${config.ErrorProneCmd} ` + `$javacFlags $bootClasspath $classpath ` + `-source $javaVersion -target $javaVersion ` + @@ -93,7 +93,7 @@ var ( "${config.ErrorProneJavacJar}", "${config.ErrorProneJar}", "${config.SoongZipCmd}", - "${config.ExtractSrcJarsCmd}", + "${config.ZipSyncCmd}", }, CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, Rspfile: "$out.rsp", diff --git a/java/config/config.go b/java/config/config.go index fc2444f4e..3d7f910f6 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -82,13 +82,13 @@ func init() { pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar") pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime") - pctx.SourcePathVariable("ExtractSrcJarsCmd", "build/soong/scripts/extract-srcjars.sh") pctx.SourcePathVariable("GenKotlinBuildFileCmd", "build/soong/scripts/gen-kotlin-build-file.sh") pctx.SourcePathVariable("JarArgsCmd", "build/soong/scripts/jar-args.sh") pctx.HostBinToolVariable("SoongZipCmd", "soong_zip") pctx.HostBinToolVariable("MergeZipsCmd", "merge_zips") pctx.HostBinToolVariable("Zip2ZipCmd", "zip2zip") + pctx.HostBinToolVariable("ZipSyncCmd", "zipsync") pctx.VariableFunc("DxCmd", func(ctx android.PackageVarContext) string { config := ctx.Config() if config.IsEnvFalse("USE_D8") { diff --git a/java/config/makevars.go b/java/config/makevars.go index 7e125d59d..5210f2022 100644 --- a/java/config/makevars.go +++ b/java/config/makevars.go @@ -73,7 +73,7 @@ func makeVarsProvider(ctx android.MakeVarsContext) { } ctx.Strict("SOONG_JAVAC_WRAPPER", "${SoongJavacWrapper}") - ctx.Strict("EXTRACT_SRCJARS", "${ExtractSrcJarsCmd}") + ctx.Strict("ZIPSYNC", "${ZipSyncCmd}") ctx.Strict("JACOCO_CLI_JAR", "${JacocoCLIJar}") ctx.Strict("DEFAULT_JACOCO_EXCLUDE_FILTER", strings.Join(DefaultJacocoExcludeFilter, ",")) diff --git a/java/droiddoc.go b/java/droiddoc.go index 8c605355c..9a86ab55b 100644 --- a/java/droiddoc.go +++ b/java/droiddoc.go @@ -28,14 +28,14 @@ var ( javadoc = pctx.AndroidStaticRule("javadoc", blueprint.RuleParams{ Command: `rm -rf "$outDir" "$srcJarDir" "$stubsDir" && mkdir -p "$outDir" "$srcJarDir" "$stubsDir" && ` + - `${config.ExtractSrcJarsCmd} $srcJarDir $srcJarDir/list $srcJars && ` + + `${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` + `${config.JavadocCmd} -encoding UTF-8 @$out.rsp @$srcJarDir/list ` + `$opts $bootclasspathArgs $classpathArgs -sourcepath $sourcepath ` + `-d $outDir -quiet && ` + `${config.SoongZipCmd} -write_if_changed -d -o $docZip -C $outDir -D $outDir && ` + `${config.SoongZipCmd} -write_if_changed -jar -o $out -C $stubsDir -D $stubsDir`, CommandDeps: []string{ - "${config.ExtractSrcJarsCmd}", + "${config.ZipSyncCmd}", "${config.JavadocCmd}", "${config.SoongZipCmd}", "$JsilverJar", diff --git a/scripts/extract-srcjars.sh b/scripts/extract-srcjars.sh deleted file mode 100755 index f81032b67..000000000 --- a/scripts/extract-srcjars.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -e - -# Copyright 2017 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Extracts .java files from source jars in a specified directory and writes out a list of the files - -if [ -z "$1" -o -z "$2" ]; then - echo "usage: $0 [ ...]" >&2 - exit 1 -fi - -output_dir=$1 -shift -output_file=$1 -shift - -rm -f $output_file -touch $output_file - -for j in "$@"; do - for f in $(zipinfo -1 $j '*.java'); do - echo $output_dir/$f >> $output_file - done - unzip -qn -d $output_dir $j '*.java' -done - -duplicates=$(cat $output_file | sort | uniq -d | uniq) -if [ -n "$duplicates" ]; then - echo Duplicate source files: - echo $duplicates - exit 1 -fi