From 2a4549ec9859e184c0c6e980646f117f6afa83a4 Mon Sep 17 00:00:00 2001 From: Sasha Smundak Date: Mon, 5 Nov 2018 16:49:08 -0800 Subject: [PATCH] Support source code cross-referencing for C++ and Java Use Kythe (https://kythe.io) to build cross reference for the Android source code. ~generate the input for it during the build. This is done on demand: if XREF_CORPUS environment variable is set, build emits a Ninja rule to generate Kythe input for each compilation rule. It also emits two consolidation rules (`xref_cxx` and `xref_java`), that depend on all Kythe input generation rules for C++ and Java. The value of the XREF_CORPUS environment variable is recorded in the generated files and thus passed to Kythe. For the AOSP master branch it is `android.googlesource.com/platform/superproject`, so the command to build all input for Kythe on that branch is: ``` XREF_CORPUS=android.googlesource.com/platform/superproject m xref_cxx xref_java ``` Each Kythe input generation rule generates a single file with .kzip extension. Individual .kzip files have a lot of common information, so there will be a post-build consolidation step run to combine them. The consolidated .kzip file is then passed to Kythe backend. The tools to generate .kzip files are provided by Kythe (it calls them 'extractors'). We are going to build them in toolbuilding branches (clang-tools and build-tools) and check them in as binaries into master and other PDK branches: For C++, `prebuilts/clang-tools/linux-x86/bin/cxx_extractor` for Java, `prebuilts/build-tools/common/framework/javac_extractor.jar` Bug: 121267023 Test: 1) When XREF_CORPUS is set, build generates Ninja rules to create .kzip files; 2) When XREF_CORPUS is set, building `xref_cxx`/`xref_java` creates .kzip files; 3) Unless XREF_CORPUS is set, build generates the same Ninja rules as before Change-Id: If957b35d7abc82dbfbb3665980e7c34afe7c789e --- android/config.go | 8 ++++ cc/builder.go | 38 +++++++++++++++++++ cc/cc.go | 39 ++++++++++++++++++++ cc/util.go | 1 + java/builder.go | 86 +++++++++++++++++++++++++++++++++++++++++++ java/config/config.go | 1 + java/java.go | 42 +++++++++++++++++++++ 7 files changed, 215 insertions(+) diff --git a/android/config.go b/android/config.go index 556ddcc29..074dfc729 100644 --- a/android/config.go +++ b/android/config.go @@ -756,6 +756,14 @@ func (c *config) RunErrorProne() bool { return c.IsEnvTrue("RUN_ERROR_PRONE") } +func (c *config) XrefCorpusName() string { + return c.Getenv("XREF_CORPUS") +} + +func (c *config) EmitXrefRules() bool { + return c.XrefCorpusName() != "" +} + // Returns true if -source 1.9 -target 1.9 is being passed to javac func (c *config) TargetOpenJDK9() bool { return c.targetOpenJDK9 diff --git a/cc/builder.go b/cc/builder.go index cbe2c8860..2909d51bd 100644 --- a/cc/builder.go +++ b/cc/builder.go @@ -221,6 +221,17 @@ var ( Rspfile: "$out.rsp", RspfileContent: "$in", }) + + _ = pctx.SourcePathVariable("cxxExtractor", + "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor") + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + kytheExtract = pctx.StaticRule("kythe", + blueprint.RuleParams{ + Command: "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out $cxxExtractor $cFlags $in ", + CommandDeps: []string{"$cxxExtractor"}, + }, + "cFlags") ) func init() { @@ -257,6 +268,7 @@ type builderFlags struct { tidy bool coverage bool sAbiDump bool + emitXrefs bool systemIncludeFlags string @@ -281,6 +293,7 @@ type Objects struct { tidyFiles android.Paths coverageFiles android.Paths sAbiDumpFiles android.Paths + kytheFiles android.Paths } func (a Objects) Copy() Objects { @@ -289,6 +302,7 @@ func (a Objects) Copy() Objects { tidyFiles: append(android.Paths{}, a.tidyFiles...), coverageFiles: append(android.Paths{}, a.coverageFiles...), sAbiDumpFiles: append(android.Paths{}, a.sAbiDumpFiles...), + kytheFiles: append(android.Paths{}, a.kytheFiles...), } } @@ -298,6 +312,7 @@ func (a Objects) Append(b Objects) Objects { tidyFiles: append(a.tidyFiles, b.tidyFiles...), coverageFiles: append(a.coverageFiles, b.coverageFiles...), sAbiDumpFiles: append(a.sAbiDumpFiles, b.sAbiDumpFiles...), + kytheFiles: append(a.kytheFiles, b.kytheFiles...), } } @@ -314,6 +329,10 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and if flags.coverage { coverageFiles = make(android.Paths, 0, len(srcFiles)) } + var kytheFiles android.Paths + if flags.emitXrefs { + kytheFiles = make(android.Paths, 0, len(srcFiles)) + } commonFlags := strings.Join([]string{ flags.globalFlags, @@ -401,6 +420,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and coverage := flags.coverage dump := flags.sAbiDump rule := cc + emitXref := flags.emitXrefs switch srcFile.Ext() { case ".s": @@ -412,6 +432,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and tidy = false coverage = false dump = false + emitXref = false case ".c": ccCmd = "clang" moduleCflags = cflags @@ -450,6 +471,22 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and }, }) + if emitXref { + kytheFile := android.ObjPathWithExt(ctx, subdir, srcFile, "kzip") + ctx.Build(pctx, android.BuildParams{ + Rule: kytheExtract, + Description: "Xref C++ extractor " + srcFile.Rel(), + Output: kytheFile, + Input: srcFile, + Implicits: cFlagsDeps, + OrderOnly: pathDeps, + Args: map[string]string{ + "cFlags": moduleCflags, + }, + }) + kytheFiles = append(kytheFiles, kytheFile) + } + if tidy { tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy") tidyFiles = append(tidyFiles, tidyFile) @@ -493,6 +530,7 @@ func TransformSourceToObj(ctx android.ModuleContext, subdir string, srcFiles and tidyFiles: tidyFiles, coverageFiles: coverageFiles, sAbiDumpFiles: sAbiDumpFiles, + kytheFiles: kytheFiles, } } diff --git a/cc/cc.go b/cc/cc.go index 2cee80717..d22363e0e 100644 --- a/cc/cc.go +++ b/cc/cc.go @@ -77,6 +77,7 @@ func init() { ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel() }) + android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory) pctx.Import("android/soong/cc/config") } @@ -162,6 +163,7 @@ type Flags struct { Tidy bool Coverage bool SAbiDump bool + EmitXrefs bool // If true, generate Ninja rules to generate emitXrefs input files for Kythe RequiredInstructionSet string DynamicLinker string @@ -346,6 +348,10 @@ type dependencyTag struct { explicitlyVersioned bool } +type xref interface { + XrefCcFiles() android.Paths +} + var ( sharedDepTag = dependencyTag{name: "shared", library: true} sharedExportDepTag = dependencyTag{name: "shared", library: true, reexportFlags: true} @@ -427,6 +433,8 @@ type Module struct { staticVariant *Module makeLinkType string + // Kythe (source file indexer) paths for this compilation module + kytheFiles android.Paths } func (c *Module) OutputFile() android.OptionalPath { @@ -657,6 +665,10 @@ func installToBootstrap(name string, config android.Config) bool { return isBionic(name) } +func (c *Module) XrefCcFiles() android.Paths { + return c.kytheFiles +} + type baseModuleContext struct { android.BaseModuleContext moduleContextImpl @@ -995,6 +1007,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { flags := Flags{ Toolchain: c.toolchain(ctx), + EmitXrefs: ctx.Config().EmitXrefRules(), } if c.compiler != nil { flags = c.compiler.compilerFlags(ctx, flags, deps) @@ -1060,6 +1073,7 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) { if ctx.Failed() { return } + c.kytheFiles = objs.kytheFiles } if c.linker != nil { @@ -2366,6 +2380,31 @@ func getCurrentNdkPrebuiltVersion(ctx DepsContext) string { return ctx.Config().PlatformSdkVersion() } +func kytheExtractAllFactory() android.Singleton { + return &kytheExtractAllSingleton{} +} + +type kytheExtractAllSingleton struct { +} + +func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if ccModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...) + } + }) + // TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets + if len(xrefTargets) > 0 { + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, "xref_cxx"), + Inputs: xrefTargets, + //Default: true, + }) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var BoolPtr = proptools.BoolPtr diff --git a/cc/util.go b/cc/util.go index 2e1bb2590..0d1b2f049 100644 --- a/cc/util.go +++ b/cc/util.go @@ -74,6 +74,7 @@ func flagsToBuilderFlags(in Flags) builderFlags { coverage: in.Coverage, tidy: in.Tidy, sAbiDump: in.SAbiDump, + emitXrefs: in.EmitXrefs, systemIncludeFlags: strings.Join(in.SystemIncludeFlags, " "), diff --git a/java/builder.go b/java/builder.go index 22eff7c4b..f174cf0c6 100644 --- a/java/builder.go +++ b/java/builder.go @@ -62,6 +62,37 @@ var ( "javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", "outDir", "annoDir", "javaVersion") + _ = pctx.VariableFunc("kytheCorpus", + func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() }) + // Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about + // "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ... + // to field java.nio.Buffer.address" + kytheExtract = pctx.AndroidStaticRule("kythe", + blueprint.RuleParams{ + Command: `${config.ZipSyncCmd} -d $srcJarDir ` + + `-l $srcJarDir/list -f "*.java" $srcJars && ` + + `( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` + + `KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` + + `KYTHE_CORPUS=${kytheCorpus} ` + + `${config.SoongJavacWrapper} ${config.JavaCmd} ` + + `--add-opens=java.base/java.nio=ALL-UNNAMED ` + + `-jar ${config.JavaKytheExtractorJar} ` + + `${config.JavacHeapFlags} ${config.CommonJdkFlags} ` + + `$processorpath $processor $javacFlags $bootClasspath $classpath ` + + `-source $javaVersion -target $javaVersion ` + + `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list)`, + CommandDeps: []string{ + "${config.JavaCmd}", + "${config.JavaKytheExtractorJar}", + "${config.ZipSyncCmd}", + }, + CommandOrderOnly: []string{"${config.SoongJavacWrapper}"}, + Rspfile: "$out.rsp", + RspfileContent: "$in", + }, + "javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir", + "outDir", "annoDir", "javaVersion") + turbine = pctx.AndroidStaticRule("turbine", blueprint.RuleParams{ Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + @@ -196,6 +227,61 @@ func RunErrorProne(ctx android.ModuleContext, outputFile android.WritablePath, "errorprone", "errorprone") } +// Emits the rule to generate Xref input file (.kzip file) for the given set of source files and source jars +// to compile with given set of builder flags, etc. +func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath, + srcFiles, srcJars android.Paths, + flags javaBuilderFlags, deps android.Paths, + intermediatesDir string) { + + deps = append(deps, srcJars...) + + var bootClasspath string + if flags.javaVersion == "1.9" { + var systemModuleDeps android.Paths + bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device()) + deps = append(deps, systemModuleDeps...) + } else { + deps = append(deps, flags.bootClasspath...) + if len(flags.bootClasspath) == 0 && ctx.Device() { + // explicitly specify -bootclasspath "" if the bootclasspath is empty to + // ensure java does not fall back to the default bootclasspath. + bootClasspath = `-bootclasspath ""` + } else { + bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath") + } + } + + deps = append(deps, flags.classpath...) + deps = append(deps, flags.processorPath...) + + processor := "-proc:none" + if flags.processor != "" { + processor = "-processor " + flags.processor + } + + ctx.Build(pctx, + android.BuildParams{ + Rule: kytheExtract, + Description: "Xref Java extractor", + Output: xrefFile, + Inputs: srcFiles, + Implicits: deps, + Args: map[string]string{ + "annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(), + "bootClasspath": bootClasspath, + "classpath": flags.classpath.FormJavaClassPath("-classpath"), + "javacFlags": flags.javacFlags, + "javaVersion": flags.javaVersion, + "outDir": android.PathForModuleOut(ctx, "javac", "classes.xref").String(), + "processorpath": flags.processorPath.FormJavaClassPath("-processorpath"), + "processor": processor, + "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars.xref").String(), + "srcJars": strings.Join(srcJars.Strings(), " "), + }, + }) +} + func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath, srcFiles, srcJars android.Paths, flags javaBuilderFlags) { diff --git a/java/config/config.go b/java/config/config.go index 6a0a10a86..cb137447c 100644 --- a/java/config/config.go +++ b/java/config/config.go @@ -94,6 +94,7 @@ func init() { pctx.SourcePathVariable("JlinkCmd", "${JavaToolchain}/jlink") pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod") pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar") + pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar") pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime") pctx.SourcePathVariable("GenKotlinBuildFileCmd", "build/soong/scripts/gen-kotlin-build-file.sh") diff --git a/java/java.go b/java/java.go index f3e10bebd..c6f6a3455 100644 --- a/java/java.go +++ b/java/java.go @@ -50,6 +50,7 @@ func init() { android.RegisterModuleType("dex_import", DexImportFactory) android.RegisterSingletonType("logtags", LogtagsSingleton) + android.RegisterSingletonType("kythe_java_extract", kytheExtractJavaFactory) } // TODO: @@ -342,6 +343,9 @@ type Module struct { hiddenAPI dexpreopter + + // list of the xref extraction files + kytheFiles android.Paths } func (j *Module) OutputFiles(tag string) (android.Paths, error) { @@ -382,6 +386,10 @@ type SrcDependency interface { CompiledSrcJars() android.Paths } +type xref interface { + XrefJavaFiles() android.Paths +} + func (j *Module) CompiledSrcs() android.Paths { return j.compiledJavaSrcs } @@ -390,6 +398,10 @@ func (j *Module) CompiledSrcJars() android.Paths { return j.compiledSrcJars } +func (j *Module) XrefJavaFiles() android.Paths { + return j.kytheFiles +} + var _ SrcDependency = (*Module)(nil) func InitJavaModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) { @@ -1138,6 +1150,12 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps) jars = append(jars, classes) } + if ctx.Config().EmitXrefRules() { + extractionFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".kzip") + emitXrefRule(ctx, extractionFile, uniqueSrcFiles, srcJars, flags, extraJarDeps, "xref") + j.kytheFiles = append(j.kytheFiles, extractionFile) + + } if ctx.Failed() { return } @@ -2203,6 +2221,30 @@ func DefaultsFactory(props ...interface{}) android.Module { return module } +func kytheExtractJavaFactory() android.Singleton { + return &kytheExtractJavaSingleton{} +} + +type kytheExtractJavaSingleton struct { +} + +func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) { + var xrefTargets android.Paths + ctx.VisitAllModules(func(module android.Module) { + if javaModule, ok := module.(xref); ok { + xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...) + } + }) + // TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets + if len(xrefTargets) > 0 { + ctx.Build(pctx, android.BuildParams{ + Rule: blueprint.Phony, + Output: android.PathForPhony(ctx, "xref_java"), + Inputs: xrefTargets, + }) + } +} + var Bool = proptools.Bool var BoolDefault = proptools.BoolDefault var String = proptools.String