From c651b5295b6a59a43b2930188449fa6b98c754ff Mon Sep 17 00:00:00 2001 From: hamzeh Date: Thu, 29 Apr 2021 12:50:47 -0700 Subject: [PATCH] Add support for packaging rust fuzzers Test: make haiku-rust Change-Id: Idd4d836d11e0ae615b59c6648d49348449589787 --- rust/androidmk.go | 34 +++++++- rust/fuzz.go | 206 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 1 deletion(-) diff --git a/rust/androidmk.go b/rust/androidmk.go index 5f89d73c7..ea45ebd01 100644 --- a/rust/androidmk.go +++ b/rust/androidmk.go @@ -91,7 +91,6 @@ func (binary *binaryDecorator) AndroidMk(ctx AndroidMkContext, ret *android.Andr if binary.distFile.Valid() { ret.DistFiles = android.MakeDefaultDistFiles(binary.distFile.Path()) } - ret.Class = "EXECUTABLES" } @@ -201,3 +200,36 @@ func (compiler *baseCompiler) AndroidMk(ctx AndroidMkContext, ret *android.Andro entries.SetString("LOCAL_MODULE_STEM", stem) }) } + +func (fuzz *fuzzDecorator) AndroidMkEntries(ctx AndroidMkContext, entries *android.AndroidMkEntries) { + ctx.SubAndroidMk(entries, fuzz.binaryDecorator) + + var fuzzFiles []string + for _, d := range fuzz.corpus { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.corpusIntermediateDir.String())+":corpus/"+d.Base()) + } + + for _, d := range fuzz.data { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.dataIntermediateDir.String())+":data/"+d.Rel()) + } + + if fuzz.dictionary != nil { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.dictionary.String())+":"+fuzz.dictionary.Base()) + } + + if fuzz.config != nil { + fuzzFiles = append(fuzzFiles, + filepath.Dir(fuzz.config.String())+":config.json") + } + + entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, + entries *android.AndroidMkEntries) { + entries.SetBool("LOCAL_IS_FUZZ_TARGET", true) + if len(fuzzFiles) > 0 { + entries.AddStrings("LOCAL_TEST_DATA", fuzzFiles...) + } + }) +} diff --git a/rust/fuzz.go b/rust/fuzz.go index d69997114..7e1c55a50 100644 --- a/rust/fuzz.go +++ b/rust/fuzz.go @@ -15,6 +15,10 @@ package rust import ( + "path/filepath" + "sort" + "strings" + "android/soong/android" "android/soong/cc" "android/soong/rust/config" @@ -22,6 +26,7 @@ import ( func init() { android.RegisterModuleType("rust_fuzz", RustFuzzFactory) + android.RegisterSingletonType("rust_fuzz_packaging", rustFuzzPackagingFactory) } type fuzzDecorator struct { @@ -93,3 +98,204 @@ func (fuzzer *fuzzDecorator) stdLinkage(ctx *depsContext) RustLinkage { func (fuzzer *fuzzDecorator) autoDep(ctx android.BottomUpMutatorContext) autoDep { return rlibAutoDep } + +// Responsible for generating GNU Make rules that package fuzz targets into +// their architecture & target/host specific zip file. +type rustFuzzPackager struct { + packages android.Paths + fuzzTargets map[string]bool +} + +func rustFuzzPackagingFactory() android.Singleton { + return &rustFuzzPackager{} +} + +type fileToZip struct { + SourceFilePath android.Path + DestinationPathPrefix string +} + +type archOs struct { + hostOrTarget string + arch string + dir string +} + +func (s *rustFuzzPackager) GenerateBuildActions(ctx android.SingletonContext) { + + // Map between each architecture + host/device combination. + archDirs := make(map[archOs][]fileToZip) + + // List of individual fuzz targets. + s.fuzzTargets = make(map[string]bool) + + ctx.VisitAllModules(func(module android.Module) { + // Discard non-fuzz targets. + rustModule, ok := module.(*Module) + if !ok { + return + } + + fuzzModule, ok := rustModule.compiler.(*fuzzDecorator) + if !ok { + return + } + + // Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of + // fuzz targets we're going to package anyway. + if !rustModule.Enabled() || rustModule.Properties.PreventInstall || + rustModule.InRamdisk() || rustModule.InVendorRamdisk() || rustModule.InRecovery() { + return + } + + // Discard modules that are in an unavailable namespace. + if !rustModule.ExportedToMake() { + return + } + + hostOrTargetString := "target" + if rustModule.Host() { + hostOrTargetString = "host" + } + + archString := rustModule.Arch().ArchType.String() + archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString) + archOs := archOs{hostOrTarget: hostOrTargetString, arch: archString, dir: archDir.String()} + + var files []fileToZip + builder := android.NewRuleBuilder(pctx, ctx) + + // Package the corpora into a zipfile. + if fuzzModule.corpus != nil { + corpusZip := archDir.Join(ctx, module.Name()+"_seed_corpus.zip") + command := builder.Command().BuiltTool("soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", corpusZip) + rspFile := corpusZip.ReplaceExtension(ctx, "rsp") + command.FlagWithRspFileInputList("-r ", rspFile, fuzzModule.corpus) + files = append(files, fileToZip{corpusZip, ""}) + } + + // Package the data into a zipfile. + if fuzzModule.data != nil { + dataZip := archDir.Join(ctx, module.Name()+"_data.zip") + command := builder.Command().BuiltTool("soong_zip"). + FlagWithOutput("-o ", dataZip) + for _, f := range fuzzModule.data { + intermediateDir := strings.TrimSuffix(f.String(), f.Rel()) + command.FlagWithArg("-C ", intermediateDir) + command.FlagWithInput("-f ", f) + } + files = append(files, fileToZip{dataZip, ""}) + } + + // The executable. + files = append(files, fileToZip{rustModule.unstrippedOutputFile.Path(), ""}) + + // The dictionary. + if fuzzModule.dictionary != nil { + files = append(files, fileToZip{fuzzModule.dictionary, ""}) + } + + // Additional fuzz config. + if fuzzModule.config != nil { + files = append(files, fileToZip{fuzzModule.config, ""}) + } + + fuzzZip := archDir.Join(ctx, module.Name()+".zip") + + command := builder.Command().BuiltTool("soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", fuzzZip) + + for _, file := range files { + if file.DestinationPathPrefix != "" { + command.FlagWithArg("-P ", file.DestinationPathPrefix) + } else { + command.Flag("-P ''") + } + command.FlagWithInput("-f ", file.SourceFilePath) + } + + builder.Build("create-"+fuzzZip.String(), + "Package "+module.Name()+" for "+archString+"-"+hostOrTargetString) + + // Don't add modules to 'make haiku-rust' that are set to not be + // exported to the fuzzing infrastructure. + if config := fuzzModule.Properties.Fuzz_config; config != nil { + if rustModule.Host() && !BoolDefault(config.Fuzz_on_haiku_host, true) { + return + } else if !BoolDefault(config.Fuzz_on_haiku_device, true) { + return + } + } + + s.fuzzTargets[module.Name()] = true + archDirs[archOs] = append(archDirs[archOs], fileToZip{fuzzZip, ""}) + }) + + var archOsList []archOs + for archOs := range archDirs { + archOsList = append(archOsList, archOs) + } + sort.Slice(archOsList, func(i, j int) bool { return archOsList[i].dir < archOsList[j].dir }) + + for _, archOs := range archOsList { + filesToZip := archDirs[archOs] + arch := archOs.arch + hostOrTarget := archOs.hostOrTarget + builder := android.NewRuleBuilder(pctx, ctx) + outputFile := android.PathForOutput(ctx, "fuzz-rust-"+hostOrTarget+"-"+arch+".zip") + s.packages = append(s.packages, outputFile) + + command := builder.Command().BuiltTool("soong_zip"). + Flag("-j"). + FlagWithOutput("-o ", outputFile). + Flag("-L 0") // No need to try and re-compress the zipfiles. + + for _, fileToZip := range filesToZip { + if fileToZip.DestinationPathPrefix != "" { + command.FlagWithArg("-P ", fileToZip.DestinationPathPrefix) + } else { + command.Flag("-P ''") + } + command.FlagWithInput("-f ", fileToZip.SourceFilePath) + } + builder.Build("create-fuzz-package-"+arch+"-"+hostOrTarget, + "Create fuzz target packages for "+arch+"-"+hostOrTarget) + } + +} + +func (s *rustFuzzPackager) MakeVars(ctx android.MakeVarsContext) { + packages := s.packages.Strings() + sort.Strings(packages) + + ctx.Strict("SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES", strings.Join(packages, " ")) + + // Preallocate the slice of fuzz targets to minimise memory allocations. + fuzzTargets := make([]string, 0, len(s.fuzzTargets)) + for target, _ := range s.fuzzTargets { + fuzzTargets = append(fuzzTargets, target) + } + sort.Strings(fuzzTargets) + ctx.Strict("ALL_RUST_FUZZ_TARGETS", strings.Join(fuzzTargets, " ")) +} + +func (fuzz *fuzzDecorator) install(ctx ModuleContext) { + fuzz.binaryDecorator.baseCompiler.dir = filepath.Join( + "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) + fuzz.binaryDecorator.baseCompiler.dir64 = filepath.Join( + "fuzz", ctx.Target().Arch.ArchType.String(), ctx.ModuleName()) + fuzz.binaryDecorator.baseCompiler.install(ctx) + + if fuzz.Properties.Corpus != nil { + fuzz.corpus = android.PathsForModuleSrc(ctx, fuzz.Properties.Corpus) + } + if fuzz.Properties.Data != nil { + fuzz.data = android.PathsForModuleSrc(ctx, fuzz.Properties.Data) + } + if fuzz.Properties.Dictionary != nil { + fuzz.dictionary = android.PathForModuleSrc(ctx, *fuzz.Properties.Dictionary) + } +}