Supported python build in host side.
The base module handles all the common functionalites, such as version compatibilty check, version variations split, source file format check, source/data file duplicate check. The library/binary module focuses on how to generate binary build actions, such as setting up stub script, zipping, filling in __init__.py in runfiles dir tree. Bug: b/31676493 Test: go test under python package Change-Id: I06608369f350f7195873d459e1c8d1bdb811e77e
This commit is contained in:
Nan Zhang
parent
7c34c4c8eb
commit
db0b9a3cf3
19
Android.bp
19
Android.bp
|
|
@ -209,6 +209,25 @@ bootstrap_go_package {
|
|||
pluginFor: ["soong_build"],
|
||||
}
|
||||
|
||||
bootstrap_go_package {
|
||||
name: "soong-python",
|
||||
pkgPath: "android/soong/python",
|
||||
deps: [
|
||||
"blueprint",
|
||||
"soong-android",
|
||||
],
|
||||
srcs: [
|
||||
"python/binary.go",
|
||||
"python/builder.go",
|
||||
"python/library.go",
|
||||
"python/python.go",
|
||||
],
|
||||
testSrcs: [
|
||||
"python/python_test.go",
|
||||
],
|
||||
pluginFor: ["soong_build"],
|
||||
}
|
||||
|
||||
//
|
||||
// Defaults to enable various configurations of host bionic
|
||||
//
|
||||
|
|
|
|||
|
|
@ -217,13 +217,22 @@ func translateAndroidMkModule(ctx blueprint.SingletonContext, w io.Writer, mod b
|
|||
host := false
|
||||
switch amod.Os().Class {
|
||||
case Host:
|
||||
// Make cannot identify LOCAL_MODULE_HOST_ARCH:= common.
|
||||
if archStr != "common" {
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_HOST_ARCH :=", archStr)
|
||||
}
|
||||
host = true
|
||||
case HostCross:
|
||||
// Make cannot identify LOCAL_MODULE_HOST_CROSS_ARCH:= common.
|
||||
if archStr != "common" {
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_HOST_CROSS_ARCH :=", archStr)
|
||||
}
|
||||
host = true
|
||||
case Device:
|
||||
// Make cannot identify LOCAL_MODULE_TARGET_ARCH:= common.
|
||||
if archStr != "common" {
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_TARGET_ARCH :=", archStr)
|
||||
}
|
||||
|
||||
if len(amod.commonProperties.Logtags) > 0 {
|
||||
fmt.Fprintln(w, "LOCAL_LOGTAGS_FILES := ", strings.Join(amod.commonProperties.Logtags, " "))
|
||||
|
|
|
|||
|
|
@ -190,6 +190,7 @@ var BuildOs = func() OsType {
|
|||
|
||||
var (
|
||||
osTypeList []OsType
|
||||
commonTargetMap = make(map[string]Target)
|
||||
|
||||
NoOsType OsType
|
||||
Linux = NewOsType("linux", Host, false)
|
||||
|
|
@ -236,6 +237,13 @@ func NewOsType(name string, class OsClass, defDisabled bool) OsType {
|
|||
DefaultDisabled: defDisabled,
|
||||
}
|
||||
osTypeList = append(osTypeList, os)
|
||||
|
||||
if _, found := commonTargetMap[name]; found {
|
||||
panic(fmt.Errorf("Found Os type duplicate during OsType registration: %q", name))
|
||||
} else {
|
||||
commonTargetMap[name] = Target{Os: os, Arch: Arch{ArchType: Common}}
|
||||
}
|
||||
|
||||
return os
|
||||
}
|
||||
|
||||
|
|
@ -249,15 +257,6 @@ func osByName(name string) OsType {
|
|||
return NoOsType
|
||||
}
|
||||
|
||||
var (
|
||||
commonTarget = Target{
|
||||
Os: Android,
|
||||
Arch: Arch{
|
||||
ArchType: Common,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
type Target struct {
|
||||
Os OsType
|
||||
Arch Arch
|
||||
|
|
@ -989,6 +988,20 @@ func filterMultilibTargets(targets []Target, multilib string) []Target {
|
|||
return ret
|
||||
}
|
||||
|
||||
func getCommonTargets(targets []Target) []Target {
|
||||
var ret []Target
|
||||
set := make(map[string]bool)
|
||||
|
||||
for _, t := range targets {
|
||||
if _, found := set[t.Os.String()]; !found {
|
||||
set[t.Os.String()] = true
|
||||
ret = append(ret, commonTargetMap[t.Os.String()])
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Use the module multilib setting to select one or more targets from a target list
|
||||
func decodeMultilib(multilib string, targets []Target, prefer32 bool) ([]Target, error) {
|
||||
buildTargets := []Target{}
|
||||
|
|
@ -1001,7 +1014,7 @@ func decodeMultilib(multilib string, targets []Target, prefer32 bool) ([]Target,
|
|||
}
|
||||
switch multilib {
|
||||
case "common":
|
||||
buildTargets = append(buildTargets, commonTarget)
|
||||
buildTargets = append(buildTargets, getCommonTargets(targets)...)
|
||||
case "both":
|
||||
if prefer32 {
|
||||
buildTargets = append(buildTargets, filterMultilibTargets(targets, "lib32")...)
|
||||
|
|
|
|||
|
|
@ -81,8 +81,18 @@ func registerMutators(ctx *blueprint.Context) {
|
|||
}
|
||||
|
||||
func RegisterTestMutators(ctx *blueprint.Context) {
|
||||
mutators := registerMutatorsContext{}
|
||||
mutators := ®isterMutatorsContext{}
|
||||
|
||||
register := func(funcs []RegisterMutatorFunc) {
|
||||
for _, f := range funcs {
|
||||
f(mutators)
|
||||
}
|
||||
}
|
||||
|
||||
register(testPreDeps)
|
||||
mutators.BottomUp("deps", depsMutator).Parallel()
|
||||
register(testPostDeps)
|
||||
|
||||
registerMutatorsToContext(ctx, mutators.mutators)
|
||||
}
|
||||
|
||||
|
|
@ -97,7 +107,7 @@ type RegisterMutatorsContext interface {
|
|||
|
||||
type RegisterMutatorFunc func(RegisterMutatorsContext)
|
||||
|
||||
var preArch, preDeps, postDeps []RegisterMutatorFunc
|
||||
var preArch, preDeps, postDeps, testPreDeps, testPostDeps []RegisterMutatorFunc
|
||||
|
||||
func PreArchMutators(f RegisterMutatorFunc) {
|
||||
preArch = append(preArch, f)
|
||||
|
|
@ -111,6 +121,14 @@ func PostDepsMutators(f RegisterMutatorFunc) {
|
|||
postDeps = append(postDeps, f)
|
||||
}
|
||||
|
||||
func TestPreDepsMutators(f RegisterMutatorFunc) {
|
||||
testPreDeps = append(testPreDeps, f)
|
||||
}
|
||||
|
||||
func TeststPostDepsMutators(f RegisterMutatorFunc) {
|
||||
testPostDeps = append(testPostDeps, f)
|
||||
}
|
||||
|
||||
type AndroidTopDownMutator func(TopDownMutatorContext)
|
||||
|
||||
type TopDownMutatorContext interface {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
// Copyright 2017 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package python
|
||||
|
||||
// This file contains the module types for building Python binary.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
|
||||
"android/soong/android"
|
||||
)
|
||||
|
||||
func init() {
|
||||
android.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
|
||||
}
|
||||
|
||||
type PythonBinaryProperties struct {
|
||||
// the name of the source file that is the main entry point of the program.
|
||||
// this file must also be listed in srcs.
|
||||
// If left unspecified, module name is used instead.
|
||||
// If name doesn’t match any filename in srcs, main must be specified.
|
||||
Main string
|
||||
|
||||
// set the name of the output binary.
|
||||
Stem string
|
||||
|
||||
// append to the name of the output binary.
|
||||
Suffix string
|
||||
}
|
||||
|
||||
type PythonBinary struct {
|
||||
pythonBaseModule
|
||||
|
||||
binaryProperties PythonBinaryProperties
|
||||
|
||||
// soong_zip arguments from all its dependencies.
|
||||
depsParSpecs []parSpec
|
||||
|
||||
// Python runfiles paths from all its dependencies.
|
||||
depsPyRunfiles []string
|
||||
|
||||
// the installation path for Python binary.
|
||||
installPath android.OutputPath
|
||||
}
|
||||
|
||||
var _ PythonSubModule = (*PythonBinary)(nil)
|
||||
|
||||
var (
|
||||
stubTemplateHost = "build/soong/python/scripts/stub_template_host.txt"
|
||||
)
|
||||
|
||||
func PythonBinaryHostFactory() (blueprint.Module, []interface{}) {
|
||||
module := &PythonBinary{}
|
||||
|
||||
return InitPythonBaseModule(&module.pythonBaseModule, module, android.HostSupportedNoCross,
|
||||
&module.binaryProperties)
|
||||
}
|
||||
|
||||
func (p *PythonBinary) GeneratePythonBuildActions(ctx android.ModuleContext) {
|
||||
p.pythonBaseModule.GeneratePythonBuildActions(ctx)
|
||||
|
||||
// no Python source file for compiling par file.
|
||||
if len(p.pythonBaseModule.srcsPathMappings) == 0 && len(p.depsPyRunfiles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// the runfiles packages needs to be populated with "__init__.py".
|
||||
newPyPkgs := []string{}
|
||||
// the set to de-duplicate the new Python packages above.
|
||||
newPyPkgSet := make(map[string]bool)
|
||||
// the runfiles dirs have been treated as packages.
|
||||
existingPyPkgSet := make(map[string]bool)
|
||||
|
||||
wholePyRunfiles := []string{}
|
||||
for _, path := range p.pythonBaseModule.srcsPathMappings {
|
||||
wholePyRunfiles = append(wholePyRunfiles, path.dest)
|
||||
}
|
||||
wholePyRunfiles = append(wholePyRunfiles, p.depsPyRunfiles...)
|
||||
|
||||
// find all the runfiles dirs which have been treated as packages.
|
||||
for _, path := range wholePyRunfiles {
|
||||
if filepath.Base(path) != initFileName {
|
||||
continue
|
||||
}
|
||||
existingPyPkg := PathBeforeLastSlash(path)
|
||||
if _, found := existingPyPkgSet[existingPyPkg]; found {
|
||||
panic(fmt.Errorf("found init file path duplicates: %q for module: %q.",
|
||||
path, ctx.ModuleName()))
|
||||
} else {
|
||||
existingPyPkgSet[existingPyPkg] = true
|
||||
}
|
||||
parentPath := PathBeforeLastSlash(existingPyPkg)
|
||||
populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs)
|
||||
}
|
||||
|
||||
// create new packages under runfiles tree.
|
||||
for _, path := range wholePyRunfiles {
|
||||
if filepath.Base(path) == initFileName {
|
||||
continue
|
||||
}
|
||||
parentPath := PathBeforeLastSlash(path)
|
||||
populateNewPyPkgs(parentPath, existingPyPkgSet, newPyPkgSet, &newPyPkgs)
|
||||
}
|
||||
|
||||
main := p.getPyMainFile(ctx)
|
||||
if main == "" {
|
||||
return
|
||||
}
|
||||
interp := p.getInterpreter(ctx)
|
||||
if interp == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// we need remove "runfiles/" suffix since stub script starts
|
||||
// searching for main file in each sub-dir of "runfiles" directory tree.
|
||||
binFile := registerBuildActionForParFile(ctx, p.getInterpreter(ctx),
|
||||
strings.TrimPrefix(main, runFiles+"/"), p.getStem(ctx),
|
||||
newPyPkgs, append(p.depsParSpecs, p.pythonBaseModule.parSpec))
|
||||
|
||||
// install par file.
|
||||
p.installPath = ctx.InstallFile(
|
||||
android.PathForModuleInstall(ctx, "bin"), binFile)
|
||||
}
|
||||
|
||||
// get interpreter path.
|
||||
func (p *PythonBinary) getInterpreter(ctx android.ModuleContext) string {
|
||||
var interp string
|
||||
switch p.pythonBaseModule.properties.ActualVersion {
|
||||
case pyVersion2:
|
||||
interp = "python2"
|
||||
case pyVersion3:
|
||||
interp = "python3"
|
||||
default:
|
||||
panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
|
||||
p.properties.ActualVersion, ctx.ModuleName()))
|
||||
}
|
||||
|
||||
return interp
|
||||
}
|
||||
|
||||
// find main program path within runfiles tree.
|
||||
func (p *PythonBinary) getPyMainFile(ctx android.ModuleContext) string {
|
||||
var main string
|
||||
if p.binaryProperties.Main == "" {
|
||||
main = p.BaseModuleName() + pyExt
|
||||
} else {
|
||||
main = p.binaryProperties.Main
|
||||
}
|
||||
|
||||
for _, path := range p.pythonBaseModule.srcsPathMappings {
|
||||
if main == path.src.Rel() {
|
||||
return path.dest
|
||||
}
|
||||
}
|
||||
ctx.PropertyErrorf("main", "%q is not listed in srcs.", main)
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (p *PythonBinary) getStem(ctx android.ModuleContext) string {
|
||||
stem := ctx.ModuleName()
|
||||
if p.binaryProperties.Stem != "" {
|
||||
stem = p.binaryProperties.Stem
|
||||
}
|
||||
|
||||
return stem + p.binaryProperties.Suffix
|
||||
}
|
||||
|
||||
// Sets the given directory and all its ancestor directories as Python packages.
|
||||
func populateNewPyPkgs(pkgPath string, existingPyPkgSet,
|
||||
newPyPkgSet map[string]bool, newPyPkgs *[]string) {
|
||||
for pkgPath != "" {
|
||||
if _, found := existingPyPkgSet[pkgPath]; found {
|
||||
break
|
||||
}
|
||||
if _, found := newPyPkgSet[pkgPath]; !found {
|
||||
newPyPkgSet[pkgPath] = true
|
||||
*newPyPkgs = append(*newPyPkgs, pkgPath)
|
||||
// Gets its ancestor directory by trimming last slash.
|
||||
pkgPath = PathBeforeLastSlash(pkgPath)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// filepath.Dir("abc") -> "." and filepath.Dir("/abc") -> "/". However,
|
||||
// the PathBeforeLastSlash() will return "" for both cases above.
|
||||
func PathBeforeLastSlash(path string) string {
|
||||
if idx := strings.LastIndex(path, "/"); idx != -1 {
|
||||
return path[:idx]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (p *PythonBinary) GeneratePythonAndroidMk() (ret android.AndroidMkData, err error) {
|
||||
// Soong installation is only supported for host modules. Have Make
|
||||
// installation trigger Soong installation.
|
||||
if p.pythonBaseModule.Target().Os.Class == android.Host {
|
||||
ret.OutputFile = android.OptionalPathForPath(p.installPath)
|
||||
}
|
||||
ret.Class = "EXECUTABLES"
|
||||
|
||||
ret.Extra = append(ret.Extra, func(w io.Writer, outputFile android.Path) error {
|
||||
path := p.installPath.RelPathString()
|
||||
dir, file := filepath.Split(path)
|
||||
stem := strings.TrimSuffix(file, filepath.Ext(file))
|
||||
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(file))
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_PATH := $(OUT_DIR)/"+filepath.Clean(dir))
|
||||
fmt.Fprintln(w, "LOCAL_MODULE_STEM := "+stem)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
// Copyright 2017 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package python
|
||||
|
||||
// This file contains Ninja build actions for building Python program.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"android/soong/android"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
_ "github.com/google/blueprint/bootstrap"
|
||||
)
|
||||
|
||||
var (
|
||||
pctx = android.NewPackageContext("android/soong/python")
|
||||
|
||||
par = pctx.AndroidStaticRule("par",
|
||||
blueprint.RuleParams{
|
||||
Command: `touch $initFile && ` +
|
||||
`sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` +
|
||||
`$parCmd -o $parFile $parArgs && echo '#!/usr/bin/env python' | cat - $parFile > $out && ` +
|
||||
`chmod +x $out && (rm -f $initFile; rm -f $stub; rm -f $parFile)`,
|
||||
CommandDeps: []string{"$parCmd", "$template"},
|
||||
Description: "build par $out",
|
||||
},
|
||||
"initFile", "interp", "main", "template", "stub", "parCmd", "parFile", "parArgs")
|
||||
)
|
||||
|
||||
func init() {
|
||||
pctx.Import("github.com/google/blueprint/bootstrap")
|
||||
pctx.Import("android/soong/common")
|
||||
|
||||
pctx.HostBinToolVariable("parCmd", "soong_zip")
|
||||
}
|
||||
|
||||
type fileListSpec struct {
|
||||
fileList android.Path
|
||||
relativeRoot string
|
||||
}
|
||||
|
||||
type parSpec struct {
|
||||
rootPrefix string
|
||||
|
||||
fileListSpecs []fileListSpec
|
||||
}
|
||||
|
||||
func (p parSpec) soongParArgs() string {
|
||||
ret := "-P " + p.rootPrefix
|
||||
|
||||
for _, spec := range p.fileListSpecs {
|
||||
ret += " -C " + spec.relativeRoot + " -l " + spec.fileList.String()
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func registerBuildActionForModuleFileList(ctx android.ModuleContext,
|
||||
name string, files android.Paths) android.Path {
|
||||
fileList := android.PathForModuleOut(ctx, name+".list")
|
||||
|
||||
content := []string{}
|
||||
for _, file := range files {
|
||||
content = append(content, file.String())
|
||||
}
|
||||
|
||||
ctx.ModuleBuild(pctx, android.ModuleBuildParams{
|
||||
Rule: android.WriteFile,
|
||||
Output: fileList,
|
||||
Implicits: files,
|
||||
Args: map[string]string{
|
||||
"content": strings.Join(content, "\n"),
|
||||
},
|
||||
})
|
||||
|
||||
return fileList
|
||||
}
|
||||
|
||||
func registerBuildActionForParFile(ctx android.ModuleContext,
|
||||
interpreter, main, binName string, newPyPkgs []string, parSpecs []parSpec) android.Path {
|
||||
|
||||
// intermediate output path for __init__.py
|
||||
initFile := android.PathForModuleOut(ctx, initFileName).String()
|
||||
|
||||
// the path of stub_template_host.txt from source tree.
|
||||
template := android.PathForSource(ctx, stubTemplateHost)
|
||||
|
||||
// intermediate output path for __main__.py
|
||||
stub := android.PathForModuleOut(ctx, mainFileName).String()
|
||||
|
||||
// intermediate output path for par file.
|
||||
parFile := android.PathForModuleOut(ctx, binName+parFileExt)
|
||||
|
||||
// intermediate output path for bin executable.
|
||||
binFile := android.PathForModuleOut(ctx, binName)
|
||||
|
||||
// implicit dependency for parFile build action.
|
||||
implicits := android.Paths{}
|
||||
for _, p := range parSpecs {
|
||||
for _, f := range p.fileListSpecs {
|
||||
implicits = append(implicits, f.fileList)
|
||||
}
|
||||
}
|
||||
|
||||
parArgs := []string{}
|
||||
parArgs = append(parArgs, "-C "+strings.TrimSuffix(stub, mainFileName)+" -f "+stub)
|
||||
parArgs = append(parArgs, "-C "+strings.TrimSuffix(initFile, initFileName)+" -f "+initFile)
|
||||
for _, pkg := range newPyPkgs {
|
||||
parArgs = append(parArgs, "-P "+pkg+" -f "+initFile)
|
||||
}
|
||||
for _, p := range parSpecs {
|
||||
parArgs = append(parArgs, p.soongParArgs())
|
||||
}
|
||||
|
||||
ctx.ModuleBuild(pctx, android.ModuleBuildParams{
|
||||
Rule: par,
|
||||
Output: binFile,
|
||||
Implicits: implicits,
|
||||
Args: map[string]string{
|
||||
"initFile": initFile,
|
||||
// the "\" isn't being interpreted by regex parser, it's being
|
||||
// interpreted in the string literal.
|
||||
"interp": strings.Replace(interpreter, "/", `\/`, -1),
|
||||
"main": strings.Replace(main, "/", `\/`, -1),
|
||||
"template": template.String(),
|
||||
"stub": stub,
|
||||
"parFile": parFile.String(),
|
||||
"parArgs": strings.Join(parArgs, " "),
|
||||
},
|
||||
})
|
||||
|
||||
return binFile
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
// Copyright 2017 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package python
|
||||
|
||||
// This file contains the module types for building Python library.
|
||||
|
||||
import (
|
||||
"github.com/google/blueprint"
|
||||
|
||||
"android/soong/android"
|
||||
)
|
||||
|
||||
func init() {
|
||||
android.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
|
||||
}
|
||||
|
||||
type PythonLibrary struct {
|
||||
pythonBaseModule
|
||||
}
|
||||
|
||||
var _ PythonSubModule = (*PythonLibrary)(nil)
|
||||
|
||||
func PythonLibraryHostFactory() (blueprint.Module, []interface{}) {
|
||||
module := &PythonLibrary{}
|
||||
|
||||
return InitPythonBaseModule(&module.pythonBaseModule, module, android.HostSupportedNoCross)
|
||||
}
|
||||
|
||||
func (p *PythonLibrary) GeneratePythonAndroidMk() (ret android.AndroidMkData, err error) {
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,448 @@
|
|||
// Copyright 2017 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package python
|
||||
|
||||
// This file contains the "Base" module type for building Python program.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
|
||||
"android/soong/android"
|
||||
)
|
||||
|
||||
func init() {
|
||||
android.PreDepsMutators(func(ctx android.RegisterMutatorsContext) {
|
||||
ctx.BottomUp("version_split", versionSplitMutator()).Parallel()
|
||||
})
|
||||
}
|
||||
|
||||
// the version properties that apply to python libraries and binaries.
|
||||
type PythonVersionProperties struct {
|
||||
// true, if the module is required to be built with this version.
|
||||
Enabled *bool
|
||||
|
||||
// if specified, common src files are converted to specific version with converter tool.
|
||||
// Converter bool
|
||||
|
||||
// non-empty list of .py files under this strict Python version.
|
||||
// srcs may reference the outputs of other modules that produce source files like genrule
|
||||
// or filegroup using the syntax ":module".
|
||||
Srcs []string
|
||||
|
||||
// list of the Python libraries under this Python version.
|
||||
Libs []string
|
||||
}
|
||||
|
||||
// properties that apply to python libraries and binaries.
|
||||
type PythonBaseModuleProperties struct {
|
||||
// the package path prefix within the output artifact at which to place the source/data
|
||||
// files of the current module.
|
||||
// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
|
||||
// (from a.b.c import ...) statement.
|
||||
// if left unspecified, all the source/data files of current module are copied to
|
||||
// "runfiles/" tree directory directly.
|
||||
Pkg_path string
|
||||
|
||||
// list of source (.py) files compatible both with Python2 and Python3 used to compile the
|
||||
// Python module.
|
||||
// srcs may reference the outputs of other modules that produce source files like genrule
|
||||
// or filegroup using the syntax ":module".
|
||||
// Srcs has to be non-empty.
|
||||
Srcs []string
|
||||
|
||||
// list of files or filegroup modules that provide data that should be installed alongside
|
||||
// the test. the file extension can be arbitrary except for (.py).
|
||||
Data []string
|
||||
|
||||
// list of the Python libraries compatible both with Python2 and Python3.
|
||||
Libs []string
|
||||
|
||||
Version struct {
|
||||
// all the "srcs" or Python dependencies that are to be used only for Python2.
|
||||
Py2 PythonVersionProperties
|
||||
|
||||
// all the "srcs" or Python dependencies that are to be used only for Python3.
|
||||
Py3 PythonVersionProperties
|
||||
}
|
||||
|
||||
// the actual version each module uses after variations created.
|
||||
// this property name is hidden from users' perspectives, and soong will populate it during
|
||||
// runtime.
|
||||
ActualVersion string `blueprint:"mutated"`
|
||||
}
|
||||
|
||||
type pathMapping struct {
|
||||
dest string
|
||||
src android.Path
|
||||
}
|
||||
|
||||
type pythonBaseModule struct {
|
||||
android.ModuleBase
|
||||
subModule PythonSubModule
|
||||
|
||||
properties PythonBaseModuleProperties
|
||||
|
||||
// the Python files of current module after expanding source dependencies.
|
||||
// pathMapping: <dest: runfile_path, src: source_path>
|
||||
srcsPathMappings []pathMapping
|
||||
|
||||
// the data files of current module after expanding source dependencies.
|
||||
// pathMapping: <dest: runfile_path, src: source_path>
|
||||
dataPathMappings []pathMapping
|
||||
|
||||
// the soong_zip arguments for zipping current module source/data files.
|
||||
parSpec parSpec
|
||||
}
|
||||
|
||||
type PythonSubModule interface {
|
||||
GeneratePythonBuildActions(ctx android.ModuleContext)
|
||||
GeneratePythonAndroidMk() (ret android.AndroidMkData, err error)
|
||||
}
|
||||
|
||||
type PythonDependency interface {
|
||||
GetSrcsPathMappings() []pathMapping
|
||||
GetDataPathMappings() []pathMapping
|
||||
GetParSpec() parSpec
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping {
|
||||
return p.srcsPathMappings
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) GetDataPathMappings() []pathMapping {
|
||||
return p.dataPathMappings
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) GetParSpec() parSpec {
|
||||
return p.parSpec
|
||||
}
|
||||
|
||||
var _ PythonDependency = (*pythonBaseModule)(nil)
|
||||
|
||||
var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil)
|
||||
|
||||
func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule,
|
||||
hod android.HostOrDeviceSupported,
|
||||
props ...interface{}) (blueprint.Module, []interface{}) {
|
||||
|
||||
baseModule.subModule = subModule
|
||||
|
||||
props = append(props, &baseModule.properties)
|
||||
|
||||
return android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon, props...)
|
||||
}
|
||||
|
||||
// the tag used to mark dependencies within "py_libs" attribute.
|
||||
type pythonDependencyTag struct {
|
||||
blueprint.BaseDependencyTag
|
||||
}
|
||||
|
||||
var pyDependencyTag pythonDependencyTag
|
||||
|
||||
var (
|
||||
pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
|
||||
pyExt = ".py"
|
||||
pyVersion2 = "PY2"
|
||||
pyVersion3 = "PY3"
|
||||
initFileName = "__init__.py"
|
||||
mainFileName = "__main__.py"
|
||||
parFileExt = ".zip"
|
||||
runFiles = "runfiles"
|
||||
)
|
||||
|
||||
// create version variants for modules.
|
||||
func versionSplitMutator() func(android.BottomUpMutatorContext) {
|
||||
return func(mctx android.BottomUpMutatorContext) {
|
||||
if base, ok := mctx.Module().(*pythonBaseModule); ok {
|
||||
versionNames := []string{}
|
||||
if base.properties.Version.Py2.Enabled != nil &&
|
||||
*(base.properties.Version.Py2.Enabled) == true {
|
||||
versionNames = append(versionNames, pyVersion2)
|
||||
}
|
||||
if !(base.properties.Version.Py3.Enabled != nil &&
|
||||
*(base.properties.Version.Py3.Enabled) == false) {
|
||||
versionNames = append(versionNames, pyVersion3)
|
||||
}
|
||||
modules := mctx.CreateVariations(versionNames...)
|
||||
for i, v := range versionNames {
|
||||
// set the actual version for Python module.
|
||||
modules[i].(*pythonBaseModule).properties.ActualVersion = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) {
|
||||
// deps from "data".
|
||||
android.ExtractSourcesDeps(ctx, p.properties.Data)
|
||||
// deps from "srcs".
|
||||
android.ExtractSourcesDeps(ctx, p.properties.Srcs)
|
||||
|
||||
switch p.properties.ActualVersion {
|
||||
case pyVersion2:
|
||||
// deps from "version.py2.srcs" property.
|
||||
android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)
|
||||
|
||||
ctx.AddVariationDependencies(nil, pyDependencyTag,
|
||||
uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
|
||||
p.properties.Version.Py2.Libs)...)
|
||||
case pyVersion3:
|
||||
// deps from "version.py3.srcs" property.
|
||||
android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)
|
||||
|
||||
ctx.AddVariationDependencies(nil, pyDependencyTag,
|
||||
uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
|
||||
p.properties.Version.Py3.Libs)...)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
|
||||
p.properties.ActualVersion, ctx.ModuleName()))
|
||||
}
|
||||
}
|
||||
|
||||
// check "libs" duplicates from current module dependencies.
|
||||
func uniqueLibs(ctx android.BottomUpMutatorContext,
|
||||
commonLibs []string, versionProp string, versionLibs []string) []string {
|
||||
set := make(map[string]string)
|
||||
ret := []string{}
|
||||
|
||||
// deps from "libs" property.
|
||||
for _, l := range commonLibs {
|
||||
if _, found := set[l]; found {
|
||||
ctx.PropertyErrorf("libs", "%q has duplicates within libs.", l)
|
||||
} else {
|
||||
set[l] = "libs"
|
||||
ret = append(ret, l)
|
||||
}
|
||||
}
|
||||
// deps from "version.pyX.libs" property.
|
||||
for _, l := range versionLibs {
|
||||
if _, found := set[l]; found {
|
||||
ctx.PropertyErrorf(versionProp, "%q has duplicates within %q.", set[l])
|
||||
} else {
|
||||
set[l] = versionProp
|
||||
ret = append(ret, l)
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
|
||||
p.subModule.GeneratePythonBuildActions(ctx)
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) {
|
||||
// expand python files from "srcs" property.
|
||||
srcs := p.properties.Srcs
|
||||
switch p.properties.ActualVersion {
|
||||
case pyVersion2:
|
||||
srcs = append(srcs, p.properties.Version.Py2.Srcs...)
|
||||
case pyVersion3:
|
||||
srcs = append(srcs, p.properties.Version.Py3.Srcs...)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
|
||||
p.properties.ActualVersion, ctx.ModuleName()))
|
||||
}
|
||||
expandedSrcs := ctx.ExpandSources(srcs, nil)
|
||||
if len(expandedSrcs) == 0 {
|
||||
ctx.ModuleErrorf("doesn't have any source files!")
|
||||
}
|
||||
|
||||
// expand data files from "data" property.
|
||||
expandedData := ctx.ExpandSources(p.properties.Data, nil)
|
||||
|
||||
// sanitize pkg_path.
|
||||
pkg_path := p.properties.Pkg_path
|
||||
if pkg_path != "" {
|
||||
pkg_path = filepath.Clean(p.properties.Pkg_path)
|
||||
if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
|
||||
strings.HasPrefix(pkg_path, "/") {
|
||||
ctx.PropertyErrorf("pkg_path", "%q is not a valid format.",
|
||||
p.properties.Pkg_path)
|
||||
return
|
||||
}
|
||||
// pkg_path starts from "runfiles/" implicitly.
|
||||
pkg_path = filepath.Join(runFiles, pkg_path)
|
||||
} else {
|
||||
// pkg_path starts from "runfiles/" implicitly.
|
||||
pkg_path = runFiles
|
||||
}
|
||||
|
||||
p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
|
||||
|
||||
p.parSpec = p.dumpFileList(ctx, pkg_path)
|
||||
|
||||
p.uniqWholeRunfilesTree(ctx)
|
||||
}
|
||||
|
||||
// generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
|
||||
// for python/data files.
|
||||
func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
|
||||
expandedSrcs, expandedData android.Paths) {
|
||||
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
|
||||
// check duplicates.
|
||||
destToPySrcs := make(map[string]string)
|
||||
destToPyData := make(map[string]string)
|
||||
|
||||
for _, s := range expandedSrcs {
|
||||
if s.Ext() != pyExt {
|
||||
ctx.PropertyErrorf("srcs", "found non (.py) file: %q!", s.String())
|
||||
continue
|
||||
}
|
||||
runfilesPath := filepath.Join(pkg_path, s.Rel())
|
||||
identifiers := strings.Split(strings.TrimSuffix(runfilesPath, pyExt), "/")
|
||||
for _, token := range identifiers {
|
||||
if !pyIdentifierRegexp.MatchString(token) {
|
||||
ctx.PropertyErrorf("srcs", "the path %q contains invalid token %q.",
|
||||
runfilesPath, token)
|
||||
}
|
||||
}
|
||||
if fillInMap(ctx, destToPySrcs, runfilesPath, s.String(), p.Name(), p.Name()) {
|
||||
p.srcsPathMappings = append(p.srcsPathMappings,
|
||||
pathMapping{dest: runfilesPath, src: s})
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range expandedData {
|
||||
if d.Ext() == pyExt {
|
||||
ctx.PropertyErrorf("data", "found (.py) file: %q!", d.String())
|
||||
continue
|
||||
}
|
||||
runfilesPath := filepath.Join(pkg_path, d.Rel())
|
||||
if fillInMap(ctx, destToPyData, runfilesPath, d.String(), p.Name(), p.Name()) {
|
||||
p.dataPathMappings = append(p.dataPathMappings,
|
||||
pathMapping{dest: runfilesPath, src: d})
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// register build actions to dump filelist to disk.
|
||||
func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
|
||||
relativeRootMap := make(map[string]android.Paths)
|
||||
// the soong_zip params in order to pack current module's Python/data files.
|
||||
ret := parSpec{rootPrefix: pkg_path}
|
||||
|
||||
pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
|
||||
|
||||
// "srcs" or "data" properties may have filegroup so it might happen that
|
||||
// the relative root for each source path is different.
|
||||
for _, path := range pathMappings {
|
||||
relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
|
||||
if v, found := relativeRootMap[relativeRoot]; found {
|
||||
relativeRootMap[relativeRoot] = append(v, path.src)
|
||||
} else {
|
||||
relativeRootMap[relativeRoot] = android.Paths{path.src}
|
||||
}
|
||||
}
|
||||
|
||||
var keys []string
|
||||
|
||||
// in order to keep stable order of soong_zip params, we sort the keys here.
|
||||
for k := range relativeRootMap {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
for _, k := range keys {
|
||||
// use relative root as filelist name.
|
||||
fileListPath := registerBuildActionForModuleFileList(
|
||||
ctx, strings.Replace(k, "/", "_", -1), relativeRootMap[k])
|
||||
ret.fileListSpecs = append(ret.fileListSpecs,
|
||||
fileListSpec{fileList: fileListPath, relativeRoot: k})
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// check Python/data files duplicates from current module and its whole dependencies.
|
||||
func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
|
||||
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
|
||||
// check duplicates.
|
||||
destToPySrcs := make(map[string]string)
|
||||
destToPyData := make(map[string]string)
|
||||
|
||||
for _, path := range p.srcsPathMappings {
|
||||
destToPySrcs[path.dest] = path.src.String()
|
||||
}
|
||||
for _, path := range p.dataPathMappings {
|
||||
destToPyData[path.dest] = path.src.String()
|
||||
}
|
||||
|
||||
// visit all its dependencies in depth first.
|
||||
ctx.VisitDepsDepthFirst(func(module blueprint.Module) {
|
||||
// module can only depend on Python library.
|
||||
if base, ok := module.(*pythonBaseModule); ok {
|
||||
if _, ok := base.subModule.(*PythonLibrary); !ok {
|
||||
panic(fmt.Errorf(
|
||||
"the dependency %q of module %q is not Python library!",
|
||||
ctx.ModuleName(), ctx.OtherModuleName(module)))
|
||||
}
|
||||
} else {
|
||||
return
|
||||
}
|
||||
if dep, ok := module.(PythonDependency); ok {
|
||||
srcs := dep.GetSrcsPathMappings()
|
||||
for _, path := range srcs {
|
||||
if !fillInMap(ctx, destToPySrcs,
|
||||
path.dest, path.src.String(), ctx.ModuleName(),
|
||||
ctx.OtherModuleName(module)) {
|
||||
continue
|
||||
}
|
||||
// binary needs the Python runfiles paths from all its
|
||||
// dependencies to fill __init__.py in each runfiles dir.
|
||||
if sub, ok := p.subModule.(*PythonBinary); ok {
|
||||
sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest)
|
||||
}
|
||||
}
|
||||
data := dep.GetDataPathMappings()
|
||||
for _, path := range data {
|
||||
fillInMap(ctx, destToPyData,
|
||||
path.dest, path.src.String(), ctx.ModuleName(),
|
||||
ctx.OtherModuleName(module))
|
||||
}
|
||||
// binary needs the soong_zip arguments from all its
|
||||
// dependencies to generate executable par file.
|
||||
if sub, ok := p.subModule.(*PythonBinary); ok {
|
||||
sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func fillInMap(ctx android.ModuleContext, m map[string]string,
|
||||
key, value, curModule, otherModule string) bool {
|
||||
if oldValue, found := m[key]; found {
|
||||
ctx.ModuleErrorf("found two files to be placed at the same runfiles location %q."+
|
||||
" First file: in module %s at path %q."+
|
||||
" Second file: in module %s at path %q.",
|
||||
key, curModule, oldValue, otherModule, value)
|
||||
return false
|
||||
} else {
|
||||
m[key] = value
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *pythonBaseModule) AndroidMk() (ret android.AndroidMkData, err error) {
|
||||
return p.subModule.GeneratePythonAndroidMk()
|
||||
}
|
||||
|
|
@ -0,0 +1,456 @@
|
|||
// Copyright 2017 Google Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package python
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"android/soong/android"
|
||||
|
||||
"github.com/google/blueprint"
|
||||
)
|
||||
|
||||
type pyBinary struct {
|
||||
name string
|
||||
actualVersion string
|
||||
pyRunfiles []string
|
||||
depsPyRunfiles []string
|
||||
parSpec string
|
||||
depsParSpecs []string
|
||||
}
|
||||
|
||||
var (
|
||||
buildNamePrefix = "soong_python_test"
|
||||
moduleVariantErrTemplate = "%s: module %q variant %q: "
|
||||
pkgPathErrTemplate = moduleVariantErrTemplate +
|
||||
"pkg_path: %q is not a valid format."
|
||||
badIdentifierErrTemplate = moduleVariantErrTemplate +
|
||||
"srcs: the path %q contains invalid token %q."
|
||||
dupRunfileErrTemplate = moduleVariantErrTemplate +
|
||||
"found two files to be placed at the same runfiles location %q." +
|
||||
" First file: in module %s at path %q." +
|
||||
" Second file: in module %s at path %q."
|
||||
noSrcFileErr = moduleVariantErrTemplate + "doesn't have any source files!"
|
||||
badSrcFileExtErr = moduleVariantErrTemplate + "srcs: found non (.py) file: %q!"
|
||||
badDataFileExtErr = moduleVariantErrTemplate + "data: found (.py) file: %q!"
|
||||
bpFile = "Blueprints"
|
||||
|
||||
data = []struct {
|
||||
desc string
|
||||
mockFiles map[string][]byte
|
||||
|
||||
errors []string
|
||||
expectedBinaries []pyBinary
|
||||
}{
|
||||
{
|
||||
desc: "module without any src files",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
}`,
|
||||
),
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(noSrcFileErr,
|
||||
"dir/Blueprints:1:1", "lib1", "PY3"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module with bad src file ext",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
srcs: [
|
||||
"file1.exe",
|
||||
],
|
||||
}`,
|
||||
),
|
||||
"dir/file1.exe": nil,
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(badSrcFileExtErr,
|
||||
"dir/Blueprints:3:11", "lib1", "PY3", "dir/file1.exe"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module with bad data file ext",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
data: [
|
||||
"file2.py",
|
||||
],
|
||||
}`,
|
||||
),
|
||||
"dir/file1.py": nil,
|
||||
"dir/file2.py": nil,
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(badDataFileExtErr,
|
||||
"dir/Blueprints:6:11", "lib1", "PY3", "dir/file2.py"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module with bad pkg_path format",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
pkg_path: "a/c/../../",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
}
|
||||
|
||||
python_library_host {
|
||||
name: "lib2",
|
||||
pkg_path: "a/c/../../../",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
}
|
||||
|
||||
python_library_host {
|
||||
name: "lib3",
|
||||
pkg_path: "/a/c/../../",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
}`,
|
||||
),
|
||||
"dir/file1.py": nil,
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(pkgPathErrTemplate,
|
||||
"dir/Blueprints:11:15", "lib2", "PY3", "a/c/../../../"),
|
||||
fmt.Sprintf(pkgPathErrTemplate,
|
||||
"dir/Blueprints:19:15", "lib3", "PY3", "/a/c/../../"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module with bad runfile src path format",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
pkg_path: "a/b/c/",
|
||||
srcs: [
|
||||
".file1.py",
|
||||
"123/file1.py",
|
||||
"-e/f/file1.py",
|
||||
],
|
||||
}`,
|
||||
),
|
||||
"dir/.file1.py": nil,
|
||||
"dir/123/file1.py": nil,
|
||||
"dir/-e/f/file1.py": nil,
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
|
||||
"lib1", "PY3", "runfiles/a/b/c/-e/f/file1.py", "-e"),
|
||||
fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
|
||||
"lib1", "PY3", "runfiles/a/b/c/.file1.py", ".file1"),
|
||||
fmt.Sprintf(badIdentifierErrTemplate, "dir/Blueprints:4:11",
|
||||
"lib1", "PY3", "runfiles/a/b/c/123/file1.py", "123"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module with duplicate runfile path",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib1",
|
||||
pkg_path: "a/b/",
|
||||
srcs: [
|
||||
"c/file1.py",
|
||||
],
|
||||
}
|
||||
|
||||
python_library_host {
|
||||
name: "lib2",
|
||||
pkg_path: "a/b/c/",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
libs: [
|
||||
"lib1",
|
||||
],
|
||||
}
|
||||
`,
|
||||
),
|
||||
"dir/c/file1.py": nil,
|
||||
"dir/file1.py": nil,
|
||||
},
|
||||
errors: []string{
|
||||
fmt.Sprintf(dupRunfileErrTemplate, "dir/Blueprints:9:6",
|
||||
"lib2", "PY3", "runfiles/a/b/c/file1.py", "lib2", "dir/file1.py",
|
||||
"lib1", "dir/c/file1.py"),
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "module for testing dependencies",
|
||||
mockFiles: map[string][]byte{
|
||||
bpFile: []byte(`subdirs = ["dir"]`),
|
||||
filepath.Join("dir", bpFile): []byte(
|
||||
`python_library_host {
|
||||
name: "lib5",
|
||||
pkg_path: "a/b/",
|
||||
srcs: [
|
||||
"file1.py",
|
||||
],
|
||||
version: {
|
||||
py2: {
|
||||
enabled: true,
|
||||
},
|
||||
py3: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
python_library_host {
|
||||
name: "lib6",
|
||||
pkg_path: "c/d/",
|
||||
srcs: [
|
||||
"file2.py",
|
||||
],
|
||||
libs: [
|
||||
"lib5",
|
||||
],
|
||||
}
|
||||
|
||||
python_binary_host {
|
||||
name: "bin",
|
||||
pkg_path: "e/",
|
||||
srcs: [
|
||||
"bin.py",
|
||||
],
|
||||
libs: [
|
||||
"lib5",
|
||||
],
|
||||
version: {
|
||||
py3: {
|
||||
enabled: true,
|
||||
srcs: [
|
||||
"file4.py",
|
||||
],
|
||||
libs: [
|
||||
"lib6",
|
||||
],
|
||||
},
|
||||
},
|
||||
}`,
|
||||
),
|
||||
filepath.Join("dir", "file1.py"): nil,
|
||||
filepath.Join("dir", "file2.py"): nil,
|
||||
filepath.Join("dir", "bin.py"): nil,
|
||||
filepath.Join("dir", "file4.py"): nil,
|
||||
stubTemplateHost: []byte(`PYTHON_BINARY = '%interpreter%'
|
||||
MAIN_FILE = '%main%'`),
|
||||
},
|
||||
expectedBinaries: []pyBinary{
|
||||
{
|
||||
name: "bin",
|
||||
actualVersion: "PY3",
|
||||
pyRunfiles: []string{
|
||||
"runfiles/e/bin.py",
|
||||
"runfiles/e/file4.py",
|
||||
},
|
||||
depsPyRunfiles: []string{
|
||||
"runfiles/a/b/file1.py",
|
||||
"runfiles/c/d/file2.py",
|
||||
},
|
||||
parSpec: "-P runfiles/e -C dir/ -l @prefix@/.intermediates/dir/bin/PY3/dir_.list",
|
||||
depsParSpecs: []string{
|
||||
"-P runfiles/a/b -C dir/ -l @prefix@/.intermediates/dir/lib5/PY3/dir_.list",
|
||||
"-P runfiles/c/d -C dir/ -l @prefix@/.intermediates/dir/lib6/PY3/dir_.list",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func TestPythonModule(t *testing.T) {
|
||||
config, buildDir := setupBuildEnv(t)
|
||||
defer tearDownBuildEnv()
|
||||
android.TestPreDepsMutators(func(ctx android.RegisterMutatorsContext) {
|
||||
ctx.BottomUp("version_split", versionSplitMutator()).Parallel()
|
||||
})
|
||||
for _, d := range data {
|
||||
t.Run(d.desc, func(t *testing.T) {
|
||||
ctx := blueprint.NewContext()
|
||||
android.RegisterTestMutators(ctx)
|
||||
ctx.RegisterModuleType("python_library_host", PythonLibraryHostFactory)
|
||||
ctx.RegisterModuleType("python_binary_host", PythonBinaryHostFactory)
|
||||
ctx.MockFileSystem(d.mockFiles)
|
||||
_, testErrs := ctx.ParseBlueprintsFiles(bpFile)
|
||||
fail(t, testErrs)
|
||||
_, actErrs := ctx.PrepareBuildActions(config)
|
||||
if len(actErrs) > 0 {
|
||||
testErrs = append(testErrs, expectErrors(t, actErrs, d.errors)...)
|
||||
} else {
|
||||
for _, e := range d.expectedBinaries {
|
||||
testErrs = append(testErrs,
|
||||
expectModule(t, ctx, buildDir, e.name,
|
||||
e.actualVersion,
|
||||
e.pyRunfiles, e.depsPyRunfiles,
|
||||
e.parSpec, e.depsParSpecs)...)
|
||||
}
|
||||
}
|
||||
fail(t, testErrs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func expectErrors(t *testing.T, actErrs []error, expErrs []string) (testErrs []error) {
|
||||
actErrStrs := []string{}
|
||||
for _, v := range actErrs {
|
||||
actErrStrs = append(actErrStrs, v.Error())
|
||||
}
|
||||
sort.Strings(actErrStrs)
|
||||
if len(actErrStrs) != len(expErrs) {
|
||||
t.Errorf("got (%d) errors, expected (%d) errors!", len(actErrStrs), len(expErrs))
|
||||
for _, v := range actErrStrs {
|
||||
testErrs = append(testErrs, errors.New(v))
|
||||
}
|
||||
} else {
|
||||
sort.Strings(expErrs)
|
||||
for i, v := range actErrStrs {
|
||||
if v != expErrs[i] {
|
||||
testErrs = append(testErrs, errors.New(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func expectModule(t *testing.T, ctx *blueprint.Context, buildDir, name, variant string,
|
||||
expPyRunfiles, expDepsPyRunfiles []string,
|
||||
expParSpec string, expDepsParSpecs []string) (testErrs []error) {
|
||||
module := findModule(ctx, name, variant)
|
||||
if module == nil {
|
||||
t.Fatalf("failed to find module %s!", name)
|
||||
}
|
||||
|
||||
base, baseOk := module.(*pythonBaseModule)
|
||||
if !baseOk {
|
||||
t.Fatalf("%s is not Python module!", name)
|
||||
}
|
||||
sub, subOk := base.subModule.(*PythonBinary)
|
||||
if !subOk {
|
||||
t.Fatalf("%s is not Python binary!", name)
|
||||
}
|
||||
|
||||
actPyRunfiles := []string{}
|
||||
for _, path := range base.srcsPathMappings {
|
||||
actPyRunfiles = append(actPyRunfiles, path.dest)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(actPyRunfiles, expPyRunfiles) {
|
||||
testErrs = append(testErrs, errors.New(fmt.Sprintf(
|
||||
`binary "%s" variant "%s" has unexpected pyRunfiles: %q!`,
|
||||
base.Name(),
|
||||
base.properties.ActualVersion,
|
||||
actPyRunfiles)))
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(sub.depsPyRunfiles, expDepsPyRunfiles) {
|
||||
testErrs = append(testErrs, errors.New(fmt.Sprintf(
|
||||
`binary "%s" variant "%s" has unexpected depsPyRunfiles: %q!`,
|
||||
base.Name(),
|
||||
base.properties.ActualVersion,
|
||||
sub.depsPyRunfiles)))
|
||||
}
|
||||
|
||||
if base.parSpec.soongParArgs() != strings.Replace(expParSpec, "@prefix@", buildDir, 1) {
|
||||
testErrs = append(testErrs, errors.New(fmt.Sprintf(
|
||||
`binary "%s" variant "%s" has unexpected parSpec: %q!`,
|
||||
base.Name(),
|
||||
base.properties.ActualVersion,
|
||||
base.parSpec.soongParArgs())))
|
||||
}
|
||||
|
||||
actDepsParSpecs := []string{}
|
||||
for i, p := range sub.depsParSpecs {
|
||||
actDepsParSpecs = append(actDepsParSpecs, p.soongParArgs())
|
||||
expDepsParSpecs[i] = strings.Replace(expDepsParSpecs[i], "@prefix@", buildDir, 1)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(actDepsParSpecs, expDepsParSpecs) {
|
||||
testErrs = append(testErrs, errors.New(fmt.Sprintf(
|
||||
`binary "%s" variant "%s" has unexpected depsParSpecs: %q!`,
|
||||
base.Name(),
|
||||
base.properties.ActualVersion,
|
||||
actDepsParSpecs)))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func setupBuildEnv(t *testing.T) (config android.Config, buildDir string) {
|
||||
buildDir, err := ioutil.TempDir("", buildNamePrefix)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config = android.TestConfig(buildDir)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func tearDownBuildEnv() {
|
||||
os.RemoveAll(buildNamePrefix)
|
||||
}
|
||||
|
||||
func findModule(ctx *blueprint.Context, name, variant string) blueprint.Module {
|
||||
var ret blueprint.Module
|
||||
ctx.VisitAllModules(func(m blueprint.Module) {
|
||||
if ctx.ModuleName(m) == name && ctx.ModuleSubDir(m) == variant {
|
||||
ret = m
|
||||
}
|
||||
})
|
||||
return ret
|
||||
}
|
||||
|
||||
func fail(t *testing.T, errs []error) {
|
||||
if len(errs) > 0 {
|
||||
for _, err := range errs {
|
||||
t.Error(err)
|
||||
}
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import shutil
|
||||
import sys
|
||||
import subprocess
|
||||
import zipfile
|
||||
|
||||
PYTHON_BINARY = '%interpreter%'
|
||||
MAIN_FILE = '%main%'
|
||||
PYTHON_PATH = 'PYTHONPATH'
|
||||
ZIP_RUNFILES_DIRECTORY_NAME = 'runfiles'
|
||||
|
||||
def SearchPathEnv(name):
|
||||
search_path = os.getenv('PATH', os.defpath).split(os.pathsep)
|
||||
for directory in search_path:
|
||||
if directory == '': continue
|
||||
path = os.path.join(directory, name)
|
||||
if os.path.islink(path):
|
||||
path = os.path.realpath(path)
|
||||
# Check if path is actual executable file.
|
||||
if os.path.isfile(path) and os.access(path, os.X_OK):
|
||||
return path
|
||||
return None
|
||||
|
||||
def FindPythonBinary():
|
||||
if PYTHON_BINARY.startswith('/'):
|
||||
# Case 1: Python interpreter is directly provided with absolute path.
|
||||
return PYTHON_BINARY
|
||||
else:
|
||||
# Case 2: Find Python interpreter through environment variable: PATH.
|
||||
return SearchPathEnv(PYTHON_BINARY)
|
||||
|
||||
# Create the runfiles tree by extracting the zip file
|
||||
def ExtractRunfiles():
|
||||
temp_dir = tempfile.mkdtemp("", "Soong.python_")
|
||||
zf = zipfile.ZipFile(os.path.dirname(__file__))
|
||||
zf.extractall(temp_dir)
|
||||
return os.path.join(temp_dir, ZIP_RUNFILES_DIRECTORY_NAME)
|
||||
|
||||
def Main():
|
||||
args = sys.argv[1:]
|
||||
|
||||
new_env = {}
|
||||
|
||||
try:
|
||||
runfiles_path = ExtractRunfiles()
|
||||
|
||||
# Add runfiles path to PYTHONPATH.
|
||||
python_path_entries = [runfiles_path]
|
||||
|
||||
# Add top dirs within runfiles path to PYTHONPATH.
|
||||
top_entries = [os.path.join(runfiles_path, i) for i in os.listdir(runfiles_path)]
|
||||
top_pkg_dirs = [i for i in top_entries if os.path.isdir(i)]
|
||||
python_path_entries += top_pkg_dirs
|
||||
|
||||
old_python_path = os.environ.get(PYTHON_PATH)
|
||||
separator = ':'
|
||||
new_python_path = separator.join(python_path_entries)
|
||||
|
||||
# Copy old PYTHONPATH.
|
||||
if old_python_path:
|
||||
new_python_path += separator + old_python_path
|
||||
new_env[PYTHON_PATH] = new_python_path
|
||||
|
||||
# Now look for main python source file.
|
||||
main_filepath = os.path.join(runfiles_path, MAIN_FILE)
|
||||
assert os.path.exists(main_filepath), \
|
||||
'Cannot exec() %r: file not found.' % main_filepath
|
||||
assert os.access(main_filepath, os.R_OK), \
|
||||
'Cannot exec() %r: file not readable.' % main_filepath
|
||||
|
||||
python_program = FindPythonBinary()
|
||||
if python_program is None:
|
||||
raise AssertionError('Could not find python binary: ' + PYTHON_BINARY)
|
||||
args = [python_program, main_filepath] + args
|
||||
|
||||
os.environ.update(new_env)
|
||||
|
||||
sys.stdout.flush()
|
||||
retCode = subprocess.call(args)
|
||||
exit(retCode)
|
||||
except:
|
||||
raise
|
||||
finally:
|
||||
shutil.rmtree(os.path.dirname(runfiles_path), True)
|
||||
|
||||
if __name__ == '__main__':
|
||||
Main()
|
||||
Loading…
Reference in New Issue