514 lines
17 KiB
Python
514 lines
17 KiB
Python
# Copyright 2020 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Parses config file and provides various ways of using it."""
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import collections
|
|
|
|
# The config file must be in XML with a structure as descibed below.
|
|
#
|
|
# The top level config element shall contain one or more "target" child
|
|
# elements. Each of these may contain one or more build_config child elements.
|
|
# The build_config child elements will inherit the properties of the target
|
|
# parent.
|
|
#
|
|
# Each "target" and "build_config" may contain the following:
|
|
#
|
|
# Attributes:
|
|
#
|
|
# name: The name of the target.
|
|
#
|
|
# android_target: The name of the android target used with lunch
|
|
#
|
|
# allow_readwrite_all: "true" if the full source folder shall be mounted as
|
|
# read/write. It should be accompanied by a comment with the bug describing
|
|
# why it was required.
|
|
#
|
|
# tags: A comma-separated list of strings to be associated with the target
|
|
# and any of its nested build_targets. You can use a tag to associate
|
|
# information with a target in your configuration file, and retrieve that
|
|
# information using the get_tags API or the has_tag API.
|
|
#
|
|
# Child elements:
|
|
#
|
|
# config: A generic name-value configuration element.
|
|
#
|
|
# Attributes:
|
|
# name: Name of the configuration
|
|
# value: Value of the configuration
|
|
#
|
|
# overlay: An overlay to be mounted while building the target.
|
|
#
|
|
# Attributes:
|
|
#
|
|
# name: The name of the overlay.
|
|
#
|
|
# Child elements:
|
|
#
|
|
# replacement_path: An overlay path that supersedes any conflicts
|
|
# after it.
|
|
#
|
|
# Properties:
|
|
#
|
|
# name: The name of the replacement path. This path will will
|
|
# superced the same path for any subsequent conflicts. If two
|
|
# overlays have the same replacement path an error will occur.
|
|
#
|
|
#
|
|
# view: A map (optionally) specifying a filesystem view mapping for each
|
|
# target.
|
|
#
|
|
# Attributes:
|
|
#
|
|
# name: The name of the view.
|
|
#
|
|
# allow_readwrite: A folder to mount read/write
|
|
# inside the Android build nsjail. Each allowed read-write entry should be
|
|
# accompanied by a bug that indicates why it was required and tracks the
|
|
# progress to a fix.
|
|
#
|
|
# Attributes:
|
|
#
|
|
# path: The path to be allowed read-write mounting.
|
|
#
|
|
# build_config: A list of goals to be used while building the target.
|
|
#
|
|
# Attributes:
|
|
#
|
|
# name: The name of the build config. Defaults to the target name
|
|
# if not set.
|
|
#
|
|
# Child elements:
|
|
#
|
|
# goal: A build goal.
|
|
#
|
|
# Properties:
|
|
#
|
|
# name: The name of the build goal. The build tools pass the name
|
|
# attribute as a parameter to make. This can have a value like
|
|
# "droid" or "VAR=value".
|
|
#
|
|
# contexts: A comma-separated list of the contexts in which this
|
|
# goal applies. If this attribute is missing or blank, the goal
|
|
# applies to all contexts. Otherwise, it applies only in the
|
|
# requested contexts (see get_build_goals).
|
|
|
|
Overlay = collections.namedtuple('Overlay', ['name', 'replacement_paths'])
|
|
|
|
class BuildConfig(object):
|
|
"""Represents configuration of a build_target.
|
|
|
|
Attributes:
|
|
name: name of the build_target used to pull the configuration.
|
|
android_target: The name of the android target used with lunch.
|
|
tags: List of tags associated with the build target config
|
|
build_goals: List of goals to be used while building the target.
|
|
overlays: List of overlays to be mounted.
|
|
views: A list of (source, destination) string path tuple to be mounted.
|
|
See view nodes in XML.
|
|
allow_readwrite_all: If true, mount source tree as rw.
|
|
allow_readwrite: List of directories to be mounted as rw.
|
|
allowed_projects_file: a string path name of a file with a containing
|
|
allowed projects.
|
|
configurations: a map of name to value configurations
|
|
"""
|
|
|
|
def __init__(self,
|
|
name,
|
|
android_target,
|
|
tags=frozenset(),
|
|
build_goals=(),
|
|
overlays=(),
|
|
views=(),
|
|
allow_readwrite_all=False,
|
|
allow_readwrite=(),
|
|
allowed_projects_file=None,
|
|
configurations=None):
|
|
super().__init__()
|
|
self.name = name
|
|
self.android_target = android_target
|
|
self.tags = tags
|
|
self.build_goals = list(build_goals)
|
|
self.overlays = list(overlays)
|
|
self.views = list(views)
|
|
self.allow_readwrite_all = allow_readwrite_all
|
|
self.allow_readwrite = list(allow_readwrite)
|
|
self.allowed_projects_file = allowed_projects_file
|
|
self.configurations = configurations or {}
|
|
|
|
def validate(self):
|
|
"""Run tests to validate build configuration"""
|
|
if not self.name:
|
|
raise ValueError('Error build_config must have a name.')
|
|
# Validate that a build config does not contain an overlay with
|
|
# conflicting replacement paths.
|
|
if len(self.overlays) > 1 and set.intersection(
|
|
*[o.replacement_paths for o in self.overlays]):
|
|
raise ValueError(
|
|
'Error build_config overlays have conflicting replacement_paths.')
|
|
|
|
@classmethod
|
|
def from_config(cls, config_elem, fs_view_map, base_config=None):
|
|
"""Creates a BuildConfig from a config XML element and an optional
|
|
base_config.
|
|
|
|
Args:
|
|
config_elem: the config XML node element to build the configuration
|
|
fs_view_map: A map of view names to list of tuple(source, destination)
|
|
paths.
|
|
base_config: the base BuildConfig to use
|
|
|
|
Returns:
|
|
A build config generated from the config element and the base
|
|
configuration if provided.
|
|
"""
|
|
if base_config is None:
|
|
# Build a base_config with required elements from the new config_elem
|
|
name = config_elem.get('name')
|
|
base_config = cls(
|
|
name=name, android_target=config_elem.get('android_target', name))
|
|
|
|
return cls(
|
|
android_target=config_elem.get('android_target',
|
|
base_config.android_target),
|
|
name=config_elem.get('name', base_config.name),
|
|
allowed_projects_file=config_elem.get(
|
|
'allowed_projects_file', base_config.allowed_projects_file),
|
|
build_goals=_get_build_config_goals(config_elem,
|
|
base_config.build_goals),
|
|
tags=_get_config_tags(config_elem, base_config.tags),
|
|
overlays=_get_overlays(config_elem, base_config.overlays),
|
|
allow_readwrite=_get_allow_readwrite(config_elem,
|
|
base_config.allow_readwrite),
|
|
views=_get_views(config_elem, fs_view_map, base_config.views),
|
|
allow_readwrite_all=_get_allowed_readwrite_all(
|
|
config_elem, base_config.allow_readwrite_all),
|
|
configurations=_get_configurations(config_elem,
|
|
base_config.configurations)
|
|
)
|
|
|
|
|
|
def _get_configurations(config_elem, base):
|
|
configs = dict(base)
|
|
configs.update({
|
|
config.get('name'): config.get('value')
|
|
for config in config_elem.findall('config')
|
|
})
|
|
return configs
|
|
|
|
|
|
def _get_build_config_goals(config_elem, base=None):
|
|
"""Retrieves goals from build_config or target.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
base: Initial list of goals to prepend to the list
|
|
|
|
Returns:
|
|
A list of tuples where the first element of the tuple is the build goal
|
|
name, and the second is a list of the contexts to which this goal applies.
|
|
"""
|
|
|
|
return base + [(goal.get('name'), set(goal.get('contexts').split(','))
|
|
if goal.get('contexts') else None)
|
|
for goal in config_elem.findall('goal')]
|
|
|
|
|
|
def _get_config_tags(config_elem, base=frozenset()):
|
|
"""Retrieves tags from build_config or target.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
base: Initial list of tags to seed the set
|
|
|
|
Returns:
|
|
A set of tags for a build_config.
|
|
"""
|
|
tags = config_elem.get('tags')
|
|
return base.union(set(tags.split(',')) if tags else set())
|
|
|
|
|
|
def _get_allowed_readwrite_all(config_elem, default=False):
|
|
"""Determines if build_config or target is set to allow readwrite for all
|
|
source paths.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
default: Value to use if element doesn't contain the
|
|
allow_readwrite_all attribute.
|
|
|
|
Returns:
|
|
True if build config is set to allow readwrite for all sorce paths
|
|
"""
|
|
value = config_elem.get('allow_readwrite_all')
|
|
return value == 'true' if value else default
|
|
|
|
|
|
def _get_overlays(config_elem, base=None):
|
|
"""Retrieves list of overlays from build_config or target.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
base: Initial list of overlays to prepend to the list
|
|
|
|
Returns:
|
|
A list of tuples of overlays and replacement paths to mount for a build_config or target.
|
|
"""
|
|
overlays = []
|
|
for overlay in config_elem.findall('overlay'):
|
|
overlays.append(
|
|
Overlay(
|
|
name=overlay.get('name'),
|
|
replacement_paths=set([
|
|
path.get('path') for path in overlay.findall('replacement_path')
|
|
])))
|
|
return base + overlays
|
|
|
|
def _get_views(config_elem, fs_view_map, base=None):
|
|
"""Retrieves list of views from build_config or target.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
base: Initial list of views to prepend to the list
|
|
|
|
Returns:
|
|
A list of (source, destination) string path tuple to be mounted. See view
|
|
nodes in XML.
|
|
"""
|
|
return base + [fs for o in config_elem.findall('view')
|
|
for fs in fs_view_map[o.get('name')]]
|
|
|
|
|
|
def _get_allow_readwrite(config_elem, base=None):
|
|
"""Retrieves list of directories to be mounted rw from build_config or
|
|
target.
|
|
|
|
Args:
|
|
config_elem: A build_config or target xml element.
|
|
base: Initial list of rw directories to prepend to the list
|
|
|
|
Returns:
|
|
A list of directories to be mounted rw.
|
|
"""
|
|
return (base +
|
|
[o.get('path') for o in config_elem.findall('allow_readwrite')])
|
|
|
|
|
|
def _get_fs_view_map(config):
|
|
"""Retrieves the map of filesystem views.
|
|
|
|
Args:
|
|
config: An XML Element that is the root of the config XML tree.
|
|
|
|
Returns:
|
|
A dict of filesystem views keyed by view name. A filesystem view is a
|
|
list of (source, destination) string path tuples.
|
|
"""
|
|
# A valid config file is not required to include FS Views, only overlay
|
|
# targets.
|
|
return {
|
|
view.get('name'): [(path.get('source'), path.get('destination'))
|
|
for path in view.findall('path')
|
|
] for view in config.findall('view')
|
|
}
|
|
|
|
|
|
def _get_build_config_map(config):
|
|
"""Retrieves a map of all build config.
|
|
|
|
Args:
|
|
config: An XML Element that is the root of the config XML tree.
|
|
|
|
Returns:
|
|
A dict of BuildConfig keyed by build_target.
|
|
"""
|
|
fs_view_map = _get_fs_view_map(config)
|
|
build_config_map = {}
|
|
for target_config in config.findall('target'):
|
|
base_target = BuildConfig.from_config(target_config, fs_view_map)
|
|
|
|
for build_config in target_config.findall('build_config'):
|
|
build_target = BuildConfig.from_config(build_config, fs_view_map,
|
|
base_target)
|
|
build_target.validate()
|
|
build_config_map[build_target.name] = build_target
|
|
|
|
return build_config_map
|
|
|
|
|
|
class Config:
|
|
"""Presents an API to the static XML configuration."""
|
|
|
|
def __init__(self, config_filename):
|
|
"""Initializes a Config instance from the specificed filename
|
|
|
|
This method parses the XML content of the file named by config_filename
|
|
into internal data structures. You can then use various methods to query
|
|
the static config.
|
|
|
|
Args:
|
|
config_filename: The name of the file from which to load the config.
|
|
"""
|
|
|
|
tree = ET.parse(config_filename)
|
|
config = tree.getroot()
|
|
self._build_config_map = _get_build_config_map(config)
|
|
|
|
def get_available_build_targets(self):
|
|
"""Return a list of available build targets."""
|
|
return sorted(self._build_config_map.keys())
|
|
|
|
def get_tags(self, build_target):
|
|
"""Given a build_target, return the (possibly empty) set of tags."""
|
|
return self._build_config_map[build_target].tags
|
|
|
|
def has_tag(self, build_target, tag):
|
|
"""Return true if build_target has tag.
|
|
|
|
Args:
|
|
build_target: A string build_target to be queried.
|
|
tag: A string tag that this target may have.
|
|
|
|
Returns:
|
|
If the build_target has the tag, True. Otherwise, False.
|
|
"""
|
|
return tag in self._build_config_map[build_target].tags
|
|
|
|
def get_allowed_projects_file(self, build_target):
|
|
"""Given a build_target, return a string with the allowed projects file."""
|
|
return self._build_config_map[build_target].allowed_projects_file
|
|
|
|
def get_build_config_android_target(self, build_target):
|
|
"""Given a build_target, return an android_target.
|
|
|
|
Generally a build_target maps directory to the android_target of the same
|
|
name, but they can differ. In a config.xml file, the name attribute of a
|
|
target element is the android_target (which is used for lunch). The name
|
|
attribute (if any) of a build_config element is the build_target. If a
|
|
build_config element does not have a name attribute, then the build_target
|
|
is the android_target.
|
|
|
|
Args:
|
|
build_target: A string build_target to be queried.
|
|
|
|
Returns:
|
|
A string android_target that can be used for lunch.
|
|
"""
|
|
return self._build_config_map[build_target].android_target
|
|
|
|
def get_build_goals(self, build_target, contexts=frozenset()):
|
|
"""Given a build_target and a context, return a list of build goals.
|
|
|
|
For a given build_target, we may build in a variety of contexts. For
|
|
example we might build in continuous integration, or we might build
|
|
locally, or other contexts defined by the configuration file and scripts
|
|
that use it. The contexts parameter is a set of strings that specify the
|
|
contexts for which this function should retrieve goals.
|
|
|
|
In the configuration file, each goal has a contexts attribute, which
|
|
specifies the contexts to which the goal applies. We treat a goal with no
|
|
contexts attribute as applying to all contexts.
|
|
|
|
Example:
|
|
|
|
<build_config>
|
|
<goal name="droid"/>
|
|
<goal name="dist" contexts="ota"/>
|
|
</build_config>
|
|
|
|
Here we have the goal "droid", which matches all contexts, and the goal
|
|
"dist", which matches the "ota" context. Invoking this method with the
|
|
set(['ota']) would return ['droid', 'dist'].
|
|
|
|
Args:
|
|
build_target: A string build_target to be queried.
|
|
context: A set of contexts for which to retrieve goals.
|
|
|
|
Returns:
|
|
A list of strings, where each string is a goal to be passed to make.
|
|
"""
|
|
|
|
build_goals = []
|
|
for goal, build_contexts in self._build_config_map[
|
|
build_target].build_goals:
|
|
if not build_contexts:
|
|
build_goals.append(goal)
|
|
elif build_contexts.intersection(contexts):
|
|
build_goals.append(goal)
|
|
|
|
return build_goals
|
|
|
|
def get_rw_allowlist_map(self):
|
|
"""Return read-write allowlist map.
|
|
|
|
Returns:
|
|
A dict of string lists of keyed by target name. Each value in the dict is
|
|
a list of allowed read-write paths corresponding to the target.
|
|
"""
|
|
return {b.name: b.allow_readwrite for b in self._build_config_map.values()}
|
|
|
|
def get_allow_readwrite_all(self, build_target):
|
|
"""Return True if the target should mount all its source as read-write.
|
|
|
|
Args:
|
|
build_target: A string build_target to be queried.
|
|
|
|
Returns:
|
|
True if the target should mount all its source as read-write.
|
|
"""
|
|
return self._build_config_map[build_target].allow_readwrite_all
|
|
|
|
def get_overlay_map(self):
|
|
"""Return the overlay map.
|
|
|
|
Returns:
|
|
A dict of keyed by target name. Each value in the dict is a list of
|
|
overlay names corresponding to the target.
|
|
"""
|
|
return {
|
|
b.name : [o.name for o in b.overlays
|
|
] for b in self._build_config_map.values()
|
|
}
|
|
|
|
|
|
def get_fs_view_map(self):
|
|
"""Return the filesystem view map.
|
|
Returns:
|
|
A dict of filesystem views keyed by target name. A filesystem view is a
|
|
list of (source, destination) string path tuples.
|
|
"""
|
|
return {b.name : b.views for b in self._build_config_map.values()}
|
|
|
|
|
|
def get_build_config(self, build_target):
|
|
return self._build_config_map[build_target]
|
|
|
|
|
|
def factory(config_filename):
|
|
"""Create an instance of a Config class.
|
|
|
|
Args:
|
|
config_filename: The name of the file from which to load the config. This
|
|
can be None, which results in this function returning None.
|
|
|
|
Returns:
|
|
If config_filename is None, returns None. Otherwise, a new instance of a
|
|
Config class containing the configuration parsed from config_filename.
|
|
"""
|
|
if config_filename is None:
|
|
return None
|
|
|
|
return Config(config_filename)
|