p71924506
/
meta-ros
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

DRAFT mcf: Recognize "-f CONFIGFILE" option, v2 of CONFIGFILE, and more (v7.0.0) 

- Also allow CONFIGFILE to have a ".mcf" extension.

- Drop support for broken tags=

- Support CONFIGFILE version 2 (McfFileVersion = 2)

  - Rename "webos_layers" to "Layers", but continue to recognize old name.

  - Recognize dict-s for fetch and options, but continue to recognize
    file version 1 strings.

  - Recognize an optional BblayersConfExtraLines variable in CONFIGFILE
    that contains a list of strings to be appended as separate lines to the generated
    conf/bblayers.conf .
  - Rename "submission" item to "fetch".

- Fix error message to mention conf/layer.conf instead of incorrect local.conf .
- Correct/improve messages

- Requires Python 3.5 => major version bump.
This commit is contained in:
Herb Kuta 2019-05-30 17:15:45 -07:00
parent f586a9cd53
commit e23d42f253
1 changed files with 104 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
scripts/mcf
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
# Copyright (c) 2008-2017 LG Electronics, Inc.
# Copyright (c) 2008-2019 LG Electronics, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import errno
import logging
@ -24,8 +23,10 @@ import re
from time import gmtime, strftime, sleep
import shutil
import glob
import importlib.util
import importlib.machinery
__version__ = "6.2.3"
__version__ = "7.0.0"
logger = logging.getLogger(__name__)
@ -34,6 +35,7 @@ TRACE = False
REMOTE = "origin"
SSTATE_MIRRORS = ''
LAYERS = {}
MCFFILEVERSION = None
DISTRO = None
SUPPORTED_MACHINES = []
@ -116,14 +118,16 @@ def getopts():
general.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default,
help='command to mcf; if update is given, none of the remaining options nor MACHINE can be specified (default: %(default)s)')
general.add_argument('-f', '--config-file', dest='configfile', help='.mcf/.py file specifying the build configuration (default: weboslayers.py in same directory as this script)')
if mcfcommand in ('configure','update+configure'):
variations = parser.add_argument_group('Build Instructions')
variations.add_argument('-p', '--enable-parallel-make', dest='parallel_make', type=int, default=0,
help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 = 2x the number of processor cores)')
help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 means 2x the number of processor cores)')
variations.add_argument('-b', '--enable-bb-number-threads', dest='bb_number_threads', type=int, default=0,
help='maximum number of bitbake tasks to spawn (default: 0 = 2x the number of processor cores))')
help='maximum number of bitbake tasks to spawn (default: 0 means 2x the number of processor cores))')
icecc = parser.add_argument_group('ICECC Configuration')
@ -263,53 +267,91 @@ def location_to_dirname(location):
str1 = location.split('/')
return os.path.splitext(str1[len(str1)-1])[0]
def read_weboslayers(path):
sys.path.insert(0,path)
if not os.path.isfile(os.path.join(path,'weboslayers.py')):
raise Exception("Error: Configuration file %s does not exist!" % os.path.join(path,'weboslayers.py'))
def read_mcfconfigfile(configfile):
if not os.path.isfile(configfile):
raise Exception("Error: Configuration file %s does not exist!" % configfile)
from weboslayers import webos_layers
importlib.machinery.SOURCE_SUFFIXES.append(".mcf")
spec = importlib.util.spec_from_file_location("mcfconfigfile", configfile)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules["mcfconfigfile"] = module
for p in webos_layers:
layer = {"name":p[0], "priority":p[1], "url":p[2], "submission":p[3], "location":p[4]}
try:
from mcfconfigfile import McfFileVersion
except ImportError:
McfFileVersion = 1
global MCFFILEVERSION
MCFFILEVERSION = McfFileVersion
logger.debug("Config file version: %d" % MCFFILEVERSION)
if MCFFILEVERSION > 1:
try:
from mcfconfigfile import Layers
except ImportError:
from mcfconfigfile import webos_layers
Layers = webos_layers
else:
from mcfconfigfile import webos_layers
Layers = webos_layers
for p in Layers:
if MCFFILEVERSION > 1 and isinstance(p[4], dict):
options = p[4]
try:
location = options["location"]
except KeyError:
location = ''
else:
# It's expected to be a string.
location = p[4]
layer = {"name":p[0], "priority":p[1], "url":p[2], "fetch":p[3], "location":location}
LAYERS[layer["name"]] = layer
parsesubmissions(layer)
parsefetch(layer)
if not layer["url"] and not layer["location"]:
raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in weboslayers.py" % layer["name"])
raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in %s" % (layer["name"], configfile))
if not layer["location"]:
layer["location"] = location_to_dirname(layer["url"])
from weboslayers import Distribution
from mcfconfigfile import Distribution
global DISTRO
DISTRO = Distribution
from weboslayers import Machines
from mcfconfigfile import Machines
global SUPPORTED_MACHINES
SUPPORTED_MACHINES = Machines
def parsesubmissions(layer):
def parsefetch(layer):
fetch = layer["fetch"]
branch = ''
commit = ''
tag = ''
for vgit in layer["submission"].split(','):
if not vgit:
continue
str1, str2 = vgit.split('=')
if str1.lower() == 'commit':
if not commit:
commit = str2
elif str1.lower() == 'branch':
branch = str2
elif str1.lower() == 'tag':
if not tag:
tag = str2
if MCFFILEVERSION > 1 and isinstance(fetch, dict):
try:
branch = fetch["branch"]
except KeyError:
branch = 'master'
try:
commit = fetch["commit"]
except KeyError:
pass
else:
# It's expected to be a string.
for vgit in fetch.split(','):
if not vgit:
continue
str1, str2 = vgit.split('=')
if str1.lower() == 'commit':
if not commit:
commit = str2
elif str1.lower() == 'branch':
branch = str2
if not branch:
branch = 'master'
if not branch:
branch = 'master'
layer["branch_new"] = branch
layer["commit_new"] = commit
layer["tag_new"] = tag
def wait_for_git_mirror(newcommitid):
repodir=os.getcwd()
@ -386,18 +428,6 @@ def downloadrepo(layer):
cmd ='git checkout %s' % newcommitid
echo_check_call(cmd)
newtag = layer["tag_new"]
if newtag:
if newbranch and newbranch != currentbranch:
# older git doesn't allow to update reference on currently checked out branch
cmd ='git checkout -B %s %s' % (newbranch,newtag)
elif newbranch:
# we're already on requested branch
cmd ='git reset --hard %s' % newtag
else:
cmd ='git checkout %s' % newtag
echo_check_call(cmd)
os.chdir(olddir)
def parselayerconffile(layer, layerconffile):
@ -415,6 +445,7 @@ def traversedir(layer):
for path, dirs, files in os.walk(layer["location"]):
if os.path.basename(os.path.dirname(path)) == layer["name"]:
for filename in files:
# XXX Should check that it's under a "conf" subdirectory.
if filename == 'layer.conf':
layer["collection_path"] = os.path.relpath(os.path.dirname(path), os.path.dirname(layer["location"]))
logger.debug("traversedir(%s,%s) -> %s" % (layer["name"], layer["location"], layer["collection_path"]))
@ -436,7 +467,7 @@ def write_bblayers_conf(sourcedir):
priorities = ""
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"], reverse=True):
if layer["priority"] == -1:
# bitbake is not metadata layer, skip it
# Layer is not metadata layer, skip it
continue
if os.path.isabs(layer["location"]):
@ -446,7 +477,7 @@ def write_bblayers_conf(sourcedir):
layer_name = layer["name"].replace('-','_').upper()
if "collection_path" not in layer:
logger.error("Layer %s doesn't exist at all or local.conf file wasn't found inside" % layer["name"])
logger.error("Layer %s doesn't exist or no conf/layer.conf file was found inside" % layer["name"])
continue
locations += "%s_LAYER ?= \"%s/%s\"\n" % (layer_name, topdir, layer["collection_path"])
@ -462,13 +493,20 @@ def write_bblayers_conf(sourcedir):
f.write(bblayers)
f.write('"\n')
f.write(priorities)
try:
from mcfconfigfile import BblayersConfExtraLines
if BblayersConfExtraLines:
f.write('\n# Lines from the BblayersConfExtraLines setting:\n')
f.writelines('\n'.join(BblayersConfExtraLines) + '\n')
except ImportError:
pass
def update_layers(sourcedir):
logger.info('MCF-%s: Updating build directory' % __version__)
logger.info('MCF-%s: Updating layers' % __version__)
layers_sanity = list()
update_location = list()
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
if layer["submission"] and layer["location"] not in update_location:
if layer["fetch"] and layer["location"] not in update_location:
update_location.append(layer["location"])
if not os.path.exists(os.path.abspath(layer["location"])):
# downloadrepo
@ -838,6 +876,7 @@ def configure_build(srcdir, options):
['@prog@', progname],
['@srcdir@', srcdir],
['@abs_srcdir@', abs_srcdir],
['@configfileoption@', ('--config-file=%s' % options.configfile) if options.configfile else '' ],
]
# if icecc is not installed, or version does not match requirements, then disabling icecc is the correct action.
@ -874,12 +913,25 @@ if __name__ == '__main__':
# NB. The exec done by mcf.status causes argv[0] to be an absolute pathname
progname = sys.argv[0]
# Uses importlib.util.module_from_spec => requires Python 3.5 or later (see
# https://docs.python.org/3.5/library/importlib.html?highlight=importlib%20util%20module_from_spec#importlib.util.module_from_spec).
if sys.hexversion < 0x03050000:
logger.error("%s requires Python 3.5 or later" % progname)
sys.exit(1)
# Use the same timestamp for everything created by this invocation of mcf
timestamp = strftime("%Y%m%d%H%M%S", gmtime())
options = getopts()
srcdir = os.path.dirname(progname)
if options.configfile:
srcdir = "."
configfile = options.configfile
else:
srcdir = os.path.dirname(progname)
configfile = os.path.join(srcdir, "weboslayers.py")
abs_srcdir = os.path.abspath(srcdir)
if options.mcfcommand == 'update':
@ -891,10 +943,10 @@ if __name__ == '__main__':
if options.clean:
enable_clean()
read_weboslayers(srcdir)
read_mcfconfigfile(configfile)
for M in options.MACHINE:
if M not in SUPPORTED_MACHINES:
logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in weboslayers.py '%s')" % (M, SUPPORTED_MACHINES))
logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in %s: '%s')" % (M, options.configfile, SUPPORTED_MACHINES))
sys.exit(1)
if options.mcfcommand != 'configure':