DRAFT mcf: Recognize "-f CONFIGFILE" option, v2 of CONFIGFILE, and more (v7.0.0)
- Also allow CONFIGFILE to have a ".mcf" extension.
- Drop support for broken tags=
- Support CONFIGFILE version 2 (McfFileVersion = 2)
- Rename "webos_layers" to "Layers", but continue to recognize old name.
- Recognize dict-s for fetch and options, but continue to recognize
file version 1 strings.
- Recognize an optional BblayersConfExtraLines variable in CONFIGFILE
that contains a list of strings to be appended as separate lines to the generated
conf/bblayers.conf .
- Rename "submission" item to "fetch".
- Fix error message to mention conf/layer.conf instead of incorrect local.conf .
- Correct/improve messages
- Requires Python 3.5 => major version bump.
This commit is contained in:
Herb Kuta
parent
f586a9cd53
commit
e23d42f253
156
scripts/mcf
156
scripts/mcf
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2008-2017 LG Electronics, Inc.
|
||||
# Copyright (c) 2008-2019 LG Electronics, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -13,7 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import logging
|
||||
|
|
@ -24,8 +23,10 @@ import re
|
|||
from time import gmtime, strftime, sleep
|
||||
import shutil
|
||||
import glob
|
||||
import importlib.util
|
||||
import importlib.machinery
|
||||
|
||||
__version__ = "6.2.3"
|
||||
__version__ = "7.0.0"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ TRACE = False
|
|||
REMOTE = "origin"
|
||||
SSTATE_MIRRORS = ''
|
||||
LAYERS = {}
|
||||
MCFFILEVERSION = None
|
||||
DISTRO = None
|
||||
SUPPORTED_MACHINES = []
|
||||
|
||||
|
|
@ -116,14 +118,16 @@ def getopts():
|
|||
general.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default,
|
||||
help='command to mcf; if update is given, none of the remaining options nor MACHINE can be specified (default: %(default)s)')
|
||||
|
||||
general.add_argument('-f', '--config-file', dest='configfile', help='.mcf/.py file specifying the build configuration (default: weboslayers.py in same directory as this script)')
|
||||
|
||||
if mcfcommand in ('configure','update+configure'):
|
||||
variations = parser.add_argument_group('Build Instructions')
|
||||
|
||||
variations.add_argument('-p', '--enable-parallel-make', dest='parallel_make', type=int, default=0,
|
||||
help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 = 2x the number of processor cores)')
|
||||
help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 means 2x the number of processor cores)')
|
||||
|
||||
variations.add_argument('-b', '--enable-bb-number-threads', dest='bb_number_threads', type=int, default=0,
|
||||
help='maximum number of bitbake tasks to spawn (default: 0 = 2x the number of processor cores))')
|
||||
help='maximum number of bitbake tasks to spawn (default: 0 means 2x the number of processor cores))')
|
||||
|
||||
icecc = parser.add_argument_group('ICECC Configuration')
|
||||
|
||||
|
|
@ -263,53 +267,91 @@ def location_to_dirname(location):
|
|||
str1 = location.split('/')
|
||||
return os.path.splitext(str1[len(str1)-1])[0]
|
||||
|
||||
def read_weboslayers(path):
|
||||
sys.path.insert(0,path)
|
||||
if not os.path.isfile(os.path.join(path,'weboslayers.py')):
|
||||
raise Exception("Error: Configuration file %s does not exist!" % os.path.join(path,'weboslayers.py'))
|
||||
def read_mcfconfigfile(configfile):
|
||||
if not os.path.isfile(configfile):
|
||||
raise Exception("Error: Configuration file %s does not exist!" % configfile)
|
||||
|
||||
from weboslayers import webos_layers
|
||||
importlib.machinery.SOURCE_SUFFIXES.append(".mcf")
|
||||
spec = importlib.util.spec_from_file_location("mcfconfigfile", configfile)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
sys.modules["mcfconfigfile"] = module
|
||||
|
||||
for p in webos_layers:
|
||||
layer = {"name":p[0], "priority":p[1], "url":p[2], "submission":p[3], "location":p[4]}
|
||||
try:
|
||||
from mcfconfigfile import McfFileVersion
|
||||
except ImportError:
|
||||
McfFileVersion = 1
|
||||
global MCFFILEVERSION
|
||||
MCFFILEVERSION = McfFileVersion
|
||||
logger.debug("Config file version: %d" % MCFFILEVERSION)
|
||||
|
||||
if MCFFILEVERSION > 1:
|
||||
try:
|
||||
from mcfconfigfile import Layers
|
||||
except ImportError:
|
||||
from mcfconfigfile import webos_layers
|
||||
Layers = webos_layers
|
||||
else:
|
||||
from mcfconfigfile import webos_layers
|
||||
Layers = webos_layers
|
||||
|
||||
for p in Layers:
|
||||
if MCFFILEVERSION > 1 and isinstance(p[4], dict):
|
||||
options = p[4]
|
||||
try:
|
||||
location = options["location"]
|
||||
except KeyError:
|
||||
location = ''
|
||||
else:
|
||||
# It's expected to be a string.
|
||||
location = p[4]
|
||||
|
||||
layer = {"name":p[0], "priority":p[1], "url":p[2], "fetch":p[3], "location":location}
|
||||
LAYERS[layer["name"]] = layer
|
||||
parsesubmissions(layer)
|
||||
parsefetch(layer)
|
||||
if not layer["url"] and not layer["location"]:
|
||||
raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in weboslayers.py" % layer["name"])
|
||||
raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in %s" % (layer["name"], configfile))
|
||||
if not layer["location"]:
|
||||
layer["location"] = location_to_dirname(layer["url"])
|
||||
|
||||
from weboslayers import Distribution
|
||||
from mcfconfigfile import Distribution
|
||||
global DISTRO
|
||||
DISTRO = Distribution
|
||||
|
||||
from weboslayers import Machines
|
||||
from mcfconfigfile import Machines
|
||||
global SUPPORTED_MACHINES
|
||||
SUPPORTED_MACHINES = Machines
|
||||
|
||||
def parsesubmissions(layer):
|
||||
def parsefetch(layer):
|
||||
fetch = layer["fetch"]
|
||||
branch = ''
|
||||
commit = ''
|
||||
tag = ''
|
||||
for vgit in layer["submission"].split(','):
|
||||
if not vgit:
|
||||
continue
|
||||
str1, str2 = vgit.split('=')
|
||||
if str1.lower() == 'commit':
|
||||
if not commit:
|
||||
commit = str2
|
||||
elif str1.lower() == 'branch':
|
||||
branch = str2
|
||||
elif str1.lower() == 'tag':
|
||||
if not tag:
|
||||
tag = str2
|
||||
if MCFFILEVERSION > 1 and isinstance(fetch, dict):
|
||||
try:
|
||||
branch = fetch["branch"]
|
||||
except KeyError:
|
||||
branch = 'master'
|
||||
try:
|
||||
commit = fetch["commit"]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
# It's expected to be a string.
|
||||
for vgit in fetch.split(','):
|
||||
if not vgit:
|
||||
continue
|
||||
str1, str2 = vgit.split('=')
|
||||
if str1.lower() == 'commit':
|
||||
if not commit:
|
||||
commit = str2
|
||||
elif str1.lower() == 'branch':
|
||||
branch = str2
|
||||
|
||||
if not branch:
|
||||
branch = 'master'
|
||||
if not branch:
|
||||
branch = 'master'
|
||||
|
||||
layer["branch_new"] = branch
|
||||
layer["commit_new"] = commit
|
||||
layer["tag_new"] = tag
|
||||
|
||||
def wait_for_git_mirror(newcommitid):
|
||||
repodir=os.getcwd()
|
||||
|
|
@ -386,18 +428,6 @@ def downloadrepo(layer):
|
|||
cmd ='git checkout %s' % newcommitid
|
||||
echo_check_call(cmd)
|
||||
|
||||
newtag = layer["tag_new"]
|
||||
if newtag:
|
||||
if newbranch and newbranch != currentbranch:
|
||||
# older git doesn't allow to update reference on currently checked out branch
|
||||
cmd ='git checkout -B %s %s' % (newbranch,newtag)
|
||||
elif newbranch:
|
||||
# we're already on requested branch
|
||||
cmd ='git reset --hard %s' % newtag
|
||||
else:
|
||||
cmd ='git checkout %s' % newtag
|
||||
echo_check_call(cmd)
|
||||
|
||||
os.chdir(olddir)
|
||||
|
||||
def parselayerconffile(layer, layerconffile):
|
||||
|
|
@ -415,6 +445,7 @@ def traversedir(layer):
|
|||
for path, dirs, files in os.walk(layer["location"]):
|
||||
if os.path.basename(os.path.dirname(path)) == layer["name"]:
|
||||
for filename in files:
|
||||
# XXX Should check that it's under a "conf" subdirectory.
|
||||
if filename == 'layer.conf':
|
||||
layer["collection_path"] = os.path.relpath(os.path.dirname(path), os.path.dirname(layer["location"]))
|
||||
logger.debug("traversedir(%s,%s) -> %s" % (layer["name"], layer["location"], layer["collection_path"]))
|
||||
|
|
@ -436,7 +467,7 @@ def write_bblayers_conf(sourcedir):
|
|||
priorities = ""
|
||||
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"], reverse=True):
|
||||
if layer["priority"] == -1:
|
||||
# bitbake is not metadata layer, skip it
|
||||
# Layer is not metadata layer, skip it
|
||||
continue
|
||||
|
||||
if os.path.isabs(layer["location"]):
|
||||
|
|
@ -446,7 +477,7 @@ def write_bblayers_conf(sourcedir):
|
|||
layer_name = layer["name"].replace('-','_').upper()
|
||||
|
||||
if "collection_path" not in layer:
|
||||
logger.error("Layer %s doesn't exist at all or local.conf file wasn't found inside" % layer["name"])
|
||||
logger.error("Layer %s doesn't exist or no conf/layer.conf file was found inside" % layer["name"])
|
||||
continue
|
||||
|
||||
locations += "%s_LAYER ?= \"%s/%s\"\n" % (layer_name, topdir, layer["collection_path"])
|
||||
|
|
@ -462,13 +493,20 @@ def write_bblayers_conf(sourcedir):
|
|||
f.write(bblayers)
|
||||
f.write('"\n')
|
||||
f.write(priorities)
|
||||
try:
|
||||
from mcfconfigfile import BblayersConfExtraLines
|
||||
if BblayersConfExtraLines:
|
||||
f.write('\n# Lines from the BblayersConfExtraLines setting:\n')
|
||||
f.writelines('\n'.join(BblayersConfExtraLines) + '\n')
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
def update_layers(sourcedir):
|
||||
logger.info('MCF-%s: Updating build directory' % __version__)
|
||||
logger.info('MCF-%s: Updating layers' % __version__)
|
||||
layers_sanity = list()
|
||||
update_location = list()
|
||||
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
|
||||
if layer["submission"] and layer["location"] not in update_location:
|
||||
if layer["fetch"] and layer["location"] not in update_location:
|
||||
update_location.append(layer["location"])
|
||||
if not os.path.exists(os.path.abspath(layer["location"])):
|
||||
# downloadrepo
|
||||
|
|
@ -838,6 +876,7 @@ def configure_build(srcdir, options):
|
|||
['@prog@', progname],
|
||||
['@srcdir@', srcdir],
|
||||
['@abs_srcdir@', abs_srcdir],
|
||||
['@configfileoption@', ('--config-file=%s' % options.configfile) if options.configfile else '' ],
|
||||
]
|
||||
|
||||
# if icecc is not installed, or version does not match requirements, then disabling icecc is the correct action.
|
||||
|
|
@ -874,12 +913,25 @@ if __name__ == '__main__':
|
|||
# NB. The exec done by mcf.status causes argv[0] to be an absolute pathname
|
||||
progname = sys.argv[0]
|
||||
|
||||
|
||||
# Uses importlib.util.module_from_spec => requires Python 3.5 or later (see
|
||||
# https://docs.python.org/3.5/library/importlib.html?highlight=importlib%20util%20module_from_spec#importlib.util.module_from_spec).
|
||||
if sys.hexversion < 0x03050000:
|
||||
logger.error("%s requires Python 3.5 or later" % progname)
|
||||
sys.exit(1)
|
||||
|
||||
# Use the same timestamp for everything created by this invocation of mcf
|
||||
timestamp = strftime("%Y%m%d%H%M%S", gmtime())
|
||||
|
||||
options = getopts()
|
||||
|
||||
srcdir = os.path.dirname(progname)
|
||||
if options.configfile:
|
||||
srcdir = "."
|
||||
configfile = options.configfile
|
||||
else:
|
||||
srcdir = os.path.dirname(progname)
|
||||
configfile = os.path.join(srcdir, "weboslayers.py")
|
||||
|
||||
abs_srcdir = os.path.abspath(srcdir)
|
||||
|
||||
if options.mcfcommand == 'update':
|
||||
|
|
@ -891,10 +943,10 @@ if __name__ == '__main__':
|
|||
if options.clean:
|
||||
enable_clean()
|
||||
|
||||
read_weboslayers(srcdir)
|
||||
read_mcfconfigfile(configfile)
|
||||
for M in options.MACHINE:
|
||||
if M not in SUPPORTED_MACHINES:
|
||||
logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in weboslayers.py '%s')" % (M, SUPPORTED_MACHINES))
|
||||
logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in %s: '%s')" % (M, options.configfile, SUPPORTED_MACHINES))
|
||||
sys.exit(1)
|
||||
|
||||
if options.mcfcommand != 'configure':
|
||||
|
|
|
|||
Loading…
Reference in New Issue