From e23d42f253a2fb955abdf4d419b8e214ddd8132f Mon Sep 17 00:00:00 2001 From: Herb Kuta Date: Thu, 30 May 2019 17:15:45 -0700 Subject: [PATCH] DRAFT mcf: Recognize "-f CONFIGFILE" option, v2 of CONFIGFILE, and more (v7.0.0) - Also allow CONFIGFILE to have a ".mcf" extension. - Drop support for broken tags= - Support CONFIGFILE version 2 (McfFileVersion = 2) - Rename "webos_layers" to "Layers", but continue to recognize old name. - Recognize dict-s for fetch and options, but continue to recognize file version 1 strings. - Recognize an optional BblayersConfExtraLines variable in CONFIGFILE that contains a list of strings to be appended as separate lines to the generated conf/bblayers.conf . - Rename "submission" item to "fetch". - Fix error message to mention conf/layer.conf instead of incorrect local.conf . - Correct/improve messages - Requires Python 3.5 => major version bump. --- scripts/mcf | 156 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 104 insertions(+), 52 deletions(-) diff --git a/scripts/mcf b/scripts/mcf index 66b6e4d..f5ce9d2 100755 --- a/scripts/mcf +++ b/scripts/mcf @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2008-2017 LG Electronics, Inc. +# Copyright (c) 2008-2019 LG Electronics, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import argparse import errno import logging @@ -24,8 +23,10 @@ import re from time import gmtime, strftime, sleep import shutil import glob +import importlib.util +import importlib.machinery -__version__ = "6.2.3" +__version__ = "7.0.0" logger = logging.getLogger(__name__) @@ -34,6 +35,7 @@ TRACE = False REMOTE = "origin" SSTATE_MIRRORS = '' LAYERS = {} +MCFFILEVERSION = None DISTRO = None SUPPORTED_MACHINES = [] @@ -116,14 +118,16 @@ def getopts(): general.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default, help='command to mcf; if update is given, none of the remaining options nor MACHINE can be specified (default: %(default)s)') + general.add_argument('-f', '--config-file', dest='configfile', help='.mcf/.py file specifying the build configuration (default: weboslayers.py in same directory as this script)') + if mcfcommand in ('configure','update+configure'): variations = parser.add_argument_group('Build Instructions') variations.add_argument('-p', '--enable-parallel-make', dest='parallel_make', type=int, default=0, - help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 = 2x the number of processor cores)') + help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 means 2x the number of processor cores)') variations.add_argument('-b', '--enable-bb-number-threads', dest='bb_number_threads', type=int, default=0, - help='maximum number of bitbake tasks to spawn (default: 0 = 2x the number of processor cores))') + help='maximum number of bitbake tasks to spawn (default: 0 means 2x the number of processor cores))') icecc = parser.add_argument_group('ICECC Configuration') @@ -263,53 +267,91 @@ def location_to_dirname(location): str1 = location.split('/') return os.path.splitext(str1[len(str1)-1])[0] -def read_weboslayers(path): - sys.path.insert(0,path) - if not os.path.isfile(os.path.join(path,'weboslayers.py')): - raise Exception("Error: Configuration file %s does not exist!" % os.path.join(path,'weboslayers.py')) +def read_mcfconfigfile(configfile): + if not os.path.isfile(configfile): + raise Exception("Error: Configuration file %s does not exist!" % configfile) - from weboslayers import webos_layers + importlib.machinery.SOURCE_SUFFIXES.append(".mcf") + spec = importlib.util.spec_from_file_location("mcfconfigfile", configfile) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules["mcfconfigfile"] = module - for p in webos_layers: - layer = {"name":p[0], "priority":p[1], "url":p[2], "submission":p[3], "location":p[4]} + try: + from mcfconfigfile import McfFileVersion + except ImportError: + McfFileVersion = 1 + global MCFFILEVERSION + MCFFILEVERSION = McfFileVersion + logger.debug("Config file version: %d" % MCFFILEVERSION) + + if MCFFILEVERSION > 1: + try: + from mcfconfigfile import Layers + except ImportError: + from mcfconfigfile import webos_layers + Layers = webos_layers + else: + from mcfconfigfile import webos_layers + Layers = webos_layers + + for p in Layers: + if MCFFILEVERSION > 1 and isinstance(p[4], dict): + options = p[4] + try: + location = options["location"] + except KeyError: + location = '' + else: + # It's expected to be a string. + location = p[4] + + layer = {"name":p[0], "priority":p[1], "url":p[2], "fetch":p[3], "location":location} LAYERS[layer["name"]] = layer - parsesubmissions(layer) + parsefetch(layer) if not layer["url"] and not layer["location"]: - raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in weboslayers.py" % layer["name"]) + raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in %s" % (layer["name"], configfile)) if not layer["location"]: layer["location"] = location_to_dirname(layer["url"]) - from weboslayers import Distribution + from mcfconfigfile import Distribution global DISTRO DISTRO = Distribution - from weboslayers import Machines + from mcfconfigfile import Machines global SUPPORTED_MACHINES SUPPORTED_MACHINES = Machines -def parsesubmissions(layer): +def parsefetch(layer): + fetch = layer["fetch"] branch = '' commit = '' - tag = '' - for vgit in layer["submission"].split(','): - if not vgit: - continue - str1, str2 = vgit.split('=') - if str1.lower() == 'commit': - if not commit: - commit = str2 - elif str1.lower() == 'branch': - branch = str2 - elif str1.lower() == 'tag': - if not tag: - tag = str2 + if MCFFILEVERSION > 1 and isinstance(fetch, dict): + try: + branch = fetch["branch"] + except KeyError: + branch = 'master' + try: + commit = fetch["commit"] + except KeyError: + pass + else: + # It's expected to be a string. + for vgit in fetch.split(','): + if not vgit: + continue + str1, str2 = vgit.split('=') + if str1.lower() == 'commit': + if not commit: + commit = str2 + elif str1.lower() == 'branch': + branch = str2 - if not branch: - branch = 'master' + if not branch: + branch = 'master' layer["branch_new"] = branch layer["commit_new"] = commit - layer["tag_new"] = tag def wait_for_git_mirror(newcommitid): repodir=os.getcwd() @@ -386,18 +428,6 @@ def downloadrepo(layer): cmd ='git checkout %s' % newcommitid echo_check_call(cmd) - newtag = layer["tag_new"] - if newtag: - if newbranch and newbranch != currentbranch: - # older git doesn't allow to update reference on currently checked out branch - cmd ='git checkout -B %s %s' % (newbranch,newtag) - elif newbranch: - # we're already on requested branch - cmd ='git reset --hard %s' % newtag - else: - cmd ='git checkout %s' % newtag - echo_check_call(cmd) - os.chdir(olddir) def parselayerconffile(layer, layerconffile): @@ -415,6 +445,7 @@ def traversedir(layer): for path, dirs, files in os.walk(layer["location"]): if os.path.basename(os.path.dirname(path)) == layer["name"]: for filename in files: + # XXX Should check that it's under a "conf" subdirectory. if filename == 'layer.conf': layer["collection_path"] = os.path.relpath(os.path.dirname(path), os.path.dirname(layer["location"])) logger.debug("traversedir(%s,%s) -> %s" % (layer["name"], layer["location"], layer["collection_path"])) @@ -436,7 +467,7 @@ def write_bblayers_conf(sourcedir): priorities = "" for layer in sorted(LAYERS.values(), key=lambda l: l["priority"], reverse=True): if layer["priority"] == -1: - # bitbake is not metadata layer, skip it + # Layer is not metadata layer, skip it continue if os.path.isabs(layer["location"]): @@ -446,7 +477,7 @@ def write_bblayers_conf(sourcedir): layer_name = layer["name"].replace('-','_').upper() if "collection_path" not in layer: - logger.error("Layer %s doesn't exist at all or local.conf file wasn't found inside" % layer["name"]) + logger.error("Layer %s doesn't exist or no conf/layer.conf file was found inside" % layer["name"]) continue locations += "%s_LAYER ?= \"%s/%s\"\n" % (layer_name, topdir, layer["collection_path"]) @@ -462,13 +493,20 @@ def write_bblayers_conf(sourcedir): f.write(bblayers) f.write('"\n') f.write(priorities) + try: + from mcfconfigfile import BblayersConfExtraLines + if BblayersConfExtraLines: + f.write('\n# Lines from the BblayersConfExtraLines setting:\n') + f.writelines('\n'.join(BblayersConfExtraLines) + '\n') + except ImportError: + pass def update_layers(sourcedir): - logger.info('MCF-%s: Updating build directory' % __version__) + logger.info('MCF-%s: Updating layers' % __version__) layers_sanity = list() update_location = list() for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]): - if layer["submission"] and layer["location"] not in update_location: + if layer["fetch"] and layer["location"] not in update_location: update_location.append(layer["location"]) if not os.path.exists(os.path.abspath(layer["location"])): # downloadrepo @@ -838,6 +876,7 @@ def configure_build(srcdir, options): ['@prog@', progname], ['@srcdir@', srcdir], ['@abs_srcdir@', abs_srcdir], + ['@configfileoption@', ('--config-file=%s' % options.configfile) if options.configfile else '' ], ] # if icecc is not installed, or version does not match requirements, then disabling icecc is the correct action. @@ -874,12 +913,25 @@ if __name__ == '__main__': # NB. The exec done by mcf.status causes argv[0] to be an absolute pathname progname = sys.argv[0] + + # Uses importlib.util.module_from_spec => requires Python 3.5 or later (see + # https://docs.python.org/3.5/library/importlib.html?highlight=importlib%20util%20module_from_spec#importlib.util.module_from_spec). + if sys.hexversion < 0x03050000: + logger.error("%s requires Python 3.5 or later" % progname) + sys.exit(1) + # Use the same timestamp for everything created by this invocation of mcf timestamp = strftime("%Y%m%d%H%M%S", gmtime()) options = getopts() - srcdir = os.path.dirname(progname) + if options.configfile: + srcdir = "." + configfile = options.configfile + else: + srcdir = os.path.dirname(progname) + configfile = os.path.join(srcdir, "weboslayers.py") + abs_srcdir = os.path.abspath(srcdir) if options.mcfcommand == 'update': @@ -891,10 +943,10 @@ if __name__ == '__main__': if options.clean: enable_clean() - read_weboslayers(srcdir) + read_mcfconfigfile(configfile) for M in options.MACHINE: if M not in SUPPORTED_MACHINES: - logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in weboslayers.py '%s')" % (M, SUPPORTED_MACHINES)) + logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in %s: '%s')" % (M, options.configfile, SUPPORTED_MACHINES)) sys.exit(1) if options.mcfcommand != 'configure':