DRAFT mcf: Recognize "-f CONFIGFILE" option, v2 of CONFIGFILE, and more (v7.0.0)

- Also allow CONFIGFILE to have a ".mcf" extension.

- Drop support for broken tags=

- Support CONFIGFILE version 2 (McfFileVersion = 2)

  - Rename "webos_layers" to "Layers", but continue to recognize old name.

  - Recognize dict-s for fetch and options, but continue to recognize
    file version 1 strings.

  - Recognize an optional BblayersConfExtraLines variable in CONFIGFILE
    that contains a list of strings to be appended as separate lines to the generated
    conf/bblayers.conf .
  - Rename "submission" item to "fetch".

- Fix error message to mention conf/layer.conf instead of incorrect local.conf .
- Correct/improve messages

- Requires Python 3.5 => major version bump.

scripts/mcf

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2008-2017 LG Electronics, Inc.
+# Copyright (c) 2008-2019 LG Electronics, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import argparse
 import errno
 import logging
@@ -24,8 +23,10 @@ import re
 from time import gmtime, strftime, sleep
 import shutil
 import glob
+import importlib.util
+import importlib.machinery
 
-__version__ = "6.2.3"
+__version__ = "7.0.0"
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +35,7 @@ TRACE = False
 REMOTE = "origin"
 SSTATE_MIRRORS = ''
 LAYERS = {}
+MCFFILEVERSION = None
 DISTRO = None
 SUPPORTED_MACHINES = []
 
@@ -116,14 +118,16 @@ def getopts():
     general.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default,
                               help='command to mcf; if update is given, none of the remaining options nor MACHINE can be specified (default: %(default)s)')
 
+    general.add_argument('-f', '--config-file', dest='configfile', help='.mcf/.py file specifying the build configuration (default: weboslayers.py in same directory as this script)')
+
     if mcfcommand in ('configure','update+configure'):
         variations = parser.add_argument_group('Build Instructions')
 
         variations.add_argument('-p', '--enable-parallel-make', dest='parallel_make', type=int, default=0,
-                                help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 = 2x the number of processor cores)')
+                                help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 means 2x the number of processor cores)')
 
         variations.add_argument('-b', '--enable-bb-number-threads', dest='bb_number_threads', type=int, default=0,
-                                help='maximum number of bitbake tasks to spawn (default: 0 = 2x the number of processor cores))')
+                                help='maximum number of bitbake tasks to spawn (default: 0 means 2x the number of processor cores))')
 
         icecc = parser.add_argument_group('ICECC Configuration')
 
@@ -263,53 +267,91 @@ def location_to_dirname(location):
     str1 = location.split('/')
     return os.path.splitext(str1[len(str1)-1])[0]
 
-def read_weboslayers(path):
+def read_mcfconfigfile(configfile):
-    sys.path.insert(0,path)
+    if not os.path.isfile(configfile):
-    if not os.path.isfile(os.path.join(path,'weboslayers.py')):
+        raise Exception("Error: Configuration file %s does not exist!" % configfile)
-        raise Exception("Error: Configuration file %s does not exist!" % os.path.join(path,'weboslayers.py'))
 
-    from weboslayers import webos_layers
+    importlib.machinery.SOURCE_SUFFIXES.append(".mcf")
+    spec = importlib.util.spec_from_file_location("mcfconfigfile", configfile)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules["mcfconfigfile"] = module
 
-    for p in webos_layers:
+    try:
-        layer = {"name":p[0], "priority":p[1], "url":p[2], "submission":p[3], "location":p[4]}
+        from mcfconfigfile import McfFileVersion
+    except ImportError:
+        McfFileVersion = 1
+    global MCFFILEVERSION
+    MCFFILEVERSION = McfFileVersion
+    logger.debug("Config file version: %d" % MCFFILEVERSION)
+
+    if MCFFILEVERSION > 1:
+        try:
+            from mcfconfigfile import Layers
+        except ImportError:
+            from mcfconfigfile import webos_layers
+            Layers = webos_layers
+    else:
+        from mcfconfigfile import webos_layers
+        Layers = webos_layers
+
+    for p in Layers:
+        if MCFFILEVERSION > 1 and isinstance(p[4], dict):
+            options = p[4]
+            try:
+                 location = options["location"]
+            except KeyError:
+                location = ''
+        else:
+            # It's expected to be a string.
+            location = p[4]
+
+        layer = {"name":p[0], "priority":p[1], "url":p[2], "fetch":p[3], "location":location}
         LAYERS[layer["name"]] = layer
-        parsesubmissions(layer)
+        parsefetch(layer)
         if not layer["url"] and not layer["location"]:
-            raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in weboslayers.py" % layer["name"])
+            raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in %s" % (layer["name"], configfile))
         if not layer["location"]:
             layer["location"] = location_to_dirname(layer["url"])
 
-    from weboslayers import Distribution
+    from mcfconfigfile import Distribution
     global DISTRO
     DISTRO = Distribution
 
-    from weboslayers import Machines
+    from mcfconfigfile import Machines
     global SUPPORTED_MACHINES
     SUPPORTED_MACHINES = Machines
 
-def parsesubmissions(layer):
+def parsefetch(layer):
+    fetch = layer["fetch"]
     branch = ''
     commit = ''
-    tag = ''
+    if MCFFILEVERSION > 1 and isinstance(fetch, dict):
-    for vgit in layer["submission"].split(','):
+        try:
-        if not vgit:
+            branch = fetch["branch"]
-            continue
+        except KeyError:
-        str1, str2 = vgit.split('=')
+            branch = 'master'
-        if str1.lower() == 'commit':
+        try:
-            if not commit:
+            commit = fetch["commit"]
-                commit = str2
+        except KeyError:
-        elif str1.lower() == 'branch':
+            pass
-            branch = str2
+    else:
-        elif str1.lower() == 'tag':
+        # It's expected to be a string.
-            if not tag:
+        for vgit in fetch.split(','):
-                tag = str2
+            if not vgit:
+                continue
+            str1, str2 = vgit.split('=')
+            if str1.lower() == 'commit':
+                if not commit:
+                    commit = str2
+            elif str1.lower() == 'branch':
+                branch = str2
 
-    if not branch:
+        if not branch:
-        branch = 'master'
+            branch = 'master'
 
     layer["branch_new"] = branch
     layer["commit_new"] = commit
-    layer["tag_new"] = tag
 
 def wait_for_git_mirror(newcommitid):
     repodir=os.getcwd()
@@ -386,18 +428,6 @@ def downloadrepo(layer):
                 cmd ='git checkout %s' % newcommitid
             echo_check_call(cmd)
 
-    newtag = layer["tag_new"]
-    if newtag:
-        if newbranch and newbranch != currentbranch:
-            # older git doesn't allow to update reference on currently checked out branch
-            cmd ='git checkout -B %s %s' % (newbranch,newtag)
-        elif newbranch:
-            # we're already on requested branch
-            cmd ='git reset --hard %s' % newtag
-        else:
-            cmd ='git checkout %s' % newtag
-        echo_check_call(cmd)
-
     os.chdir(olddir)
 
 def parselayerconffile(layer, layerconffile):
@@ -415,6 +445,7 @@ def traversedir(layer):
     for path, dirs, files in os.walk(layer["location"]):
         if os.path.basename(os.path.dirname(path)) == layer["name"]:
             for filename in files:
+                # XXX Should check that it's under a "conf" subdirectory.
                 if filename == 'layer.conf':
                     layer["collection_path"] = os.path.relpath(os.path.dirname(path), os.path.dirname(layer["location"]))
                     logger.debug("traversedir(%s,%s) -> %s" % (layer["name"], layer["location"], layer["collection_path"]))
@@ -436,7 +467,7 @@ def write_bblayers_conf(sourcedir):
     priorities = ""
     for layer in sorted(LAYERS.values(), key=lambda l: l["priority"], reverse=True):
         if layer["priority"] == -1:
-            # bitbake is not metadata layer, skip it
+            # Layer is not metadata layer, skip it
             continue
 
         if os.path.isabs(layer["location"]):
@@ -446,7 +477,7 @@ def write_bblayers_conf(sourcedir):
         layer_name = layer["name"].replace('-','_').upper()
 
         if "collection_path" not in layer:
-            logger.error("Layer %s doesn't exist at all or local.conf file wasn't found inside" % layer["name"])
+            logger.error("Layer %s doesn't exist or no conf/layer.conf file was found inside" % layer["name"])
             continue
 
         locations  += "%s_LAYER ?= \"%s/%s\"\n" % (layer_name, topdir, layer["collection_path"])
@@ -462,13 +493,20 @@ def write_bblayers_conf(sourcedir):
         f.write(bblayers)
         f.write('"\n')
         f.write(priorities)
+        try:
+            from mcfconfigfile import BblayersConfExtraLines
+            if BblayersConfExtraLines:
+                f.write('\n# Lines from the BblayersConfExtraLines setting:\n')
+                f.writelines('\n'.join(BblayersConfExtraLines) + '\n')
+        except ImportError:
+            pass
 
 def update_layers(sourcedir):
-    logger.info('MCF-%s: Updating build directory' % __version__)
+    logger.info('MCF-%s: Updating layers' % __version__)
     layers_sanity = list()
     update_location = list()
     for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
-        if layer["submission"] and layer["location"] not in update_location:
+        if layer["fetch"] and layer["location"] not in update_location:
             update_location.append(layer["location"])
             if not os.path.exists(os.path.abspath(layer["location"])):
                 # downloadrepo
@@ -838,6 +876,7 @@ def configure_build(srcdir, options):
         ['@prog@', progname],
         ['@srcdir@', srcdir],
         ['@abs_srcdir@', abs_srcdir],
+        ['@configfileoption@', ('--config-file=%s' % options.configfile) if options.configfile else '' ],
     ]
 
     # if icecc is not installed, or version does not match requirements, then disabling icecc is the correct action.
@@ -874,12 +913,25 @@ if __name__ == '__main__':
     # NB. The exec done by mcf.status causes argv[0] to be an absolute pathname
     progname = sys.argv[0]
 
+
+    # Uses importlib.util.module_from_spec => requires Python 3.5 or later (see
+    # https://docs.python.org/3.5/library/importlib.html?highlight=importlib%20util%20module_from_spec#importlib.util.module_from_spec).
+    if sys.hexversion < 0x03050000:
+        logger.error("%s requires Python 3.5 or later" % progname)
+        sys.exit(1)
+
     # Use the same timestamp for everything created by this invocation of mcf
     timestamp = strftime("%Y%m%d%H%M%S", gmtime())
 
     options = getopts()
 
-    srcdir = os.path.dirname(progname)
+    if options.configfile:
+        srcdir = "."
+        configfile = options.configfile
+    else:
+        srcdir = os.path.dirname(progname)
+        configfile = os.path.join(srcdir, "weboslayers.py")
+
     abs_srcdir = os.path.abspath(srcdir)
 
     if options.mcfcommand == 'update':
@@ -891,10 +943,10 @@ if __name__ == '__main__':
     if options.clean:
         enable_clean()
 
-    read_weboslayers(srcdir)
+    read_mcfconfigfile(configfile)
     for M in options.MACHINE:
         if M not in SUPPORTED_MACHINES:
-            logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in weboslayers.py '%s')" % (M, SUPPORTED_MACHINES))
+            logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in %s: '%s')" % (M, options.configfile, SUPPORTED_MACHINES))
             sys.exit(1)
 
     if options.mcfcommand != 'configure':