2019-05-30 00:50:47 +08:00
#!/usr/bin/env python3
2019-05-31 08:15:45 +08:00
# Copyright (c) 2008-2019 LG Electronics, Inc.
2019-05-30 00:50:47 +08:00
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import errno
import logging
import os
import subprocess
import sys
import re
from time import gmtime, strftime, sleep
import shutil
import glob
2019-05-31 08:15:45 +08:00
import importlib.util
import importlib.machinery
2019-05-30 00:50:47 +08:00
2019-05-31 08:15:45 +08:00
__version__ = "7.0.0"
2019-05-30 00:50:47 +08:00
logger = logging.getLogger(__name__)
CLEAN = False
TRACE = False
REMOTE = "origin"
SSTATE_MIRRORS = ''
LAYERS = {}
2019-05-31 08:15:45 +08:00
MCFFILEVERSION = None
2019-05-30 00:50:47 +08:00
DISTRO = None
SUPPORTED_MACHINES = []
def echo_check_call(todo, verbosity=False):
if verbosity or TRACE:
cmd = 'set -x; ' + todo
else:
cmd = todo
logger.debug(cmd)
return str(subprocess.check_output(cmd, shell=True), encoding='utf-8', errors='strict')
def enable_trace():
global TRACE
TRACE = True
def enable_clean():
logger.warn('Running in clean non-interactive mode, all possible local changes and untracked files will be removed')
global CLEAN
CLEAN = True
def set_log_level(level):
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
f = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s', datefmt='%Y-%m-%dT%H:%M:%S')
s = logging.StreamHandler()
s.setLevel(level)
s.setFormatter(f)
logging.getLogger('').addHandler(s)
# Essentially, mcf parses options, creates mcf.status, and runs mcf.status.
def process_file(f, replacements):
(ifile, ofile) = f
with open(ifile, 'r') as f:
status = f.read()
for i, j in replacements:
status = status.replace(i, j)
odir = os.path.dirname(ofile)
if odir and not os.path.isdir(odir):
os.mkdir(odir)
with open(ofile, 'w') as f:
f.write(status)
def getopts():
mcfcommand_option = '--command'
mcfcommand_dest = 'mcfcommand'
# be careful when changing this, jenkins-job.sh is doing
# grep "mcfcommand_choices = \['configure', 'update', "
# to detect if it needs to explicitly run --command update after default action
mcfcommand_choices = ['configure', 'update', 'update+configure']
mcfcommand_default = 'update+configure'
# Just parse the --command argument here, so that we can select a parser
mcfcommand_parser = argparse.ArgumentParser(add_help=False)
mcfcommand_parser.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default)
mcfcommand_parser_result = mcfcommand_parser.parse_known_args()
mcfcommand = mcfcommand_parser_result[0].mcfcommand
# Put --command back in (as the first option) so that the main parser sees everything
arglist = [mcfcommand_option, mcfcommand ] + mcfcommand_parser_result[1]
parser = argparse.ArgumentParser()
general = parser.add_argument_group('General Options')
verbosity = general.add_mutually_exclusive_group()
verbosity.add_argument('-s', '--silent', action='count', help='work silently, repeat the option twice to hide also the warnings, tree times to hide the errors as well')
verbosity.add_argument('-v', '--verbose', action='count', help='work verbosely, repeat the option twice for more debug output')
general.add_argument('-c', '--clean', dest='clean', action='store_true', default=False, help='clean checkout - WARN: removes all local changes')
general.add_argument('-V', '--version', action='version', version='%(prog)s {0}'.format(__version__), help='print version and exit')
general.add_argument(mcfcommand_option, dest=mcfcommand_dest, choices=mcfcommand_choices, default=mcfcommand_default,
help='command to mcf; if update is given, none of the remaining options nor MACHINE can be specified (default: %(default)s)')
2019-05-31 08:15:45 +08:00
general.add_argument('-f', '--config-file', dest='configfile', help='.mcf/.py file specifying the build configuration (default: weboslayers.py in same directory as this script)')
2019-05-30 00:50:47 +08:00
if mcfcommand in ('configure','update+configure'):
variations = parser.add_argument_group('Build Instructions')
variations.add_argument('-p', '--enable-parallel-make', dest='parallel_make', type=int, default=0,
2019-05-31 08:15:45 +08:00
help='maximum number of parallel tasks each submake of bitbake should spawn (default: 0 means 2x the number of processor cores)')
2019-05-30 00:50:47 +08:00
variations.add_argument('-b', '--enable-bb-number-threads', dest='bb_number_threads', type=int, default=0,
2019-05-31 08:15:45 +08:00
help='maximum number of bitbake tasks to spawn (default: 0 means 2x the number of processor cores))')
2019-05-30 00:50:47 +08:00
icecc = parser.add_argument_group('ICECC Configuration')
icecc_enable = icecc.add_mutually_exclusive_group()
# This can be changed to enabled by default when ES-1618 is fixed
icecc_enable.add_argument('--enable-icecc', dest='enable_icecc', action='store_true', default=False,
help='enable build to use ICECC, causes the shared state from the build artifacts not to be used (default: False)')
icecc_enable.add_argument('--disable-icecc', dest='enable_icecc', action='store_false', default=True,
help='disable build from using ICECC (default: True)')
icecc.add_argument('--enable-icecc-parallel-make', dest='icecc_parallel_make', type=int, default=0,
help='Number of parallel threads for ICECC build (default: 0 = 4x the number of processor cores))')
icecc_advanced = parser.add_argument_group('ICECC Advanced Configuration')
icecc_advanced.add_argument('--enable-icecc-user-package-blacklist', dest='icecc_user_package_blacklist', action='append',
help='Space separated list of components/recipes to be excluded from using ICECC (default: None)')
icecc_advanced.add_argument('--enable-icecc-user-class-blacklist', dest='icecc_user_class_blacklist', action='append',
help='Space separated list of components/recipes class to be excluded from using ICECC (default: None)')
icecc_advanced.add_argument('--enable-icecc-user-package-whitelist', dest='icecc_user_package_whitelist', action='append',
help='Space separated list of components/recipes to be forced to use ICECC (default: None)')
icecc_advanced.add_argument('--enable-icecc-location', dest='icecc_location', default='',
help='location of ICECC tool (default: None)')
icecc_advanced.add_argument('--enable-icecc-env-exec', dest='icecc_env_exec', default='',
help='location of ICECC environment script (default: None)')
partitions = parser.add_argument_group('Source Identification')
mirrors = parser.add_argument_group('Networking and Mirrors')
network = mirrors.add_mutually_exclusive_group()
network.add_argument('--disable-network', dest='network', action='store_false', default=True,
help='disable fetching through the network (default: False)')
network.add_argument('--enable-network', dest='network', action='store_true', default=True,
help='enable fetching through the network (default: True)')
mirrors.add_argument('--sstatemirror', dest='sstatemirror', action='append',
help='set sstatemirror to specified URL, repeat this option if you want multiple sstate mirrors (default: None)')
premirrorurl = mirrors.add_mutually_exclusive_group()
default_premirror = 'http://downloads.yoctoproject.org/mirror/sources'
premirrorurl.add_argument('--enable-default-premirror', dest='premirror', action='store_const', const=default_premirror, default="",
help='enable default premirror URL (default: False)')
premirrorurl.add_argument('--premirror', '--enable-premirror', dest='premirror', default='',
help='set premirror to specified URL (default: None)')
premirroronly = mirrors.add_mutually_exclusive_group()
premirroronly.add_argument('--disable-fetch-premirror-only', dest='fetchpremirroronly', action='store_false', default=False,
help='disable fetching through the network (default: False)')
premirroronly.add_argument('--enable-fetch-premirror-only', dest='fetchpremirroronly', action='store_true', default=False,
help='enable fetching through the network (default: True)')
tarballs = mirrors.add_mutually_exclusive_group()
tarballs.add_argument('--disable-generate-mirror-tarballs', dest='generatemirrortarballs', action='store_false', default=False,
help='disable tarball generation of fetched components (default: True)')
tarballs.add_argument('--enable-generate-mirror-tarballs', dest='generatemirrortarballs', action='store_true', default=False,
help='generate tarballs suitable for mirroring (default: False)')
buildhistory = parser.add_argument_group('Buildhistory')
buildhistory1 = buildhistory.add_mutually_exclusive_group()
buildhistory1.add_argument('--disable-buildhistory', dest='buildhistory', action='store_false', default=True,
help='disable buildhistory functionality (default: False)')
buildhistory1.add_argument('--enable-buildhistory', dest='buildhistory', action='store_true', default=True,
help='enable buildhistory functionality (default: True)')
buildhistory.add_argument('--enable-buildhistoryauthor', dest='buildhistoryauthor', default='', help='specify name and email used in buildhistory git commits (default: none, will use author from git global config)')
parser.add_argument('MACHINE', nargs='+')
options = parser.parse_args(arglist)
if mcfcommand in ('configure','update+configure') and options.sstatemirror:
process_sstatemirror_option(options)
return options
def process_sstatemirror_option(options):
"""
Sets global variable SSTATE_MIRRORS based on list of mirrors in options.sstatemirror
/PATH suffix is automatically added when generating SSTATE_MIRRORS value
verify that user didn't already include it and show error if he did
"""
sstate_mirrors = ''
for m in options.sstatemirror:
if not m:
continue
if m.endswith("/PATH"):
logger.error("sstatemirror entry '%s', already ends with '/PATH', remove that" % m)
sys.exit(1)
if m.endswith("/"):
logger.error("sstatemirror entry '%s', ends with '/', remove that" % m)
sys.exit(1)
if len(m) <= 7:
logger.error("sstatemirror entry '%s', is incorrect, we expect at least 7 characters for protocol" % m)
sys.exit(1)
sstate_mirrors += "file://.* %s/PATH \\n \\\n" % m
if sstate_mirrors:
global SSTATE_MIRRORS
SSTATE_MIRRORS = "SSTATE_MIRRORS ?= \" \\\n%s\"\n" % sstate_mirrors
def _icecc_installed():
try:
# Note that if package is not installed following call will throw an exception
iceinstallstatus,iceversion = subprocess.check_output("dpkg-query -W icecc 2>&1" ,
shell=True,
universal_newlines=True).split()
# We are expecting icecc for the name
if 'icecc' == iceinstallstatus:
if '1.0.1-1' == iceversion:
return True
else:
logger.warn("WARNING: Wrong icecc package version {} is installed, disabling build from using ICECC.\n".format(iceversion) + \
"Please check 'How To Install ICECC on Your Workstation (Client)'\n" + \
"http://wiki.lgsvl.com/pages/viewpage.action?pageId=96175316")
return False
else:
logger.warn('WARNING: ICECC package installation check failed, disabling build from using ICECC.')
return False
except:
logger.warn('WARNING: ICECC package installation check failed, disabling build from using ICECC.')
return False
def location_to_dirname(location):
str1 = location.split('/')
return os.path.splitext(str1[len(str1)-1])[0]
2019-05-31 08:15:45 +08:00
def read_mcfconfigfile(configfile):
if not os.path.isfile(configfile):
raise Exception("Error: Configuration file %s does not exist!" % configfile)
importlib.machinery.SOURCE_SUFFIXES.append(".mcf")
spec = importlib.util.spec_from_file_location("mcfconfigfile", configfile)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules["mcfconfigfile"] = module
try:
from mcfconfigfile import McfFileVersion
except ImportError:
McfFileVersion = 1
global MCFFILEVERSION
MCFFILEVERSION = McfFileVersion
logger.debug("Config file version: %d" % MCFFILEVERSION)
if MCFFILEVERSION > 1:
try:
from mcfconfigfile import Layers
except ImportError:
from mcfconfigfile import webos_layers
Layers = webos_layers
else:
from mcfconfigfile import webos_layers
Layers = webos_layers
2019-05-30 00:50:47 +08:00
2019-05-31 08:15:45 +08:00
for p in Layers:
if MCFFILEVERSION > 1 and isinstance(p[4], dict):
options = p[4]
try:
location = options["location"]
except KeyError:
location = ''
else:
# It's expected to be a string.
location = p[4]
2019-05-30 00:50:47 +08:00
2019-05-31 08:15:45 +08:00
layer = {"name":p[0], "priority":p[1], "url":p[2], "fetch":p[3], "location":location}
2019-05-30 00:50:47 +08:00
LAYERS[layer["name"]] = layer
2019-05-31 08:15:45 +08:00
parsefetch(layer)
2019-05-30 00:50:47 +08:00
if not layer["url"] and not layer["location"]:
2019-05-31 08:15:45 +08:00
raise Exception("Error: Layer '%s' does not have either URL or alternative working-dir defined in %s" % (layer["name"], configfile))
2019-05-30 00:50:47 +08:00
if not layer["location"]:
layer["location"] = location_to_dirname(layer["url"])
2019-05-31 08:15:45 +08:00
from mcfconfigfile import Distribution
2019-05-30 00:50:47 +08:00
global DISTRO
DISTRO = Distribution
2019-05-31 08:15:45 +08:00
from mcfconfigfile import Machines
2019-05-30 00:50:47 +08:00
global SUPPORTED_MACHINES
SUPPORTED_MACHINES = Machines
2019-05-31 08:15:45 +08:00
def parsefetch(layer):
fetch = layer["fetch"]
2019-05-30 00:50:47 +08:00
branch = ''
commit = ''
2019-05-31 08:15:45 +08:00
if MCFFILEVERSION > 1 and isinstance(fetch, dict):
try:
branch = fetch["branch"]
except KeyError:
branch = 'master'
try:
commit = fetch["commit"]
except KeyError:
pass
else:
# It's expected to be a string.
for vgit in fetch.split(','):
if not vgit:
continue
str1, str2 = vgit.split('=')
if str1.lower() == 'commit':
if not commit:
commit = str2
elif str1.lower() == 'branch':
branch = str2
if not branch:
branch = 'master'
2019-05-30 00:50:47 +08:00
layer["branch_new"] = branch
layer["commit_new"] = commit
def wait_for_git_mirror(newcommitid):
repodir=os.getcwd()
cmd = 'git fetch %s %s >&2' % (REMOTE, newcommitid)
success = False
nr_of_retries = 30
for i in range(1, nr_of_retries+1):
logger.info('MCF-%s: trying to fetch revision %s in %s attempt %s from %s' % (__version__, newcommitid, repodir, i, nr_of_retries))
try:
if newcommitid.startswith('refs/changes/'):
echo_check_call(cmd)
elif not contains_ref(newcommitid):
echo_check_call('git remote update && git fetch %s --tags' % REMOTE)
success = True
break
except subprocess.CalledProcessError:
sleep(30)
if not success:
logger.error("MCF-%s Cannot checkout %s in %s" % (__version__, newcommitid, repodir))
sys.exit(1)
def downloadrepo(layer):
cmd = 'git clone %s %s' % (layer["url"], layer["location"])
echo_check_call(cmd)
olddir = os.getcwd()
os.chdir(layer["location"])
newbranch = layer["branch_new"]
if newbranch:
refbranchlist = echo_check_call("git branch")
refbranch = refbranchlist.splitlines()
foundbranch = False
for ibranch in refbranch:
if newbranch in ibranch:
foundbranch = True
if not foundbranch:
refbranchlist = echo_check_call("git branch -r")
refbranch = refbranchlist.splitlines()
for ibranch in refbranch:
if ibranch == " %s/%s" % (REMOTE, newbranch):
foundbranch = True
logger.info( " found %s " % ibranch )
cmd ='git checkout -B %s %s' % (newbranch,ibranch)
echo_check_call(cmd)
break
currentbranch = echo_check_call("git rev-parse --abbrev-ref HEAD").rstrip()
newcommitid = layer["commit_new"]
if newcommitid:
if newcommitid.startswith('refs/changes/'):
wait_for_git_mirror(newcommitid)
if newbranch and newbranch != currentbranch:
# older git doesn't allow to update reference on currently checked out branch
cmd ='git checkout -B %s FETCH_HEAD' % (newbranch)
elif newbranch:
# we're already on requested branch
cmd ='git reset --hard FETCH_HEAD'
else:
# we don't have any branch preference use detached
cmd ='git checkout FETCH_HEAD'
echo_check_call(cmd)
else:
if not contains_ref(newcommitid):
wait_for_git_mirror(newcommitid)
if newbranch and newbranch != currentbranch:
# older git doesn't allow to update reference on currently checked out branch
cmd ='git checkout -B %s %s' % (newbranch,newcommitid)
elif newbranch:
# we're already on requested branch
cmd ='git reset --hard %s' % newcommitid
else:
# we don't have any branch preference use detached
cmd ='git checkout %s' % newcommitid
echo_check_call(cmd)
os.chdir(olddir)
def parselayerconffile(layer, layerconffile):
with open(layerconffile, 'r') as f:
lines = f.readlines()
for line in lines:
if re.search( 'BBFILE_COLLECTIONS.*=' , line):
(dummy, collectionname) = line.rsplit('=')
collectionname = collectionname.strip()
collectionname = collectionname.strip("\"")
layer["collection_name"] = collectionname
logger.debug("parselayerconffile(%s,%s) -> %s" % (layer["name"], layerconffile, layer["collection_name"]))
def traversedir(layer):
for path, dirs, files in os.walk(layer["location"]):
if os.path.basename(os.path.dirname(path)) == layer["name"]:
for filename in files:
2019-05-31 08:15:45 +08:00
# XXX Should check that it's under a "conf" subdirectory.
2019-05-30 00:50:47 +08:00
if filename == 'layer.conf':
layer["collection_path"] = os.path.relpath(os.path.dirname(path), os.path.dirname(layer["location"]))
logger.debug("traversedir(%s,%s) -> %s" % (layer["name"], layer["location"], layer["collection_path"]))
layerconffile = os.path.join(path, filename)
parselayerconffile(layer, layerconffile)
break
def parse_collections(srcdir):
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
if os.path.exists(layer["location"]):
traversedir(layer)
else:
raise Exception("Error: Directory '%s' does not exist, you probably need to call update" % layer["location"])
def write_bblayers_conf(sourcedir):
locations = ""
bblayers = ""
priorities = ""
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"], reverse=True):
if layer["priority"] == -1:
2019-05-31 08:15:45 +08:00
# Layer is not metadata layer, skip it
2019-05-30 00:50:47 +08:00
continue
if os.path.isabs(layer["location"]):
topdir = layer["location"]
else:
topdir = "${TOPDIR}"
layer_name = layer["name"].replace('-','_').upper()
if "collection_path" not in layer:
2019-05-31 08:15:45 +08:00
logger.error("Layer %s doesn't exist or no conf/layer.conf file was found inside" % layer["name"])
2019-05-30 00:50:47 +08:00
continue
locations += "%s_LAYER ?= \"%s/%s\"\n" % (layer_name, topdir, layer["collection_path"])
bblayers += " ${%s_LAYER} \\\n" % layer_name
priorities += "BBFILE_PRIORITY_%s_forcevariable = \"%s\"\n" % (layer["collection_name"], layer["priority"])
with open(os.path.join(sourcedir, "conf", "bblayers.conf"), 'a') as f:
f.write('\n')
f.write(locations)
f.write('\n')
f.write('BBFILES ?= ""\n')
f.write('BBLAYERS ?= " \\\n')
f.write(bblayers)
f.write('"\n')
f.write(priorities)
2019-05-31 08:15:45 +08:00
try:
from mcfconfigfile import BblayersConfExtraLines
if BblayersConfExtraLines:
f.write('\n# Lines from the BblayersConfExtraLines setting:\n')
f.writelines('\n'.join(BblayersConfExtraLines) + '\n')
except ImportError:
pass
2019-05-30 00:50:47 +08:00
def update_layers(sourcedir):
2019-05-31 08:15:45 +08:00
logger.info('MCF-%s: Updating layers' % __version__)
2019-05-30 00:50:47 +08:00
layers_sanity = list()
update_location = list()
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
2019-05-31 08:15:45 +08:00
if layer["fetch"] and layer["location"] not in update_location:
2019-05-30 00:50:47 +08:00
update_location.append(layer["location"])
if not os.path.exists(os.path.abspath(layer["location"])):
# downloadrepo
downloadrepo(layer)
else:
# run sanity check on repo
if reposanitycheck(layer) != 0:
layers_sanity.append(layer["location"])
# update layers
updaterepo(layer)
if layers_sanity:
logger.info('Found local changes for repos(s) %s' % layers_sanity)
printupdatesummary()
def printupdatesummary ():
logger.info('Repo Update Summary')
logger.info('===================')
found = False
for layer in sorted(LAYERS.values(), key=lambda l: l["priority"]):
if "sanity_uncommitted_clean" in layer and layer["sanity_uncommitted_clean"]:
logger.info(' *) local uncommitted changes were removed because of --clean parameter')
found = True
if "sanity_uncommitted_changes" in layer and layer["sanity_uncommitted_changes"]:
logger.info(' *) local uncommitted changes, use \'git stash pop\' to retrieve')
found = True
if "sanity_dumped_changes" in layer and layer["sanity_dumped_changes"]:
logger.info(' *) local committed changes, patches are backed up in %s/' % layer["repo_patch_dir"])
found = True
if "sanity_untracked_changes" in layer and layer["sanity_untracked_changes"]:
logger.info(' *) local untracked changes')
found = True
if "branch_new" in layer and "branch_current" in layer and layer["branch_new"] != layer["branch_current"]:
logger.info(' *) switched branches from %s to %s' % (layer["branch_current"], layer["branch_new"]))
found = True
if not found:
logger.info('No local changes found')
def get_remote_branch(newbranch, second_call = False):
remotebranch = None
refbranchlist = echo_check_call("git branch -r")
refbranch = refbranchlist.splitlines()
for ibranch in refbranch:
if ibranch == " %s/%s" % (REMOTE, newbranch):
remotebranch = ibranch.strip()
break
if remotebranch or second_call:
return remotebranch
else:
# try it again after "git remote update"
echo_check_call("git remote update")
return get_remote_branch(newbranch, True)
def reposanitycheck(layer):
olddir = os.getcwd()
os.chdir(layer["location"])
layer["branch_current"] = echo_check_call("git rev-parse --abbrev-ref HEAD").rstrip()
res = False
if CLEAN:
if echo_check_call("git status --porcelain -s"):
layer["sanity_uncommitted_clean"] = True
logger.warn('Removing all local changes and untracked files in [%s]' % layer["location"])
# abort rebase if git pull --rebase from update_layers got stuck on some local commit
try:
echo_check_call("git rebase --abort 2>/dev/null")
except subprocess.CalledProcessError:
# we can ignore this one
pass
echo_check_call("git stash clear")
echo_check_call("git clean -fdx")
echo_check_call("git reset --hard")
else:
logger.info('Checking for local changes in [%s]' % layer["location"])
if echo_check_call("git status --porcelain --u=no -s"):
logger.warn('Found local uncommitted changes in [%s]' % layer["location"])
layer["sanity_uncommitted_changes"] = True
echo_check_call("git stash")
res = True
if echo_check_call("git status --porcelain -s | grep -v '^?? MCF-PATCHES_' || true"):
logger.warn('Found local untracked changes in [%s]' % layer["location"])
layer["sanity_untracked_changes"] = True
res = True
try:
remote = echo_check_call('git remote | grep "^%s$"' % REMOTE)
except subprocess.CalledProcessError:
remote = ''
if not remote:
logger.error("Checkout %s doesn't have the remote '%s'" % (layer["location"], REMOTE))
raise Exception("Checkout %s doesn't have the remote '%s'" % (layer["location"], REMOTE))
try:
urlcurrent = echo_check_call("git config remote.%s.url" % REMOTE)
except subprocess.CalledProcessError:
# git config returns 1 when the option isn't set
urlcurrent = ''
# there is extra newline at the end
urlcurrent = urlcurrent.strip()
logger.debug("reposanitycheck(%s) dir %s, branchinfo %s, branchinfonew %s, url %s, urlnew %s" % (layer["name"], layer["location"], layer["branch_current"], layer["branch_new"], layer["url"], urlcurrent))
if urlcurrent != layer["url"]:
logger.warn("Changing url for remote '%s' from '%s' to '%s'" % (REMOTE, urlcurrent, layer["url"]))
echo_check_call("git remote set-url %s %s" % (REMOTE, layer["url"]))
# Sync with new remote repo
try:
echo_check_call('git remote update')
except subprocess.CalledProcessError:
raise Exception('Failed to fetch %s repo' % LOCATIONS[layer])
newbranch = layer["branch_new"]
if newbranch:
refbranchlist = echo_check_call("git branch")
refbranch = refbranchlist.splitlines()
foundlocalbranch = False
needcheckout = True
for ibranch in refbranch:
if ibranch == " %s" % newbranch:
foundlocalbranch = True
break
if ibranch == "* %s" % newbranch:
foundlocalbranch = True
needcheckout = False
break
remotebranch = get_remote_branch(newbranch)
if foundlocalbranch and remotebranch:
if needcheckout:
echo_check_call('git checkout %s' % newbranch)
head = echo_check_call("git rev-parse --abbrev-ref HEAD").rstrip()
patchdir = './MCF-PATCHES_%s-%s' % (head.replace('/','_'), timestamp)
layer["repo_patch_dir"] = "%s/%s" % (layer["location"], patchdir)
cmd ='git format-patch %s..%s -o %s' % (remotebranch,newbranch,patchdir)
rawpatches = echo_check_call(cmd)
patches = rawpatches.splitlines()
num = len(patches)
# logger.info( ' info: number of patches: %s ' % num)
if num > 0:
layer["sanity_dumped_changes"] = True
res = True
else:
# remove empty dir if there weren't any patches created by format-patch
cmd ='rmdir --ignore-fail-on-non-empty %s' % patchdir
echo_check_call(cmd)
try:
trackingbranch = echo_check_call("git config --get branch.%s.merge" % newbranch)
except subprocess.CalledProcessError:
# git config returns 1 when the option isn't set
trackingbranch = ''
try:
trackingremote = echo_check_call("git config --get branch.%s.remote" % newbranch)
except subprocess.CalledProcessError:
# git config returns 1 when the option isn't set
trackingremote = ''
# there is extra newline at the end
trackingbranch = trackingbranch.strip()
trackingremote = trackingremote.strip()
if not trackingbranch or not trackingremote or trackingbranch.replace('refs/heads',trackingremote) != remotebranch:
logger.warn("checkout %s was tracking '%s/%s' changing it to track '%s'" % (layer["location"], trackingremote, trackingbranch, remotebranch))
# to ensure we are tracking remote
echo_check_call('git branch %s --set-upstream-to %s' % (newbranch, remotebranch))
elif not foundlocalbranch and remotebranch:
echo_check_call('git checkout -b %s %s' % (newbranch, remotebranch))
else:
# anything else is failure
raise Exception('Could not find local and remote branches for %s' % newbranch)
else:
raise Exception('Undefined branch name')
newdir = os.chdir(olddir)
return res
# Taken from bitbake/lib/bb/fetch2/git.py with modifications for mcf usage
def contains_ref(tag):
cmd = "git log --pretty=oneline -n 1 %s -- 2>/dev/null | wc -l" % (tag)
output = echo_check_call(cmd)
if len(output.split()) > 1:
raise Exception("Error: '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
return output.split()[0] != "0"
def updaterepo(layer):
olddir = os.getcwd()
os.chdir(layer["location"])
layer["commit_current"] = echo_check_call("git log --pretty=format:%h -1")
newcommitid = layer["commit_new"]
currentcommitid = layer["commit_current"]
newbranch = layer["branch_new"]
currentbranch = layer["branch_current"]
logger.debug("updaterepo(%s) dir %s, id %s, newid %s, branch %s, newbranch %s" % (layer["name"], layer["location"], currentcommitid, newcommitid, currentbranch, newbranch))
if newcommitid != currentcommitid:
logger.info('Updating [%s]' % layer["location"])
if newcommitid:
if newcommitid.startswith('refs/changes/'):
wait_for_git_mirror(newcommitid)
if newbranch and newbranch != currentbranch:
# older git doesn't allow to update reference on currently checked out branch
cmd ='git checkout -B %s FETCH_HEAD' % (newbranch)
elif newbranch:
# we're already on requested branch
cmd ='git reset --hard FETCH_HEAD'
else:
# we don't have any branch preference use detached
cmd ='git checkout FETCH_HEAD'
echo_check_call(cmd)
else:
if not contains_ref(newcommitid):
wait_for_git_mirror(newcommitid)
if newbranch and newbranch != currentbranch:
# older git doesn't allow to update reference on currently checked out branch
cmd ='git checkout -B %s %s' % (newbranch,newcommitid)
elif newbranch:
# we're already on requested branch
cmd ='git reset --hard %s' % newcommitid
else:
# we don't have any branch preference use detached
cmd ='git checkout %s' % newcommitid
echo_check_call(cmd)
else:
if CLEAN:
echo_check_call("git remote update")
echo_check_call('git reset --hard %s/%s' % (REMOTE, newbranch))
else:
# current branch always tracks a remote one
echo_check_call('git pull %s' % REMOTE)
logger.info('Done updating [%s]' % layer["location"])
else:
logger.info(('[%s] is up-to-date.' % layer["location"]))
newdir = os.chdir(olddir)
os.getcwd()
def set_verbosity(options):
if options.silent and options.silent == 1:
set_log_level('WARNING')
elif options.silent and options.silent == 2:
set_log_level('ERROR')
elif options.silent and options.silent >= 3:
set_log_level('CRITICAL')
elif options.verbose and options.verbose == 1:
set_log_level('DEBUG')
elif options.verbose and options.verbose >= 2:
set_log_level('DEBUG')
# but also run every system command with set -x
enable_trace()
else:
set_log_level('INFO')
def recover_current_mcf_state(srcdir, origoptions):
mcfstatusfile = os.path.join(srcdir, "mcf.status")
if not os.path.exists(mcfstatusfile):
raise Exception("mcf.status does not exist.")
commandlinereconstructed = list()
commandlinereconstructed.append('ignored-argv-0')
start = False
with open(mcfstatusfile, 'r') as f:
for line in f.readlines():
line = line.strip()
if not start:
start = line.startswith("exec")
continue
if start:
if line.startswith('--command'):
# skip --command configure
continue
elif line.startswith('--'):
line = line.rstrip('\\')
line = line.strip(' ')
line = line.replace('\"','')
line = line.replace('\'','')
commandlinereconstructed.append(line)
else:
lines = line.rstrip('\\')
lines = lines.lstrip()
lines = lines.rstrip()
lines = lines.split()
for lline in lines:
commandlinereconstructed.append(lline)
sys.argv = commandlinereconstructed
options = getopts()
# always use clean/verbose/silent flags from origoptions not mcf.status
options.clean = origoptions.clean
options.verbose = origoptions.verbose
options.silent = origoptions.silent
return options
def checkmirror(name, url):
if url.startswith('file://'):
pathstr = url[7:]
if not os.path.isdir(pathstr):
logger.warn("%s parameter '%s' points to non-existent directory" % (name, url))
elif not os.listdir(pathstr):
logger.warn("%s parameter '%s' points to empty directory, did you forgot to mount it?" % (name, url))
elif len(url) <= 7:
logger.error("%s parameter '%s' is incorrect, we expect at least 7 characters for protocol" % (name, url))
def sanitycheck(options):
try:
mirror = echo_check_call('git config -l | grep "^url\..*insteadof=github.com/"')
except subprocess.CalledProcessError:
# git config returns 1 when the option isn't set
mirror = ''
pass
if not mirror:
logger.warn('No mirror for github.com was detected, please define mirrors in ~/.gitconfig if some are available')
if options.sstatemirror:
for m in options.sstatemirror:
if not m:
continue
checkmirror('sstatemirror', m)
if options.premirror:
checkmirror('premirror', options.premirror)
def configure_build(srcdir, options):
files = [
[os.path.join(srcdir, 'build-templates', 'mcf-status.in'), 'mcf.status' ],
[os.path.join(srcdir, 'build-templates', 'oe-init-build-env.in'), 'oe-init-build-env' ],
[os.path.join(srcdir, 'build-templates', 'Makefile.in'), 'Makefile' ],
[os.path.join(srcdir, 'build-templates', 'bblayers-conf.in'), 'conf/bblayers.conf'],
[os.path.join(srcdir, 'build-templates', 'local-conf.in'), 'conf/local.conf' ],
]
replacements = [
['@bb_number_threads@', str(options.bb_number_threads)],
['@parallel_make@', str(options.parallel_make)],
['@no_network@', '0' if options.network else '1'],
['@fetchpremirroronly@', '1' if options.fetchpremirroronly else '0'],
['@generatemirrortarballs@', '1' if options.generatemirrortarballs else '0'],
['@buildhistory_enabled@', '1' if options.buildhistory else '0'],
['@buildhistory_class@', 'buildhistory' if options.buildhistory else '' ],
['@buildhistory_author_assignment@', 'BUILDHISTORY_COMMIT_AUTHOR ?= "%s"' % options.buildhistoryauthor if options.buildhistoryauthor else ''],
['@premirror_assignment@', 'SOURCE_MIRROR_URL ?= "%s"' % options.premirror if options.premirror else ''],
['@premirror_inherit@', 'INHERIT += "own-mirrors"' if options.premirror else ''],
['@sstatemirror_assignment@', SSTATE_MIRRORS if options.sstatemirror else ''],
['@premirror@', options.premirror],
['@sstatemirror@', ' --sstatemirror='.join(options.sstatemirror) if options.sstatemirror else ''],
['@buildhistoryauthor@', options.buildhistoryauthor],
['@buildhistory@', '--%s-buildhistory' % ('enable' if options.buildhistory else 'disable')],
['@network@', '--%s-network' % ('enable' if options.network else 'disable')],
['@fetchpremirroronlyoption@', '--%s-fetch-premirror-only' % ('enable' if options.fetchpremirroronly else 'disable')],
['@generatemirrortarballsoption@', '--%s-generate-mirror-tarballs' % ('enable' if options.generatemirrortarballs else 'disable')],
['@machine@', options.MACHINE[0]],
['@machines@', ' '.join(options.MACHINE)],
['@distro@', DISTRO],
['@prog@', progname],
['@srcdir@', srcdir],
['@abs_srcdir@', abs_srcdir],
2019-05-31 08:15:45 +08:00
['@configfileoption@', ('--config-file=%s' % options.configfile) if options.configfile else '' ],
2019-05-30 00:50:47 +08:00
]
# if icecc is not installed, or version does not match requirements, then disabling icecc is the correct action.
icestate = _icecc_installed()
icecc_replacements = [
['@icecc_disable_enable@', '1' if not icestate or not options.enable_icecc else ''],
['@icecc_parallel_make@', '%s' % options.icecc_parallel_make],
['@alternative_icecc_installation@', ('ICECC_PATH ?= "%s"' % options.icecc_location) if options.icecc_location else ''],
['@icecc_user_package_blacklist@', ('ICECC_USER_PACKAGE_BL ?= "%s"' % ' '.join(options.icecc_user_package_blacklist)) if options.icecc_user_package_blacklist else ''],
['@icecc_user_class_blacklist@', ('ICECC_USER_CLASS_BL ?= "%s"' % ' '.join(options.icecc_user_class_blacklist)) if options.icecc_user_class_blacklist else ''],
['@icecc_user_package_whitelist@', ('ICECC_USER_PACKAGE_WL ?= "%s"' % ' '.join(options.icecc_user_package_whitelist)) if options.icecc_user_package_whitelist else ''],
['@icecc_environment_script@', 'ICECC_ENV_EXEC ?= "%s"' % options.icecc_env_exec if options.icecc_location else ''],
['@icecc_disable_enable_mcf@', '--%s-icecc' % ('disable' if not icestate or not options.enable_icecc else 'enable')],
['@alternative_icecc_installation_mcf@', options.icecc_location if options.icecc_location else ''],
['@icecc_environment_script_mcf@', options.icecc_env_exec if options.icecc_location else ''],
['@icecc_user_package_blacklist_mcf@', (' '.join(options.icecc_user_package_blacklist)) if options.icecc_user_package_blacklist else ''],
['@icecc_user_class_blacklist_mcf@', (' '.join(options.icecc_user_class_blacklist)) if options.icecc_user_class_blacklist else ''],
['@icecc_user_package_whitelist_mcf@', (' '.join(options.icecc_user_package_whitelist)) if options.icecc_user_package_whitelist else ''],
]
replacements = replacements + icecc_replacements
logger.info('MCF-%s: Configuring build directory BUILD' % __version__)
for f in files:
process_file(f, replacements)
parse_collections(srcdir)
write_bblayers_conf(srcdir)
logger.info('MCF-%s: Done configuring build directory BUILD' % __version__)
echo_check_call('/bin/chmod a+x mcf.status', options.verbose)
if __name__ == '__main__':
# NB. The exec done by mcf.status causes argv[0] to be an absolute pathname
progname = sys.argv[0]
2019-05-31 08:15:45 +08:00
# Uses importlib.util.module_from_spec => requires Python 3.5 or later (see
# https://docs.python.org/3.5/library/importlib.html?highlight=importlib%20util%20module_from_spec#importlib.util.module_from_spec).
if sys.hexversion < 0x03050000:
logger.error("%s requires Python 3.5 or later" % progname)
sys.exit(1)
2019-05-30 00:50:47 +08:00
# Use the same timestamp for everything created by this invocation of mcf
timestamp = strftime("%Y%m%d%H%M%S", gmtime())
options = getopts()
2019-05-31 08:15:45 +08:00
if options.configfile:
srcdir = "."
configfile = options.configfile
else:
srcdir = os.path.dirname(progname)
configfile = os.path.join(srcdir, "weboslayers.py")
2019-05-30 00:50:47 +08:00
abs_srcdir = os.path.abspath(srcdir)
if options.mcfcommand == 'update':
# recover current mcf state
options = recover_current_mcf_state(srcdir, options)
set_verbosity(options)
if options.clean:
enable_clean()
2019-05-31 08:15:45 +08:00
read_mcfconfigfile(configfile)
2019-05-30 00:50:47 +08:00
for M in options.MACHINE:
if M not in SUPPORTED_MACHINES:
2019-05-31 08:15:45 +08:00
logger.error("MACHINE argument '%s' isn't supported (does not appear in Machines in %s: '%s')" % (M, options.configfile, SUPPORTED_MACHINES))
2019-05-30 00:50:47 +08:00
sys.exit(1)
if options.mcfcommand != 'configure':
update_layers(srcdir)
configure_build(srcdir, options)
sanitycheck(options)
logger.info('Done.')
