From 54fac3216d31c878daabac4dc1d85a375c7bdae7 Mon Sep 17 00:00:00 2001 From: Herb Kuta Date: Wed, 10 Apr 2019 17:03:25 -0700 Subject: [PATCH] DRAFT Drop various scripts obsolesced by superflore --- scripts/check-versions.py | 302 --------------------------------- scripts/create-packagegroup.py | 55 ------ scripts/create-recipe.py | 90 ---------- scripts/create-ros-recipe.py | 231 ------------------------- 4 files changed, 678 deletions(-) delete mode 100755 scripts/check-versions.py delete mode 100755 scripts/create-packagegroup.py delete mode 100755 scripts/create-recipe.py delete mode 100755 scripts/create-ros-recipe.py diff --git a/scripts/check-versions.py b/scripts/check-versions.py deleted file mode 100755 index 0803d5f..0000000 --- a/scripts/check-versions.py +++ /dev/null @@ -1,302 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2017, Blueye Robotics AS -# -# check-version.py is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# check-version.py is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# See for full license text. - -import os -import urllib.request -import sys -import re -import hashlib -from distutils.version import LooseVersion -import yaml - -__author__ = "Johannes Schrimpf" -__copyright__ = "Copyright 2017, Blueye Robotics AS" -__credits__ = ["Johannes Schrimpf"] -__license__ = "GPLv3" - -BASE_DIR = "../recipes-ros" -DEBUG = False -EXCLUDE = ["packagegroups"] - - -class MoveRepoExcetion(Exception): - pass - - -class DistroUrlException(Exception): - pass - - -class Distribution(): - data = None - - @staticmethod - def get_version(package): - if Distribution.data is None: - Distribution.data = yaml.load(urllib.request.urlopen(DIST_FILE).read()) - package = package.replace("-", "_") - return Distribution.data["repositories"][package]["release"]["version"] - - @staticmethod - def get_url(package): - if Distribution.data is None: - Distribution.data = yaml.load(urllib.request.urlopen(DIST_FILE).read()) - package = package.replace("-", "_") - try: - return Distribution.data["repositories"][package]["source"]["url"].split(".git")[0] - except KeyError: - raise DistroUrlException() - - -def print_debug(text): - if DEBUG: - print(text) - - -def print_err(text): - pre = '\033[91m' - post = '\033[0m' - print(pre + text + post) - - -def print_ok(text): - pre = '\033[92m' - post = '\033[0m' - print(pre + text + post) - - -def check_version(package, print_info="none", details=False): - printlist = [] - package_dir = os.path.join(BASE_DIR, package) - versions = set() - version = None - try: - dist_ver = Distribution.get_version(package) - dist_ver = dist_ver.split("-")[0] - except KeyError: - dist_ver = "" - for filename in (os.path.join(package_dir, fn) for fn in os.listdir(package_dir) - if fn.endswith(".bb")): - version = filename.split("_")[1].split(".bb")[0] - versions.update([version]) - - if details: - printlist.append(" - " + os.path.basename(filename)) - if len(versions) > 1: - print("Package: %s" % package) - print(versions) - raise Exception("Multiple versions per package not supported at this time") - - if version is None: - version = "n.a." - match = dist_ver == version - - if print_info == "all" or print_info == "mismatch" and not match: - if dist_ver == "": - pre = post = "" - mid = " " - elif version == "git": - mid = " " - pre = post = "" - else: - pre = '\033[92m' if match else '\033[91m' - post = '\033[0m' - if match: - mid = " = " - else: - assert LooseVersion(version) != LooseVersion(dist_ver) - if LooseVersion(version) > LooseVersion(dist_ver): - mid = " > " - elif LooseVersion(dist_ver) > LooseVersion(version): - mid = " < " - print(package.ljust(35) + pre + version.ljust(10) + mid + dist_ver.ljust(10) + post) - for line in printlist: - print(line) - return match, version, dist_ver - - -def get_checksums_from_url(url): - data = urllib.request.urlopen(url).read() - md5sum = hashlib.md5() - md5sum.update(data) - md5sum_hex = md5sum.hexdigest() - sha256sum = hashlib.sha256() - sha256sum.update(data) - sha256sum_hex = sha256sum.hexdigest() - return md5sum_hex, sha256sum_hex - - -def update_checksums_in_file(package, filename, dist_version): - with open(filename) as recipe_file: - data = recipe_file.read() - - ros_pv = dist_version - ros_spn = package.replace('-', '_') - ros_sp = "%s-%s" % (ros_spn, ros_pv) - - url = re.search(r'SRC_URI\s*=\s*"(\S*)\s*["\\]', data).group(1) - url = url.replace("${ROS_SPN}", ros_spn) - url = url.replace("${ROS_SP}", ros_sp) - url = url.replace("${PV}", ros_pv) - - if "protocol=git" in url: - print_err("Using git protocol. Please update manually.") - - url = url.split(";")[0] - - repo = Distribution.get_url(package) - - if repo not in url: - print(url) - print(repo) - raise MoveRepoExcetion() - - try: - md5sum = re.search(r'SRC_URI\[md5sum\]\s*=\s*"(\S*)"', data).group(1) - except AttributeError: - print_err("Error reading md5sum in package %s. Please update manually." % package) - return False - try: - sha256sum = re.search(r'SRC_URI\[sha256sum\]\s*=\s*"(\S*)"', data).group(1) - except AttributeError: - print_err("Error reading sha256sum in package %s. Please update manually." % package) - return False - - if len(md5sum) != 32 and len(sha256sum) != 64: - print_err("Failed reading checksums.") - return False - - md5sum_new, sha256sum_new = get_checksums_from_url(url) - print_debug("Updating checksums in file %s" % filename) - print_debug("old md5: %s" % md5sum) - print_debug("new md5: %s" % md5sum_new) - print_debug("old sha256: %s" % sha256sum) - print_debug("new sha256: %s" % sha256sum_new) - with open(filename) as recipe_file: - recipe_data = recipe_file.read() - recipe_data = recipe_data.replace(md5sum, md5sum_new) - recipe_data = recipe_data.replace(sha256sum, sha256sum_new) - with open(filename, 'w') as recipe_file: - recipe_file.write(recipe_data) - return True - - -def update_all_packages(): - for package in (x for x in sorted(os.listdir(BASE_DIR)) if x not in EXCLUDE and - os.path.isdir(os.path.join(BASE_DIR, x))): - if not check_version(package)[0]: - update_package(package) - - -def update_package(package): - print_debug("Updating %s" % package) - print_header() - match, version, dist_version = check_version(package, print_info="all", details=True) - if match: - print("Packet is already in newest version") - return - if dist_version == "": - print("Packet not found in dist file") - return - - if version == "git": - print("Layer uses git version. Please update manually") - return - elif LooseVersion(version) > LooseVersion(dist_version) and "--downgrade" not in sys.argv: - print("Layer version is newer than dist version, " + - "use --downgrade as last argument to downgrade") - return - - try: - path = os.path.join(BASE_DIR, package) - - update_include = False - rename_requests = [] - for recipe in (os.path.join(path, fn) for fn in os.listdir(path) if fn.endswith(".bb")): - with open(recipe) as recipe_file: - data = recipe_file.read() - - old_fn = os.path.join(recipe) - new_fn = os.path.join(recipe.replace(version, dist_version)) - rename_requests.append([old_fn, new_fn]) - - if "SRC_URI[md5sum]" not in data and "SRC_URI[sha256sum]" not in data: - update_include = True - else: - update_checksums_in_file(package, old_fn, dist_version) - - if update_include: - inc_fn = os.path.join(BASE_DIR, package, package + ".inc") - update_checksums_in_file(package, inc_fn, dist_version) - except MoveRepoExcetion: - print_err("Repo moved %s" % package) - except DistroUrlException: - print_err("Cannot retrieve source url for %s" % package) - else: - print_debug("Renaming files for %s" % package) - for [old_fn, new_fn] in rename_requests: - print_debug("old: %s" % old_fn) - print_debug("new: %s" % new_fn) - os.rename(old_fn, new_fn) - print_ok("Updated") - - -def print_header(): - print("\033[1m\033[4m" + "package".ljust(35) + "layer".ljust(13) + - "distro".ljust(10) + '\033[0m') - - -def print_list(details=False): - print_header() - for package in (x for x in sorted(os.listdir(BASE_DIR)) if x not in EXCLUDE and - os.path.isdir(os.path.join(BASE_DIR, x))): - check_version(package, details=details, print_info="all") - - -def print_mismatch(details=False): - print_header() - for package in (x for x in sorted(os.listdir(BASE_DIR)) if x not in EXCLUDE and - os.path.isdir(os.path.join(BASE_DIR, x))): - check_version(package, details=details, print_info="mismatch") - - -def print_help(): - filename = sys.argv[0] - print("Usage:") - print("List all versions: %s list (--details)" % filename) - print("List all versions that don't match: %s mismatch (--details)" % filename) - print("Update recipe to dist version: %s update (--downgrade)" % filename) - print("Update all recipes to dist version: %s update-all (--downgrade)" % filename) - - -if __name__ == "__main__": - with open("../conf/layer.conf") as layerfile: - DISTRO = re.search(r'ROSDISTRO\s:=\s*"(\S*)\s*["\\]', layerfile.read()).group(1) - DIST_FILE = "https://raw.githubusercontent.com/ros/rosdistro/master/" +\ - DISTRO + "/distribution.yaml" - if len(sys.argv) == 1: - print_help() - else: - if sys.argv[1] == "list": - print_list(details="--details" in sys.argv) - elif sys.argv[1] == "mismatch": - print_mismatch(details="--details" in sys.argv) - elif sys.argv[1] == "update" and len(sys.argv) >= 3: - update_package(sys.argv[2]) - elif sys.argv[1] == "update-all": - update_all_packages() - else: - print_help() diff --git a/scripts/create-packagegroup.py b/scripts/create-packagegroup.py deleted file mode 100755 index 1f58111..0000000 --- a/scripts/create-packagegroup.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2013 Stefan Herbrechtsmeier, Bielefeld University -# -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import sys -import httplib -import re - -name = 'ros_comm' - -if len(sys.argv) > 1: - name = sys.argv[1] - -conn = httplib.HTTPConnection('packages.ros.org') -conn.request('GET', '/web/rosinstall/generate/raw/groovy/' + name) -res = conn.getresponse() -data = res.read() -conn.close() - -packages = [] -for p in re.findall('local-name:.(\w+)' ,data): - packages.append(p.replace('_', '-')) - -name = name.replace('_', '-') - -print('packagegroup-' + name + '.bb') -print('') -print('DESCRIPTION = "' + name + ' package group"'); -print('LICENSE = "MIT"') -print('') -print('inherit packagegroup') -print('') -print('PACKAGES = "${PN}"') -print('') -print('RDEPENDS_${PN} = "\\\n ' + ' \\\n '.join(packages) + ' \\\n "') diff --git a/scripts/create-recipe.py b/scripts/create-recipe.py deleted file mode 100755 index d4caf09..0000000 --- a/scripts/create-recipe.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2013 Stefan Herbrechtsmeier, Bielefeld University -# -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import sys -import os -import xml.etree.ElementTree as ElementTree -import hashlib - -filename = 'package.xml' - -if len(sys.argv) > 1: - filename = sys.argv[1] - -if not os.path.isfile(filename): - exit(-1) - -tree = ElementTree.parse(filename) -root = tree.getroot() - -name = tree.findtext('name', '') -version = tree.findtext('version', '') -description = tree.findtext('description', '').strip() -license = tree.findtext('license', '') -homepage = tree.findtext('url', '') - -f = file(filename) -license_line = '' -license_md5 = '' -i = 0 -for line in f: - i += 1 - if 'license' in line: - license_line = str(i) - md5 = hashlib.md5() - md5.update(line) - license_md5 = md5.hexdigest() - break - - -buildtools = [] -for e in root.findall('buildtool_depend'): - buildtools.append(e.text) - -depends = [] -for e in root.findall('build_depend'): - depends.append(e.text.replace('_', '-')) - -print(name + '_' + version + ".bb") -print('') -print('DESCRIPTION = "' + description + '"') -if homepage: - print('HOMEPAGE = "' + homepage + '"') -print('SECTION = "devel"') -print('LICENSE = "' + license + '"') -print('LIC_FILES_CHKSUM = "file://package.xml;beginline=' + license_line + - ';endline=' + license_line + ';md5=' + license_md5 + '"') -if depends: - print('') - print('DEPENDS = "' + ' '.join(depends) + '"') -print('') -print('SRC_URI = ";downloadfilename=${P}.tar.gz"') -print('SRC_URI[md5sum] = ""') -print('SRC_URI[sha256sum] = ""') -print('') -print('S = "${WORKDIR}/-${PV}"') -print('') -if buildtools: - print('inherit ' + ' '.join(buildtools)) - diff --git a/scripts/create-ros-recipe.py b/scripts/create-ros-recipe.py deleted file mode 100755 index 64bb299..0000000 --- a/scripts/create-ros-recipe.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2016 David Bensoussan, Synapticon GmbH -# -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import re -import hashlib -import urllib -import tarfile -import sys - -from lxml import etree - - -def printUsage(): - print("Usage of commands:\n") - print "-h/--help" - print "-g " - print " Example: python create_ros_recipe.py -g OctoMap octomap-ros 0.4.0" - - -def printCommands(): - print("Summary of commands:\n") - print "-h/--help - prints commands and example" - print "-g/--generates - generates recipe of ros package" - - -def correct(string): - return re.sub(r'\.(?! )', '. ', re.sub(r' +', ' ', - re.sub(r'\n', '', string))) - - -class xmlParser: - - def __init__(self, xml_path): - self.xml_path = xml_path - self.tree = etree.parse(self.xml_path) - - def getName(self): - return self.tree.xpath("/package/name")[0].text - - def getVersion(self): - return self.tree.xpath("/package/version")[0].text - - # Sometimes the description has - def getDescription(self): - return correct( - self.tree.xpath("/package/description")[0].text).lstrip(' ') - - def getAuthorName(self): - return self.tree.xpath("/package/author")[0].text - - def getLicense(self): - return self.tree.xpath("/package/license")[0].text - - def getDependencies(self): - dependencies = [] - for dependency in self.tree.xpath("/package/build_depend"): - dependencies.append(dependency.text.replace("_", "-")) - return dependencies - - def getLicenseLineNumber(self): - with open(self.xml_path) as file: - for num, line in enumerate(file, 1): - if 'license' in line: - return num - return 'CLOSED' - - -class yoctoRecipe: - - def __init__(self, repository, name, version): - self.name = name - self.repository = repository - self.version = version - self.description = None - self.url = None - self.author = None - self.license = None - self.dependencies = None - self.license_line = None - self.license_md5 = None - self.src_md5 = None - self.src_sha256 = None - - def getLicenseMD5(self, license): - if license == "BSD": - return "d566ef916e9dedc494f5f793a6690ba5" - elif license == "Mozilla Public License Version 1.1": - return "e1b5a50d4dd59d8102e41a7a2254462d" - elif license == "CC-BY-NC-SA-2.0": - return "11e24f757f025b2cbebd5b14b4a7ca19" - elif license == "LGPL-2.1": - return "184dd1523b9a109aead3fbbe0b4262e0" - elif license == "GPL": - return "162b49cfbae9eadf37c9b89b2d2ac6be" - elif license == "LGPL-2.1+": - return "58d727014cda5ed405b7fb52666a1f97" - elif license == "LGPLv2": - return "46ee8693f40a89a31023e97ae17ecf19" - elif license == "MIT": - return "58e54c03ca7f821dd3967e2a2cd1596e" - - def getSrcMD5(self): - return hashlib.md5( - open("./" + self.getArchiveName(), 'rb').read()).hexdigest() - - def getSrcSha256(self): - return hashlib.sha256( - open("./" + self.getArchiveName(), 'rb').read()).hexdigest() - - def importXML(self): - xml = xmlParser(self.getFolderName() + "/package.xml") - self.description = xml.getDescription() - self.author = xml.getAuthorName() - self.license = xml.getLicense() - self.dependencies = xml.getDependencies() - self.license_line = xml.getLicenseLineNumber() - self.license_md5 = self.getLicenseMD5(self.license) - - def getURL(self): - return "https://github.com/" + \ - self.repository + "/" + \ - self.name.replace("-", "_") + \ - "/archive/" + str(self.version) + \ - ".tar.gz" - - def getFolderName(self): - return self.name.replace("-", "_") + "-" + str(self.version) - - def getArchiveName(self): - return self.name.replace("-", "_") + \ - "-" + str(self.version) + \ - ".tar.gz" - - # To debug - def printXML(self): - print self.name - print self.version - print self.description - print self.author - print self.license - print self.dependencies - - def parseDependencies(self): - dependencies_ = 'DEPENDS = \"' - for f in self.dependencies: - dependencies_ = dependencies_ + f + " " - dependencies_ = dependencies_.rstrip() + "\"" - return dependencies_ + "\n" - - def downloadArchive(self): - urllib.urlretrieve(self.getURL(), self.getArchiveName()) - - def extractArchive(self): - tar = tarfile.open(self.getArchiveName(), "r:gz") - tar.extractall() - tar.close() - - def createRecipe(self): - filename = self.name.replace("_", "-") + "_" + self.version + ".bb" - print "Recipe generated:\n" + filename - file = open(self.name.replace("_", "-") + - "_" + self.version + ".bb", "w") - - file.write('DESCRIPTION = \"' + self.description.rstrip() + "\"\n") - file.write('AUTHOR = \"' + self.author + '\"\n') - file.write('SECTION = \"devel\"\n') - file.write('LICENSE = \"' + self.license + "\"\n") - file.write('LIC_FILES_CHKSUM = file://package.xml;beginline=' + - str(self.license_line) + - ";endline=" + - str(self.license_line) + - ";md5=" + - self.license_md5 + - "\"\n") - file.write('\n') - - file.write(self.parseDependencies() + "\n") - file.write('SRC_URI = ' + - "https://github.com/" + self.repository + - "/${ROS_SPN}/archive/${PV}.tar.gz;" + - "downloadfilename=${ROS_SP}.tar.gz\"\n ") - file.write('SRC_URI[md5sum] = \"' + self.getSrcMD5() + "\"\n") - file.write('SRC_URI[sha256sum] = ' + self.getSrcSha256() + "\"\n") - - file.write('\n') - file.write('S = \"${WORKDIR}/${ROS_SP}\"\n') - file.write('\n') - - file.write('inherit catkin\n') - -if __name__ == '__main__': - if sys.argv[1] == "-h" or sys.argv[1] == "--help": - printCommands() - printUsage() - if sys.argv[1] == "-g" or sys.argv[1] == "--generate": - if len(sys.argv) == 5: - recipe = yoctoRecipe(sys.argv[2], sys.argv[3], sys.argv[4]) - recipe.downloadArchive() - recipe.extractArchive() - recipe.importXML() - recipe.createRecipe() - else: - print "Please provide 3 arguments" - printUsage() - printCommands() - else: - print "Please provide 3 arguments" - printUsage() - printCommands()