From b6c650d438ec068364d130bdc183292b0a170da9 Mon Sep 17 00:00:00 2001 From: Axel Date: Thu, 1 Oct 2020 16:12:15 +0200 Subject: [PATCH] Python 2 returns. --- Jenkinsfile | 17 ++-- Util/BuildTools/BuildPythonAPI.sh | 13 ++- Util/BuildTools/Check.sh | 15 +++- Util/BuildTools/Setup.sh | 140 ++++++++++++++++-------------- 4 files changed, 101 insertions(+), 84 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ccde049cf..9411a4b2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { steps { - sh 'make setup ARGS="--python-version=3.7"' + sh 'make setup ARGS="--python-version=3.7,2"' } } stage('ubuntu build') @@ -51,8 +51,7 @@ pipeline steps { sh 'make LibCarla' - sh 'make PythonAPI ARGS="--python-version=3.7"' - sh 'make PythonAPI ARGS="--python-version=2"' + sh 'make PythonAPI ARGS="--python-version=3.7,2"' sh 'make CarlaUE4Editor' sh 'make examples' } @@ -69,7 +68,7 @@ pipeline { steps { - sh 'make check ARGS="--all --xml --python-version=3.7"' + sh 'make check ARGS="--all --xml --python-version=3.7,2"' } post { @@ -91,8 +90,8 @@ pipeline { steps { - sh 'make package ARGS="--python-version=3.7"' - sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate --python-version=3.7"' + sh 'make package ARGS="--python-version=3.7,2"' + sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate --python-version=3.7,2"' sh 'make examples ARGS="localhost 3654"' } post @@ -111,7 +110,7 @@ pipeline { JOB_ID = "${env.BUILD_TAG}" jenkinsLib = load("/home/jenkins/jenkins.groovy") - + jenkinsLib.CreateUbuntuTestNode(JOB_ID) } } @@ -144,11 +143,11 @@ pipeline { JOB_ID = "${env.BUILD_TAG}" jenkinsLib = load("/home/jenkins/jenkins.groovy") - + jenkinsLib.DeleteUbuntuTestNode(JOB_ID) } } - } + } } } stage('ubuntu deploy dev') diff --git a/Util/BuildTools/BuildPythonAPI.sh b/Util/BuildTools/BuildPythonAPI.sh index 00bd3a10c..a7f64bcb5 100755 --- a/Util/BuildTools/BuildPythonAPI.sh +++ b/Util/BuildTools/BuildPythonAPI.sh @@ -21,7 +21,7 @@ OPTS=`getopt -o h --long help,rebuild,clean,rss,python-version:,packages:,clean- eval set -- "$OPTS" -PY_VERSION=3 +PY_VERSION_LIST=3 while [[ $# -gt 0 ]]; do case "$1" in @@ -30,7 +30,7 @@ while [[ $# -gt 0 ]]; do BUILD_PYTHONAPI=true; shift ;; --python-version ) - PY_VERSION="$2" + PY_VERSION_LIST="$2" shift 2 ;; --rss ) BUILD_RSS_VARIANT=true; @@ -53,6 +53,9 @@ if ! { ${REMOVE_INTERMEDIATE} || ${BUILD_PYTHONAPI} ; }; then fatal_error "Nothing selected to be done." fi +# Convert comma-separated string to array of unique elements. +IFS="," read -r -a PY_VERSION_LIST <<< "${PY_VERSION_LIST}" + pushd "${CARLA_PYTHONAPI_SOURCE_FOLDER}" >/dev/null # ============================================================================== @@ -80,9 +83,11 @@ fi if ${BUILD_PYTHONAPI} ; then - log "Building Python API for Python 3." + for PY_VERSION in ${PY_VERSION_LIST[@]} ; do + log "Building Python API for Python 3." - /usr/bin/env python${PY_VERSION} setup.py bdist_egg + /usr/bin/env python${PY_VERSION} setup.py bdist_egg + done fi diff --git a/Util/BuildTools/Check.sh b/Util/BuildTools/Check.sh index 248e0b0bb..ed28f4aaf 100755 --- a/Util/BuildTools/Check.sh +++ b/Util/BuildTools/Check.sh @@ -42,7 +42,7 @@ OPTS=`getopt -o h --long help,gdb,xml,gtest_args:,all,libcarla-release,libcarla- eval set -- "$OPTS" -PY_VERSION=3 +PY_VERSION_LIST=3 while [[ $# -gt 0 ]]; do case "$1" in @@ -80,7 +80,7 @@ while [[ $# -gt 0 ]]; do GTEST_ARGS="--gtest_filter=benchmark*"; shift ;; --python-version ) - PY_VERSION="$2" + PY_VERSION_LIST="$2" shift 2 ;; -h | --help ) echo "$DOC_STRING" @@ -96,6 +96,9 @@ if ! { ${LIBCARLA_RELEASE} || ${LIBCARLA_DEBUG} || ${PYTHON_API} || ${SMOKE_TEST fatal_error "Nothing selected to be done." fi +# Convert comma-separated string to array of unique elements. +IFS="," read -r -a PY_VERSION_LIST <<< "${PY_VERSION_LIST}" + # ============================================================================== # -- Download Content need it by the tests ------------------------------------- # ============================================================================== @@ -176,9 +179,13 @@ fi if ${PYTHON_API} ; then - log "Running Python API for Python 3 unit tests." + for PY_VERSION in ${PY_VERSION_LIST[@]} ; do - /usr/bin/env python${PY_VERSION} -m nose2 ${EXTRA_ARGS} + log "Running Python API for Python ${PY_VERSION} unit tests." + + /usr/bin/env python${PY_VERSION} -m nose2 ${EXTRA_ARGS} + + done if ${XML_OUTPUT} ; then mv test-results.xml ${CARLA_TEST_RESULTS_FOLDER}/python-api-3.xml diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 995822c5b..c88d93e6f 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -12,12 +12,12 @@ OPTS=`getopt -o h --long help,python-version: -n 'parse-options' -- "$@"` eval set -- "$OPTS" -PY_VERSION=3 +PY_VERSION_LIST=3 while [[ $# -gt 0 ]]; do case "$1" in --python-version ) - PY_VERSION="$2"; + PY_VERSION_LIST="$2"; shift 2 ;; -h | --help ) echo "$DOC_STRING" @@ -44,6 +44,9 @@ export CXX=/usr/bin/clang++-8 source $(dirname "$0")/Environment.sh +# Convert comma-separated string to array of unique elements. +IFS="," read -r -a PY_VERSION_LIST <<< "${PY_VERSION_LIST}" + mkdir -p ${CARLA_BUILD_FOLDER} pushd ${CARLA_BUILD_FOLDER} >/dev/null @@ -104,80 +107,83 @@ BOOST_BASENAME="boost-${BOOST_VERSION}-${CXX_TAG}" BOOST_INCLUDE=${PWD}/${BOOST_BASENAME}-install/include BOOST_LIBPATH=${PWD}/${BOOST_BASENAME}-install/lib -SHOULD_BUILD_BOOST=true +for PY_VERSION in ${PY_VERSION_LIST[@]} ; do -PYTHON_VERSION=$(/usr/bin/env python${PY_VERSION} -V) -LIB_NAME=${PYTHON_VERSION:7:3} -LIB_NAME=${LIB_NAME//.} -if [[ -d "${BOOST_BASENAME}-install" ]] ; then - if [ -f "${BOOST_BASENAME}-install/lib/libboost_python${LIB_NAME}.a" ] ; then - SHOULD_BUILD_BOOST=false - log "${BOOST_BASENAME} already installed." - fi -fi - -if { ${SHOULD_BUILD_BOOST} ; } ; then - rm -Rf ${BOOST_BASENAME}-source - - BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_} - - log "Retrieving boost." - wget "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true - # try to use the backup boost we have in Jenkins - if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then - log "Using boost backup" - wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + SHOULD_BUILD_BOOST=true + PYTHON_VERSION=$(/usr/bin/env python${PY_VERSION} -V) + LIB_NAME=${PYTHON_VERSION:7:3} + LIB_NAME=${LIB_NAME//.} + if [[ -d "${BOOST_BASENAME}-install" ]] ; then + if [ -f "${BOOST_BASENAME}-install/lib/libboost_python${LIB_NAME}.a" ] ; then + SHOULD_BUILD_BOOST=false + log "${BOOST_BASENAME} already installed." + fi fi - log "Extracting boost for Python ${PY_VERSION}." - tar -xzf ${BOOST_PACKAGE_BASENAME}.tar.gz - mkdir -p ${BOOST_BASENAME}-install/include - mv ${BOOST_PACKAGE_BASENAME} ${BOOST_BASENAME}-source - # Boost patch for exception handling - cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/rational.hpp" "${BOOST_BASENAME}-source/boost/rational.hpp" - cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/read.hpp" "${BOOST_BASENAME}-source/boost/geometry/io/wkt/read.hpp" - # --- + if { ${SHOULD_BUILD_BOOST} ; } ; then + rm -Rf ${BOOST_BASENAME}-source - pushd ${BOOST_BASENAME}-source >/dev/null + BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_} - BOOST_TOOLSET="clang-8.0" - BOOST_CFLAGS="-fPIC -std=c++14 -DBOOST_ERROR_CODE_HEADER_ONLY" + log "Retrieving boost." + wget "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + # try to use the backup boost we have in Jenkins + if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then + log "Using boost backup" + wget "https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + fi - py3="/usr/bin/env python${PY_VERSION}" - py3_root=`${py3} -c "import sys; print(sys.prefix)"` - pyv=`$py3 -c "import sys;x='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(x)";` - ./bootstrap.sh \ - --with-toolset=clang \ - --prefix=../boost-install \ - --with-libraries=python,filesystem,system,program_options \ - --with-python=${py3} --with-python-root=${py3_root} + log "Extracting boost for Python ${PY_VERSION}." + tar -xzf ${BOOST_PACKAGE_BASENAME}.tar.gz + mkdir -p ${BOOST_BASENAME}-install/include + mv ${BOOST_PACKAGE_BASENAME} ${BOOST_BASENAME}-source + # Boost patch for exception handling + cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/rational.hpp" "${BOOST_BASENAME}-source/boost/rational.hpp" + cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/read.hpp" "${BOOST_BASENAME}-source/boost/geometry/io/wkt/read.hpp" + # --- + + pushd ${BOOST_BASENAME}-source >/dev/null + + BOOST_TOOLSET="clang-8.0" + BOOST_CFLAGS="-fPIC -std=c++14 -DBOOST_ERROR_CODE_HEADER_ONLY" + + py3="/usr/bin/env python${PY_VERSION}" + py3_root=`${py3} -c "import sys; print(sys.prefix)"` + pyv=`$py3 -c "import sys;x='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(x)";` + ./bootstrap.sh \ + --with-toolset=clang \ + --prefix=../boost-install \ + --with-libraries=python,filesystem,system,program_options \ + --with-python=${py3} --with-python-root=${py3_root} + + if ${TRAVIS} ; then + echo "using python : ${pyv} : ${py3_root}/bin/python${PY_VERSION} ;" > ${HOME}/user-config.jam + else + echo "using python : ${pyv} : ${py3_root}/bin/python${PY_VERSION} ;" > project-config.jam + fi + + ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} stage release + ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} install + + popd >/dev/null + + rm -Rf ${BOOST_BASENAME}-source + rm ${BOOST_PACKAGE_BASENAME}.tar.gz + + # Boost patch for exception handling + cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/rational.hpp" "${BOOST_BASENAME}-install/include/boost/rational.hpp" + cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/read.hpp" "${BOOST_BASENAME}-install/include/boost/geometry/io/wkt/read.hpp" + # --- + + # Install boost dependencies + mkdir -p "${LIBCARLA_INSTALL_CLIENT_FOLDER}/include/system" + mkdir -p "${LIBCARLA_INSTALL_CLIENT_FOLDER}/lib" + cp -rf ${BOOST_BASENAME}-install/include/* ${LIBCARLA_INSTALL_CLIENT_FOLDER}/include/system + cp -rf ${BOOST_BASENAME}-install/lib/* ${LIBCARLA_INSTALL_CLIENT_FOLDER}/lib - if ${TRAVIS} ; then - echo "using python : ${pyv} : ${py3_root}/bin/python${PY_VERSION} ;" > ${HOME}/user-config.jam - else - echo "using python : ${pyv} : ${py3_root}/bin/python${PY_VERSION} ;" > project-config.jam fi - ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} stage release - ./b2 toolset="${BOOST_TOOLSET}" cxxflags="${BOOST_CFLAGS}" --prefix="../${BOOST_BASENAME}-install" -j ${CARLA_BUILD_CONCURRENCY} install - - popd >/dev/null - - rm -Rf ${BOOST_BASENAME}-source - rm ${BOOST_PACKAGE_BASENAME}.tar.gz - - # Boost patch for exception handling - cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/rational.hpp" "${BOOST_BASENAME}-install/include/boost/rational.hpp" - cp "${CARLA_BUILD_FOLDER}/../Util/BoostFiles/read.hpp" "${BOOST_BASENAME}-install/include/boost/geometry/io/wkt/read.hpp" - # --- - - # Install boost dependencies - mkdir -p "${LIBCARLA_INSTALL_CLIENT_FOLDER}/include/system" - mkdir -p "${LIBCARLA_INSTALL_CLIENT_FOLDER}/lib" - cp -rf ${BOOST_BASENAME}-install/include/* ${LIBCARLA_INSTALL_CLIENT_FOLDER}/include/system - cp -rf ${BOOST_BASENAME}-install/lib/* ${LIBCARLA_INSTALL_CLIENT_FOLDER}/lib - -fi +done unset BOOST_BASENAME