diff --git a/Jenkinsfile b/Jenkinsfile index 04849a567..f1a1fad98 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,28 +11,13 @@ pipeline stages { - stage('Creating nodes') - { - agent { label "master" } - steps - { - script - { - JOB_ID = "${env.BUILD_TAG}" - jenkinsLib = load("/home/jenkins/jenkins_426.groovy") - - jenkinsLib.CreateUbuntuBuildNode(JOB_ID) - jenkinsLib.CreateWindowsBuildNode(JOB_ID) - } - } - } stage('Building CARLA') { parallel { stage('ubuntu') { - agent { label "ubuntu && build && ${JOB_ID}" } + agent { label "ubuntu" } environment { UE4_ROOT = '/home/jenkins/UnrealEngine_4.26' @@ -44,7 +29,7 @@ pipeline steps { sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' - sh 'make setup ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' + sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' } } stage('ubuntu build') @@ -52,7 +37,7 @@ pipeline steps { sh 'make LibCarla' - sh 'make PythonAPI ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' sh 'make CarlaUE4Editor ARGS="--chrono"' sh 'make plugins' sh 'make examples' @@ -72,7 +57,7 @@ pipeline { steps { - sh 'make check ARGS="--all --xml --python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' } post { @@ -94,8 +79,8 @@ pipeline { steps { - sh 'make package ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' - sh 'make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate --python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make package ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' + sh 'make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' sh 'make examples ARGS="localhost 3654"' } post @@ -107,24 +92,11 @@ pipeline // stash includes: 'Dist/AdditionalMaps*.tar.gz', name: 'ubuntu_package2' stash includes: 'Examples/', name: 'ubuntu_examples' } - success - { - node('master') - { - script - { - JOB_ID = "${env.BUILD_TAG}" - jenkinsLib = load("/home/jenkins/jenkins_426.groovy") - - jenkinsLib.CreateUbuntuTestNode(JOB_ID) - } - } - } } } stage('ubuntu smoke tests') { - agent { label "ubuntu && gpu && ${JOB_ID}" } + agent { label "ubuntu && gpu" } steps { unstash name: 'ubuntu_eggs' @@ -135,7 +107,7 @@ pipeline sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/' // sh 'tar -xvzf Dist/AdditionalMaps*.tar.gz -C Dist/' sh 'DISPLAY= ./Dist/CarlaUE4.sh -nullrhi -RenderOffScreen --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' - sh 'make smoke_tests ARGS="--xml --python-version=3.7 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make smoke_tests ARGS="--xml --python-version=3.8 --target-wheel-platform=manylinux_2_27_x86_64"' sh 'make run-examples ARGS="localhost 3654"' } post @@ -145,19 +117,21 @@ pipeline archiveArtifacts 'CarlaUE4.log' junit 'Build/test-results/smoke-tests-*.xml' deleteDir() - node('master') - { - script - { - JOB_ID = "${env.BUILD_TAG}" - jenkinsLib = load("/home/jenkins/jenkins_426.groovy") - - jenkinsLib.DeleteUbuntuTestNode(JOB_ID) - } - } } } } + + stage('TEST: ubuntu deploy sim') + { + when { branch "ruben/jenkins_migration"; } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--test"' + } + + } + stage('ubuntu deploy dev') { when { branch "dev"; } @@ -176,26 +150,88 @@ pipeline sh 'make deploy ARGS="--replace-latest --docker-push"' } } - stage('ubuntu Doxygen') + + stage('ubuntu Doxygen generation') { when { anyOf { branch "master"; branch "dev"; buildingTag() } } steps { - sh 'rm -rf ~/carla-simulator.github.io/Doxygen' - sh ''' - cd ~/carla-simulator.github.io - git remote set-url origin git@docs:carla-simulator/carla-simulator.github.io.git - git fetch - git checkout -B master origin/master - ''' sh 'make docs' - sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/' - sh ''' - cd ~/carla-simulator.github.io - git add Doxygen - git commit -m "Updated c++ docs" || true - git push - ''' + sh 'tar -czf carla_doc.tar.gz ./Doxygen' + stash includes: 'carla_doc.tar.gz', name: 'carla_docs' + } + } + + stage('ubuntu Doxygen upload') + { + when { anyOf { branch "master"; branch "dev"; buildingTag() } } + steps + { + checkout scmGit(branches: [[name: '*/master']], extensions: [checkoutOption(120), cloneOption(noTags:false, reference:'', shallow: false, timeout:120)], userRemoteConfigs: [[credentialsId: 'github_token_as_pwd_2', url: 'https://github.com/carla-simulator/carla-simulator.github.io.git']]) + unstash name: 'carla_docs' + + withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { + sh ''' + tar -xvzf carla_doc.tar.gz + git add Doxygen + git commit -m "Updated c++ docs" || true + git push + ''' + } + } + post + { + always + { + deleteDir() + } + } + } + stage('TEST: ubuntu Doxygen generation') + { + when { branch "ruben/jenkins_migration"; } + steps + { + sh 'make docs' + sh 'tar -czf carla_doc.tar.gz ./Doxygen' + stash includes: 'carla_doc.tar.gz', name: 'carla_docs' + } + } + + stage('TEST: ubuntu Doxygen upload') + { + when { branch "ruben/jenkins_migration"; } + steps + { + dir('doc_repo') + { + checkout scmGit( + branches: [[name: '*/ruben/jenkins_migration']], + extensions: [ + checkoutOption(120), + localBranch("**"), + cloneOption(noTags:false, reference:'', shallow: false, timeout:120) + ], + userRemoteConfigs: [ + [ + credentialsId: 'github_token_as_pwd_2', + url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' + ] + ] + ) + unstash name: 'carla_docs' + withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { + sh ''' + pwd + ls -lh + tar -xvzf carla_doc.tar.gz + git add Doxygen + git commit -m "Updated c++ docs" || true + git push --set-upstream origin ruben/jenkins_migration + ''' + } + } + } post { @@ -211,23 +247,14 @@ pipeline always { deleteDir() - - node('master') - { - script - { - JOB_ID = "${env.BUILD_TAG}" - jenkinsLib = load("/home/jenkins/jenkins_426.groovy") - - jenkinsLib.DeleteUbuntuBuildNode(JOB_ID) - } - } } } } + /* stage('windows') { - agent { label "windows && build && ${JOB_ID}" } + + agent { label "windows" } environment { UE4_ROOT = 'C:\\UE_4.26' @@ -239,11 +266,11 @@ pipeline steps { bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject """ bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make setup ARGS="--chrono" """ } @@ -253,19 +280,19 @@ pipeline steps { bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make LibCarla """ bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make PythonAPI """ bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make CarlaUE4Editor ARGS="--chrono" """ bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make plugins """ } @@ -283,7 +310,7 @@ pipeline steps { bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat call Update.bat """ } @@ -293,11 +320,11 @@ pipeline steps { bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make package ARGS="--chrono" """ bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate" """ } @@ -307,37 +334,29 @@ pipeline } } } + stage('windows deploy') { when { anyOf { branch "master"; branch "dev"; buildingTag() } } steps { bat """ - call ../setEnv64.bat + call C:\\Users\\jenkins\\setEnv64.bat git checkout . - make deploy ARGS="--replace-latest" + REM make deploy ARGS="--replace-latest" """ } } + } post { always { deleteDir() - - node('master') - { - script - { - JOB_ID = "${env.BUILD_TAG}" - jenkinsLib = load("/home/jenkins/jenkins_426.groovy") - - jenkinsLib.DeleteWindowsBuildNode(JOB_ID) - } - } } } - } + }*/ + } } } diff --git a/Update.sh b/Update.sh index 7d4e3cd10..3c4d47e2d 100755 --- a/Update.sh +++ b/Update.sh @@ -41,6 +41,9 @@ done # -- Set up environment -------------------------------------------------------- # ============================================================================== +MAX_PARALLELL_DOWNLOADS=16 +MAX_CONNECTIONS_PER_SERVER=16 + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" pushd "$SCRIPT_DIR" >/dev/null @@ -60,7 +63,7 @@ function download_content { mkdir -p Content if hash aria2c 2>/dev/null; then echo -e "${CONTENT_LINK}\n\tout=Content.tar.gz" > .aria2c.input - aria2c -j16 -x16 --input-file=.aria2c.input + aria2c -j${MAX_PARALLELL_DOWNLOADS} -x${MAX_CONNECTIONS_PER_SERVER} --input-file=.aria2c.input rm -f .aria2c.input else wget -c ${CONTENT_LINK} -O Content.tar.gz diff --git a/Util/BuildTools/Deploy.sh b/Util/BuildTools/Deploy.sh index 55fb6beba..3d3e45a80 100755 --- a/Util/BuildTools/Deploy.sh +++ b/Util/BuildTools/Deploy.sh @@ -12,6 +12,10 @@ AWS_COPY="aws s3 cp" DOCKER="docker" UNTAR="tar -xvzf" UPLOAD_MAPS=true +PROFILE="--profile Jenkins-CVC" +ENDPOINT="--endpoint-url=https://s3.us-east-005.backblazeb2.com/" +TEST=false + # ============================================================================== # -- Parse arguments ----------------------------------------------------------- @@ -21,12 +25,15 @@ DOC_STRING="Upload latest build to S3." USAGE_STRING="Usage: $0 [-h|--help] [--replace-latest] [--docker-push] [--dry-run]" -OPTS=`getopt -o h --long help,replace-latest,docker-push,dry-run -n 'parse-options' -- "$@"` +OPTS=`getopt -o h --long help,replace-latest,docker-push,dry-run,test -n 'parse-options' -- "$@"` eval set -- "$OPTS" while [[ $# -gt 0 ]]; do case "$1" in + --test ) + TEST=true + shift ;; --replace-latest ) REPLACE_LATEST=true; shift ;; @@ -86,6 +93,25 @@ if [ ! -f ${LATEST_PACKAGE_PATH} ]; then fatal_error "Latest package not found, please run 'make package'." fi + +# ============================================================================== +# -- TEST -------------------------------------------------------------------- +# ============================================================================== + +if ${TEST} ; then + LATEST_PACKAGE=test_CARLA_${REPOSITORY_TAG}.tar.gz + LATEST_PACKAGE_PATH=./${LATEST_PACKAGE} + LATEST_PACKAGE2=test_AdditionalMaps_${REPOSITORY_TAG}.tar.gz + LATEST_PACKAGE_PATH2=./${LATEST_PACKAGE2} + + DEPLOY_NAME=test_CARLA_${REPOSITORY_TAG}.tar.gz + DEPLOY_NAME2=test_AdditionalMaps_${REPOSITORY_TAG}.tar.gz + + touch ${LATEST_PACKAGE} + touch ${LATEST_PACKAGE2} + +fi + # ============================================================================== # -- Upload -------------------------------------------------------------------- # ============================================================================== @@ -93,10 +119,10 @@ fi DEPLOY_URI=${S3_PREFIX}/${DEPLOY_NAME} DEPLOY_URI2=${S3_PREFIX}/${DEPLOY_NAME2} -${AWS_COPY} ${LATEST_PACKAGE_PATH} ${DEPLOY_URI} +${AWS_COPY} ${LATEST_PACKAGE_PATH} ${DEPLOY_URI} ${ENDPOINT} ${PROFILE} log "Latest build uploaded to ${DEPLOY_URI}." -${AWS_COPY} ${LATEST_PACKAGE_PATH2} ${DEPLOY_URI2} +${AWS_COPY} ${LATEST_PACKAGE_PATH2} ${DEPLOY_URI2} ${ENDPOINT} ${PROFILE} log "Latest build uploaded to ${DEPLOY_URI2}." # ============================================================================== @@ -105,10 +131,10 @@ log "Latest build uploaded to ${DEPLOY_URI2}." if ${REPLACE_LATEST} ; then - ${AWS_COPY} ${DEPLOY_URI} ${LATEST_DEPLOY_URI} + ${AWS_COPY} ${DEPLOY_URI} ${LATEST_DEPLOY_URI} ${ENDPOINT} ${PROFILE} log "Latest build uploaded to ${LATEST_DEPLOY_URI}." - ${AWS_COPY} ${DEPLOY_URI2} ${LATEST_DEPLOY_URI2} + ${AWS_COPY} ${DEPLOY_URI2} ${LATEST_DEPLOY_URI2} ${ENDPOINT} ${PROFILE} log "Latest build uploaded to ${LATEST_DEPLOY_URI2}." fi diff --git a/Util/BuildTools/Setup.sh b/Util/BuildTools/Setup.sh index 5277db2ac..0a46ae555 100755 --- a/Util/BuildTools/Setup.sh +++ b/Util/BuildTools/Setup.sh @@ -69,6 +69,7 @@ UNREAL_HOSTED_CFLAGS="--sysroot=$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/Hos BOOST_VERSION=1.80.0 BOOST_BASENAME="boost-${BOOST_VERSION}-${CXX_TAG}" +BOOST_SHA256SUM="4b2136f98bdd1f5857f1c3dea9ac2018effe65286cf251534b6ae20cc45e1847" BOOST_INCLUDE=${PWD}/${BOOST_BASENAME}-install/include BOOST_LIBPATH=${PWD}/${BOOST_BASENAME}-install/lib @@ -91,9 +92,9 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_} log "Retrieving boost." - wget "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true + wget "https://archives.boost.io/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true # try to use the backup boost we have in Jenkins - if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then + if [ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ] || [[ $(sha256sum "${BOOST_PACKAGE_BASENAME}.tar.gz") != "${BOOST_SHA256SUM}" ]] ; then log "Using boost backup" wget "https://carla-releases.s3.us-east-005.backblazeb2.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true fi diff --git a/Util/InstallersWin/install_boost.bat b/Util/InstallersWin/install_boost.bat index 441ece291..ebd467462 100644 --- a/Util/InstallersWin/install_boost.bat +++ b/Util/InstallersWin/install_boost.bat @@ -1,5 +1,5 @@ @echo off -setlocal +setlocal enabledelayedexpansion rem BAT script that downloads and installs a ready to use rem boost build for CARLA (carla.org). @@ -66,12 +66,13 @@ rem -- Local Variables --------------------------------------------------------- rem ============================================================================ set BOOST_BASENAME=boost-%BOOST_VERSION% +set BOOST_SHA256SUM="e34756f63abe8ac34b35352743f17d061fcc825969a2dd8458264edb38781782" set BOOST_TEMP_FOLDER=boost_%BOOST_VERSION:.=_% set BOOST_TEMP_FILE=%BOOST_TEMP_FOLDER%.zip set BOOST_TEMP_FILE_DIR=%BUILD_DIR%%BOOST_TEMP_FILE% -set BOOST_REPO=https://boostorg.jfrog.io/artifactory/main/release/%BOOST_VERSION%/source/%BOOST_TEMP_FILE% +set BOOST_REPO=https://archives.boost.io/release/%BOOST_VERSION%/source/%BOOST_TEMP_FILE% set BOOST_SRC_DIR=%BUILD_DIR%%BOOST_BASENAME%-source\ set BOOST_INSTALL_DIR=%BUILD_DIR%%BOOST_BASENAME%-install\ set BOOST_LIB_DIR=%BOOST_INSTALL_DIR%lib\ @@ -84,16 +85,20 @@ if exist "%BOOST_INSTALL_DIR%" ( goto already_build ) +set _checksum="" + if not exist "%BOOST_SRC_DIR%" ( if not exist "%BOOST_TEMP_FILE_DIR%" ( echo %FILE_N% Retrieving boost. powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%BOOST_REPO%', '%BOOST_TEMP_FILE_DIR%')" + call :CheckSumEvaluate %BOOST_TEMP_FILE_DIR%,%BOOST_SHA256SUM%,_checksum ) - if not exist "%BOOST_TEMP_FILE_DIR%" ( + if "!_checksum!" == "1" ( echo %FILE_N% Using Boost backup powershell -Command "(New-Object System.Net.WebClient).DownloadFile('https://carla-releases.s3.us-east-005.backblazeb2.com/Backup/%BOOST_TEMP_FILE%', '%BOOST_TEMP_FILE_DIR%')" + call :CheckSumEvaluate %BOOST_TEMP_FILE_DIR%,%BOOST_SHA256SUM%,_checksum ) - if %errorlevel% neq 0 goto error_download + if "!_checksum!" == "1" goto error_download echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while... if exist "%ProgramW6432%/7-Zip/7z.exe" ( "%ProgramW6432%/7-Zip/7z.exe" x "%BOOST_TEMP_FILE_DIR%" -o"%BUILD_DIR%" -y @@ -197,3 +202,25 @@ rem ============================================================================ echo %FILE_N% Exiting with error... endlocal exit /b %errorlevel% + +:CheckSumEvaluate +set filepath=%1 +set checksum=%2 + +echo %FILE_N% calculating %filepath% checksum... + +set PsCommand="(Get-FileHash %filepath%).Hash -eq '%checksum%'" + +for /f %%F in ('Powershell -C %PsCommand%') do ( + set filechecksum=%%F +) + +if %filechecksum% == True ( + echo %FILE_N% %filepath% checksum OK + set "%~3=0" + exit /b 0 +) else ( + echo %FILE_N% %filepath% BAD SHA256 checksum + set "%~3=1" + exit /b 1 +) \ No newline at end of file