Merge branch 'ruben/jenkins_migration' of github.com:carla-simulator/carla into dev

This commit is contained in:
Ruben Abad 2024-02-06 11:15:20 +01:00
commit 11f22c0e85
5 changed files with 184 additions and 108 deletions

211
Jenkinsfile vendored
View File

@ -11,28 +11,13 @@ pipeline
stages
{
stage('Creating nodes')
{
agent { label "master" }
steps
{
script
{
JOB_ID = "${env.BUILD_TAG}"
jenkinsLib = load("/home/jenkins/jenkins_426.groovy")
jenkinsLib.CreateUbuntuBuildNode(JOB_ID)
jenkinsLib.CreateWindowsBuildNode(JOB_ID)
}
}
}
stage('Building CARLA')
{
parallel
{
stage('ubuntu')
{
agent { label "ubuntu && build && ${JOB_ID}" }
agent { label "ubuntu" }
environment
{
UE4_ROOT = '/home/jenkins/UnrealEngine_4.26'
@ -44,7 +29,7 @@ pipeline
steps
{
sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject'
sh 'make setup ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"'
sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"'
}
}
stage('ubuntu build')
@ -52,7 +37,7 @@ pipeline
steps
{
sh 'make LibCarla'
sh 'make PythonAPI ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make CarlaUE4Editor ARGS="--chrono"'
sh 'make plugins'
sh 'make examples'
@ -72,7 +57,7 @@ pipeline
{
steps
{
sh 'make check ARGS="--all --xml --python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"'
}
post
{
@ -94,8 +79,8 @@ pipeline
{
steps
{
sh 'make package ARGS="--python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"'
sh 'make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate --python-version=3.7,2 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make package ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"'
sh 'make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make examples ARGS="localhost 3654"'
}
post
@ -107,24 +92,11 @@ pipeline
// stash includes: 'Dist/AdditionalMaps*.tar.gz', name: 'ubuntu_package2'
stash includes: 'Examples/', name: 'ubuntu_examples'
}
success
{
node('master')
{
script
{
JOB_ID = "${env.BUILD_TAG}"
jenkinsLib = load("/home/jenkins/jenkins_426.groovy")
jenkinsLib.CreateUbuntuTestNode(JOB_ID)
}
}
}
}
}
stage('ubuntu smoke tests')
{
agent { label "ubuntu && gpu && ${JOB_ID}" }
agent { label "ubuntu && gpu" }
steps
{
unstash name: 'ubuntu_eggs'
@ -135,7 +107,7 @@ pipeline
sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/'
// sh 'tar -xvzf Dist/AdditionalMaps*.tar.gz -C Dist/'
sh 'DISPLAY= ./Dist/CarlaUE4.sh -nullrhi -RenderOffScreen --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &'
sh 'make smoke_tests ARGS="--xml --python-version=3.7 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make smoke_tests ARGS="--xml --python-version=3.8 --target-wheel-platform=manylinux_2_27_x86_64"'
sh 'make run-examples ARGS="localhost 3654"'
}
post
@ -145,19 +117,21 @@ pipeline
archiveArtifacts 'CarlaUE4.log'
junit 'Build/test-results/smoke-tests-*.xml'
deleteDir()
node('master')
{
script
{
JOB_ID = "${env.BUILD_TAG}"
jenkinsLib = load("/home/jenkins/jenkins_426.groovy")
jenkinsLib.DeleteUbuntuTestNode(JOB_ID)
}
}
}
}
}
stage('TEST: ubuntu deploy sim')
{
when { branch "ruben/jenkins_migration"; }
steps
{
sh 'git checkout .'
sh 'make deploy ARGS="--test"'
}
}
stage('ubuntu deploy dev')
{
when { branch "dev"; }
@ -176,26 +150,88 @@ pipeline
sh 'make deploy ARGS="--replace-latest --docker-push"'
}
}
stage('ubuntu Doxygen')
stage('ubuntu Doxygen generation')
{
when { anyOf { branch "master"; branch "dev"; buildingTag() } }
steps
{
sh 'rm -rf ~/carla-simulator.github.io/Doxygen'
sh '''
cd ~/carla-simulator.github.io
git remote set-url origin git@docs:carla-simulator/carla-simulator.github.io.git
git fetch
git checkout -B master origin/master
'''
sh 'make docs'
sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/'
sh '''
cd ~/carla-simulator.github.io
git add Doxygen
git commit -m "Updated c++ docs" || true
git push
'''
sh 'tar -czf carla_doc.tar.gz ./Doxygen'
stash includes: 'carla_doc.tar.gz', name: 'carla_docs'
}
}
stage('ubuntu Doxygen upload')
{
when { anyOf { branch "master"; branch "dev"; buildingTag() } }
steps
{
checkout scmGit(branches: [[name: '*/master']], extensions: [checkoutOption(120), cloneOption(noTags:false, reference:'', shallow: false, timeout:120)], userRemoteConfigs: [[credentialsId: 'github_token_as_pwd_2', url: 'https://github.com/carla-simulator/carla-simulator.github.io.git']])
unstash name: 'carla_docs'
withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) {
sh '''
tar -xvzf carla_doc.tar.gz
git add Doxygen
git commit -m "Updated c++ docs" || true
git push
'''
}
}
post
{
always
{
deleteDir()
}
}
}
stage('TEST: ubuntu Doxygen generation')
{
when { branch "ruben/jenkins_migration"; }
steps
{
sh 'make docs'
sh 'tar -czf carla_doc.tar.gz ./Doxygen'
stash includes: 'carla_doc.tar.gz', name: 'carla_docs'
}
}
stage('TEST: ubuntu Doxygen upload')
{
when { branch "ruben/jenkins_migration"; }
steps
{
dir('doc_repo')
{
checkout scmGit(
branches: [[name: '*/ruben/jenkins_migration']],
extensions: [
checkoutOption(120),
localBranch("**"),
cloneOption(noTags:false, reference:'', shallow: false, timeout:120)
],
userRemoteConfigs: [
[
credentialsId: 'github_token_as_pwd_2',
url: 'https://github.com/carla-simulator/carla-simulator.github.io.git'
]
]
)
unstash name: 'carla_docs'
withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) {
sh '''
pwd
ls -lh
tar -xvzf carla_doc.tar.gz
git add Doxygen
git commit -m "Updated c++ docs" || true
git push --set-upstream origin ruben/jenkins_migration
'''
}
}
}
post
{
@ -211,23 +247,14 @@ pipeline
always
{
deleteDir()
node('master')
{
script
{
JOB_ID = "${env.BUILD_TAG}"
jenkinsLib = load("/home/jenkins/jenkins_426.groovy")
jenkinsLib.DeleteUbuntuBuildNode(JOB_ID)
}
}
}
}
}
/*
stage('windows')
{
agent { label "windows && build && ${JOB_ID}" }
agent { label "windows" }
environment
{
UE4_ROOT = 'C:\\UE_4.26'
@ -239,11 +266,11 @@ pipeline
steps
{
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject
"""
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make setup ARGS="--chrono"
"""
}
@ -253,19 +280,19 @@ pipeline
steps
{
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make LibCarla
"""
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make PythonAPI
"""
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make CarlaUE4Editor ARGS="--chrono"
"""
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make plugins
"""
}
@ -283,7 +310,7 @@ pipeline
steps
{
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
call Update.bat
"""
}
@ -293,11 +320,11 @@ pipeline
steps
{
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make package ARGS="--chrono"
"""
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
make package ARGS="--packages=AdditionalMaps,Town06_Opt,Town07_Opt,Town11,Town12,Town13,Town15 --target-archive=AdditionalMaps --clean-intermediate"
"""
}
@ -307,37 +334,29 @@ pipeline
}
}
}
stage('windows deploy')
{
when { anyOf { branch "master"; branch "dev"; buildingTag() } }
steps {
bat """
call ../setEnv64.bat
call C:\\Users\\jenkins\\setEnv64.bat
git checkout .
make deploy ARGS="--replace-latest"
REM make deploy ARGS="--replace-latest"
"""
}
}
}
post
{
always
{
deleteDir()
node('master')
{
script
{
JOB_ID = "${env.BUILD_TAG}"
jenkinsLib = load("/home/jenkins/jenkins_426.groovy")
jenkinsLib.DeleteWindowsBuildNode(JOB_ID)
}
}
}
}
}
}*/
}
}
}

View File

@ -41,6 +41,9 @@ done
# -- Set up environment --------------------------------------------------------
# ==============================================================================
MAX_PARALLELL_DOWNLOADS=16
MAX_CONNECTIONS_PER_SERVER=16
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
pushd "$SCRIPT_DIR" >/dev/null
@ -60,7 +63,7 @@ function download_content {
mkdir -p Content
if hash aria2c 2>/dev/null; then
echo -e "${CONTENT_LINK}\n\tout=Content.tar.gz" > .aria2c.input
aria2c -j16 -x16 --input-file=.aria2c.input
aria2c -j${MAX_PARALLELL_DOWNLOADS} -x${MAX_CONNECTIONS_PER_SERVER} --input-file=.aria2c.input
rm -f .aria2c.input
else
wget -c ${CONTENT_LINK} -O Content.tar.gz

View File

@ -12,6 +12,10 @@ AWS_COPY="aws s3 cp"
DOCKER="docker"
UNTAR="tar -xvzf"
UPLOAD_MAPS=true
PROFILE="--profile Jenkins-CVC"
ENDPOINT="--endpoint-url=https://s3.us-east-005.backblazeb2.com/"
TEST=false
# ==============================================================================
# -- Parse arguments -----------------------------------------------------------
@ -21,12 +25,15 @@ DOC_STRING="Upload latest build to S3."
USAGE_STRING="Usage: $0 [-h|--help] [--replace-latest] [--docker-push] [--dry-run]"
OPTS=`getopt -o h --long help,replace-latest,docker-push,dry-run -n 'parse-options' -- "$@"`
OPTS=`getopt -o h --long help,replace-latest,docker-push,dry-run,test -n 'parse-options' -- "$@"`
eval set -- "$OPTS"
while [[ $# -gt 0 ]]; do
case "$1" in
--test )
TEST=true
shift ;;
--replace-latest )
REPLACE_LATEST=true;
shift ;;
@ -86,6 +93,25 @@ if [ ! -f ${LATEST_PACKAGE_PATH} ]; then
fatal_error "Latest package not found, please run 'make package'."
fi
# ==============================================================================
# -- TEST --------------------------------------------------------------------
# ==============================================================================
if ${TEST} ; then
LATEST_PACKAGE=test_CARLA_${REPOSITORY_TAG}.tar.gz
LATEST_PACKAGE_PATH=./${LATEST_PACKAGE}
LATEST_PACKAGE2=test_AdditionalMaps_${REPOSITORY_TAG}.tar.gz
LATEST_PACKAGE_PATH2=./${LATEST_PACKAGE2}
DEPLOY_NAME=test_CARLA_${REPOSITORY_TAG}.tar.gz
DEPLOY_NAME2=test_AdditionalMaps_${REPOSITORY_TAG}.tar.gz
touch ${LATEST_PACKAGE}
touch ${LATEST_PACKAGE2}
fi
# ==============================================================================
# -- Upload --------------------------------------------------------------------
# ==============================================================================
@ -93,10 +119,10 @@ fi
DEPLOY_URI=${S3_PREFIX}/${DEPLOY_NAME}
DEPLOY_URI2=${S3_PREFIX}/${DEPLOY_NAME2}
${AWS_COPY} ${LATEST_PACKAGE_PATH} ${DEPLOY_URI}
${AWS_COPY} ${LATEST_PACKAGE_PATH} ${DEPLOY_URI} ${ENDPOINT} ${PROFILE}
log "Latest build uploaded to ${DEPLOY_URI}."
${AWS_COPY} ${LATEST_PACKAGE_PATH2} ${DEPLOY_URI2}
${AWS_COPY} ${LATEST_PACKAGE_PATH2} ${DEPLOY_URI2} ${ENDPOINT} ${PROFILE}
log "Latest build uploaded to ${DEPLOY_URI2}."
# ==============================================================================
@ -105,10 +131,10 @@ log "Latest build uploaded to ${DEPLOY_URI2}."
if ${REPLACE_LATEST} ; then
${AWS_COPY} ${DEPLOY_URI} ${LATEST_DEPLOY_URI}
${AWS_COPY} ${DEPLOY_URI} ${LATEST_DEPLOY_URI} ${ENDPOINT} ${PROFILE}
log "Latest build uploaded to ${LATEST_DEPLOY_URI}."
${AWS_COPY} ${DEPLOY_URI2} ${LATEST_DEPLOY_URI2}
${AWS_COPY} ${DEPLOY_URI2} ${LATEST_DEPLOY_URI2} ${ENDPOINT} ${PROFILE}
log "Latest build uploaded to ${LATEST_DEPLOY_URI2}."
fi

View File

@ -69,6 +69,7 @@ UNREAL_HOSTED_CFLAGS="--sysroot=$UE4_ROOT/Engine/Extras/ThirdPartyNotUE/SDKs/Hos
BOOST_VERSION=1.80.0
BOOST_BASENAME="boost-${BOOST_VERSION}-${CXX_TAG}"
BOOST_SHA256SUM="4b2136f98bdd1f5857f1c3dea9ac2018effe65286cf251534b6ae20cc45e1847"
BOOST_INCLUDE=${PWD}/${BOOST_BASENAME}-install/include
BOOST_LIBPATH=${PWD}/${BOOST_BASENAME}-install/lib
@ -91,9 +92,9 @@ for PY_VERSION in ${PY_VERSION_LIST[@]} ; do
BOOST_PACKAGE_BASENAME=boost_${BOOST_VERSION//./_}
log "Retrieving boost."
wget "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true
wget "https://archives.boost.io/release/${BOOST_VERSION}/source/${BOOST_PACKAGE_BASENAME}.tar.gz" || true
# try to use the backup boost we have in Jenkins
if [[ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ]] ; then
if [ ! -f "${BOOST_PACKAGE_BASENAME}.tar.gz" ] || [[ $(sha256sum "${BOOST_PACKAGE_BASENAME}.tar.gz") != "${BOOST_SHA256SUM}" ]] ; then
log "Using boost backup"
wget "https://carla-releases.s3.us-east-005.backblazeb2.com/Backup/${BOOST_PACKAGE_BASENAME}.tar.gz" || true
fi

View File

@ -1,5 +1,5 @@
@echo off
setlocal
setlocal enabledelayedexpansion
rem BAT script that downloads and installs a ready to use
rem boost build for CARLA (carla.org).
@ -66,12 +66,13 @@ rem -- Local Variables ---------------------------------------------------------
rem ============================================================================
set BOOST_BASENAME=boost-%BOOST_VERSION%
set BOOST_SHA256SUM="e34756f63abe8ac34b35352743f17d061fcc825969a2dd8458264edb38781782"
set BOOST_TEMP_FOLDER=boost_%BOOST_VERSION:.=_%
set BOOST_TEMP_FILE=%BOOST_TEMP_FOLDER%.zip
set BOOST_TEMP_FILE_DIR=%BUILD_DIR%%BOOST_TEMP_FILE%
set BOOST_REPO=https://boostorg.jfrog.io/artifactory/main/release/%BOOST_VERSION%/source/%BOOST_TEMP_FILE%
set BOOST_REPO=https://archives.boost.io/release/%BOOST_VERSION%/source/%BOOST_TEMP_FILE%
set BOOST_SRC_DIR=%BUILD_DIR%%BOOST_BASENAME%-source\
set BOOST_INSTALL_DIR=%BUILD_DIR%%BOOST_BASENAME%-install\
set BOOST_LIB_DIR=%BOOST_INSTALL_DIR%lib\
@ -84,16 +85,20 @@ if exist "%BOOST_INSTALL_DIR%" (
goto already_build
)
set _checksum=""
if not exist "%BOOST_SRC_DIR%" (
if not exist "%BOOST_TEMP_FILE_DIR%" (
echo %FILE_N% Retrieving boost.
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%BOOST_REPO%', '%BOOST_TEMP_FILE_DIR%')"
call :CheckSumEvaluate %BOOST_TEMP_FILE_DIR%,%BOOST_SHA256SUM%,_checksum
)
if not exist "%BOOST_TEMP_FILE_DIR%" (
if "!_checksum!" == "1" (
echo %FILE_N% Using Boost backup
powershell -Command "(New-Object System.Net.WebClient).DownloadFile('https://carla-releases.s3.us-east-005.backblazeb2.com/Backup/%BOOST_TEMP_FILE%', '%BOOST_TEMP_FILE_DIR%')"
call :CheckSumEvaluate %BOOST_TEMP_FILE_DIR%,%BOOST_SHA256SUM%,_checksum
)
if %errorlevel% neq 0 goto error_download
if "!_checksum!" == "1" goto error_download
echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while...
if exist "%ProgramW6432%/7-Zip/7z.exe" (
"%ProgramW6432%/7-Zip/7z.exe" x "%BOOST_TEMP_FILE_DIR%" -o"%BUILD_DIR%" -y
@ -197,3 +202,25 @@ rem ============================================================================
echo %FILE_N% Exiting with error...
endlocal
exit /b %errorlevel%
:CheckSumEvaluate
set filepath=%1
set checksum=%2
echo %FILE_N% calculating %filepath% checksum...
set PsCommand="(Get-FileHash %filepath%).Hash -eq '%checksum%'"
for /f %%F in ('Powershell -C %PsCommand%') do (
set filechecksum=%%F
)
if %filechecksum% == True (
echo %FILE_N% %filepath% checksum OK
set "%~3=0"
exit /b 0
) else (
echo %FILE_N% %filepath% BAD SHA256 checksum
set "%~3=1"
exit /b 1
)