From ccf76e0f329af445a26e11cc610c2f6a0d68b336 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Tue, 6 Feb 2024 11:32:32 +0100 Subject: [PATCH 01/16] run entire pipeline in gpu machine --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f1a1fad98..e12e20345 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,7 @@ pipeline { stage('ubuntu') { - agent { label "ubuntu" } + agent { label "gpu" } environment { UE4_ROOT = '/home/jenkins/UnrealEngine_4.26' @@ -96,7 +96,6 @@ pipeline } stage('ubuntu smoke tests') { - agent { label "ubuntu && gpu" } steps { unstash name: 'ubuntu_eggs' From 1c8b906b6a9ca7af3c2fb23b85473cdce96911c9 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Tue, 6 Feb 2024 12:01:20 +0100 Subject: [PATCH 02/16] incremental repo --- Jenkinsfile | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e12e20345..0021960ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -202,11 +202,13 @@ pipeline when { branch "ruben/jenkins_migration"; } steps { - dir('doc_repo') + + dir('${env.WORKSPACE}/doc_repo') { checkout scmGit( branches: [[name: '*/ruben/jenkins_migration']], extensions: [ + cleanBeforeCheckout(), checkoutOption(120), localBranch("**"), cloneOption(noTags:false, reference:'', shallow: false, timeout:120) @@ -232,20 +234,6 @@ pipeline } } - post - { - always - { - deleteDir() - } - } - } - } - post - { - always - { - deleteDir() } } } From 61bc48c1e132c47ee372dabd1d96722bfb2a0cb6 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Tue, 6 Feb 2024 17:06:12 +0100 Subject: [PATCH 03/16] show clone command to debug --- Util/BuildTools/BuildOSM2ODR.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Util/BuildTools/BuildOSM2ODR.sh b/Util/BuildTools/BuildOSM2ODR.sh index 61e7d449f..1477292df 100755 --- a/Util/BuildTools/BuildOSM2ODR.sh +++ b/Util/BuildTools/BuildOSM2ODR.sh @@ -79,6 +79,7 @@ if ${BUILD_OSM2ODR} ; then # [ ! -d ${OSM2ODR_BUILD_FOLDER} ] && mkdir ${OSM2ODR_BUILD_FOLDER} if ${GIT_PULL} ; then if [ ! -d ${OSM2ODR_SOURCE_FOLDER} ] ; then + echo "git clone -b ${OSM2ODR_BRANCH} ${OSM2ODR_REPO} ${OSM2ODR_SOURCE_FOLDER}" git clone -b ${OSM2ODR_BRANCH} ${OSM2ODR_REPO} ${OSM2ODR_SOURCE_FOLDER} fi cd ${OSM2ODR_SOURCE_FOLDER} From 5a256cb28bcb0010e2cbba94120508c15956ddac Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Tue, 6 Feb 2024 17:52:10 +0100 Subject: [PATCH 04/16] reduce number of stash calls --- Jenkinsfile | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0021960ac..4e1e0b48b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,8 +48,6 @@ pipeline { archiveArtifacts 'PythonAPI/carla/dist/*.egg' archiveArtifacts 'PythonAPI/carla/dist/*.whl' - stash includes: 'PythonAPI/carla/dist/*.egg', name: 'ubuntu_eggs' - stash includes: 'PythonAPI/carla/dist/*.whl', name: 'ubuntu_wheels' } } } @@ -88,9 +86,6 @@ pipeline always { archiveArtifacts 'Dist/*.tar.gz' - stash includes: 'Dist/CARLA*.tar.gz', name: 'ubuntu_package' - // stash includes: 'Dist/AdditionalMaps*.tar.gz', name: 'ubuntu_package2' - stash includes: 'Examples/', name: 'ubuntu_examples' } } } @@ -98,13 +93,6 @@ pipeline { steps { - unstash name: 'ubuntu_eggs' - unstash name: 'ubuntu_wheels' - unstash name: 'ubuntu_package' - // unstash name: 'ubuntu_package2' - unstash name: 'ubuntu_examples' - sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/' - // sh 'tar -xvzf Dist/AdditionalMaps*.tar.gz -C Dist/' sh 'DISPLAY= ./Dist/CarlaUE4.sh -nullrhi -RenderOffScreen --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' sh 'make smoke_tests ARGS="--xml --python-version=3.8 --target-wheel-platform=manylinux_2_27_x86_64"' sh 'make run-examples ARGS="localhost 3654"' @@ -115,7 +103,6 @@ pipeline { archiveArtifacts 'CarlaUE4.log' junit 'Build/test-results/smoke-tests-*.xml' - deleteDir() } } } @@ -178,13 +165,6 @@ pipeline ''' } } - post - { - always - { - deleteDir() - } - } } stage('TEST: ubuntu Doxygen generation') { From 17aeeff3c13c47df3d9401835cf27fe5f014ca45 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Tue, 6 Feb 2024 17:53:13 +0100 Subject: [PATCH 05/16] clean echo --- Util/BuildTools/BuildOSM2ODR.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Util/BuildTools/BuildOSM2ODR.sh b/Util/BuildTools/BuildOSM2ODR.sh index 1477292df..61e7d449f 100755 --- a/Util/BuildTools/BuildOSM2ODR.sh +++ b/Util/BuildTools/BuildOSM2ODR.sh @@ -79,7 +79,6 @@ if ${BUILD_OSM2ODR} ; then # [ ! -d ${OSM2ODR_BUILD_FOLDER} ] && mkdir ${OSM2ODR_BUILD_FOLDER} if ${GIT_PULL} ; then if [ ! -d ${OSM2ODR_SOURCE_FOLDER} ] ; then - echo "git clone -b ${OSM2ODR_BRANCH} ${OSM2ODR_REPO} ${OSM2ODR_SOURCE_FOLDER}" git clone -b ${OSM2ODR_BRANCH} ${OSM2ODR_REPO} ${OSM2ODR_SOURCE_FOLDER} fi cd ${OSM2ODR_SOURCE_FOLDER} From 1d77cec6ef0e19e2e33f16683d9b33236f7c1296 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 02:42:50 +0100 Subject: [PATCH 06/16] prevent delete dir content --- Jenkinsfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0021960ac..44f1fa7d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -115,7 +115,6 @@ pipeline { archiveArtifacts 'CarlaUE4.log' junit 'Build/test-results/smoke-tests-*.xml' - deleteDir() } } } @@ -178,13 +177,6 @@ pipeline ''' } } - post - { - always - { - deleteDir() - } - } } stage('TEST: ubuntu Doxygen generation') { From 61cc60ffe0856d640f5995b75950ef1e75c1ab23 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 13:29:18 +0100 Subject: [PATCH 07/16] fix workspace --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 44f1fa7d4..95bcc18f1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { steps { + sh 'echo "Current workspace is ${env.WORKSPACE}"' sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' } @@ -195,7 +196,7 @@ pipeline steps { - dir('${env.WORKSPACE}/doc_repo') + dir("${env.WORKSPACE}/doc_repo") { checkout scmGit( branches: [[name: '*/ruben/jenkins_migration']], From 4fdd8d73f2e7c2792c87d5568b766593d83dbee4 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 13:33:57 +0100 Subject: [PATCH 08/16] minor change --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 95bcc18f1..ce7abbfaf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { steps { - sh 'echo "Current workspace is ${env.WORKSPACE}"' + sh 'echo "Current workspace is ${WORKSPACE}"' sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' } From 0ba17bf334918661ab661f473dbb293f571bda55 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 16:45:11 +0100 Subject: [PATCH 09/16] pipeline in parallel --- Jenkinsfile | 139 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 84 insertions(+), 55 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4e1e0b48b..ec42a9946 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,55 +24,100 @@ pipeline } stages { - stage('ubuntu setup') + parallel { - steps + stage('generate libs') { - sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' - sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' - } - } - stage('ubuntu build') - { - steps - { - sh 'make LibCarla' - sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' - sh 'make CarlaUE4Editor ARGS="--chrono"' - sh 'make plugins' - sh 'make examples' - } - post - { - always + stages { - archiveArtifacts 'PythonAPI/carla/dist/*.egg' - archiveArtifacts 'PythonAPI/carla/dist/*.whl' + stage('ubuntu setup') + { + steps + { + sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' + sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' + } + } + stage('ubuntu build') + { + steps + { + sh 'make LibCarla' + sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make CarlaUE4Editor ARGS="--chrono"' + sh 'make plugins' + sh 'make examples' + } + post + { + always + { + archiveArtifacts 'PythonAPI/carla/dist/*.egg' + archiveArtifacts 'PythonAPI/carla/dist/*.whl' + } + } + } + stage('ubuntu unit tests') + { + steps + { + sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' + } + post + { + always + { + junit 'Build/test-results/*.xml' + archiveArtifacts 'profiler.csv' + } + } + } } } - } - stage('ubuntu unit tests') - { - steps + stage('Download additional resources') { - sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' - } - post - { - always + stages { - junit 'Build/test-results/*.xml' - archiveArtifacts 'profiler.csv' + stage('TEST: Checkout Doxygen repo') + { + when { branch "ruben/jenkins_migration"; } + steps + { + + dir('${env.WORKSPACE}/doc_repo') + { + checkout scmGit( + branches: [[name: '*/ruben/jenkins_migration']], + extensions: [ + cleanBeforeCheckout(), + checkoutOption(120), + localBranch("**"), + cloneOption(noTags:false, reference:'', shallow: false, timeout:120) + ], + userRemoteConfigs: [ + [ + credentialsId: 'github_token_as_pwd_2', + url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' + ] + ] + ) + } + + } + } + + stage('ubuntu retrieve content') + { + steps + { + sh './Update.sh' + } + } } } + } - stage('ubuntu retrieve content') - { - steps - { - sh './Update.sh' - } - } + stage('ubuntu package') { steps @@ -182,24 +227,8 @@ pipeline when { branch "ruben/jenkins_migration"; } steps { - dir('${env.WORKSPACE}/doc_repo') { - checkout scmGit( - branches: [[name: '*/ruben/jenkins_migration']], - extensions: [ - cleanBeforeCheckout(), - checkoutOption(120), - localBranch("**"), - cloneOption(noTags:false, reference:'', shallow: false, timeout:120) - ], - userRemoteConfigs: [ - [ - credentialsId: 'github_token_as_pwd_2', - url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' - ] - ] - ) unstash name: 'carla_docs' withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { sh ''' From 2525e355af7ef1acc466312b5f85cc9208456501 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 17:07:56 +0100 Subject: [PATCH 10/16] parrallel version --- Jenkinsfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index df6526c6f..c113e0bb9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ pipeline steps { - dir('${env.WORKSPACE}/doc_repo') + dir('doc_repo') { checkout scmGit( branches: [[name: '*/ruben/jenkins_migration']], @@ -227,14 +227,11 @@ pipeline when { branch "ruben/jenkins_migration"; } steps { - - dir("${env.WORKSPACE}/doc_repo") + dir('doc_repo') { unstash name: 'carla_docs' withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { sh ''' - pwd - ls -lh tar -xvzf carla_doc.tar.gz git add Doxygen git commit -m "Updated c++ docs" || true From ee647027fe6f5ec0661492c4ade73f0e0d10c360 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 17:16:52 +0100 Subject: [PATCH 11/16] fix jenkinsfile syntax error --- Jenkinsfile | 133 +++++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c113e0bb9..3f4d6dcd5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,98 +24,101 @@ pipeline } stages { - parallel + stage('prepare environment') { - stage('generate libs') + parallel { - stages + stage('generate libs') { - stage('ubuntu setup') + stages { - steps + stage('ubuntu setup') { - sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' - sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' - } - } - stage('ubuntu build') - { - steps - { - sh 'make LibCarla' - sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' - sh 'make CarlaUE4Editor ARGS="--chrono"' - sh 'make plugins' - sh 'make examples' - } - post - { - always + steps { - archiveArtifacts 'PythonAPI/carla/dist/*.egg' - archiveArtifacts 'PythonAPI/carla/dist/*.whl' + sh 'git update-index --skip-worktree Unreal/CarlaUE4/CarlaUE4.uproject' + sh 'make setup ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64 --chrono"' } } - } - stage('ubuntu unit tests') - { - steps + stage('ubuntu build') { - sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' - } - post - { - always + steps { - junit 'Build/test-results/*.xml' - archiveArtifacts 'profiler.csv' + sh 'make LibCarla' + sh 'make PythonAPI ARGS="--python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make CarlaUE4Editor ARGS="--chrono"' + sh 'make plugins' + sh 'make examples' + } + post + { + always + { + archiveArtifacts 'PythonAPI/carla/dist/*.egg' + archiveArtifacts 'PythonAPI/carla/dist/*.whl' + } + } + } + stage('ubuntu unit tests') + { + steps + { + sh 'make check ARGS="--all --xml --python-version=3.8,2 --target-wheel-platform=manylinux_2_27_x86_64"' + } + post + { + always + { + junit 'Build/test-results/*.xml' + archiveArtifacts 'profiler.csv' + } } } } } - } - stage('Download additional resources') - { - stages + stage('Download additional resources') { - stage('TEST: Checkout Doxygen repo') + stages { - when { branch "ruben/jenkins_migration"; } - steps + stage('TEST: Checkout Doxygen repo') { - - dir('doc_repo') + when { branch "ruben/jenkins_migration"; } + steps { - checkout scmGit( - branches: [[name: '*/ruben/jenkins_migration']], - extensions: [ - cleanBeforeCheckout(), - checkoutOption(120), - localBranch("**"), - cloneOption(noTags:false, reference:'', shallow: false, timeout:120) - ], - userRemoteConfigs: [ - [ - credentialsId: 'github_token_as_pwd_2', - url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' + + dir('doc_repo') + { + checkout scmGit( + branches: [[name: '*/ruben/jenkins_migration']], + extensions: [ + cleanBeforeCheckout(), + checkoutOption(120), + localBranch("**"), + cloneOption(noTags:false, reference:'', shallow: false, timeout:120) + ], + userRemoteConfigs: [ + [ + credentialsId: 'github_token_as_pwd_2', + url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' + ] ] - ] - ) + ) + } + } - } - } - stage('ubuntu retrieve content') - { - steps + stage('ubuntu retrieve content') { - sh './Update.sh' + steps + { + sh './Update.sh' + } } } } - } + } } stage('ubuntu package') From 4b0b3be85226627479464d4d142025cedb7845d3 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 17:32:31 +0100 Subject: [PATCH 12/16] test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3f4d6dcd5..e18f0e633 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ pipeline { stage('Building CARLA') { - parallel + //parallel { stage('ubuntu') { From 8b588452afd3608c32b62d044d909f5f8b179713 Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 17:34:12 +0100 Subject: [PATCH 13/16] syntax error --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e18f0e633..ef80078e8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline stage('Building CARLA') { //parallel - { + //{ stage('ubuntu') { agent { label "gpu" } @@ -354,7 +354,7 @@ pipeline } }*/ - } + //} } } } From a68fee0260e1ea75588e02e492bdba7557db560c Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Wed, 7 Feb 2024 17:35:48 +0100 Subject: [PATCH 14/16] syntax error --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ef80078e8..d9c1dfdb4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,8 +11,8 @@ pipeline stages { - stage('Building CARLA') - { + //stage('Building CARLA') + //{ //parallel //{ stage('ubuntu') @@ -355,6 +355,6 @@ pipeline }*/ //} - } + //} } } From 5f7330979308176621a0075f6776505e378748bf Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Thu, 8 Feb 2024 00:57:50 +0100 Subject: [PATCH 15/16] paralelize uploads --- Jenkinsfile | 247 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 146 insertions(+), 101 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d9c1dfdb4..460b3405e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,6 +56,8 @@ pipeline { archiveArtifacts 'PythonAPI/carla/dist/*.egg' archiveArtifacts 'PythonAPI/carla/dist/*.whl' + stash includes: 'PythonAPI/carla/dist/*.egg', name: 'ubuntu_eggs' + stash includes: 'PythonAPI/carla/dist/*.whl', name: 'ubuntu_wheels' } } } @@ -134,115 +136,158 @@ pipeline always { archiveArtifacts 'Dist/*.tar.gz' + stash includes: 'Dist/CARLA*.tar.gz', name: 'ubuntu_package' + stash includes: 'Examples/', name: 'ubuntu_examples' } } } - stage('ubuntu smoke tests') + + stage('Testing and documentation') { - steps + parallel { - sh 'DISPLAY= ./Dist/CarlaUE4.sh -nullrhi -RenderOffScreen --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' - sh 'make smoke_tests ARGS="--xml --python-version=3.8 --target-wheel-platform=manylinux_2_27_x86_64"' - sh 'make run-examples ARGS="localhost 3654"' - } - post - { - always + stage('Testing') { - archiveArtifacts 'CarlaUE4.log' - junit 'Build/test-results/smoke-tests-*.xml' - } - } - } - - stage('TEST: ubuntu deploy sim') - { - when { branch "ruben/jenkins_migration"; } - steps - { - sh 'git checkout .' - sh 'make deploy ARGS="--test"' - } - - } - - stage('ubuntu deploy dev') - { - when { branch "dev"; } - steps - { - sh 'git checkout .' - sh 'make deploy ARGS="--replace-latest"' - } - } - stage('ubuntu deploy master') - { - when { anyOf { branch "master"; buildingTag() } } - steps - { - sh 'git checkout .' - sh 'make deploy ARGS="--replace-latest --docker-push"' - } - } - - stage('ubuntu Doxygen generation') - { - when { anyOf { branch "master"; branch "dev"; buildingTag() } } - steps - { - sh 'make docs' - sh 'tar -czf carla_doc.tar.gz ./Doxygen' - stash includes: 'carla_doc.tar.gz', name: 'carla_docs' - } - } - - stage('ubuntu Doxygen upload') - { - when { anyOf { branch "master"; branch "dev"; buildingTag() } } - steps - { - checkout scmGit(branches: [[name: '*/master']], extensions: [checkoutOption(120), cloneOption(noTags:false, reference:'', shallow: false, timeout:120)], userRemoteConfigs: [[credentialsId: 'github_token_as_pwd_2', url: 'https://github.com/carla-simulator/carla-simulator.github.io.git']]) - unstash name: 'carla_docs' - - withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { - sh ''' - tar -xvzf carla_doc.tar.gz - git add Doxygen - git commit -m "Updated c++ docs" || true - git push - ''' - } - } - } - stage('TEST: ubuntu Doxygen generation') - { - when { branch "ruben/jenkins_migration"; } - steps - { - sh 'make docs' - sh 'tar -czf carla_doc.tar.gz ./Doxygen' - stash includes: 'carla_doc.tar.gz', name: 'carla_docs' - } - } - - stage('TEST: ubuntu Doxygen upload') - { - when { branch "ruben/jenkins_migration"; } - steps - { - dir('doc_repo') - { - unstash name: 'carla_docs' - withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { - sh ''' - tar -xvzf carla_doc.tar.gz - git add Doxygen - git commit -m "Updated c++ docs" || true - git push --set-upstream origin ruben/jenkins_migration - ''' + stages + { + stage('ubuntu smoke tests') + { + steps + { + unstash name: 'ubuntu_eggs' + unstash name: 'ubuntu_wheels' + unstash name: 'ubuntu_package' + unstash name: 'ubuntu_examples' + sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/' + sh 'DISPLAY= ./Dist/CarlaUE4.sh -nullrhi -RenderOffScreen --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' + sh 'make smoke_tests ARGS="--xml --python-version=3.8 --target-wheel-platform=manylinux_2_27_x86_64"' + sh 'make run-examples ARGS="localhost 3654"' + } + post + { + always + { + archiveArtifacts 'CarlaUE4.log' + junit 'Build/test-results/smoke-tests-*.xml' + } + } + } + } + } + stage('Generate documentation') + { + stages + { + stage('ubuntu Doxygen generation') + { + when { anyOf { branch "master"; branch "dev"; buildingTag() } } + steps + { + sh 'make docs' + sh 'tar -czf carla_doc.tar.gz ./Doxygen' + stash includes: 'carla_doc.tar.gz', name: 'carla_docs' + } + } + + stage('TEST: ubuntu Doxygen generation') + { + when { branch "ruben/jenkins_migration"; } + steps + { + sh 'make docs' + sh 'tar -czf carla_doc.tar.gz ./Doxygen' + stash includes: 'carla_doc.tar.gz', name: 'carla_docs' + } + } + } + } + } + } + + stage('Deployment and documentation publishing') + { + parallel + { + stage('Release Deployment') + { + stages + { + stage('TEST: ubuntu deploy sim') + { + when { branch "ruben/jenkins_migration"; } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--test"' + } + + } + + stage('ubuntu deploy dev') + { + when { branch "dev"; } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--replace-latest"' + } + } + stage('ubuntu deploy master') + { + when { anyOf { branch "master"; buildingTag() } } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--replace-latest --docker-push"' + } + } + } + } + stage('Publish documentation') + { + stages + { + stage('ubuntu Doxygen upload') + { + when { anyOf { branch "master"; branch "dev"; buildingTag() } } + steps + { + checkout scmGit(branches: [[name: '*/master']], extensions: [checkoutOption(120), cloneOption(noTags:false, reference:'', shallow: false, timeout:120)], userRemoteConfigs: [[credentialsId: 'github_token_as_pwd_2', url: 'https://github.com/carla-simulator/carla-simulator.github.io.git']]) + unstash name: 'carla_docs' + + withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { + sh ''' + tar -xvzf carla_doc.tar.gz + git add Doxygen + git commit -m "Updated c++ docs" || true + git push + ''' + } + } + } + stage('TEST: ubuntu Doxygen upload') + { + when { branch "ruben/jenkins_migration"; } + steps + { + dir('doc_repo') + { + unstash name: 'carla_docs' + withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { + sh ''' + tar -xvzf carla_doc.tar.gz + git add Doxygen + git commit -m "Updated c++ docs" || true + git push --set-upstream origin ruben/jenkins_migration + ''' + } + } + + } + } + } } - } } } From 41a2655c24a85bb1b319ebf4f739c133eeb6ecec Mon Sep 17 00:00:00 2001 From: Ruben Abad Date: Thu, 8 Feb 2024 12:29:11 +0100 Subject: [PATCH 16/16] propagate changes to dev and master cases --- Jenkinsfile | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 460b3405e..1ba592edd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,6 +109,33 @@ pipeline } } + stage('Checkout Doxygen repo') + { + when { anyOf { branch "master"; branch "dev"; buildingTag() } } + steps + { + + dir('doc_repo') + { + checkout scmGit( + branches: [[name: '*/master']], + extensions: [ + cleanBeforeCheckout(), + checkoutOption(120), + localBranch("**"), + cloneOption(noTags:false, reference:'', shallow: false, timeout:120) + ], + userRemoteConfigs: [ + [ + credentialsId: 'github_token_as_pwd_2', + url: 'https://github.com/carla-simulator/carla-simulator.github.io.git' + ] + ] + ) + } + + } + } stage('ubuntu retrieve content') { @@ -252,17 +279,19 @@ pipeline when { anyOf { branch "master"; branch "dev"; buildingTag() } } steps { - checkout scmGit(branches: [[name: '*/master']], extensions: [checkoutOption(120), cloneOption(noTags:false, reference:'', shallow: false, timeout:120)], userRemoteConfigs: [[credentialsId: 'github_token_as_pwd_2', url: 'https://github.com/carla-simulator/carla-simulator.github.io.git']]) - unstash name: 'carla_docs' - - withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { - sh ''' - tar -xvzf carla_doc.tar.gz - git add Doxygen - git commit -m "Updated c++ docs" || true - git push - ''' + dir('doc_repo') + { + unstash name: 'carla_docs' + withCredentials([gitUsernamePassword(credentialsId: 'github_token_as_pwd_2', gitToolName: 'git-tool')]) { + sh ''' + tar -xvzf carla_doc.tar.gz + git add Doxygen + git commit -m "Updated c++ docs" || true + git push --set-upstream origin ruben/jenkins_migration + ''' + } } + } } stage('TEST: ubuntu Doxygen upload')