From 90284dc91e426273bea6e577a71ee15abd057209 Mon Sep 17 00:00:00 2001 From: Marc Garcia Puig Date: Thu, 5 Mar 2020 13:16:18 +0100 Subject: [PATCH] Modifying pipeline for use AWS cloud (#2504) Build success Added package Remove delete workspace for testing Clean workspace after build Add deploy for testing Relocation of Unreal Engine Using absolute path for UnrealEngine Execute shell Test without remove at end Using bash instead of shell New jenkins file for testing nodes new tag modified modify 'test' stage all in GPU tags all in 'build' and 'gpu' Adjust some scripts to run smooth on GPU slave force deploy always Disable docker push Remove all on slaves at end add some locks add some parallel stages Fix windows package, now it copies all required files Add 7zip option to compress package for windows Add 'make CarlaUE4Editor' to windows Fix the program files path for windows 64 More jobs on parallel Add Update.bat to windows pipeline Fixing make CarlaUE4Editor Commenting stages in windows Enable again download of content and fix artifact Build only Ubuntu Enable deploy of Docker Make packaging of additional maps Enabling windows build again Adding Deploy for windows Change URL to the new S3 buckets Changed the Jenkins public IP Fixes from Codacy Disable Windows build Add more time to smoke tests for connecting Add 'run' option of examples without compilation Enabled delete all after job Adds a lock for the packaging Removing all locks Update the Doxygen step in Ubuntu Fix nodes to use --- Docs/build_update.md | 2 +- Docs/tuto_D_contribute_assets.md | 2 +- Examples/CppClient/Makefile | 6 +- Jenkinsfile | 333 +++++++++++++++++++-------- PythonAPI/test/smoke/__init__.py | 2 +- README.md | 2 +- Update.bat | 76 ++++++ Update.sh | 2 +- Util/BuildTools/BuildCarlaUE4.bat | 39 +++- Util/BuildTools/Check.sh | 2 + Util/BuildTools/Deploy.bat | 121 ++++++++++ Util/BuildTools/Deploy.sh | 2 +- Util/BuildTools/Linux.mk | 2 +- Util/BuildTools/Package.bat | 198 ++++++++++++++-- Util/BuildTools/Setup.bat | 2 +- Util/BuildTools/Windows.mk | 6 + Util/ContentVersions.txt | 2 +- Util/DockerUtils/dist/build.sh | 0 Util/InstallersWin/install_boost.bat | 8 +- 19 files changed, 673 insertions(+), 134 deletions(-) create mode 100644 Update.bat create mode 100644 Util/BuildTools/Deploy.bat mode change 100755 => 100644 Util/DockerUtils/dist/build.sh diff --git a/Docs/build_update.md b/Docs/build_update.md index 93ac576f4..433f76477 100644 --- a/Docs/build_update.md +++ b/Docs/build_update.md @@ -32,7 +32,7 @@ Binary releases are prepackaged and thus, tied to a specific version of CARLA. I

- + Get the nightly build

diff --git a/Docs/tuto_D_contribute_assets.md b/Docs/tuto_D_contribute_assets.md index 8b9a4fdc2..9b3f52fab 100644 --- a/Docs/tuto_D_contribute_assets.md +++ b/Docs/tuto_D_contribute_assets.md @@ -21,5 +21,5 @@ update this link to the latest version. [contentrepolink]: https://bitbucket.org/carla-simulator/carla-content [carlarepolink]: https://github.com/carla-simulator/carla -[jenkinslink]: http://158.109.9.218:8080/blue/organizations/jenkins/carla-content/activity +[jenkinslink]: http://35.181.165.160:8080/blue/organizations/jenkins/carla-content/activity [cvlink]: https://github.com/carla-simulator/carla/edit/master/Util/ContentVersions.txt diff --git a/Examples/CppClient/Makefile b/Examples/CppClient/Makefile index 1223e374d..05713a94c 100644 --- a/Examples/CppClient/Makefile +++ b/Examples/CppClient/Makefile @@ -22,9 +22,13 @@ run: build $(call log,Running C++ Client...) @$(BINDIR)/cpp_client $(ARGS) +run.only: + $(call log,Running C++ Client...) + @$(BINDIR)/cpp_client $(ARGS) + build: $(BINDIR)/cpp_client -$(BINDIR)/cpp_client: build_libcarla +$(BINDIR)/cpp_client: | build_libcarla $(call log,Compiling C++ Client...) @mkdir -p $(BINDIR) @$(CXX) $(CXXFLAGS) -I$(INSTALLDIR)/include -isystem $(INSTALLDIR)/include/system -L$(INSTALLDIR)/lib \ diff --git a/Jenkinsfile b/Jenkinsfile index ef4440377..7b329ad17 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,106 +1,251 @@ -pipeline { - agent any +pipeline +{ + agent none - environment { - UE4_ROOT = '/var/lib/jenkins/UnrealEngine_4.22' - } - - options { + options + { buildDiscarder(logRotator(numToKeepStr: '3', artifactNumToKeepStr: '3')) } - stages { - - stage('Setup') { - steps { - sh 'make setup' - } - } - - stage('Build') { - steps { - sh 'make LibCarla' - sh 'make PythonAPI' - sh 'make CarlaUE4Editor' - sh 'make examples' - } - post { - always { - archiveArtifacts 'PythonAPI/carla/dist/*.egg' + stages + { + stage('windows and ubuntu in parallel') + { + parallel + { + stage('ubuntu') + { + agent { label 'ubuntu && build' } + environment + { + UE4_ROOT = '/home/jenkins/UnrealEngine_4.22' + } + stages + { + stage('ubuntu setup') + { + steps + { + sh 'make setup' + } + } + stage('ubuntu build') + { + steps + { + sh 'make LibCarla' + sh 'make PythonAPI' + sh 'make CarlaUE4Editor' + sh 'make examples' + } + post + { + always + { + archiveArtifacts 'PythonAPI/carla/dist/*.egg' + stash includes: 'PythonAPI/carla/dist/*.egg', name: 'ubuntu_eggs' + } + } + } + stage('ubuntu unit tests') + { + steps + { + sh 'make check ARGS="--all --xml"' + } + post + { + always + { + junit 'Build/test-results/*.xml' + archiveArtifacts 'profiler.csv' + } + } + } + stage('ubuntu retrieve content') + { + steps + { + sh './Update.sh' + } + } + stage('ubuntu package') + { + steps + { + sh 'make package' + sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate"' + sh 'make examples ARGS="localhost 3654"' + } + post { + always { + archiveArtifacts 'Dist/*.tar.gz' + stash includes: 'Dist/CARLA*.tar.gz', name: 'ubuntu_package' + stash includes: 'Examples/', name: 'ubuntu_examples' + } + } + } + stage('ubuntu smoke tests') + { + agent { label 'ubuntu && gpu' } + steps + { + unstash name: 'ubuntu_eggs' + unstash name: 'ubuntu_package' + unstash name: 'ubuntu_examples' + sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/' + sh 'DISPLAY= ./Dist/CarlaUE4.sh -opengl --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' + sh 'make smoke_tests ARGS="--xml"' + sh 'make run-examples ARGS="localhost 3654"' + } + post + { + always + { + archiveArtifacts 'CarlaUE4.log' + junit 'Build/test-results/smoke-tests-*.xml' + } + } + } + stage('ubuntu deploy') + { + when { anyOf { branch "master"; buildingTag() } } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--replace-latest --docker-push"' + } + } + stage('ubuntu Doxygen') + { + when { anyOf { branch "master"; buildingTag() } } + steps + { + sh 'make docs' + sh 'rm -rf ~/carla-simulator.github.io/Doxygen' + sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/' + sh ''' + cd ~/carla-simulator.github.io + git pull + git add Doxygen + git commit -m "Updated c++ docs" || true + git push + ''' + } + } + } } - } - } - - stage('Unit Tests') { - steps { - sh 'make check ARGS="--all --xml"' - } - post { - always { - junit 'Build/test-results/*.xml' - archiveArtifacts 'profiler.csv' - } - } - } - - stage('Retrieve Content') { - steps { - sh './Update.sh' - } - } - - stage('Package') { - steps { - sh 'make package' - sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate"' - } - post { - always { - archiveArtifacts 'Dist/*.tar.gz' - } - } - } - - stage('Smoke Tests') { - steps { - sh 'DISPLAY= ./Dist/*/LinuxNoEditor/CarlaUE4.sh -opengl --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' - sh 'make smoke_tests ARGS="--xml"' - sh 'make run-examples ARGS="localhost 3654"' - } - post { - always { - archiveArtifacts 'CarlaUE4.log' - junit 'Build/test-results/smoke-tests-*.xml' - } - } - } - - stage('Deploy') { - when { anyOf { branch "master"; buildingTag() } } - steps { - sh 'git checkout .' - sh 'make deploy ARGS="--replace-latest --docker-push"' - } - } - - stage('Doxygen') { - when { anyOf { branch "master"; buildingTag() } } - steps { - sh 'make docs' - sh 'rm -rf ~/carla-simulator.github.io/Doxygen' - sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/' - sh 'cd ~/carla-simulator.github.io && \ - git pull && \ - git add Doxygen && \ - git commit -m "Updated c++ docs" || true && \ - git push' + // stage('windows') + // { + // environment + // { + // UE4_ROOT = 'C:\\Program Files\\Epic Games\\UE_4.22' + // } + // stages + // { + // stage('windows setup') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make setup + // """ + // } + // } + // stage('windows build') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make LibCarla + // make PythonAPI + // make CarlaUE4Editor + // // make examples + // // """ + // } + // post + // { + // always + // { + // archiveArtifacts 'PythonAPI/carla/dist/*.egg' + // stash includes: 'PythonAPI/carla/dist/*.egg', name: 'windows_eggs' + // } + // } + // } + // // stage('windows unit tests') + // // { + // // agent { label 'windows && build' } + // // steps { bat 'rem Not Implemented'} + // // } + // stage('windows retrieve content') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // call Update.bat + // """ + // } + // } + // stage('windows package') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make package + // make package ARGS="--packages=AdditionalMaps --clean-intermediate" + // """ + // // make examples ARGS="localhost 3654" + // } + // post { + // always { + // archiveArtifacts 'Build/UE4Carla/*.zip' + // // stash includes: 'Build/UE4Carla/CARLA*.zip', name: 'windows_package' + // // stash includes: 'Examples/', name: 'windows_examples' + // } + // } + // } + // // stage('windows smoke test') + // // { + // // agent { label 'windows && build' } + // // steps { bat 'rem Not Implemented'} + // // } + // stage('windows deploy') + // { + // agent { label 'windows && build' } + // when { anyOf { branch "master"; buildingTag() } } + // steps { + // bat """ + // call ../setEnv64.bat + // make deploy ARGS="--replace-latest" + // """ + // } + // } + // } + // } } } } - post { - always { - deleteDir() + post + { + success + { + node('build') + { + deleteDir() + } + node('gpu') + { + deleteDir() + } } } } diff --git a/PythonAPI/test/smoke/__init__.py b/PythonAPI/test/smoke/__init__.py index 1cf227b56..5b85757f0 100644 --- a/PythonAPI/test/smoke/__init__.py +++ b/PythonAPI/test/smoke/__init__.py @@ -26,7 +26,7 @@ TESTING_ADDRESS = ('localhost', 3654) class SmokeTest(unittest.TestCase): def setUp(self): self.client = carla.Client(*TESTING_ADDRESS) - self.client.set_timeout(10.0) + self.client.set_timeout(60.0) def tearDown(self): self.client = None diff --git a/README.md b/README.md index 8cdf1ec14..048457827 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ If you want to benchmark your model in the same conditions as in our CoRL’17 paper, check out [Benchmarking](https://github.com/carla-simulator/driving-benchmarks). -[**Get CARLA overnight build**](http://carla-assets-internal.s3.amazonaws.com/Releases/Linux/Dev/CARLA_Latest.tar.gz) +[**Get CARLA overnight build**](http://carla-releases.s3.amazonaws.com/Linux/Dev/CARLA_Latest.tar.gz) ## CARLA Ecosystem Repositories associated to the CARLA simulation platform: diff --git a/Update.bat b/Update.bat new file mode 100644 index 000000000..24e80732a --- /dev/null +++ b/Update.bat @@ -0,0 +1,76 @@ +@echo off + +rem ============================================================================ +rem -- Set up environment ------------------------------------------------------ +rem ============================================================================ + +set SCRIPT_DIR=%~dp0 +set CONTENT_FOLDER=%SCRIPT_DIR%Unreal/CarlaUE4/Content/Carla +set VERSION_FILE=%CONTENT_FOLDER%/.version +set CONTENT_VERSIONS=%SCRIPT_DIR%/Util/ContentVersions.txt + +rem ============================================================================ +rem -- Get the last version to download ---------------------------------------- +rem ============================================================================ + +if not exist "%CONTENT_FOLDER%" mkdir "%CONTENT_FOLDER%" + +for /F "delims=" %%a in (%CONTENT_VERSIONS%) do ( + set "lastLine=%%a" +) +set CONTENT_ID=%lastLine:~-16,16% +set CONTENT_LINK=http://carla-assets.s3.amazonaws.com/%CONTENT_ID%.tar.gz +if "%CONTENT_ID:~0,2%"=="20" ( + set CONTENT_FILE=%CONTENT_FOLDER%/%CONTENT_ID%.tar.gz + set CONTENT_FILE_TAR=%CONTENT_FOLDER%/%CONTENT_ID%.tar + echo %CONTENT_ID% + echo %CONTENT_LINK% +) else ( + echo Error reading the latest version from ContentVersions.txt, check last line of file %CONTENT_VERSIONS%' + goto error_download +) + +rem ============================================================================ +rem -- Download the content ---------------------------------------------------- +rem ============================================================================ + +echo Downloading "%CONTENT_LINK%"... +powershell -Command "Start-BitsTransfer -Source '%CONTENT_LINK%' -Destination '%CONTENT_FOLDER%'" +if %errorlevel% neq 0 goto error_download + +echo %FILE_N% Extracting content from "%CONTENT_FILE%", this can take a while... +if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" x "%CONTENT_FILE%" -o"%CONTENT_FOLDER%" -y + if %errorlevel% neq 0 goto error_download + echo Deleting %CONTENT_FILE:/=\% + del %CONTENT_FILE:/=\% + "%ProgramW6432%/7-Zip/7z.exe" x "%CONTENT_FILE_TAR%" -o"%CONTENT_FOLDER%" -y + if %errorlevel% neq 0 goto error_download + echo Deleting %CONTENT_FILE_TAR:/=\% + del %CONTENT_FILE_TAR:/=\% +) else ( + powershell -Command "Expand-Archive '%CONTENT_FILE%' -DestinationPath '%CONTENT_FOLDER%'" + if %errorlevel% neq 0 goto error_download + del %CONTENT_FILE% +) + +goto success + +:success + echo. + echo %FILE_N% Content has been successfully installed in "%CONTENT_FOLDER%"! + goto good_exit + +:error_download + goto bad_exit + +:good_exit + echo %FILE_N% Exiting... + endlocal + exit /b 0 + +:bad_exit + if exist "%CONTENT_FILE%" rd /s /q "%CONTENT_FOLDER%" + echo %FILE_N% Exiting with error... + endlocal + exit /b %errorlevel% diff --git a/Update.sh b/Update.sh index aa6baa0d7..7718c620b 100755 --- a/Update.sh +++ b/Update.sh @@ -47,7 +47,7 @@ pushd "$SCRIPT_DIR" >/dev/null CONTENT_FOLDER=$SCRIPT_DIR/Unreal/CarlaUE4/Content/Carla CONTENT_ID=$(tac $SCRIPT_DIR/Util/ContentVersions.txt | egrep -m 1 . | rev | cut -d' ' -f1 | rev) -CONTENT_LINK=http://carla-assets-internal.s3.amazonaws.com/Content/${CONTENT_ID}.tar.gz +CONTENT_LINK=http://carla-assets.s3.amazonaws.com/${CONTENT_ID}.tar.gz VERSION_FILE=${CONTENT_FOLDER}/.version diff --git a/Util/BuildTools/BuildCarlaUE4.bat b/Util/BuildTools/BuildCarlaUE4.bat index 83f67eda5..00a8cbf19 100644 --- a/Util/BuildTools/BuildCarlaUE4.bat +++ b/Util/BuildTools/BuildCarlaUE4.bat @@ -14,11 +14,16 @@ rem ============================================================================ rem -- Parse arguments --------------------------------------------------------- rem ============================================================================ +set BUILD_UE4_EDITOR=false set LAUNCH_UE4_EDITOR=false set REMOVE_INTERMEDIATE=false :arg-parse +echo %1 if not "%1"=="" ( + if "%1"=="--build" ( + set BUILD_UE4_EDITOR=true + ) if "%1"=="--launch" ( set LAUNCH_UE4_EDITOR=true ) @@ -37,9 +42,11 @@ if not "%1"=="" ( if %REMOVE_INTERMEDIATE% == false ( if %LAUNCH_UE4_EDITOR% == false ( - echo Nothing selected to be done. - echo %USAGE_STRING% - goto eof + if %BUILD_UE4_EDITOR% == false ( + echo Nothing selected to be done. + echo %USAGE_STRING% + goto eof + ) ) ) @@ -78,6 +85,30 @@ if %REMOVE_INTERMEDIATE% == true ( ) ) +rem Build Carla Editor +rem +if %BUILD_UE4_EDITOR% == true ( + echo %FILE_N% Building Unreal Editor... + + call "%UE4_ROOT%\Engine\Build\BatchFiles\Build.bat"^ + CarlaUE4Editor^ + Win64^ + Development^ + -WaitMutex^ + -FromMsBuild^ + "%ROOT_PATH%Unreal/CarlaUE4/CarlaUE4.uproject" + if errorlevel 1 goto bad_exit + + call "%UE4_ROOT%\Engine\Build\BatchFiles\Build.bat"^ + CarlaUE4^ + Win64^ + Development^ + -WaitMutex^ + -FromMsBuild^ + "%ROOT_PATH%Unreal/CarlaUE4/CarlaUE4.uproject" + if errorlevel 1 goto bad_exit +) + rem Launch Carla Editor rem if %LAUNCH_UE4_EDITOR% == true ( @@ -94,7 +125,7 @@ rem ============================================================================ :help echo Build LibCarla. - echo "Usage: %FILE_N% [-h^|--help] [--launch] [--clean]" + echo "Usage: %FILE_N% [-h^|--help] [--build] [--launch] [--clean]" goto good_exit :error_build diff --git a/Util/BuildTools/Check.sh b/Util/BuildTools/Check.sh index 7435c51af..7e7e30d67 100755 --- a/Util/BuildTools/Check.sh +++ b/Util/BuildTools/Check.sh @@ -54,6 +54,8 @@ while true; do shift ;; --xml ) XML_OUTPUT=true; + # Create the folder for the test-results + mkdir -p "${CARLA_TEST_RESULTS_FOLDER}" shift ;; --gtest_args ) GTEST_ARGS="$2"; diff --git a/Util/BuildTools/Deploy.bat b/Util/BuildTools/Deploy.bat new file mode 100644 index 000000000..e73076477 --- /dev/null +++ b/Util/BuildTools/Deploy.bat @@ -0,0 +1,121 @@ +@echo off +setlocal enabledelayedexpansion + +rem ============================================================================== +rem -- Set up environment -------------------------------------------------------- +rem ============================================================================== + +set REPLACE_LATEST=false +set AWS_COPY=aws s3 cp +set UPLOAD_MAPS=true + +rem ============================================================================== +rem -- Parse arguments ----------------------------------------------------------- +rem ============================================================================== + +set DOC_STRING=Upload latest build to S3 + +set USAGE_STRING="Usage: $0 [-h|--help] [--replace-latest] [--dry-run]" + +:arg-parse +if not "%1"=="" ( + if "%1"=="--replace-latest" ( + set REPLACE_LATEST=true + ) + + if "%1"=="--dry-run" ( + echo %AWS_COPY% + ) + + if "%1"=="--help" ( + echo %DOC_STRING% + echo %USAGE_STRING% + GOTO :eof + ) + + shift + goto :arg-parse +) + +rem Get repository version +for /f %%i in ('git describe --tags --dirty --always') do set REPOSITORY_TAG=%%i +if not defined REPOSITORY_TAG goto error_carla_version + +rem Last package data +set CARLA_DIST_FOLDER=%~dp0%\Build\UE4Carla +set LATEST_PACKAGE=CARLA_%REPOSITORY_TAG%.zip +set LATEST_PACKAGE_PATH=%CARLA_DIST_FOLDER%\%LATEST_PACKAGE% + +set S3_PREFIX=s3://carla-releases/Windows + +set LATEST_DEPLOY_URI=!S3_PREFIX!/Dev/CARLA_Latest.zip + +rem Check for TAG version +echo %REPOSITORY_TAG% | findstr /R /C:"^[0-9]*\.[0-9]*\.[0-9]*.$" 1>nul +if %errorlevel% == 0 ( + echo Detected tag %REPOSITORY_TAG% + set DEPLOY_NAME=CARLA_%REPOSITORY_TAG%.zip +) else ( + set S3_PREFIX=!S3_PREFIX!/Dev + git log --pretty=format:%%cd_%%h --date=format:%%Y%%m%%d -n 1 > tempo1234 + set /p DEPLOY_NAME= < tempo1234 + del tempo1234 + set DEPLOY_NAME=!DEPLOY_NAME!.zip +) +echo Version detected: %REPOSITORY_TAG% +echo Using package %LATEST_PACKAGE% as %DEPLOY_NAME% + +if not exist "%LATEST_PACKAGE_PATH%" ( + echo Latest package not found, please run 'make package' + goto :bad_exit +) + +rem ============================================================================== +rem -- Upload -------------------------------------------------------------------- +rem ============================================================================== + +set DEPLOY_URI=!S3_PREFIX!/%DEPLOY_NAME% +REM %AWS_COPY% %LATEST_PACKAGE_PATH% %DEPLOY_URI% +echo Latest build uploaded to %DEPLOY_URI% + +if %REPLACE_LATEST%==true ( + REM %AWS_COPY% %DEPLOY_URI% %LATEST_DEPLOY_URI% + echo Latest build uploaded to %LATEST_DEPLOY_URI% +) + +if %UPLOAD_MAPS%==true ( + + pushd "%CARLA_DIST_FOLDER%" + + for /f %%i in ('dir *_%REPOSITORY_TAG%.zip /b') do ( + if not %%i == %LATEST_PACKAGE% ( + set DEPLOY_MAP_URI=!S3_PREFIX!/%%i + %AWS_COPY% %%i !DEPLOY_MAP_URI! + echo %%i uploaded to !DEPLOY_MAP_URI! + ) + ) + popd +) + +rem ============================================================================== +rem -- ...and we are done -------------------------------------------------------- +rem ============================================================================== + +echo Success! + +:success + echo. + goto good_exit + +:error_carla_version + echo. + echo %FILE_N% [ERROR] Carla Version is not set + goto bad_exit + +:good_exit + endlocal + exit /b 0 + +:bad_exit + endlocal + exit /b 1 diff --git a/Util/BuildTools/Deploy.sh b/Util/BuildTools/Deploy.sh index f10aca18f..bf8f7caa8 100755 --- a/Util/BuildTools/Deploy.sh +++ b/Util/BuildTools/Deploy.sh @@ -55,7 +55,7 @@ REPOSITORY_TAG=$(get_git_repository_version) LATEST_PACKAGE=CARLA_${REPOSITORY_TAG}.tar.gz LATEST_PACKAGE_PATH=${CARLA_DIST_FOLDER}/${LATEST_PACKAGE} -S3_PREFIX=s3://carla-assets-internal/Releases/Linux +S3_PREFIX=s3://carla-releases/Linux LATEST_DEPLOY_URI=${S3_PREFIX}/Dev/CARLA_Latest.tar.gz diff --git a/Util/BuildTools/Linux.mk b/Util/BuildTools/Linux.mk index a593e8b39..ddd1ae563 100644 --- a/Util/BuildTools/Linux.mk +++ b/Util/BuildTools/Linux.mk @@ -70,7 +70,7 @@ examples: @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} build; done run-examples: - @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} run; done + @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} run.only; done CarlaUE4Editor: LibCarla.server.release @${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.sh --build diff --git a/Util/BuildTools/Package.bat b/Util/BuildTools/Package.bat index 5f8fa37b3..ca112c2ae 100644 --- a/Util/BuildTools/Package.bat +++ b/Util/BuildTools/Package.bat @@ -25,6 +25,7 @@ set DO_TARBALL=true set DO_CLEAN=false set UE_VERSION=4.22 +set PACKAGES=Carla :arg-parse if not "%1"=="" ( @@ -45,7 +46,10 @@ if not "%1"=="" ( if "%1"=="--ue-version" ( set UE_VERSION=%~2 - shift + ) + + if "%1"=="--packages" ( + set PACKAGES=%~2 ) if "%1"=="-h" ( @@ -78,10 +82,10 @@ for /f %%i in ('git describe --tags --dirty --always') do set CARLA_VERSION=%%i if not defined CARLA_VERSION goto error_carla_version set BUILD_FOLDER=%INSTALLATION_DIR%UE4Carla/%CARLA_VERSION%/ -if not exist "%BUILD_FOLDER%" mkdir "%BUILD_FOLDER%" +if not exist "!BUILD_FOLDER!" mkdir "!BUILD_FOLDER!" -set DESTINATION_ZIP="%BUILD_FOLDER%../CARLA_%CARLA_VERSION%.zip" -set SOURCE=%BUILD_FOLDER%WindowsNoEditor/ +set DESTINATION_ZIP=%INSTALLATION_DIR%UE4Carla/CARLA_%CARLA_VERSION%.zip +set SOURCE=!BUILD_FOLDER!WindowsNoEditor/ rem ============================================================================ rem -- Create Carla package ---------------------------------------------------- @@ -120,7 +124,7 @@ if %DO_PACKAGE%==true ( -stage^ -build^ -archive^ - -archivedirectory="%BUILD_FOLDER%"^ + -archivedirectory="!BUILD_FOLDER!"^ -package^ -clientconfig=Development @@ -137,14 +141,21 @@ if %DO_COPY_FILES%==true ( set XCOPY_FROM=%ROOT_PATH:/=\% set XCOPY_TO=%SOURCE:/=\% - echo f | xcopy /y "!XCOPY_FROM!LICENSE" "!XCOPY_TO!LICENSE" - echo f | xcopy /y "!XCOPY_FROM!CHANGELOG.md" "!XCOPY_TO!CHANGELOG" - echo f | xcopy /y "!XCOPY_FROM!Docs\release_readme.md" "!XCOPY_TO!README" - echo f | xcopy /y "!XCOPY_FROM!Util\Docker\Release.Dockerfile" "!XCOPY_TO!Dockerfile" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\dist\*.egg" "!XCOPY_TO!PythonAPI\" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\example.py" "!XCOPY_TO!example.py" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\manual_control.py" "!XCOPY_TO!manual_control.py" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\vehicle_gallery.py" "!XCOPY_TO!vehicle_gallery.py" + echo f | xcopy /y "!XCOPY_FROM!LICENSE" "!XCOPY_TO!LICENSE" + echo f | xcopy /y "!XCOPY_FROM!CHANGELOG.md" "!XCOPY_TO!CHANGELOG" + echo f | xcopy /y "!XCOPY_FROM!Docs\release_readme.md" "!XCOPY_TO!README" + echo f | xcopy /y "!XCOPY_FROM!Util\Docker\Release.Dockerfile" "!XCOPY_TO!Dockerfile" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\dist\*.egg" "!XCOPY_TO!PythonAPI\carla\dist\" + echo d | xcopy /y /s "!XCOPY_FROM!PythonAPI\carla\agents" "!XCOPY_TO!PythonAPI\carla\agents" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\scene_layout.py" "!XCOPY_TO!PythonAPI\carla\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\requirements.txt" "!XCOPY_TO!PythonAPI\carla\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\examples\*.py" "!XCOPY_TO!PythonAPI\examples\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\examples\requirements.txt" "!XCOPY_TO!PythonAPI\examples\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\util\*.py" "!XCOPY_TO!PythonAPI\util\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\util\requirements.txt" "!XCOPY_TO!PythonAPI\util\" + echo f | xcopy /y "!XCOPY_FROM!Unreal\CarlaUE4\Content\Carla\HDMaps\*.pcd" "!XCOPY_TO!HDMaps\" + echo f | xcopy /y "!XCOPY_FROM!Unreal\CarlaUE4\Content\Carla\HDMaps\Readme.md" "!XCOPY_TO!HDMaps\README" + ) rem ============================================================================== @@ -152,22 +163,25 @@ rem -- Zip the project --------------------------------------------------------- rem ============================================================================== if %DO_TARBALL%==true ( - echo "%FILE_N% Building package..." - - set DST_ZIP=%DESTINATION_ZIP:/=\% set SRC_PATH=%SOURCE:/=\% + echo %FILE_N% Building package... + if exist "!SRC_PATH!Manifest_NonUFSFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_NonUFSFiles_Win64.txt" if exist "!SRC_PATH!Manifest_DebugFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_DebugFiles_Win64.txt" if exist "!SRC_PATH!Manifest_UFSFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_UFSFiles_Win64.txt" - if exist "!SRC_PATH!CarlaUE4/Saved" rmdir /S /Q "!SRC_PATH!CarlaUE4/Saved" if exist "!SRC_PATH!Engine/Saved" rmdir /S /Q "!SRC_PATH!Engine/Saved" - pushd "!SRC_PATH!" - rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 - powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" - popd + set DST_ZIP=%DESTINATION_ZIP:/=\% + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" a "!DST_ZIP!" "!SRC_PATH!" -tzip -mmt -mx5 + ) else ( + pushd "!SRC_PATH!" + rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 + powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" + popd + ) ) rem ============================================================================== @@ -176,11 +190,147 @@ rem ============================================================================ if %DO_CLEAN%==true ( echo %FILE_N% Removing intermediate build. - - rmdir /S /Q "%BUILD_FOLDER%" + rmdir /S /Q "!BUILD_FOLDER!" goto :eof ) +rem ============================================================================== +rem -- Cook other packages ------------------------------------------------------- +rem ============================================================================== + +rem Set some file locations +set CARLAUE4_ROOT_FOLDER=%ROOT_PATH%Unreal/CarlaUE4 +set PACKAGE_PATH_FILE=%CARLAUE4_ROOT_FOLDER%/Content/PackagePath.txt +set MAP_LIST_FILE=%CARLAUE4_ROOT_FOLDER%/Content/MapPaths.txt + +rem through all maps to cook (parameter) +for %%i in (%PACKAGES%) do ( + + set PACKAGE_NAME=%%i + + if not !PACKAGE_NAME! == Carla ( + echo Preparing environment for cooking '!PACKAGE_NAME!'. + + set BUILD_FOLDER=%INSTALLATION_DIR%UE4Carla/!PACKAGE_NAME!_%CARLA_VERSION%\ + set PACKAGE_PATH=%CARLAUE4_ROOT_FOLDER%/Content/!PACKAGE_NAME! + + if not exist "!BUILD_FOLDER!" mkdir "!BUILD_FOLDER!" + + echo Cooking package '!PACKAGE_NAME!'... + + pushd "%CARLAUE4_ROOT_FOLDER%" + + REM # Prepare cooking of package + call "%UE4_ROOT%/Engine/Binaries/Win64/UE4Editor.exe "^ + "%CARLAUE4_ROOT_FOLDER%/CarlaUE4.uproject"^ + -run=PrepareAssetsForCooking^ + -PackageName=!PACKAGE_NAME!^ + -OnlyPrepareMaps=false + + set /p PACKAGE_FILE=<%PACKAGE_PATH_FILE% + set /p MAPS_TO_COOK=<%MAP_LIST_FILE% + + REM # Cook maps + call "%UE4_ROOT%/Engine/Binaries/Win64/UE4Editor.exe "^ + "%CARLAUE4_ROOT_FOLDER%/CarlaUE4.uproject"^ + -run=cook^ + -map="!MAPS_TO_COOK!"^ + -cooksinglepackage^ + -targetplatform="WindowsNoEditor"^ + -OutputDir="!BUILD_FOLDER!" + + REM remove the props folder if exist + set PROPS_MAP_FOLDER="%PACKAGE_PATH%/Maps/PropsMap" + if exist "%PROPS_MAP_FOLDER%" ( + rmdir /S /Q "%PROPS_MAP_FOLDER%" + ) + + popd + + echo Copying files to '!PACKAGE_NAME!'... + + pushd "!BUILD_FOLDER!" + + set SUBST_PATH=!BUILD_FOLDER!CarlaUE4 + + REM Copy the package config file to package + set TARGET="!SUBST_PATH!\Content\Carla\Config\" + mkdir !TARGET:/=\! + copy "!PACKAGE_FILE:/=\!" !TARGET:/=\! + + REM Copy some files for each map to the package + REM MAPS_TO_COOK is read into an array as tokens separated by '+', we replace the '+' by a new line + REM We need the blank line after this line, don't remove it + set MAPS_TO_COOK=!MAPS_TO_COOK:+=^ + + ! + set BASE_CONTENT=%INSTALLATION_DIR:/=\%..\Unreal\CarlaUE4\Content + for /f "tokens=1 delims=+" %%a in ("!MAPS_TO_COOK!") do ( + + REM Get path and name of map + for /f %%i in ("%%a") do ( + set MAP_FOLDER=%%~pi + set MAP_NAME=%%~ni + REM Remove the '/Game' string + set MAP_FOLDER=!MAP_FOLDER:~5! + ) + + REM # copy the OpenDrive file + set SRC=!BASE_CONTENT!!MAP_FOLDER!\OpenDrive\!MAP_NAME!.xodr + set TRG=!BUILD_FOLDER!\CarlaUE4\Content\!MAP_FOLDER!\OpenDrive\ + if exist "!SRC!" ( + mkdir "!TRG!" + copy "!SRC!" "!TRG!" + ) + + REM # copy the navigation file + set SRC=!BASE_CONTENT!!MAP_FOLDER!\Nav\!MAP_NAME!.bin + set TRG=!BUILD_FOLDER!\CarlaUE4\Content\!MAP_FOLDER!\Nav\ + if exist "!SRC!" ( + mkdir "!TRG!" + copy "!SRC!" "!TRG!" + ) + ) + + rmdir /S /Q "!BUILD_FOLDER!\CarlaUE4\Metadata" + rmdir /S /Q "!BUILD_FOLDER!\CarlaUE4\Plugins" + REM del "!BUILD_FOLDER!\CarlaUE4\Content\!PACKAGE_NAME!/Maps/!PROPS_MAP_NAME!" + del "!BUILD_FOLDER!\CarlaUE4\AssetRegistry.bin" + + if %DO_TARBALL%==true ( + + echo Packaging '!PACKAGE_NAME!'... + + set DESTINATION_ZIP=%INSTALLATION_DIR%UE4Carla/!PACKAGE_NAME!_%CARLA_VERSION%.zip + set SOURCE=!BUILD_FOLDER:/=\!\ + set DST_ZIP=!DESTINATION_ZIP:/=\! + + pushd "!SOURCE!" + + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" a "!DST_ZIP!" . -tzip -mmt -mx5 + ) else ( + rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 + powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" + ) + + popd + + if errorlevel 1 goto bad_exit + echo ZIP created at !DST_ZIP! + ) + popd + + if %DO_CLEAN%==true ( + echo %FILE_N% Removing intermediate build. + rmdir /S /Q "!BUILD_FOLDER!" + goto :eof + ) + ) +) + +rem ============================================================================ + goto success rem ============================================================================ @@ -189,7 +339,7 @@ rem ============================================================================ :success echo. - if %DO_PACKAGE%==true echo %FILE_N% Carla project successful exported to "%CARLA_OUTPUT_PATH%"! + if %DO_PACKAGE%==true echo %FILE_N% Carla project successful exported to "%BUILD_FOLDER:/=\%"! if %DO_TARBALL%==true echo %FILE_N% Compress carla project exported to "%DESTINATION_ZIP%"! goto good_exit diff --git a/Util/BuildTools/Setup.bat b/Util/BuildTools/Setup.bat index 21bd84260..15bc1e1c4 100644 --- a/Util/BuildTools/Setup.bat +++ b/Util/BuildTools/Setup.bat @@ -185,7 +185,7 @@ rem ============================================================================ FOR /F "tokens=2" %%i in (%VERSION_FILE%) do ( set HASH=%%i ) -set URL=http://carla-assets-internal.s3.amazonaws.com/Content/%HASH%.tar.gz +set URL=http://carla-assets.s3.amazonaws.com/%HASH%.tar.gz FOR /F "tokens=1 delims=:" %%i in (%VERSION_FILE%) do ( set ASSETS_VERSION=%%i diff --git a/Util/BuildTools/Windows.mk b/Util/BuildTools/Windows.mk index 84ca23491..f8ded768c 100644 --- a/Util/BuildTools/Windows.mk +++ b/Util/BuildTools/Windows.mk @@ -16,6 +16,9 @@ help: import: server @"${CARLA_BUILD_TOOLS_FOLDER}/Import.py" +CarlaUE4Editor: LibCarla + @"${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.bat" --build + launch: LibCarla @"${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.bat" --launch @@ -63,3 +66,6 @@ LibCarla: setup setup: @"${CARLA_BUILD_TOOLS_FOLDER}/Setup.bat" --boost-toolset msvc-14.1 + +deploy: + @"${CARLA_BUILD_TOOLS_FOLDER}/Deploy.bat" $(ARGS) diff --git a/Util/ContentVersions.txt b/Util/ContentVersions.txt index 7111572a1..f138ae4aa 100644 --- a/Util/ContentVersions.txt +++ b/Util/ContentVersions.txt @@ -25,7 +25,7 @@ # # You can download it directly from # -# http://carla-assets-internal.s3.amazonaws.com/Content/PUT_FILE_ID_HERE.tar.gz +# http://carla-assets.s3.amazonaws.com/PUT_FILE_ID_HERE.tar.gz 0.9.5: 20190404_c7b464a 0.9.6: 20190710_0097e66 diff --git a/Util/DockerUtils/dist/build.sh b/Util/DockerUtils/dist/build.sh old mode 100755 new mode 100644 diff --git a/Util/InstallersWin/install_boost.bat b/Util/InstallersWin/install_boost.bat index 1fbf6e519..a4c3a46cf 100644 --- a/Util/InstallersWin/install_boost.bat +++ b/Util/InstallersWin/install_boost.bat @@ -83,9 +83,13 @@ if not exist "%BOOST_SRC_DIR%" ( echo %FILE_N% Retrieving boost. powershell -Command "Start-BitsTransfer -Source '%BOOST_REPO%' -Destination '%BOOST_TEMP_FILE_DIR%'" ) - echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while... - powershell -Command "Expand-Archive '%BOOST_TEMP_FILE_DIR%' -DestinationPath '%BUILD_DIR%'" if %errorlevel% neq 0 goto error_download + echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while... + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" x "%BOOST_TEMP_FILE_DIR%" -o"%BUILD_DIR%" -y + ) else ( + powershell -Command "Expand-Archive '%BOOST_TEMP_FILE_DIR%' -DestinationPath '%BUILD_DIR%' -Force" + ) echo %FILE_N% Removing "%BOOST_TEMP_FILE%" del "%BOOST_TEMP_FILE_DIR:/=\%" rename "%BUILD_DIR%%BOOST_TEMP_FOLDER%" "%BOOST_BASENAME%-source"