diff --git a/Docs/build_update.md b/Docs/build_update.md index 93ac576f4..433f76477 100644 --- a/Docs/build_update.md +++ b/Docs/build_update.md @@ -32,7 +32,7 @@ Binary releases are prepackaged and thus, tied to a specific version of CARLA. I

- + Get the nightly build

diff --git a/Docs/tuto_D_contribute_assets.md b/Docs/tuto_D_contribute_assets.md index 8b9a4fdc2..9b3f52fab 100644 --- a/Docs/tuto_D_contribute_assets.md +++ b/Docs/tuto_D_contribute_assets.md @@ -21,5 +21,5 @@ update this link to the latest version. [contentrepolink]: https://bitbucket.org/carla-simulator/carla-content [carlarepolink]: https://github.com/carla-simulator/carla -[jenkinslink]: http://158.109.9.218:8080/blue/organizations/jenkins/carla-content/activity +[jenkinslink]: http://35.181.165.160:8080/blue/organizations/jenkins/carla-content/activity [cvlink]: https://github.com/carla-simulator/carla/edit/master/Util/ContentVersions.txt diff --git a/Examples/CppClient/Makefile b/Examples/CppClient/Makefile index 1223e374d..05713a94c 100644 --- a/Examples/CppClient/Makefile +++ b/Examples/CppClient/Makefile @@ -22,9 +22,13 @@ run: build $(call log,Running C++ Client...) @$(BINDIR)/cpp_client $(ARGS) +run.only: + $(call log,Running C++ Client...) + @$(BINDIR)/cpp_client $(ARGS) + build: $(BINDIR)/cpp_client -$(BINDIR)/cpp_client: build_libcarla +$(BINDIR)/cpp_client: | build_libcarla $(call log,Compiling C++ Client...) @mkdir -p $(BINDIR) @$(CXX) $(CXXFLAGS) -I$(INSTALLDIR)/include -isystem $(INSTALLDIR)/include/system -L$(INSTALLDIR)/lib \ diff --git a/Jenkinsfile b/Jenkinsfile index ef4440377..7b329ad17 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,106 +1,251 @@ -pipeline { - agent any +pipeline +{ + agent none - environment { - UE4_ROOT = '/var/lib/jenkins/UnrealEngine_4.22' - } - - options { + options + { buildDiscarder(logRotator(numToKeepStr: '3', artifactNumToKeepStr: '3')) } - stages { - - stage('Setup') { - steps { - sh 'make setup' - } - } - - stage('Build') { - steps { - sh 'make LibCarla' - sh 'make PythonAPI' - sh 'make CarlaUE4Editor' - sh 'make examples' - } - post { - always { - archiveArtifacts 'PythonAPI/carla/dist/*.egg' + stages + { + stage('windows and ubuntu in parallel') + { + parallel + { + stage('ubuntu') + { + agent { label 'ubuntu && build' } + environment + { + UE4_ROOT = '/home/jenkins/UnrealEngine_4.22' + } + stages + { + stage('ubuntu setup') + { + steps + { + sh 'make setup' + } + } + stage('ubuntu build') + { + steps + { + sh 'make LibCarla' + sh 'make PythonAPI' + sh 'make CarlaUE4Editor' + sh 'make examples' + } + post + { + always + { + archiveArtifacts 'PythonAPI/carla/dist/*.egg' + stash includes: 'PythonAPI/carla/dist/*.egg', name: 'ubuntu_eggs' + } + } + } + stage('ubuntu unit tests') + { + steps + { + sh 'make check ARGS="--all --xml"' + } + post + { + always + { + junit 'Build/test-results/*.xml' + archiveArtifacts 'profiler.csv' + } + } + } + stage('ubuntu retrieve content') + { + steps + { + sh './Update.sh' + } + } + stage('ubuntu package') + { + steps + { + sh 'make package' + sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate"' + sh 'make examples ARGS="localhost 3654"' + } + post { + always { + archiveArtifacts 'Dist/*.tar.gz' + stash includes: 'Dist/CARLA*.tar.gz', name: 'ubuntu_package' + stash includes: 'Examples/', name: 'ubuntu_examples' + } + } + } + stage('ubuntu smoke tests') + { + agent { label 'ubuntu && gpu' } + steps + { + unstash name: 'ubuntu_eggs' + unstash name: 'ubuntu_package' + unstash name: 'ubuntu_examples' + sh 'tar -xvzf Dist/CARLA*.tar.gz -C Dist/' + sh 'DISPLAY= ./Dist/CarlaUE4.sh -opengl --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' + sh 'make smoke_tests ARGS="--xml"' + sh 'make run-examples ARGS="localhost 3654"' + } + post + { + always + { + archiveArtifacts 'CarlaUE4.log' + junit 'Build/test-results/smoke-tests-*.xml' + } + } + } + stage('ubuntu deploy') + { + when { anyOf { branch "master"; buildingTag() } } + steps + { + sh 'git checkout .' + sh 'make deploy ARGS="--replace-latest --docker-push"' + } + } + stage('ubuntu Doxygen') + { + when { anyOf { branch "master"; buildingTag() } } + steps + { + sh 'make docs' + sh 'rm -rf ~/carla-simulator.github.io/Doxygen' + sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/' + sh ''' + cd ~/carla-simulator.github.io + git pull + git add Doxygen + git commit -m "Updated c++ docs" || true + git push + ''' + } + } + } } - } - } - - stage('Unit Tests') { - steps { - sh 'make check ARGS="--all --xml"' - } - post { - always { - junit 'Build/test-results/*.xml' - archiveArtifacts 'profiler.csv' - } - } - } - - stage('Retrieve Content') { - steps { - sh './Update.sh' - } - } - - stage('Package') { - steps { - sh 'make package' - sh 'make package ARGS="--packages=AdditionalMaps --clean-intermediate"' - } - post { - always { - archiveArtifacts 'Dist/*.tar.gz' - } - } - } - - stage('Smoke Tests') { - steps { - sh 'DISPLAY= ./Dist/*/LinuxNoEditor/CarlaUE4.sh -opengl --carla-rpc-port=3654 --carla-streaming-port=0 -nosound > CarlaUE4.log &' - sh 'make smoke_tests ARGS="--xml"' - sh 'make run-examples ARGS="localhost 3654"' - } - post { - always { - archiveArtifacts 'CarlaUE4.log' - junit 'Build/test-results/smoke-tests-*.xml' - } - } - } - - stage('Deploy') { - when { anyOf { branch "master"; buildingTag() } } - steps { - sh 'git checkout .' - sh 'make deploy ARGS="--replace-latest --docker-push"' - } - } - - stage('Doxygen') { - when { anyOf { branch "master"; buildingTag() } } - steps { - sh 'make docs' - sh 'rm -rf ~/carla-simulator.github.io/Doxygen' - sh 'cp -rf ./Doxygen ~/carla-simulator.github.io/' - sh 'cd ~/carla-simulator.github.io && \ - git pull && \ - git add Doxygen && \ - git commit -m "Updated c++ docs" || true && \ - git push' + // stage('windows') + // { + // environment + // { + // UE4_ROOT = 'C:\\Program Files\\Epic Games\\UE_4.22' + // } + // stages + // { + // stage('windows setup') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make setup + // """ + // } + // } + // stage('windows build') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make LibCarla + // make PythonAPI + // make CarlaUE4Editor + // // make examples + // // """ + // } + // post + // { + // always + // { + // archiveArtifacts 'PythonAPI/carla/dist/*.egg' + // stash includes: 'PythonAPI/carla/dist/*.egg', name: 'windows_eggs' + // } + // } + // } + // // stage('windows unit tests') + // // { + // // agent { label 'windows && build' } + // // steps { bat 'rem Not Implemented'} + // // } + // stage('windows retrieve content') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // call Update.bat + // """ + // } + // } + // stage('windows package') + // { + // agent { label 'windows && build' } + // steps + // { + // bat """ + // call ../setEnv64.bat + // make package + // make package ARGS="--packages=AdditionalMaps --clean-intermediate" + // """ + // // make examples ARGS="localhost 3654" + // } + // post { + // always { + // archiveArtifacts 'Build/UE4Carla/*.zip' + // // stash includes: 'Build/UE4Carla/CARLA*.zip', name: 'windows_package' + // // stash includes: 'Examples/', name: 'windows_examples' + // } + // } + // } + // // stage('windows smoke test') + // // { + // // agent { label 'windows && build' } + // // steps { bat 'rem Not Implemented'} + // // } + // stage('windows deploy') + // { + // agent { label 'windows && build' } + // when { anyOf { branch "master"; buildingTag() } } + // steps { + // bat """ + // call ../setEnv64.bat + // make deploy ARGS="--replace-latest" + // """ + // } + // } + // } + // } } } } - post { - always { - deleteDir() + post + { + success + { + node('build') + { + deleteDir() + } + node('gpu') + { + deleteDir() + } } } } diff --git a/PythonAPI/test/smoke/__init__.py b/PythonAPI/test/smoke/__init__.py index 1cf227b56..5b85757f0 100644 --- a/PythonAPI/test/smoke/__init__.py +++ b/PythonAPI/test/smoke/__init__.py @@ -26,7 +26,7 @@ TESTING_ADDRESS = ('localhost', 3654) class SmokeTest(unittest.TestCase): def setUp(self): self.client = carla.Client(*TESTING_ADDRESS) - self.client.set_timeout(10.0) + self.client.set_timeout(60.0) def tearDown(self): self.client = None diff --git a/README.md b/README.md index 8cdf1ec14..048457827 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ If you want to benchmark your model in the same conditions as in our CoRL’17 paper, check out [Benchmarking](https://github.com/carla-simulator/driving-benchmarks). -[**Get CARLA overnight build**](http://carla-assets-internal.s3.amazonaws.com/Releases/Linux/Dev/CARLA_Latest.tar.gz) +[**Get CARLA overnight build**](http://carla-releases.s3.amazonaws.com/Linux/Dev/CARLA_Latest.tar.gz) ## CARLA Ecosystem Repositories associated to the CARLA simulation platform: diff --git a/Update.bat b/Update.bat new file mode 100644 index 000000000..24e80732a --- /dev/null +++ b/Update.bat @@ -0,0 +1,76 @@ +@echo off + +rem ============================================================================ +rem -- Set up environment ------------------------------------------------------ +rem ============================================================================ + +set SCRIPT_DIR=%~dp0 +set CONTENT_FOLDER=%SCRIPT_DIR%Unreal/CarlaUE4/Content/Carla +set VERSION_FILE=%CONTENT_FOLDER%/.version +set CONTENT_VERSIONS=%SCRIPT_DIR%/Util/ContentVersions.txt + +rem ============================================================================ +rem -- Get the last version to download ---------------------------------------- +rem ============================================================================ + +if not exist "%CONTENT_FOLDER%" mkdir "%CONTENT_FOLDER%" + +for /F "delims=" %%a in (%CONTENT_VERSIONS%) do ( + set "lastLine=%%a" +) +set CONTENT_ID=%lastLine:~-16,16% +set CONTENT_LINK=http://carla-assets.s3.amazonaws.com/%CONTENT_ID%.tar.gz +if "%CONTENT_ID:~0,2%"=="20" ( + set CONTENT_FILE=%CONTENT_FOLDER%/%CONTENT_ID%.tar.gz + set CONTENT_FILE_TAR=%CONTENT_FOLDER%/%CONTENT_ID%.tar + echo %CONTENT_ID% + echo %CONTENT_LINK% +) else ( + echo Error reading the latest version from ContentVersions.txt, check last line of file %CONTENT_VERSIONS%' + goto error_download +) + +rem ============================================================================ +rem -- Download the content ---------------------------------------------------- +rem ============================================================================ + +echo Downloading "%CONTENT_LINK%"... +powershell -Command "Start-BitsTransfer -Source '%CONTENT_LINK%' -Destination '%CONTENT_FOLDER%'" +if %errorlevel% neq 0 goto error_download + +echo %FILE_N% Extracting content from "%CONTENT_FILE%", this can take a while... +if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" x "%CONTENT_FILE%" -o"%CONTENT_FOLDER%" -y + if %errorlevel% neq 0 goto error_download + echo Deleting %CONTENT_FILE:/=\% + del %CONTENT_FILE:/=\% + "%ProgramW6432%/7-Zip/7z.exe" x "%CONTENT_FILE_TAR%" -o"%CONTENT_FOLDER%" -y + if %errorlevel% neq 0 goto error_download + echo Deleting %CONTENT_FILE_TAR:/=\% + del %CONTENT_FILE_TAR:/=\% +) else ( + powershell -Command "Expand-Archive '%CONTENT_FILE%' -DestinationPath '%CONTENT_FOLDER%'" + if %errorlevel% neq 0 goto error_download + del %CONTENT_FILE% +) + +goto success + +:success + echo. + echo %FILE_N% Content has been successfully installed in "%CONTENT_FOLDER%"! + goto good_exit + +:error_download + goto bad_exit + +:good_exit + echo %FILE_N% Exiting... + endlocal + exit /b 0 + +:bad_exit + if exist "%CONTENT_FILE%" rd /s /q "%CONTENT_FOLDER%" + echo %FILE_N% Exiting with error... + endlocal + exit /b %errorlevel% diff --git a/Update.sh b/Update.sh index aa6baa0d7..7718c620b 100755 --- a/Update.sh +++ b/Update.sh @@ -47,7 +47,7 @@ pushd "$SCRIPT_DIR" >/dev/null CONTENT_FOLDER=$SCRIPT_DIR/Unreal/CarlaUE4/Content/Carla CONTENT_ID=$(tac $SCRIPT_DIR/Util/ContentVersions.txt | egrep -m 1 . | rev | cut -d' ' -f1 | rev) -CONTENT_LINK=http://carla-assets-internal.s3.amazonaws.com/Content/${CONTENT_ID}.tar.gz +CONTENT_LINK=http://carla-assets.s3.amazonaws.com/${CONTENT_ID}.tar.gz VERSION_FILE=${CONTENT_FOLDER}/.version diff --git a/Util/BuildTools/BuildCarlaUE4.bat b/Util/BuildTools/BuildCarlaUE4.bat index 83f67eda5..00a8cbf19 100644 --- a/Util/BuildTools/BuildCarlaUE4.bat +++ b/Util/BuildTools/BuildCarlaUE4.bat @@ -14,11 +14,16 @@ rem ============================================================================ rem -- Parse arguments --------------------------------------------------------- rem ============================================================================ +set BUILD_UE4_EDITOR=false set LAUNCH_UE4_EDITOR=false set REMOVE_INTERMEDIATE=false :arg-parse +echo %1 if not "%1"=="" ( + if "%1"=="--build" ( + set BUILD_UE4_EDITOR=true + ) if "%1"=="--launch" ( set LAUNCH_UE4_EDITOR=true ) @@ -37,9 +42,11 @@ if not "%1"=="" ( if %REMOVE_INTERMEDIATE% == false ( if %LAUNCH_UE4_EDITOR% == false ( - echo Nothing selected to be done. - echo %USAGE_STRING% - goto eof + if %BUILD_UE4_EDITOR% == false ( + echo Nothing selected to be done. + echo %USAGE_STRING% + goto eof + ) ) ) @@ -78,6 +85,30 @@ if %REMOVE_INTERMEDIATE% == true ( ) ) +rem Build Carla Editor +rem +if %BUILD_UE4_EDITOR% == true ( + echo %FILE_N% Building Unreal Editor... + + call "%UE4_ROOT%\Engine\Build\BatchFiles\Build.bat"^ + CarlaUE4Editor^ + Win64^ + Development^ + -WaitMutex^ + -FromMsBuild^ + "%ROOT_PATH%Unreal/CarlaUE4/CarlaUE4.uproject" + if errorlevel 1 goto bad_exit + + call "%UE4_ROOT%\Engine\Build\BatchFiles\Build.bat"^ + CarlaUE4^ + Win64^ + Development^ + -WaitMutex^ + -FromMsBuild^ + "%ROOT_PATH%Unreal/CarlaUE4/CarlaUE4.uproject" + if errorlevel 1 goto bad_exit +) + rem Launch Carla Editor rem if %LAUNCH_UE4_EDITOR% == true ( @@ -94,7 +125,7 @@ rem ============================================================================ :help echo Build LibCarla. - echo "Usage: %FILE_N% [-h^|--help] [--launch] [--clean]" + echo "Usage: %FILE_N% [-h^|--help] [--build] [--launch] [--clean]" goto good_exit :error_build diff --git a/Util/BuildTools/Check.sh b/Util/BuildTools/Check.sh index 7435c51af..7e7e30d67 100755 --- a/Util/BuildTools/Check.sh +++ b/Util/BuildTools/Check.sh @@ -54,6 +54,8 @@ while true; do shift ;; --xml ) XML_OUTPUT=true; + # Create the folder for the test-results + mkdir -p "${CARLA_TEST_RESULTS_FOLDER}" shift ;; --gtest_args ) GTEST_ARGS="$2"; diff --git a/Util/BuildTools/Deploy.bat b/Util/BuildTools/Deploy.bat new file mode 100644 index 000000000..e73076477 --- /dev/null +++ b/Util/BuildTools/Deploy.bat @@ -0,0 +1,121 @@ +@echo off +setlocal enabledelayedexpansion + +rem ============================================================================== +rem -- Set up environment -------------------------------------------------------- +rem ============================================================================== + +set REPLACE_LATEST=false +set AWS_COPY=aws s3 cp +set UPLOAD_MAPS=true + +rem ============================================================================== +rem -- Parse arguments ----------------------------------------------------------- +rem ============================================================================== + +set DOC_STRING=Upload latest build to S3 + +set USAGE_STRING="Usage: $0 [-h|--help] [--replace-latest] [--dry-run]" + +:arg-parse +if not "%1"=="" ( + if "%1"=="--replace-latest" ( + set REPLACE_LATEST=true + ) + + if "%1"=="--dry-run" ( + echo %AWS_COPY% + ) + + if "%1"=="--help" ( + echo %DOC_STRING% + echo %USAGE_STRING% + GOTO :eof + ) + + shift + goto :arg-parse +) + +rem Get repository version +for /f %%i in ('git describe --tags --dirty --always') do set REPOSITORY_TAG=%%i +if not defined REPOSITORY_TAG goto error_carla_version + +rem Last package data +set CARLA_DIST_FOLDER=%~dp0%\Build\UE4Carla +set LATEST_PACKAGE=CARLA_%REPOSITORY_TAG%.zip +set LATEST_PACKAGE_PATH=%CARLA_DIST_FOLDER%\%LATEST_PACKAGE% + +set S3_PREFIX=s3://carla-releases/Windows + +set LATEST_DEPLOY_URI=!S3_PREFIX!/Dev/CARLA_Latest.zip + +rem Check for TAG version +echo %REPOSITORY_TAG% | findstr /R /C:"^[0-9]*\.[0-9]*\.[0-9]*.$" 1>nul +if %errorlevel% == 0 ( + echo Detected tag %REPOSITORY_TAG% + set DEPLOY_NAME=CARLA_%REPOSITORY_TAG%.zip +) else ( + set S3_PREFIX=!S3_PREFIX!/Dev + git log --pretty=format:%%cd_%%h --date=format:%%Y%%m%%d -n 1 > tempo1234 + set /p DEPLOY_NAME= < tempo1234 + del tempo1234 + set DEPLOY_NAME=!DEPLOY_NAME!.zip +) +echo Version detected: %REPOSITORY_TAG% +echo Using package %LATEST_PACKAGE% as %DEPLOY_NAME% + +if not exist "%LATEST_PACKAGE_PATH%" ( + echo Latest package not found, please run 'make package' + goto :bad_exit +) + +rem ============================================================================== +rem -- Upload -------------------------------------------------------------------- +rem ============================================================================== + +set DEPLOY_URI=!S3_PREFIX!/%DEPLOY_NAME% +REM %AWS_COPY% %LATEST_PACKAGE_PATH% %DEPLOY_URI% +echo Latest build uploaded to %DEPLOY_URI% + +if %REPLACE_LATEST%==true ( + REM %AWS_COPY% %DEPLOY_URI% %LATEST_DEPLOY_URI% + echo Latest build uploaded to %LATEST_DEPLOY_URI% +) + +if %UPLOAD_MAPS%==true ( + + pushd "%CARLA_DIST_FOLDER%" + + for /f %%i in ('dir *_%REPOSITORY_TAG%.zip /b') do ( + if not %%i == %LATEST_PACKAGE% ( + set DEPLOY_MAP_URI=!S3_PREFIX!/%%i + %AWS_COPY% %%i !DEPLOY_MAP_URI! + echo %%i uploaded to !DEPLOY_MAP_URI! + ) + ) + popd +) + +rem ============================================================================== +rem -- ...and we are done -------------------------------------------------------- +rem ============================================================================== + +echo Success! + +:success + echo. + goto good_exit + +:error_carla_version + echo. + echo %FILE_N% [ERROR] Carla Version is not set + goto bad_exit + +:good_exit + endlocal + exit /b 0 + +:bad_exit + endlocal + exit /b 1 diff --git a/Util/BuildTools/Deploy.sh b/Util/BuildTools/Deploy.sh index f10aca18f..bf8f7caa8 100755 --- a/Util/BuildTools/Deploy.sh +++ b/Util/BuildTools/Deploy.sh @@ -55,7 +55,7 @@ REPOSITORY_TAG=$(get_git_repository_version) LATEST_PACKAGE=CARLA_${REPOSITORY_TAG}.tar.gz LATEST_PACKAGE_PATH=${CARLA_DIST_FOLDER}/${LATEST_PACKAGE} -S3_PREFIX=s3://carla-assets-internal/Releases/Linux +S3_PREFIX=s3://carla-releases/Linux LATEST_DEPLOY_URI=${S3_PREFIX}/Dev/CARLA_Latest.tar.gz diff --git a/Util/BuildTools/Linux.mk b/Util/BuildTools/Linux.mk index a593e8b39..ddd1ae563 100644 --- a/Util/BuildTools/Linux.mk +++ b/Util/BuildTools/Linux.mk @@ -70,7 +70,7 @@ examples: @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} build; done run-examples: - @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} run; done + @for D in ${CARLA_EXAMPLES_FOLDER}/*; do [ -d "$${D}" ] && make -C $${D} run.only; done CarlaUE4Editor: LibCarla.server.release @${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.sh --build diff --git a/Util/BuildTools/Package.bat b/Util/BuildTools/Package.bat index 5f8fa37b3..ca112c2ae 100644 --- a/Util/BuildTools/Package.bat +++ b/Util/BuildTools/Package.bat @@ -25,6 +25,7 @@ set DO_TARBALL=true set DO_CLEAN=false set UE_VERSION=4.22 +set PACKAGES=Carla :arg-parse if not "%1"=="" ( @@ -45,7 +46,10 @@ if not "%1"=="" ( if "%1"=="--ue-version" ( set UE_VERSION=%~2 - shift + ) + + if "%1"=="--packages" ( + set PACKAGES=%~2 ) if "%1"=="-h" ( @@ -78,10 +82,10 @@ for /f %%i in ('git describe --tags --dirty --always') do set CARLA_VERSION=%%i if not defined CARLA_VERSION goto error_carla_version set BUILD_FOLDER=%INSTALLATION_DIR%UE4Carla/%CARLA_VERSION%/ -if not exist "%BUILD_FOLDER%" mkdir "%BUILD_FOLDER%" +if not exist "!BUILD_FOLDER!" mkdir "!BUILD_FOLDER!" -set DESTINATION_ZIP="%BUILD_FOLDER%../CARLA_%CARLA_VERSION%.zip" -set SOURCE=%BUILD_FOLDER%WindowsNoEditor/ +set DESTINATION_ZIP=%INSTALLATION_DIR%UE4Carla/CARLA_%CARLA_VERSION%.zip +set SOURCE=!BUILD_FOLDER!WindowsNoEditor/ rem ============================================================================ rem -- Create Carla package ---------------------------------------------------- @@ -120,7 +124,7 @@ if %DO_PACKAGE%==true ( -stage^ -build^ -archive^ - -archivedirectory="%BUILD_FOLDER%"^ + -archivedirectory="!BUILD_FOLDER!"^ -package^ -clientconfig=Development @@ -137,14 +141,21 @@ if %DO_COPY_FILES%==true ( set XCOPY_FROM=%ROOT_PATH:/=\% set XCOPY_TO=%SOURCE:/=\% - echo f | xcopy /y "!XCOPY_FROM!LICENSE" "!XCOPY_TO!LICENSE" - echo f | xcopy /y "!XCOPY_FROM!CHANGELOG.md" "!XCOPY_TO!CHANGELOG" - echo f | xcopy /y "!XCOPY_FROM!Docs\release_readme.md" "!XCOPY_TO!README" - echo f | xcopy /y "!XCOPY_FROM!Util\Docker\Release.Dockerfile" "!XCOPY_TO!Dockerfile" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\dist\*.egg" "!XCOPY_TO!PythonAPI\" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\example.py" "!XCOPY_TO!example.py" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\manual_control.py" "!XCOPY_TO!manual_control.py" - echo f | xcopy /y "!XCOPY_FROM!PythonAPI\vehicle_gallery.py" "!XCOPY_TO!vehicle_gallery.py" + echo f | xcopy /y "!XCOPY_FROM!LICENSE" "!XCOPY_TO!LICENSE" + echo f | xcopy /y "!XCOPY_FROM!CHANGELOG.md" "!XCOPY_TO!CHANGELOG" + echo f | xcopy /y "!XCOPY_FROM!Docs\release_readme.md" "!XCOPY_TO!README" + echo f | xcopy /y "!XCOPY_FROM!Util\Docker\Release.Dockerfile" "!XCOPY_TO!Dockerfile" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\dist\*.egg" "!XCOPY_TO!PythonAPI\carla\dist\" + echo d | xcopy /y /s "!XCOPY_FROM!PythonAPI\carla\agents" "!XCOPY_TO!PythonAPI\carla\agents" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\scene_layout.py" "!XCOPY_TO!PythonAPI\carla\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\carla\requirements.txt" "!XCOPY_TO!PythonAPI\carla\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\examples\*.py" "!XCOPY_TO!PythonAPI\examples\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\examples\requirements.txt" "!XCOPY_TO!PythonAPI\examples\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\util\*.py" "!XCOPY_TO!PythonAPI\util\" + echo f | xcopy /y "!XCOPY_FROM!PythonAPI\util\requirements.txt" "!XCOPY_TO!PythonAPI\util\" + echo f | xcopy /y "!XCOPY_FROM!Unreal\CarlaUE4\Content\Carla\HDMaps\*.pcd" "!XCOPY_TO!HDMaps\" + echo f | xcopy /y "!XCOPY_FROM!Unreal\CarlaUE4\Content\Carla\HDMaps\Readme.md" "!XCOPY_TO!HDMaps\README" + ) rem ============================================================================== @@ -152,22 +163,25 @@ rem -- Zip the project --------------------------------------------------------- rem ============================================================================== if %DO_TARBALL%==true ( - echo "%FILE_N% Building package..." - - set DST_ZIP=%DESTINATION_ZIP:/=\% set SRC_PATH=%SOURCE:/=\% + echo %FILE_N% Building package... + if exist "!SRC_PATH!Manifest_NonUFSFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_NonUFSFiles_Win64.txt" if exist "!SRC_PATH!Manifest_DebugFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_DebugFiles_Win64.txt" if exist "!SRC_PATH!Manifest_UFSFiles_Win64.txt" del /Q "!SRC_PATH!Manifest_UFSFiles_Win64.txt" - if exist "!SRC_PATH!CarlaUE4/Saved" rmdir /S /Q "!SRC_PATH!CarlaUE4/Saved" if exist "!SRC_PATH!Engine/Saved" rmdir /S /Q "!SRC_PATH!Engine/Saved" - pushd "!SRC_PATH!" - rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 - powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" - popd + set DST_ZIP=%DESTINATION_ZIP:/=\% + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" a "!DST_ZIP!" "!SRC_PATH!" -tzip -mmt -mx5 + ) else ( + pushd "!SRC_PATH!" + rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 + powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" + popd + ) ) rem ============================================================================== @@ -176,11 +190,147 @@ rem ============================================================================ if %DO_CLEAN%==true ( echo %FILE_N% Removing intermediate build. - - rmdir /S /Q "%BUILD_FOLDER%" + rmdir /S /Q "!BUILD_FOLDER!" goto :eof ) +rem ============================================================================== +rem -- Cook other packages ------------------------------------------------------- +rem ============================================================================== + +rem Set some file locations +set CARLAUE4_ROOT_FOLDER=%ROOT_PATH%Unreal/CarlaUE4 +set PACKAGE_PATH_FILE=%CARLAUE4_ROOT_FOLDER%/Content/PackagePath.txt +set MAP_LIST_FILE=%CARLAUE4_ROOT_FOLDER%/Content/MapPaths.txt + +rem through all maps to cook (parameter) +for %%i in (%PACKAGES%) do ( + + set PACKAGE_NAME=%%i + + if not !PACKAGE_NAME! == Carla ( + echo Preparing environment for cooking '!PACKAGE_NAME!'. + + set BUILD_FOLDER=%INSTALLATION_DIR%UE4Carla/!PACKAGE_NAME!_%CARLA_VERSION%\ + set PACKAGE_PATH=%CARLAUE4_ROOT_FOLDER%/Content/!PACKAGE_NAME! + + if not exist "!BUILD_FOLDER!" mkdir "!BUILD_FOLDER!" + + echo Cooking package '!PACKAGE_NAME!'... + + pushd "%CARLAUE4_ROOT_FOLDER%" + + REM # Prepare cooking of package + call "%UE4_ROOT%/Engine/Binaries/Win64/UE4Editor.exe "^ + "%CARLAUE4_ROOT_FOLDER%/CarlaUE4.uproject"^ + -run=PrepareAssetsForCooking^ + -PackageName=!PACKAGE_NAME!^ + -OnlyPrepareMaps=false + + set /p PACKAGE_FILE=<%PACKAGE_PATH_FILE% + set /p MAPS_TO_COOK=<%MAP_LIST_FILE% + + REM # Cook maps + call "%UE4_ROOT%/Engine/Binaries/Win64/UE4Editor.exe "^ + "%CARLAUE4_ROOT_FOLDER%/CarlaUE4.uproject"^ + -run=cook^ + -map="!MAPS_TO_COOK!"^ + -cooksinglepackage^ + -targetplatform="WindowsNoEditor"^ + -OutputDir="!BUILD_FOLDER!" + + REM remove the props folder if exist + set PROPS_MAP_FOLDER="%PACKAGE_PATH%/Maps/PropsMap" + if exist "%PROPS_MAP_FOLDER%" ( + rmdir /S /Q "%PROPS_MAP_FOLDER%" + ) + + popd + + echo Copying files to '!PACKAGE_NAME!'... + + pushd "!BUILD_FOLDER!" + + set SUBST_PATH=!BUILD_FOLDER!CarlaUE4 + + REM Copy the package config file to package + set TARGET="!SUBST_PATH!\Content\Carla\Config\" + mkdir !TARGET:/=\! + copy "!PACKAGE_FILE:/=\!" !TARGET:/=\! + + REM Copy some files for each map to the package + REM MAPS_TO_COOK is read into an array as tokens separated by '+', we replace the '+' by a new line + REM We need the blank line after this line, don't remove it + set MAPS_TO_COOK=!MAPS_TO_COOK:+=^ + + ! + set BASE_CONTENT=%INSTALLATION_DIR:/=\%..\Unreal\CarlaUE4\Content + for /f "tokens=1 delims=+" %%a in ("!MAPS_TO_COOK!") do ( + + REM Get path and name of map + for /f %%i in ("%%a") do ( + set MAP_FOLDER=%%~pi + set MAP_NAME=%%~ni + REM Remove the '/Game' string + set MAP_FOLDER=!MAP_FOLDER:~5! + ) + + REM # copy the OpenDrive file + set SRC=!BASE_CONTENT!!MAP_FOLDER!\OpenDrive\!MAP_NAME!.xodr + set TRG=!BUILD_FOLDER!\CarlaUE4\Content\!MAP_FOLDER!\OpenDrive\ + if exist "!SRC!" ( + mkdir "!TRG!" + copy "!SRC!" "!TRG!" + ) + + REM # copy the navigation file + set SRC=!BASE_CONTENT!!MAP_FOLDER!\Nav\!MAP_NAME!.bin + set TRG=!BUILD_FOLDER!\CarlaUE4\Content\!MAP_FOLDER!\Nav\ + if exist "!SRC!" ( + mkdir "!TRG!" + copy "!SRC!" "!TRG!" + ) + ) + + rmdir /S /Q "!BUILD_FOLDER!\CarlaUE4\Metadata" + rmdir /S /Q "!BUILD_FOLDER!\CarlaUE4\Plugins" + REM del "!BUILD_FOLDER!\CarlaUE4\Content\!PACKAGE_NAME!/Maps/!PROPS_MAP_NAME!" + del "!BUILD_FOLDER!\CarlaUE4\AssetRegistry.bin" + + if %DO_TARBALL%==true ( + + echo Packaging '!PACKAGE_NAME!'... + + set DESTINATION_ZIP=%INSTALLATION_DIR%UE4Carla/!PACKAGE_NAME!_%CARLA_VERSION%.zip + set SOURCE=!BUILD_FOLDER:/=\!\ + set DST_ZIP=!DESTINATION_ZIP:/=\! + + pushd "!SOURCE!" + + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" a "!DST_ZIP!" . -tzip -mmt -mx5 + ) else ( + rem https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/compress-archive?view=powershell-6 + powershell -command "& { Compress-Archive -Path * -CompressionLevel Fastest -DestinationPath '!DST_ZIP!' }" + ) + + popd + + if errorlevel 1 goto bad_exit + echo ZIP created at !DST_ZIP! + ) + popd + + if %DO_CLEAN%==true ( + echo %FILE_N% Removing intermediate build. + rmdir /S /Q "!BUILD_FOLDER!" + goto :eof + ) + ) +) + +rem ============================================================================ + goto success rem ============================================================================ @@ -189,7 +339,7 @@ rem ============================================================================ :success echo. - if %DO_PACKAGE%==true echo %FILE_N% Carla project successful exported to "%CARLA_OUTPUT_PATH%"! + if %DO_PACKAGE%==true echo %FILE_N% Carla project successful exported to "%BUILD_FOLDER:/=\%"! if %DO_TARBALL%==true echo %FILE_N% Compress carla project exported to "%DESTINATION_ZIP%"! goto good_exit diff --git a/Util/BuildTools/Setup.bat b/Util/BuildTools/Setup.bat index 21bd84260..15bc1e1c4 100644 --- a/Util/BuildTools/Setup.bat +++ b/Util/BuildTools/Setup.bat @@ -185,7 +185,7 @@ rem ============================================================================ FOR /F "tokens=2" %%i in (%VERSION_FILE%) do ( set HASH=%%i ) -set URL=http://carla-assets-internal.s3.amazonaws.com/Content/%HASH%.tar.gz +set URL=http://carla-assets.s3.amazonaws.com/%HASH%.tar.gz FOR /F "tokens=1 delims=:" %%i in (%VERSION_FILE%) do ( set ASSETS_VERSION=%%i diff --git a/Util/BuildTools/Windows.mk b/Util/BuildTools/Windows.mk index 84ca23491..f8ded768c 100644 --- a/Util/BuildTools/Windows.mk +++ b/Util/BuildTools/Windows.mk @@ -16,6 +16,9 @@ help: import: server @"${CARLA_BUILD_TOOLS_FOLDER}/Import.py" +CarlaUE4Editor: LibCarla + @"${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.bat" --build + launch: LibCarla @"${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.bat" --launch @@ -63,3 +66,6 @@ LibCarla: setup setup: @"${CARLA_BUILD_TOOLS_FOLDER}/Setup.bat" --boost-toolset msvc-14.1 + +deploy: + @"${CARLA_BUILD_TOOLS_FOLDER}/Deploy.bat" $(ARGS) diff --git a/Util/ContentVersions.txt b/Util/ContentVersions.txt index 7111572a1..f138ae4aa 100644 --- a/Util/ContentVersions.txt +++ b/Util/ContentVersions.txt @@ -25,7 +25,7 @@ # # You can download it directly from # -# http://carla-assets-internal.s3.amazonaws.com/Content/PUT_FILE_ID_HERE.tar.gz +# http://carla-assets.s3.amazonaws.com/PUT_FILE_ID_HERE.tar.gz 0.9.5: 20190404_c7b464a 0.9.6: 20190710_0097e66 diff --git a/Util/DockerUtils/dist/build.sh b/Util/DockerUtils/dist/build.sh old mode 100755 new mode 100644 diff --git a/Util/InstallersWin/install_boost.bat b/Util/InstallersWin/install_boost.bat index 1fbf6e519..a4c3a46cf 100644 --- a/Util/InstallersWin/install_boost.bat +++ b/Util/InstallersWin/install_boost.bat @@ -83,9 +83,13 @@ if not exist "%BOOST_SRC_DIR%" ( echo %FILE_N% Retrieving boost. powershell -Command "Start-BitsTransfer -Source '%BOOST_REPO%' -Destination '%BOOST_TEMP_FILE_DIR%'" ) - echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while... - powershell -Command "Expand-Archive '%BOOST_TEMP_FILE_DIR%' -DestinationPath '%BUILD_DIR%'" if %errorlevel% neq 0 goto error_download + echo %FILE_N% Extracting boost from "%BOOST_TEMP_FILE%", this can take a while... + if exist "%ProgramW6432%/7-Zip/7z.exe" ( + "%ProgramW6432%/7-Zip/7z.exe" x "%BOOST_TEMP_FILE_DIR%" -o"%BUILD_DIR%" -y + ) else ( + powershell -Command "Expand-Archive '%BOOST_TEMP_FILE_DIR%' -DestinationPath '%BUILD_DIR%' -Force" + ) echo %FILE_N% Removing "%BOOST_TEMP_FILE%" del "%BOOST_TEMP_FILE_DIR:/=\%" rename "%BUILD_DIR%%BOOST_TEMP_FOLDER%" "%BOOST_BASENAME%-source"