From a3e0ea74b3e356d1d9f2bbfeffd61babbaaedc6c Mon Sep 17 00:00:00 2001 From: Marcel Pi Date: Thu, 16 Nov 2023 20:55:25 +0100 Subject: [PATCH] Switch to task graph based configure script. --- CMakeLists.txt | 78 +-- Configure.py | 1274 +++++++++++++++++++++--------------------------- 2 files changed, 590 insertions(+), 762 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0856fefb5..dc82ee9cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,45 +8,46 @@ set (CMAKE_CXX_STANDARD 20) set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_EXPORT_COMPILE_COMMANDS ON) -option (LIBCARLA_CLIENT "Whether to build the LibCarla client." ON) -option (LIBCARLA_SERVER "Whether to build the LibCarla server." ON) -option (LIBCARLA_PYTORCH_ENABLE "Whether to enable pytorch." OFF) -option (PYTHON_API "Whether to build the CARLA Python API." ON) -option (LIBCARLA_CLIENT_RSS "Whether to enable RSS components (ad-rss-lib)" OFF) -option (LIBCARLA_INSTALL "Whether to install LibCarla and all of its dependencies." ON) -option (OSM_ENABLE "" ON) +option (BUILD_LIBCARLA_CLIENT "Whether to build the LibCarla client." ON) +option (BUILD_LIBCARLA_SERVER "Whether to build the LibCarla server." ON) +option (ENABLE_LIBCARLA_PYTORCH "Whether to enable pytorch." OFF) +option (BUILD_PYTHON_API "Whether to build the CARLA Python API." ON) +option (ENABLE_LIBCARLA_CLIENT_RSS "Whether to enable RSS components (ad-rss-lib)" OFF) +option (INSTALL_LIBCARLA "Whether to install LibCarla and all of its dependencies." ON) +option (BUILD_OSM_WORLD_RENDERER "" ON) set (CARLA_WORKSPACE_PATH ${CMAKE_CURRENT_SOURCE_DIR}) set (CARLA_BUILD_PATH ${CMAKE_CURRENT_BINARY_DIR}) +set (CARLA_DEPENDENCIES_PATH ${CMAKE_CURRENT_BINARY_DIR}/Dependencies) set (LIBCARLA_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/LibCarla/source) set (LIBCARLA_THIRD_PARTY_SOURCE_PATH ${LIBCARLA_SOURCE_PATH}/third-party) set (PYTHONAPI_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/PythonAPI) -set (BOOST_INCLUDE_PATH ${CARLA_BUILD_PATH}/boost-install/include) -set (BOOST_LIBRARY_PATH ${CARLA_BUILD_PATH}/boost-install/lib) -set (CHRONO_INCLUDE_PATH ${CARLA_BUILD_PATH}/chrono-install/include) -set (CHRONO_LIBRARY_PATH ${CARLA_BUILD_PATH}/chrono-install/lib) -set (EIGEN_INCLUDE_PATH ${CARLA_BUILD_PATH}/eigen-install/include) -set (EIGEN_LIBRARY_PATH ${CARLA_BUILD_PATH}/eigen-install/lib) -set (GOOGLETEST_INCLUDE_PATH ${CARLA_BUILD_PATH}/gtest-install/include) -set (GOOGLETEST_LIBRARY_PATH ${CARLA_BUILD_PATH}/gtest-install/lib) -set (LIBPNG_INCLUDE_PATH ${CARLA_BUILD_PATH}/libpng-install/include) -set (LIBPNG_LIBRARY_PATH ${CARLA_BUILD_PATH}/libpng-install/lib) -set (PROJ_INCLUDE_PATH ${CARLA_BUILD_PATH}/proj-install/include) -set (PROJ_LIBRARY_PATH ${CARLA_BUILD_PATH}/proj-install/lib) -set (RECAST_INCLUDE_PATH ${CARLA_BUILD_PATH}/recast-install/include) -set (RECAST_LIBRARY_PATH ${CARLA_BUILD_PATH}/recast-install/lib) -set (RPCLIB_INCLUDE_PATH ${CARLA_BUILD_PATH}/rpclib-install/include) -set (RPCLIB_LIBRARY_PATH ${CARLA_BUILD_PATH}/rpclib-install/lib) -set (SQLITE_INCLUDE_PATH ${CARLA_BUILD_PATH}/sqlite-install/include) -set (SQLITE_LIBRARY_PATH ${CARLA_BUILD_PATH}/sqlite-install/lib) -set (XERCESC_INCLUDE_PATH ${CARLA_BUILD_PATH}/xercesc-install/include) -set (XERCESC_LIBRARY_PATH ${CARLA_BUILD_PATH}/xercesc-install/lib) -set (ZLIB_INCLUDE_PATH ${CARLA_BUILD_PATH}/zlib-install/include) -set (ZLIB_LIBRARY_PATH ${CARLA_BUILD_PATH}/zlib-install/lib) -set (ZLIB_INCLUDE_PATH ${CARLA_BUILD_PATH}/zlib-source) -set (ZLIB_LIBRARY_PATH ${CARLA_BUILD_PATH}/zlib-build) +set (BOOST_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/boost-install/include) +set (BOOST_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/boost-install/lib) +set (CHRONO_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/chrono-install/include) +set (CHRONO_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/chrono-install/lib) +set (EIGEN_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/eigen-install/include) +set (EIGEN_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/eigen-install/lib) +set (GOOGLETEST_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/gtest-install/include) +set (GOOGLETEST_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/gtest-install/lib) +set (LIBPNG_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/libpng-install/include) +set (LIBPNG_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/libpng-install/lib) +set (PROJ_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/proj-install/include) +set (PROJ_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/proj-install/lib) +set (RECAST_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/recast-install/include) +set (RECAST_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/recast-install/lib) +set (RPCLIB_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/rpclib-install/include) +set (RPCLIB_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/rpclib-install/lib) +set (SQLITE_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/sqlite-install/include) +set (SQLITE_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/sqlite-install/lib) +set (XERCESC_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/xercesc-install/include) +set (XERCESC_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/xercesc-install/lib) +set (ZLIB_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/zlib-install/include) +set (ZLIB_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/zlib-install/lib) +set (ZLIB_INCLUDE_PATH ${CARLA_DEPENDENCIES_PATH}/zlib-source) +set (ZLIB_LIBRARY_PATH ${CARLA_DEPENDENCIES_PATH}/zlib-build) add_compile_definitions (BOOST_ERROR_CODE_HEADER_ONLY) add_compile_definitions (LIBCARLA_IMAGE_WITH_PNG_SUPPORT) @@ -59,7 +60,7 @@ endif () -if (LIBCARLA_SERVER) +if (BUILD_LIBCARLA_SERVER) project (LibCarla-Server) @@ -115,16 +116,19 @@ if (LIBCARLA_SERVER) ${LIBCARLA_THIRD_PARTY_SOURCE_PATH}/pugixml/*.hpp ) - add_library (LibCarla-Server STATIC ${LIBCARLA_SERVER_SOURCES}) + add_library ( + LibCarla-Server STATIC ${LIBCARLA_SERVER_SOURCES}) - target_compile_options (LibCarla-Server PRIVATE /EHsc) - target_include_directories (LibCarla-Server PRIVATE ${LIBCARLA_SERVER_INCLUDE_PATHS}) + target_compile_options ( + LibCarla-Server PRIVATE /EHsc) + target_include_directories ( + LibCarla-Server PRIVATE ${LIBCARLA_SERVER_INCLUDE_PATHS}) endif () -if (LIBCARLA_CLIENT) +if (BUILD_LIBCARLA_CLIENT) project (LibCarla-Client) @@ -204,7 +208,7 @@ endif () -if (OSM_ENABLE) +if (BUILD_OSM_WORLD_RENDERER) project (OsmMapRenderer) diff --git a/Configure.py b/Configure.py index 115ee6cbc..64f3c47c9 100644 --- a/Configure.py +++ b/Configure.py @@ -3,6 +3,187 @@ from argparse import ArgumentParser from pathlib import Path import subprocess, tarfile, zipfile, requests, psutil, shutil, glob, json, sys, os +FORCE_SEQUENTIAL = False + +# Script types + +class Download: + def __init__(self, url): + self.url = url + +class GitRepository: + def __init__(self, url, tag : str = None): + self.url = url + self.tag = tag + +class Dependency: + def __init__( + self, + name : str, + *sources): + self.name = name + self.sources = [ e for e in sources ] + assert all(type(e) is Download or type(e) is GitRepository for e in self.sources) + +class Task: + def __init__( + self, + name : str, + in_edges : list, + body, + *args): + self.name = name + self.body = body + self.args = args + self.in_edges = in_edges + self.out_edges = [] # Filled right before task graph execution. + + def CreateSubprocessTask(name : str, in_edges : list, command : list): + return Task(name, in_edges, LaunchSubprocessImmediate, command) + + def CreateCMakeConfigureDefault( + name : str, + in_edges : list, + source_path : Path, + build_path : Path, + *args): + cmd = [ + 'cmake', + '-G', CMAKE_GENERATOR, + '-S', source_path, + '-B', build_path, + '-DCMAKE_C_COMPILER=' + c_compiler, + '-DCMAKE_CXX_COMPILER=' + cpp_compiler, + '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_CXX_FLAGS_RELEASE="/MD"' + ] + cmd.extend([ *args ]) + cmd.append(source_path) + return Task.CreateSubprocessTask(name, in_edges, cmd) + + def CreateCMakeBuildDefault( + name : str, + in_edges : list, + build_path : Path, + *args): + cmd = [ 'cmake', '--build', build_path ] + cmd.extend([ *args ]) + return Task.CreateSubprocessTask(name, in_edges, cmd) + + def CreateCMakeInstallDefault( + name : str, + in_edges : list, + build_path : Path, + install_path : Path, + *args): + cmd = [ 'cmake', '--install', build_path, '--prefix', install_path ] + cmd.extend([ *args ]) + return Task.CreateSubprocessTask(name, in_edges, cmd) + + def Run(self): + self.body(*self.args) + +class TaskGraph: + def __init__(self, parallelism : int = None): + self.sequential = FORCE_SEQUENTIAL + self.pool = ProcessPoolExecutor(parallelism) + self.futures = [] + self.tasks = [] + self.sources = [] + self.task_map = {} + + def Reset(self): + self.futures = [] + self.tasks = [] + self.sources = [] + self.task_map = {} + + def Add(self, task): + self.tasks.append(task) + if len(task.in_edges) == 0: + self.sources.append(task.name) + self.task_map[task.name] = self.tasks[-1] + + def Execute(self): + for e in self.tasks: + for in_edge in e.in_edges: + self.task_map[in_edge].out_edges.append(e) + for e in self.sources: + task = self.task_map.get(e, None) + assert task != None and type(task) is Task + if not self.sequential: + self.futures.append(self.pool.submit(task.Run)) + else: + task.Run() + if not self.sequential: + for e in as_completed(self.futures): + e.result() + self.Reset() + +class Context: + def __init__(self, args, parallelism): + self.task_graph = TaskGraph(parallelism) + self.args = args + pass + +# Constants: + +DEFAULT_DEPENDENCIES = [ + Dependency( + 'boost', + Download('https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.zip'), + Download('https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/boost_1_83_0.zip')), + Dependency( + 'chrono', + GitRepository('https://github.com/projectchrono/chrono.git', tag = '8.0.0')), + Dependency( + 'eigen', + GitRepository('https://gitlab.com/libeigen/eigen.git', tag = '3.4.0')), + Dependency( + 'libpng', + GitRepository('https://github.com/glennrp/libpng.git', tag = 'v1.6.40')), + Dependency( + 'proj', + GitRepository('https://github.com/OSGeo/PROJ.git', tag = '9.3.0'), + Download('https://download.osgeo.org/proj/proj-9.3.0.tar.gz')), + Dependency( + 'gtest', + GitRepository('https://github.com/google/googletest.git', tag = 'v1.14.0')), + Dependency( + 'zlib', + Download('https://zlib.net/current/zlib.tar.gz'), + GitRepository('https://github.com/madler/zlib.git', tag = 'v1.3')), + Dependency( + 'xercesc', + GitRepository('https://github.com/apache/xerces-c.git', tag = 'v3.2.4'), + Download('https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.2.3.zip'), + Download('https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/xerces-c-3.2.3.zip')), + Dependency( + 'sqlite', + Download('https://www.sqlite.org/2021/sqlite-amalgamation-3340100.zip')), + Dependency( + 'rpclib', + GitRepository('https://github.com/rpclib/rpclib.git', tag = 'v2.3.0')), + Dependency( + 'recast', + GitRepository('https://github.com/recastnavigation/recastnavigation.git', tag = 'v1.6.0')), +] + +OSM_WORLD_RENDERER_DEPENDENCIES = [ + Dependency( + 'libosmscout', + GitRepository('https://github.com/Framstag/libosmscout.git')), + Dependency( + 'lunasvg', + GitRepository('https://github.com/sammycage/lunasvg.git')), +] + +OSM2ODR_DEPENDENCIES = [ + Dependency( + 'sumo', + GitRepository('https://github.com/carla-simulator/sumo.git')), +] + def FindExecutable(candidates : list): for e in candidates: ec = subprocess.call( @@ -48,280 +229,149 @@ lib = FindExecutable([ ]) # Basic paths -workspace_path = Path(__file__).parent.resolve() -libcarla_path = workspace_path / 'LibCarla' -python_api_path = workspace_path / 'PythonAPI' -examples_path = workspace_path / 'Examples' -build_path = workspace_path / 'Build' -dist_path = workspace_path / 'Dist' -util_path = workspace_path / 'Util' -docker_utils_path = util_path / 'DockerUtils' -build_tools_path = util_path / 'BuildTools' +WORKSPACE_PATH = Path(__file__).parent.resolve() +LIBCARLA_ROOT_PATH = WORKSPACE_PATH / 'LibCarla' +PYTHON_API_PATH = WORKSPACE_PATH / 'PythonAPI' +EXAMPLES_PATH = WORKSPACE_PATH / 'Examples' +BUILD_PATH = WORKSPACE_PATH / 'Build' +DEPENDENCIES_PATH = BUILD_PATH / 'Dependencies' +DIST_PATH = WORKSPACE_PATH / 'Dist' +UTIL_PATH = WORKSPACE_PATH / 'Util' +DOCKER_UTILS_PATH = UTIL_PATH / 'DockerUtils' +BUILD_TOOLS_PATH = UTIL_PATH / 'BuildTools' # UE plugin -carla_ue_path = workspace_path / 'Unreal' / 'CarlaUE4' -carla_ue_plugin_root_path = carla_ue_path / 'Plugins' -carla_ue_plugins_path = carla_ue_plugin_root_path / 'Carla' -carla_ue_plugin_deps_path = carla_ue_plugin_root_path / 'CarlaDependencies' +CARLA_UE_PATH = WORKSPACE_PATH / 'Unreal' / 'CarlaUE4' +CARLA_UE_PLUGIN_ROOT_PATH = CARLA_UE_PATH / 'Plugins' +CARLA_UE_PLUGIN_PATH = CARLA_UE_PLUGIN_ROOT_PATH / 'Carla' +CARLA_UE_PLUGIN_DEPENDENCIES_PATH = CARLA_UE_PLUGIN_ROOT_PATH / 'CarlaDependencies' # PythonAPI -python_version_major = sys.version_info.major -python_version_minor = sys.version_info.minor -python_api_source_path = python_api_path / 'carla' +PYTHON_API_SOURCE_PATH = PYTHON_API_PATH / 'carla' # LibCarla -libcarla_build_path_server = build_path / 'libcarla-server-build' -libcarla_build_path_client = build_path / 'libcarla-client-build' -libcarla_build_path_pytorch = build_path / 'libcarla-pytorch-build' -libcarla_install_path_server = carla_ue_plugin_deps_path -libcarla_install_path_client = python_api_source_path / 'dependencies' -libcarla_test_content_path = build_path / 'test-content' +LIBCARLA_BUILD_PATH = BUILD_PATH +LIBCARLA_TEST_CONTENT_PATH = BUILD_PATH / 'test-content' # OSM2ODR -osm2odr_build_path = build_path / 'libosm2dr-build' -osm2odr_build_path_server = build_path / 'libosm2dr-build-server' -osm2odr_source_path = build_path / 'libosm2dr-source' +OSM2ODR_BUILD_PATH = BUILD_PATH / 'libosm2dr-build' +OSM2ODR_BUILD_PATH_SERVER = BUILD_PATH / 'libosm2dr-build-server' +OSM2ODR_SOURCE_PATH = BUILD_PATH / 'libosm2dr-source' # Misc -test_results_path = build_path / 'test-results' -libstdcpp_toolchain_path = build_path / 'LibStdCppToolChain.cmake' -libcpp_toolchain_path = build_path / 'LibCppToolChain.cmake' -executable_extension = '.exe' if os.name == 'nt' else '' -library_extension = '.lib' if os.name == 'nt' else '.a' -object_extension = '.obj' if os.name == 'nt' else '.o' -shell_script_extension = '.bat' if os.name == 'nt' else '' +TEST_RESULTS_PATH = BUILD_PATH / 'test-results' +LIBSTDCPP_TOOLCHAIN_PATH = BUILD_PATH / 'LibStdCppToolChain.cmake' +LIBCPP_TOOLCHAIN_PATH = BUILD_PATH / 'LibCppToolChain.cmake' +EXE_EXT = '.exe' if os.name == 'nt' else '' +LIB_EXT = '.lib' if os.name == 'nt' else '.a' +OBJ_EXT = '.obj' if os.name == 'nt' else '.o' +SHELL_EXT = '.bat' if os.name == 'nt' else '' # Unreal Engine -ue_workspace_path = os.getenv('UE4_ROOT') -if ue_workspace_path is None: +UE_WORKSPACE_PATH = Path(os.getenv('UE4_ROOT', '')) +if not UE_WORKSPACE_PATH.exists(): print('Could not find Unreal Engine workspace. Please set the environment variable UE4_ROOT as specified in the docs.') -ue_workspace_path = Path(ue_workspace_path) -# Dependencies -dependency_list_file_path = util_path / 'Dependencies.json' -# Houdini -houdini_url = 'https://github.com/sideeffects/HoudiniEngineForUnreal.git' -houdini_plugin_path = carla_ue_plugin_root_path / 'HoudiniEngine' -houdini_commit_hash = '55b6a16cdf274389687fce3019b33e3b6e92a914' -houdini_patch_path = util_path / 'Patches' / 'houdini_patch.txt' + exit(-1) # Omniverse -omniverse_plugin_path = ue_workspace_path / 'Engine' / 'Plugins' / 'Marketplace' / 'NVIDIA' / 'Omniverse' -omniverse_patch_path = util_path / 'Patches' / 'omniverse_4.26' +NV_OMNIVERSE_PLUGIN_PATH = UE_WORKSPACE_PATH / 'Engine' / 'Plugins' / 'Marketplace' / 'NVIDIA' / 'Omniverse' +NV_OMNIVERSE_PATCH_PATH = UTIL_PATH / 'Patches' / 'omniverse_4.26' # Script settings -parallelism = psutil.cpu_count(logical = True) -force_sequential = False -cmake_generator = 'Ninja' +DEFAULT_PARALLELISM = psutil.cpu_count(logical = True) +CMAKE_GENERATOR = 'Ninja' -dependencies = { - 'boost' : - { - 'from' : - [ - { - 'url': 'https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.zip', - 'type': 'download' - }, - { - 'url': 'https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/boost_1_83_0.zip', - 'type': 'download' - } - ] - }, - 'chrono' : - { - 'from' : - [ - { - 'url' : 'https://github.com/projectchrono/chrono.git', - 'type' : 'git', - 'tag-or-branch' : '8.0.0' - } - ] - }, - 'eigen' : - { - 'from' : - [ - { - 'url' : 'https://gitlab.com/libeigen/eigen.git', - 'type' : 'git', - 'branch' : '3.4.0' - } - ] - }, - 'libpng' : - { - 'from' : - [ - { - 'url' : 'https://github.com/glennrp/libpng.git', - 'type' : 'git', - 'tag-or-branch' : 'v1.6.40' - } - ] - }, - 'proj' : - { - 'from' : - [ - { - 'url' : 'https://download.osgeo.org/proj/proj-7.2.1.tar.gz', - 'type' : 'download' - }, - { - 'url' : 'https://github.com/madler/zlib.git', - 'type' : 'git' - } - ] - }, - 'gtest' : - { - 'from' : - [ - { - 'url' : 'https://github.com/google/googletest.git', - 'type' : 'git', - 'tag-or-branch' : 'v1.14.0' - } - ] - }, - 'zlib' : - { - 'from' : - [ - { - 'url' : 'https://zlib.net/current/zlib.tar.gz', - 'type' : 'download' - } - ] - }, - 'xercesc' : - { - 'from' : - [ - { - 'url' : 'https://github.com/apache/xerces-c.git', - 'type' : 'git', - 'tag-or-branch' : 'v3.2.4' - }, - { - 'url' : 'https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.2.3.zip', - 'type' : 'download' - }, - { - 'url' : 'https://carla-releases.s3.eu-west-3.amazonaws.com/Backup/xerces-c-3.2.3.zip', - 'type' : 'download' - } - ] - }, - 'sqlite' : - { - 'from' : - [ - { - 'url' : 'https://www.sqlite.org/2021/sqlite-amalgamation-3340100.zip', - 'type' : 'download' - } - ] - }, - 'rpclib' : - { - 'from' : - [ - { - 'url' : 'https://github.com/rpclib/rpclib.git', - 'type' : 'git', - 'tag-or-branch' : 'v2.3.0' - } - ] - }, - 'recast' : - { - 'from' : - [ - { - 'url' : 'https://github.com/recastnavigation/recastnavigation.git', - 'type' : 'git', - 'tag-or-branch' : '1.6.0' - } - ] - }, - 'libosmscout' : - { - 'from' : - [ - { - 'url' : 'https://github.com/Framstag/libosmscout.git', - 'type' : 'git', - 'tag-or-branch' : 'master' - } - ] - }, - 'lunasvg' : - { - 'from' : - [ - { - 'url' : 'https://github.com/sammycage/lunasvg.git', - 'type' : 'git', - 'tag-or-branch' : 'master' - } - ] - }, - 'sumo' : - { - 'from' : - [ - { - 'url' : 'https://github.com/carla-simulator/sumo.git', - 'type' : 'git', - 'tag-or-branch' : 'carla_osm2odr' - } - ] - } -} +# Dependency paths +BOOST_TOOLSET = 'msvc-14.2' +BOOST_TOOLSET_SHORT = 'vc142' +BOOST_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-source' +BOOST_BUILD_PATH = DEPENDENCIES_PATH / 'boost-build' +BOOST_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-install' +BOOST_INCLUDE_PATH = BOOST_INSTALL_PATH / 'include' +BOOST_LIBRARY_PATH = BOOST_INSTALL_PATH / 'lib' +BOOST_B2_PATH = BOOST_SOURCE_PATH / f'b2{EXE_EXT}' +EIGEN_SOURCE_PATH = DEPENDENCIES_PATH / 'eigen-source' +EIGEN_BUILD_PATH = DEPENDENCIES_PATH / 'eigen-build' +EIGEN_INSTALL_PATH = DEPENDENCIES_PATH / 'eigen-install' +CHRONO_SOURCE_PATH = DEPENDENCIES_PATH / 'chrono-source' +CHRONO_BUILD_PATH = DEPENDENCIES_PATH / 'chrono-build' +CHRONO_INSTALL_PATH = DEPENDENCIES_PATH / 'chrono-install' -class ConfigureContext: +GTEST_SOURCE_PATH = DEPENDENCIES_PATH / 'gtest-source' +GTEST_BUILD_PATH = DEPENDENCIES_PATH / 'gtest-build' +GTEST_INSTALL_PATH = DEPENDENCIES_PATH / 'gtest-install' - def __init__(self, arg): - self.pool = ProcessPoolExecutor(parallelism) - self.futures = [] - self.arg = arg - - def Dispatch(self, callable, *arg): - if len(arg) == 0: - if force_sequential: - callable() - else: - self.futures.append(self.pool.submit(callable)) - else: - if force_sequential: - callable(*arg) - else: - self.futures.append(self.pool.submit(callable, *arg)) - - def Wait(self): - if len(self.futures) == 0 or force_sequential: - return - for future in as_completed(self.futures): - try: - future.result() - except Exception as e: - Log(f'Failed to run task: {e}') - finally: - pass - self.futures = [] +ZLIB_SOURCE_PATH = DEPENDENCIES_PATH / 'zlib-source' +ZLIB_BUILD_PATH = DEPENDENCIES_PATH / 'zlib-build' +ZLIB_INSTALL_PATH = DEPENDENCIES_PATH / 'zlib-install' +ZLIB_LIBRARY_PATH = ZLIB_BUILD_PATH / f'zlib{LIB_EXT}' +LIBPNG_SOURCE_PATH = DEPENDENCIES_PATH / 'libpng-source' +LIBPNG_BUILD_PATH = DEPENDENCIES_PATH / 'libpng-build' +LIBPNG_INSTALL_PATH = DEPENDENCIES_PATH / 'libpng-install' +SQLITE_SOURCE_PATH = DEPENDENCIES_PATH / 'sqlite-source' +SQLITE_BUILD_PATH = DEPENDENCIES_PATH / 'sqlite-build' +SQLITE_INSTALL_PATH = DEPENDENCIES_PATH / 'sqlite-install' +SQLITE_EXECUTABLE_PATH = SQLITE_BUILD_PATH / f'sqlite{EXE_EXT}' +SQLITE_LIBRARY_PATH = SQLITE_BUILD_PATH / f'sqlite{LIB_EXT}' + +PROJ_SOURCE_PATH = DEPENDENCIES_PATH / 'proj-source' +PROJ_BUILD_PATH = DEPENDENCIES_PATH / 'proj-build' +PROJ_INSTALL_PATH = DEPENDENCIES_PATH / 'proj-install' + +RECAST_SOURCE_PATH = DEPENDENCIES_PATH / 'recast-source' +RECAST_BUILD_PATH = DEPENDENCIES_PATH / 'recast-build' +RECAST_INSTALL_PATH = DEPENDENCIES_PATH / 'recast-install' + +RPCLIB_SOURCE_PATH = DEPENDENCIES_PATH / 'rpclib-source' +RPCLIB_BUILD_PATH = DEPENDENCIES_PATH / 'rpclib-build' +RPCLIB_INSTALL_PATH = DEPENDENCIES_PATH / 'rpclib-install' + +XERCESC_SOURCE_PATH = DEPENDENCIES_PATH / 'xercesc-source' +XERCESC_BUILD_PATH = DEPENDENCIES_PATH / 'xercesc-build' +XERCESC_INSTALL_PATH = DEPENDENCIES_PATH / 'xercesc-install' + +LIBOSMSCOUT_SOURCE_PATH = DEPENDENCIES_PATH / 'libosmscout-source' +LIBOSMSCOUT_BUILD_PATH = DEPENDENCIES_PATH / 'libosmscout-build' +LIBOSMSCOUT_INSTALL_PATH = DEPENDENCIES_PATH / 'libosmscout-install' + +LUNASVG_SOURCE_PATH = DEPENDENCIES_PATH / 'lunasvg-source' +LUNASVG_BUILD_PATH = DEPENDENCIES_PATH / 'lunasvg-build' +LUNASVG_INSTALL_PATH = DEPENDENCIES_PATH / 'lunasvg-install' + +SUMO_SOURCE_PATH = DEPENDENCIES_PATH / 'sumo-source' +SUMO_BUILD_PATH = DEPENDENCIES_PATH / 'sumo-build' +SUMO_INSTALL_PATH = DEPENDENCIES_PATH / 'sumo-install' + +houdini_url = 'https://github.com/sideeffects/HoudiniEngineForUnreal.git' +houdini_plugin_path = CARLA_UE_PLUGIN_ROOT_PATH / 'HoudiniEngine' +houdini_commit_hash = '55b6a16cdf274389687fce3019b33e3b6e92a914' +houdini_patch_path = UTIL_PATH / 'Patches' / 'houdini_patch.txt' + +URL_SUFFIX = 'how_to_build_on_windows/\n' if os.name == "nt" else 'build_linux/\n' + +ERROR_MESSAGE = ( + '\n' + 'Ok, an error ocurred, don\'t panic!\n' + 'We have different platforms where you can find some help:\n' + '\n' + '- Make sure you have read the documentation:\n' + f' https://carla.readthedocs.io/en/latest/{URL_SUFFIX}' + '\n' + '- If the problem persists, submit an issue on our GitHub page:\n' + ' https://github.com/carla-simulator/carla/issues\n' + '\n' + '- Or just use our Discord server!\n' + ' We\'ll be glad to help you there:\n' + ' https://discord.gg/42KJdRj\n' +) def Log(message): message = str(message) message += '\n' print(message, end='') - - def LaunchSubprocess( cmd : list, display_output : bool = False, working_directory : Path = None): - if display_output or force_sequential: + if display_output or FORCE_SEQUENTIAL: return subprocess.run(cmd, cwd = working_directory) else: return subprocess.run( @@ -349,8 +399,6 @@ def LaunchSubprocessImmediate( ) print(error_message) - - def UpdateGitRepository(path : Path, url : str, branch : str = None): if path.exists(): LaunchSubprocessImmediate([ @@ -372,8 +420,6 @@ def UpdateGitRepository(path : Path, url : str, branch : str = None): cmd.append(path.stem) LaunchSubprocessImmediate(cmd) - - def DownloadDependency(name : str, path : Path, url : str): # Download: try: @@ -408,317 +454,181 @@ def DownloadDependency(name : str, path : Path, url : str): except Exception as err: Log(f'Failed to extract dependency "{name}": {err}') - - -def UpdateDependency(name : str, dep : dict): - download_path = build_path / f'{name}-source' - sources = dep.get('from', []) - assert type(sources) == type([]) - for source in sources: - url = source['url'] - source_type = source['type'] +def UpdateDependency(dep : Dependency): + name = dep.name + Log(f'Updating {name}.') + download_path = DEPENDENCIES_PATH / f'{name}-source' + for source in dep.sources: try: - if source_type == 'git': - branch = source.get('branch-or-tag', None) - UpdateGitRepository(download_path, url, branch) - elif source_type == 'download': + if type(source) is GitRepository: + branch = source.tag + UpdateGitRepository(download_path, source.url, branch) + elif type(source) is Download: if download_path.exists(): Log(f'Dependency "{name}" already present. Delete "{download_path}" if you wish for it to be downloaded again.') else: - DownloadDependency(name, download_path, url) + DownloadDependency(name, download_path, source.url) return finally: pass Log(f'Failed to update dependency "{name}".') assert False +def BuildLibCarlaMain(c : Context): + c.task_graph.Add(Task.CreateCMakeConfigureDefault('configure-libcarla', [], WORKSPACE_PATH, BUILD_PATH, + f'-DBUILD_LIBCARLA_SERVER={"ON" if c.args.build_libcarla_server else "OFF"}', + f'-DBUILD_LIBCARLA_CLIENT={"ON" if c.args.build_libcarla_client else "OFF"}', + f'-DBUILD_OSM_WORLD_RENDERER={"ON" if c.args.build_osm_world_renderer else "OFF"}', + f'-DLIBCARLA_PYTORCH={"OFF" if c.args.skip_libcarla_pytorch else "ON"}')) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-libcarla', [ 'configure-libcarla' ], BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-libcarla', [ 'build-libcarla' ], BUILD_PATH)) - -def ConfigureLibCarlaClient(): - cmd = [ 'cmake' ] - return LaunchSubprocessImmediate(cmd) - - - -def ConfigureLibCarlaServer(): - cmd = [ 'cmake' ] - return LaunchSubprocessImmediate(cmd) - - - -def BuildLibCarlaClient(): - ConfigureLibCarlaClient() - - - -def BuildLibCarlaServer(): - ConfigureLibCarlaServer() - - - -def BuildLibCarlaMain(c : ConfigureContext): - if not c.arg.skip_libcarla_server: - Log('Building LibCarla Server') - c.Dispatch(BuildLibCarlaServer) - if not c.arg.skip_libcarla_client: - Log('Building LibCarla Client') - c.Dispatch(BuildLibCarlaClient) - - - -def BuildCarlaUEMain(): +def BuildCarlaUECore(c : Context): if os.name == 'nt': LaunchSubprocessImmediate([ - ue_workspace_path / 'Engine' / 'Build' / 'BatchFiles' / 'Build.bat', + UE_WORKSPACE_PATH / 'Engine' / 'Build' / 'BatchFiles' / 'Build.bat', 'CarlaUE4', 'Win64', 'Development', '-WaitMutex', '-FromMsBuild', - carla_ue_path / 'CarlaUE4.uproject' + CARLA_UE_PATH / 'CarlaUE4.uproject' ]) else: pass +def BuildCarlaUEMain(c : Context): + Log('Building Carla Unreal Engine Editor') + # c.args.enable_carsim + # c.args.enable_chrono + # c.args.enable_unity + if c.args.enable_omniverse: + c.task_graph.Add(Task('install-nv_omniverse', [], InstallNVIDIAOmniverse)) + c.task_graph.Add(Task('build-carla_ue', [], BuildCarlaUECore)) +def BuildPythonAPIMain(c : Context): + Log('Building Python API') + pass + +def SetupUnrealEngine(c : Context): + Log('Setting up Unreal Engine.') def InstallNVIDIAOmniverse(): filename = 'USDCarlaInterface' header = f'{filename}.h' source = f'{filename}.cpp' - omniverse_usd_path = omniverse_plugin_path / 'Source' / 'OmniverseUSD' + omniverse_usd_path = NV_OMNIVERSE_PLUGIN_PATH / 'Source' / 'OmniverseUSD' files = [ - [ omniverse_usd_path / 'Public' / header, omniverse_patch_path / header ], - [ omniverse_usd_path / 'Private' / source, omniverse_patch_path / source ], + [ omniverse_usd_path / 'Public' / header, NV_OMNIVERSE_PATCH_PATH / header ], + [ omniverse_usd_path / 'Private' / source, NV_OMNIVERSE_PATCH_PATH / source ], ] for src, dst in files: shutil.copyfile(src, dst) - - -def BuildCarlaUEMain(c : ConfigureContext): - Log('Building Carla UE Editor') - # c.arg.use_carsim - # c.arg.use_chrono - # c.arg.use_unity - if c.arg.use_omniverse: - c.Dispatch(InstallNVIDIAOmniverse) - c.Wait() - c.Dispatch(BuildCarlaUEMain) - c.Wait() - - - -def BuildPythonAPIMain(c : ConfigureContext): - Log('Building Python API') - pass - - - -def SetupUnrealEngine(c : ConfigureContext): - Log('Setting up Unreal Engine.') - - - -def UpdateDependencies(c : ConfigureContext): - for name, dep in dependencies.items(): - Log(f'Updating {name}.') - try: - c.Dispatch(UpdateDependency, name, dep) - except Exception as err: - Log(f'Failed to update "{name}": {err}') - - - -def GetDefaultCMakeConfigureCommandLine(source_path : Path, build_path : Path) -> list: - return [ - 'cmake', - '-G', cmake_generator, - '-S', source_path, - '-B', build_path, - '-DCMAKE_C_COMPILER=' + c_compiler, - '-DCMAKE_CXX_COMPILER=' + cpp_compiler, - '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_CXX_FLAGS_RELEASE="/MD"' - ] - - - -def DefaultBuild(path : Path): - LaunchSubprocessImmediate([ 'cmake', '--build', path ], display_output = True) - - - -def DefaultInstall(path : Path, prefix : Path): - LaunchSubprocessImmediate([ 'cmake', '--install', path, '--prefix', prefix ], display_output = True) - - - -boost_toolset = 'msvc-14.2' -boost_toolset_short = 'vc142' -boost_source_path = build_path / 'boost-source' -boost_build_path = build_path / 'boost-build' -boost_install_path = build_path / 'boost-install' -boost_include_path = boost_install_path / 'include' -boost_library_path = boost_install_path / 'lib' -boost_b2_path = boost_source_path / f'b2{executable_extension}' +def UpdateDependencies(c : Context): + unique_dependencies = set(DEFAULT_DEPENDENCIES) + if c.args.build_osm_world_renderer: + unique_dependencies.update(OSM_WORLD_RENDERER_DEPENDENCIES) + if c.args.build_osm2odr: + unique_dependencies.update(OSM2ODR_DEPENDENCIES) + for dep in unique_dependencies: + name = dep.name + c.task_graph.Add(Task(f'update-{name}', [], UpdateDependency, dep)) + c.task_graph.Execute() def ConfigureBoost(): - if boost_b2_path.exists(): + if BOOST_B2_PATH.exists(): return LaunchSubprocessImmediate( - [ boost_source_path / f'bootstrap{shell_script_extension}' ], - working_directory = boost_source_path) + [ BOOST_SOURCE_PATH / f'bootstrap{SHELL_EXT}' ], + working_directory = BOOST_SOURCE_PATH) -def BuildBoost(): +def BuildAndInstallBoost(): LaunchSubprocessImmediate([ - boost_b2_path, - f'-j{parallelism}', - '--layout=system', - f'--build-dir={boost_build_path}', # ??? - '--with-system', - '--with-filesystem', - '--with-python', - '--with-date_time', + BOOST_B2_PATH, + f'-j{DEFAULT_PARALLELISM}', 'architecture=x86', 'address-model=64', - f'toolset={boost_toolset}', + f'toolset={BOOST_TOOLSET}', 'variant=release', 'link=static', 'runtime-link=shared', 'threading=multi', - f'--prefix={boost_build_path}', - f'--libdir={boost_library_path}', - f'--includedir={boost_include_path}', + '--layout=system', + '--with-system', + '--with-filesystem', + '--with-python', + '--with-date_time', + f'--build-dir={BOOST_BUILD_PATH}', + f'--prefix={BOOST_INSTALL_PATH}', + f'--libdir={BOOST_LIBRARY_PATH}', + f'--includedir={BOOST_INCLUDE_PATH}', 'install' - ], display_output = True, working_directory = boost_source_path) - - - -eigen_source_path = build_path / 'eigen-source' -eigen_build_path = build_path / 'eigen-build' -eigen_install_path = build_path / 'eigen-install' - -# def ConfigureEigen(): -# pass - -# def BuildEigen(): -# pass - - - -chrono_source_path = build_path / 'chrono-source' -chrono_build_path = build_path / 'chrono-build' -chrono_install_path = build_path / 'chrono-install' - -def ConfigureChrono(): - cmd = GetDefaultCMakeConfigureCommandLine(chrono_source_path, chrono_build_path) - cmd.extend([ - f'-DEIGEN3_INCLUDE_DIR={eigen_source_path}', - '-DENABLE_MODULE_VEHICLE=ON', - chrono_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildChrono(): - return DefaultBuild(chrono_build_path) - - - -gtest_source_path = build_path / 'gtest-source' -gtest_build_path = build_path / 'gtest-build' -gtest_install_path = build_path / 'gtest-install' - -def ConfigureGTest(): - cmd = GetDefaultCMakeConfigureCommandLine(gtest_source_path, gtest_build_path) - cmd.extend([ - gtest_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildGTest(): - return DefaultBuild(gtest_build_path) - - - -zlib_source_path = build_path / 'zlib-source' -zlib_build_path = build_path / 'zlib-build' -zlib_install_path = build_path / 'zlib-install' -zlib_library_path = zlib_build_path / f'zlib{library_extension}' - -def ConfigureZLib(): - cmd = GetDefaultCMakeConfigureCommandLine(zlib_source_path, zlib_build_path) - cmd.extend([ - zlib_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildZLib(): - return DefaultBuild(zlib_build_path) - - - -libpng_source_path = build_path / 'libpng-source' -libpng_build_path = build_path / 'libpng-build' -libpng_install_path = build_path / 'libpng-install' - -def ConfigureLibPNG(): - cmd = GetDefaultCMakeConfigureCommandLine(libpng_source_path, libpng_build_path) - cmd.extend([ - '-DPNG_TESTS=OFF', - '-DPNG_TOOLS=OFF', - '-DPNG_BUILD_ZLIB=ON', - f'-DZLIB_INCLUDE_DIRS={zlib_source_path};{zlib_build_path}', - f'-DZLIB_LIBRARIES={zlib_library_path}', - libpng_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildLibPNG(): - return DefaultBuild(libpng_build_path) - - - -sqlite_source_path = build_path / 'sqlite-source' -sqlite_build_path = build_path / 'sqlite-build' -sqlite_install_path = build_path / 'sqlite-install' -sqlite_executable_path = sqlite_build_path / f'sqlite{executable_extension}' -sqlite_library_path = sqlite_build_path / f'sqlite{library_extension}' + ], display_output = True, working_directory = BOOST_SOURCE_PATH) def BuildSQLite(): - sqlite_build_path.mkdir(exist_ok = True) - sqlite_sources = glob.glob(str(sqlite_source_path / '**' / '*.c'), recursive = True) - + SQLITE_BUILD_PATH.mkdir(exist_ok = True) + sqlite_sources = glob.glob(f'{SQLITE_SOURCE_PATH}/**/*.c', recursive = True) if os.name == 'nt' and 'clang' in c_compiler: - if not sqlite_executable_path.exists(): + if not SQLITE_EXECUTABLE_PATH.exists(): cmd = [ c_compiler, f'-fuse-ld={linker}', '-march=native', '/O2', '/MD', '/EHsc', ] cmd.extend(sqlite_sources) cmd.append('-o') - cmd.append(sqlite_executable_path) + cmd.append(SQLITE_EXECUTABLE_PATH) LaunchSubprocessImmediate(cmd) - if not sqlite_library_path.exists(): + if not SQLITE_LIBRARY_PATH.exists(): cmd = [ c_compiler, f'-fuse-ld={lib}', '-march=native', '/O2', '/MD', '/EHsc', ] cmd.extend(sqlite_sources) cmd.append('-o') - cmd.append(sqlite_library_path) + cmd.append(SQLITE_LIBRARY_PATH) LaunchSubprocessImmediate(cmd) else: pass +def FindXercesC(): + return glob.glob(f'{XERCESC_INSTALL_PATH}/**/xerces-c*{LIB_EXT}', recursive=True)[0] +def BuildDependencies(c : Context): -proj_source_path = build_path / 'proj-source' -proj_build_path = build_path / 'proj-build' -proj_install_path = build_path / 'proj-install' + # Configure: -def ConfigureProj(): - cmd = GetDefaultCMakeConfigureCommandLine(proj_source_path, proj_build_path) - cmd.extend([ - f'-DSQLITE3_INCLUDE_DIR={sqlite_source_path}', - f'-DSQLITE3_LIBRARY={sqlite_library_path}', - f'-DEXE_SQLITE3={sqlite_executable_path}', + c.task_graph.Add(Task('build-sqlite', [], BuildSQLite)) + + c.task_graph.Add(Task('configure-boost', [], ConfigureBoost)) + + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-zlib', + [], + ZLIB_SOURCE_PATH, + ZLIB_BUILD_PATH)) + + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-gtest', + [], + GTEST_SOURCE_PATH, + GTEST_BUILD_PATH)) + + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-libpng', + [], + LIBPNG_SOURCE_PATH, + LIBPNG_BUILD_PATH, + '-DPNG_TESTS=OFF', + '-DPNG_TOOLS=OFF', + '-DPNG_BUILD_ZLIB=ON', + f'-DZLIB_INCLUDE_DIRS={ZLIB_SOURCE_PATH};{ZLIB_BUILD_PATH}', + f'-DZLIB_LIBRARIES={ZLIB_LIBRARY_PATH}')) + + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-proj', + [], + PROJ_SOURCE_PATH, + PROJ_BUILD_PATH, + f'-DSQLITE3_INCLUDE_DIR={SQLITE_SOURCE_PATH}', + f'-DSQLITE3_LIBRARY={SQLITE_LIBRARY_PATH}', + f'-DEXE_SQLITE3={SQLITE_EXECUTABLE_PATH}', '-DWIN32_LEAN_AND_MEAN=1', '-DVC_EXTRALEAN=1', '-DNOMINMAX=1', @@ -732,275 +642,211 @@ def ConfigureProj(): '-DBUILD_GEOD=OFF', '-DBUILD_GIE=OFF', '-DBUILD_PROJ=OFF', - '-DBUILD_TESTING=OFF', - proj_source_path - ]) - return LaunchSubprocessImmediate(cmd) + '-DBUILD_TESTING=OFF')) -def BuildProj(): - return DefaultBuild(proj_build_path) - - - -recast_source_path = build_path / 'recast-source' -recast_build_path = build_path / 'recast-build' -recast_install_path = build_path / 'recast-install' - -def ConfigureRecast(): - cmd = GetDefaultCMakeConfigureCommandLine(recast_source_path, recast_build_path) - cmd.extend([ + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-recast', + [], + RECAST_SOURCE_PATH, + RECAST_BUILD_PATH, '-DRECASTNAVIGATION_DEMO=OFF', '-DRECASTNAVIGATION_TESTS=OFF', - '-DRECASTNAVIGATION_EXAMPLES=OFF', - recast_source_path - ]) - return LaunchSubprocessImmediate(cmd) + '-DRECASTNAVIGATION_EXAMPLES=OFF')) -def BuildRecast(): - return DefaultBuild(recast_build_path) - - - -rpclib_source_path = build_path / 'rpclib-source' -rpclib_build_path = build_path / 'rpclib-build' -rpclib_install_path = build_path / 'rpclib-install' - -def ConfigureRPCLib(): - cmd = GetDefaultCMakeConfigureCommandLine(rpclib_source_path, rpclib_build_path) - cmd.extend([ + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-rpclib', + [], + RPCLIB_SOURCE_PATH, + RPCLIB_BUILD_PATH, '-DRPCLIB_BUILD_TESTS=OFF', '-DRPCLIB_GENERATE_COMPDB=OFF', '-DRPCLIB_BUILD_EXAMPLES=OFF', '-DRPCLIB_ENABLE_LOGGING=OFF', '-DRPCLIB_ENABLE_COVERAGE=OFF', - '-DRPCLIB_MSVC_STATIC_RUNTIME=OFF', - rpclib_source_path - ]) - return LaunchSubprocessImmediate(cmd) + '-DRPCLIB_MSVC_STATIC_RUNTIME=OFF')) -def BuildRPCLib(): - return DefaultBuild(rpclib_build_path) + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-xercesc', + [], + XERCESC_SOURCE_PATH, + XERCESC_BUILD_PATH)) + if c.args.build_osm_world_renderer: + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-libosmscout', + [], + LIBOSMSCOUT_SOURCE_PATH, + LIBOSMSCOUT_BUILD_PATH, + '-DOSMSCOUT_BUILD_TOOL_STYLEEDITOR=OFF', + '-DOSMSCOUT_BUILD_TOOL_OSMSCOUT2=OFF', + '-DOSMSCOUT_BUILD_TESTS=OFF', + '-DOSMSCOUT_BUILD_CLIENT_QT=OFF', + '-DOSMSCOUT_BUILD_DEMOS=OFF')) + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-lunasvg', + [], + LUNASVG_SOURCE_PATH, + LUNASVG_BUILD_PATH)) -xercesc_source_path = build_path / 'xercesc-source' -xercesc_build_path = build_path / 'xercesc-build' -xercesc_install_path = build_path / 'xercesc-install' + if c.args.build_osm2odr: + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-sumo', + [], + SUMO_SOURCE_PATH, + SUMO_BUILD_PATH, + '-DSUMO_LIBRARIES=OFF', + f'-DZLIB_INCLUDE_DIR={ZLIB_SOURCE_PATH}', + f'-DZLIB_LIBRARY={ZLIB_INSTALL_PATH}/zlib{LIB_EXT}', + f'-DPROJ_INCLUDE_DIR={PROJ_INSTALL_PATH}/include', + f'-DPROJ_LIBRARY={PROJ_INSTALL_PATH}/lib/proj{LIB_EXT}', + f'-DXercesC_INCLUDE_DIR={XERCESC_INSTALL_PATH}/include', + f'-DXercesC_LIBRARY={FindXercesC()}')) -def ConfigureXercesc(): - cmd = GetDefaultCMakeConfigureCommandLine(xercesc_source_path, xercesc_build_path) - cmd.extend([ - xercesc_source_path - ]) - return LaunchSubprocessImmediate(cmd) + if c.args.enable_chrono: + c.task_graph.Add(Task.CreateCMakeConfigureDefault( + 'configure-chrono', + [], + CHRONO_SOURCE_PATH, + CHRONO_BUILD_PATH, + f'-DEIGEN3_INCLUDE_DIR={EIGEN_SOURCE_PATH}', + '-DENABLE_MODULE_VEHICLE=ON')) + c.task_graph.Execute() -def BuildXercesc(): - return DefaultBuild(xercesc_build_path) + # Build: + c.task_graph.sequential = True + c.task_graph.Add(Task('build-boost', [ 'configure-boost' ], BuildAndInstallBoost)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-zlib', [ 'configure-zlib' ], ZLIB_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-gtest', [ 'configure-gtest' ], GTEST_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-libpng', [ 'configure-libpng' ], LIBPNG_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-proj', [ 'configure-proj' ], PROJ_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-recast', [ 'configure-recast' ], RECAST_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-rpclib', [ 'configure-rpclib' ], RPCLIB_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-xercesc', [ 'configure-xercesc' ], XERCESC_BUILD_PATH)) + if c.args.build_osm_world_renderer: + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-lunasvg', [ 'configure-lunasvg' ], LUNASVG_BUILD_PATH)) + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-libosmscout', [ 'configure-libosmscout' ], LIBOSMSCOUT_BUILD_PATH)) + if c.args.build_osm2odr: + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-sumo', [ 'configure-sumo' ], SUMO_BUILD_PATH)) + if c.args.enable_chrono: + c.task_graph.Add(Task.CreateCMakeBuildDefault('build-chrono', [ 'configure-chrono' ], CHRONO_BUILD_PATH)) + c.task_graph.Execute() + c.task_graph.sequential = False + + # Install: - -libosmscout_source_path = build_path / 'libosmscout-source' -libosmscout_build_path = build_path / 'libosmscout-build' -libosmscout_install_path = build_path / 'libosmscout-install' - -def ConfigureLibOSMScout(): - cmd = GetDefaultCMakeConfigureCommandLine(libosmscout_source_path, libosmscout_build_path) - cmd.extend([ - '-DOSMSCOUT_BUILD_TOOL_STYLEEDITOR=OFF', - '-DOSMSCOUT_BUILD_TOOL_OSMSCOUT2=OFF', - '-DOSMSCOUT_BUILD_TESTS=OFF', - '-DOSMSCOUT_BUILD_CLIENT_QT=OFF', - '-DOSMSCOUT_BUILD_DEMOS=OFF', - libosmscout_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildLibOSMScout(): - return DefaultBuild(libosmscout_build_path) - - - -lunasvg_source_path = build_path / 'lunasvg-source' -lunasvg_build_path = build_path / 'lunasvg-build' -lunasvg_install_path = build_path / 'lunasvg-install' - -def ConfigureLunaSVG(): - cmd = GetDefaultCMakeConfigureCommandLine(lunasvg_source_path, lunasvg_build_path) - cmd.extend([ - lunasvg_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildLunaSVG(): - return DefaultBuild(lunasvg_build_path) - - - -sumo_source_path = build_path / 'sumo-source' -sumo_build_path = build_path / 'sumo-build' -sumo_install_path = build_path / 'sumo-install' - -def ConfigureSUMO(): - cmd = GetDefaultCMakeConfigureCommandLine(sumo_source_path, sumo_build_path) - xercesc_library = glob.glob(xercesc_install_path / 'lib' / f'*{library_extension}', recursive = True)[0] - cmd.extend([ - '-DSUMO_LIBRARIES=OFF', - f'-DZLIB_INCLUDE_DIR={zlib_source_path}', - f'-DZLIB_LIBRARY={zlib_install_path}/zlib{library_extension}', - f'-DPROJ_INCLUDE_DIR={proj_install_path}/include', - f'-DPROJ_LIBRARY={proj_install_path}/lib/proj{library_extension}', - f'-DXercesC_INCLUDE_DIR={xercesc_install_path}/include', - f'-DXercesC_LIBRARY={xercesc_library}', - sumo_source_path - ]) - return LaunchSubprocessImmediate(cmd) - -def BuildSUMO(): - return DefaultBuild(sumo_build_path) - - - -def BuildDependencies(c : ConfigureContext): - Log('--- BUILDING SQLITE ---') - c.Dispatch(BuildSQLite) - Log('--- CONFIGURING ZLIB ---') - c.Dispatch(ConfigureZLib) - Log('--- BUILDING ZLIB ---') - c.Dispatch(BuildZLib) - Log('--- CONFIGURING BOOST ---') - c.Dispatch(ConfigureBoost) - Log('--- CONFIGURING GOOGLETEST ---') - c.Dispatch(ConfigureGTest) - Log('--- CONFIGURING LIBPNG ---') - c.Dispatch(ConfigureLibPNG) - Log('--- CONFIGURING PROJ ---') - c.Dispatch(ConfigureProj) - Log('--- CONFIGURING RECAST ---') - c.Dispatch(ConfigureRecast) - Log('--- CONFIGURING RPCLIB ---') - c.Dispatch(ConfigureRPCLib) - Log('--- CONFIGURING XERCES-C ---') - c.Dispatch(ConfigureXercesc) - Log('--- CONFIGURING LIBOSMSCOUT ---') - c.Dispatch(ConfigureLibOSMScout) - Log('--- CONFIGURING LUNASVG ---') - c.Dispatch(ConfigureLunaSVG) - Log('--- CONFIGURING SUMO ---') - c.Dispatch(ConfigureSUMO) - if c.arg.use_chrono: - Log('--- CONFIGURING CHRONO ---') - c.Dispatch(ConfigureChrono) - c.Wait() - Log('--- BUILDING BOOST ---') - BuildBoost() - Log('--- BUILDING GOOGLETEST ---') - BuildGTest() - Log('--- BUILDING LIBPNG ---') - BuildLibPNG() - Log('--- BUILDING PROJ ---') - BuildProj() - Log('--- BUILDING RECAST ---') - BuildRecast() - Log('--- BUILDING RPCLIB ---') - BuildRPCLib() - Log('--- BUILDING XERCESC ---') - BuildXercesc() - Log('--- BUILDING LUNASVG ---') - BuildLunaSVG() - Log('--- BUILDING LIBOSMSCOUT ---') - BuildLibOSMScout() - Log('--- BUILDING SUMO ---') - BuildSUMO() - if c.arg.use_chrono: - Log('--- BUILDING CHRONO ---') - BuildChrono() - c.Wait() - DefaultInstall(gtest_build_path, gtest_install_path) - DefaultInstall(libpng_build_path, libpng_install_path) - DefaultInstall(proj_build_path, proj_install_path) - DefaultInstall(recast_build_path, recast_install_path) - DefaultInstall(rpclib_build_path, rpclib_install_path) - DefaultInstall(xercesc_build_path, xercesc_install_path) - DefaultInstall(libosmscout_build_path, libosmscout_install_path) - if c.arg.use_chrono: - DefaultInstall(chrono_build_path, chrono_install_path) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-zlib', [ 'build-zlib' ], ZLIB_BUILD_PATH, ZLIB_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-gtest', [ 'build-gtest' ], GTEST_BUILD_PATH, GTEST_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-libpng', [ 'build-libpng' ], LIBPNG_BUILD_PATH, LIBPNG_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-proj', [ 'build-proj' ], PROJ_BUILD_PATH, PROJ_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-recast', [ 'build-recast' ], RECAST_BUILD_PATH, RECAST_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-rpclib', [ 'build-rpclib' ], RPCLIB_BUILD_PATH, RPCLIB_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-xercesc', [ 'build-xercesc' ], XERCESC_BUILD_PATH, XERCESC_INSTALL_PATH)) + if c.args.build_osm_world_renderer: + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-lunasvg', [ 'build-lunasvg' ], LUNASVG_BUILD_PATH, LUNASVG_INSTALL_PATH)) + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-libosmscout', [ 'build-libosmscout' ], LIBOSMSCOUT_BUILD_PATH, LIBOSMSCOUT_INSTALL_PATH)) + if c.args.build_osm2odr: + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-sumo', [ 'build-sumo' ], SUMO_BUILD_PATH, SUMO_INSTALL_PATH)) + if c.args.enable_chrono: + c.task_graph.Add(Task.CreateCMakeInstallDefault('install-chrono', [ 'build-chrono' ], CHRONO_BUILD_PATH, CHRONO_INSTALL_PATH)) + c.task_graph.Execute() def ParseCommandLine(): arg_parser = ArgumentParser(description = __doc__) + + BUILD_LIBCARLA_CLIENT_OVERRIDE = True + BUILD_LIBCARLA_SERVER_OVERRIDE = True + BUILD_PYTHON_API_OVERRIDE = True + BUILD_CARLA_UE_OVERRIDE = True + UPDATE_DEPENDENCIES_OVERRIDE = True + BUILD_DEPENDENCIES_OVERRIDE = True + BUILD_OSM2ODR_OVERRIDE = False + BUILD_OSM_WORLD_RENDERER_OVERRIDE = False + BUILD_PACKAGE_OVERRIDE = True + CLEAN_OVERRIDE = False arg_parser.add_argument( - '-build-libcarla', + '-build-libcarla-client', action='store_true', - help = 'Build LibCarla.') + default = BUILD_LIBCARLA_CLIENT_OVERRIDE, + help = 'Whether to build LibCarla Client.') + + arg_parser.add_argument( + '-build-libcarla-server', + action='store_true', + default = BUILD_LIBCARLA_SERVER_OVERRIDE, + help = 'Whether to build LibCarla Server.') arg_parser.add_argument( '-build-python-api', action='store_true', - help = 'Build the CARLA Python API.') + default = BUILD_PYTHON_API_OVERRIDE, + help = 'Whether to build the CARLA Python API.') arg_parser.add_argument( - '-build-carla-ue', + '-build-carla-unreal', action='store_true', - help = 'Build Carla Unreal.') + default = BUILD_CARLA_UE_OVERRIDE, + help = 'Build to build the Unreal Engine Carla backend.') arg_parser.add_argument( '-update-dependencies', action='store_true', + default = UPDATE_DEPENDENCIES_OVERRIDE, help = 'Whether to update the CARLA dependencies.') arg_parser.add_argument( '-build-dependencies', action='store_true', + default = BUILD_DEPENDENCIES_OVERRIDE, help = 'Whether to build the CARLA dependencies.') + arg_parser.add_argument( + '-build-osm-world-renderer', + action='store_true', + default = BUILD_OSM_WORLD_RENDERER_OVERRIDE, + help = 'Whether to build OSM World Renderer.') + arg_parser.add_argument( '-build-osm2odr', action='store_true', - help = 'Build OSM2ODR.') + default = BUILD_OSM2ODR_OVERRIDE, + help = 'Whether to build OSM2ODR.') arg_parser.add_argument( '-package', action='store_true', - help = 'Build OSM2ODR.') + default = BUILD_PACKAGE_OVERRIDE, + help = 'Whether to package Carla.') arg_parser.add_argument( '-clean', action='store_true', + default = CLEAN_OVERRIDE, help = 'Clean build files.') - # LibCarla - - arg_parser.add_argument( - '-skip-libcarla-client', - action='store_true', - help = 'Whether to skip the libcarla client.') - - arg_parser.add_argument( - '-skip-libcarla-server', - action='store_true', - help = 'Whether to skip the libcarla server.') - # Carla UE: arg_parser.add_argument( - '-use-carsim', + '-enable-carsim', action='store_true', help = 'Whether to enable plugin "CarSim".') arg_parser.add_argument( - '-use-chrono', + '-enable-chrono', action='store_true', help = 'Whether to enable plugin "Chrono".') arg_parser.add_argument( - '-use-unity', + '-enable-unity', action='store_true', help = 'Whether to enable plugin "Unity".') arg_parser.add_argument( - '-use-omniverse', + '-enable-omniverse', action='store_true', help = 'Whether to enable plugin "NVIDIA Omniverse".') @@ -1011,42 +857,36 @@ def ParseCommandLine(): def Main(): Log('Started.') - build_path.mkdir(exist_ok = True) + BUILD_PATH.mkdir(exist_ok = True) + DEPENDENCIES_PATH.mkdir(exist_ok = True) arg = ParseCommandLine() - - c = ConfigureContext(arg) + c = Context(arg, DEFAULT_PARALLELISM) if arg.clean: try: - shutil.rmtree(build_path) + shutil.rmtree(BUILD_PATH) finally: - Log(f'Failed to remove {build_path}.') + Log(f'Failed to remove {BUILD_PATH}.') exit(-1) if arg.update_dependencies or True: UpdateDependencies(c) - c.Wait() for ext in [ '*.tmp', '*.zip', '*.tar.gz' ]: - for e in build_path.glob(ext): + for e in DEPENDENCIES_PATH.glob(ext): e.unlink(missing_ok = True) if arg.build_dependencies or True: BuildDependencies(c) - c.Wait() - if arg.build_libcarla: - BuildLibCarlaMain(c) - c.Wait() + BuildLibCarlaMain(c) if arg.build_python_api: BuildPythonAPIMain(c) - c.Wait() - if arg.build_carla_ue: + if arg.build_carla_unreal: BuildCarlaUEMain(c) - c.Wait() Log('Done.') @@ -1055,21 +895,5 @@ if __name__ == '__main__': Main() except Exception as err: Log(err) - URL_SUFFIX = 'how_to_build_on_windows/\n' if os.name == "nt" else 'build_linux/\n' - ERROR_MESSAGE = ( - '\n' - 'Ok, an error ocurred, don\'t panic!\n' - 'We have different platforms where you can find some help:\n' - '\n' - '- Make sure you have read the documentation:\n' - f' https://carla.readthedocs.io/en/latest/{URL_SUFFIX}' - '\n' - '- If the problem persists, submit an issue on our GitHub page:\n' - ' https://github.com/carla-simulator/carla/issues\n' - '\n' - '- Or just use our Discord server!\n' - ' We\'ll be glad to help you there:\n' - ' https://discord.gg/42KJdRj\n' - ) Log(ERROR_MESSAGE) exit(-1) \ No newline at end of file