from concurrent.futures import ProcessPoolExecutor, as_completed from pathlib import Path from collections import deque import subprocess, tarfile, zipfile, argparse, requests, psutil, shutil, glob, json, sys, os, stat from distutils.spawn import find_executable # Constants: FALLBACK_CARLA_VERSION_STRING = '0.9.15' EXE_EXT = '.exe' if os.name == 'nt' else '' LIB_PREFIX = '' if os.name == 'nt' else 'lib' LIB_EXT = '.lib' if os.name == 'nt' else '.a' OBJ_EXT = '.obj' if os.name == 'nt' else '.o' SHELL_EXT = '.bat' if os.name == 'nt' else '.sh' WORKSPACE_PATH = Path(__file__).parent.resolve() LIBCARLA_PATH = WORKSPACE_PATH / 'LibCarla' LIBCARLA_SOURCE_PATH = LIBCARLA_PATH / 'source' PYTHON_API_PATH = WORKSPACE_PATH / 'PythonAPI' / 'carla' EXAMPLES_PATH = WORKSPACE_PATH / 'Examples' UTIL_PATH = WORKSPACE_PATH / 'Util' DOCKER_UTILS_PATH = UTIL_PATH / 'DockerUtils' PATCHES_PATH = UTIL_PATH / 'Patches' CARLA_UE_PATH = WORKSPACE_PATH / 'Unreal' / 'CarlaUE4' CARLA_UE_PLUGIN_ROOT_PATH = CARLA_UE_PATH / 'Plugins' CARLA_UE_PLUGIN_PATH = CARLA_UE_PLUGIN_ROOT_PATH / 'Carla' CARLA_UE_PLUGIN_DEPENDENCIES_PATH = CARLA_UE_PLUGIN_ROOT_PATH / 'CarlaDependencies' CARLA_UE_CONTENT_PATH = CARLA_UE_PATH / 'Content' CARLA_UE_CONTENT_CARLA_PATH = CARLA_UE_CONTENT_PATH / 'Carla' CARLA_UE_CONTENT_VERSIONS_FILE_PATH = WORKSPACE_PATH / 'Util' / 'ContentVersions.json' LOGICAL_PROCESSOR_COUNT = psutil.cpu_count(logical = True) DEFAULT_PARALLELISM = LOGICAL_PROCESSOR_COUNT + (2 if LOGICAL_PROCESSOR_COUNT >= 8 else 0) READTHEDOCS_URL_SUFFIX = 'how_to_build_on_windows/\n' if os.name == "nt" else 'build_linux/\n' DEFAULT_BOOST_TOOLSET = 'msvc-14.3' if os.name == 'nt' else 'clang' DEFAULT_ERROR_MESSAGE = ( '\n' 'Ok, an error ocurred, don\'t panic!\n' 'We have different platforms where you can find some help:\n' '\n' '- Make sure you have read the documentation:\n' f' https://carla.readthedocs.io/en/latest/{READTHEDOCS_URL_SUFFIX}' '\n' '- If the problem persists, submit an issue on our GitHub page:\n' ' https://github.com/carla-simulator/carla/issues\n' '\n' '- Or just use our Discord server!\n' ' We\'ll be glad to help you there:\n' ' https://discord.gg/42KJdRj\n' ) def FindExecutable(candidates : list): for e in candidates: executable_path = find_executable(e) if executable_path: return e return None DEFAULT_C_COMPILER = FindExecutable([ 'cl', 'clang-cl', ] if os.name == 'nt' else [ 'clang-16', ]) DEFAULT_CPP_COMPILER = FindExecutable([ 'cl', 'clang-cl', ] if os.name == 'nt' else [ 'clang++-16', ]) DEFAULT_LINKER = FindExecutable([ 'link', 'llvm-link', ] if os.name == 'nt' else [ 'lld-16', ]) DEFAULT_LIB = FindExecutable([ 'lib', 'llvm-lib', 'llvm-ar' ] if os.name == 'nt' else [ 'llvm-ar-16', ]) DEFAULT_C_STANDARD = 11 DEFAULT_CPP_STANDARD = 20 argp = argparse.ArgumentParser(description = __doc__) def AddCLIFlag(name : str, help : str): argp.add_argument(f'--{name}', action = 'store_true', help = help) def AddCLIStringOption(name : str, default : str, help : str): argp.add_argument(f'--{name}', type = str, default = str(default), help = f'{help} (default = "{default}").') def ADDCLIIntOption(name : str, default : int, help : str): argp.add_argument(f'--{name}', type = int, default = int(default), help = f'{help} (default = {default}).') AddCLIFlag( 'all', 'Build all targets.') AddCLIFlag( 'import', f'Import maps and assets from "{WORKSPACE_PATH / "Import"}" into Unreal.') AddCLIFlag( 'package', 'Build a packaged version of CARLA ready for distribution.') AddCLIFlag( 'docs', 'Build the CARLA documentation, through Doxygen.') AddCLIFlag( 'clean', 'Delete all build files.') AddCLIFlag( 'rebuild', 'Delete all build files and recompiles.') AddCLIFlag( 'check-libcarla', 'Run unit the test suites for LibCarla') AddCLIFlag( 'check-python-api', 'Run unit the test suites for PythonAPI') AddCLIFlag( 'check', 'Run unit the test suites for LibCarla and PythonAPI') AddCLIFlag( 'carla-ue', 'Build the CARLA Unreal Engine plugin and project.') AddCLIFlag( 'update-ue-assets', 'Download the CARLA Unreal Engine assets.') AddCLIFlag( 'python-api', 'Build the CARLA Python API.') AddCLIFlag( 'libcarla-client', 'Build the LibCarla Client module.') AddCLIFlag( 'libcarla-server', 'Build the LibCarla Server module.') AddCLIFlag( 'update-deps', 'Download all project dependencies.') AddCLIFlag( 'build-deps', 'Build all project dependencies.') AddCLIFlag( 'configure-sequential', 'Whether to disable parallelism in the configuration script.') AddCLIFlag( 'no-log', 'Whether to disable saving logs.') AddCLIFlag( 'pytorch', 'Whether to enable PyTorch.') AddCLIFlag( 'chrono', 'Whether to enable Chrono.') AddCLIFlag( 'gtest', 'Whether to enable gtest.') AddCLIFlag( 'carsim', 'Whether to enable CarSim') AddCLIFlag( 'ros2', 'Whether to enable ROS2') AddCLIFlag( 'unity-build', 'Whether to enable Unity Build for Unreal Engine projects.') AddCLIFlag( 'osm2odr', 'Whether to enable OSM2ODR.') AddCLIFlag( 'osm-world-renderer', 'Whether to enable OSM World Renderer.') AddCLIFlag( 'nv-omniverse', 'Whether to enable the NVIDIA Omniverse Plugin.') AddCLIFlag( 'march-native', 'Whether to add "-march=native" to C/C++ compile flags.') AddCLIFlag( 'rss', 'Whether to enable RSS.') ADDCLIIntOption( 'parallelism', DEFAULT_PARALLELISM, 'Set the configure/build parallelism.') ADDCLIIntOption( 'c-standard', DEFAULT_C_STANDARD, 'Set the target C standard.') ADDCLIIntOption( 'cpp-standard', DEFAULT_CPP_STANDARD, 'Set the target C++ standard.') AddCLIStringOption( 'c-compiler', DEFAULT_C_COMPILER, 'Set the target C compiler.') AddCLIStringOption( 'cpp-compiler', DEFAULT_CPP_COMPILER, 'Set the target C++ compiler.') AddCLIStringOption( 'linker', DEFAULT_LINKER, 'Set the target linker.') AddCLIStringOption( 'ar', DEFAULT_LIB, 'Set the target ar/lib tool.') AddCLIStringOption( 'version', FALLBACK_CARLA_VERSION_STRING, 'Override the CARLA version.') AddCLIStringOption( 'generator', 'Ninja', 'Set the CMake generator.') AddCLIStringOption( 'build-path', WORKSPACE_PATH / 'Build', 'Set the CARLA build path.') AddCLIStringOption( 'boost-toolset', DEFAULT_BOOST_TOOLSET, 'Set the target boost toolset.') AddCLIStringOption( 'ue-path', os.getenv( 'UNREAL_ENGINE_PATH', ''), 'Set the path of Unreal Engine.') ARGS_SYNC_PATH = WORKSPACE_PATH / 'ArgsSync.json' def SyncArgs(): argv = argparse.Namespace() if __name__ == '__main__': argv = argp.parse_args() with open(ARGS_SYNC_PATH, 'w') as file: json.dump(argv.__dict__, file) else: with open(ARGS_SYNC_PATH, 'r') as file: argv.__dict__ = json.load(file) return argv ARGV = SyncArgs() SEQUENTIAL = ARGV.configure_sequential ENABLE_OSM2ODR = ARGV.osm2odr or ARGV.all ENABLE_GTEST = ARGV.gtest ENABLE_OSM_WORLD_RENDERER = ARGV.osm_world_renderer or ARGV.all ENABLE_CARLA_UE = ARGV.carla_ue or ARGV.all ENABLE_PYTHON_API = ARGV.python_api or ARGV.all ENABLE_LIBCARLA_CLIENT = ARGV.libcarla_client or ARGV.all ENABLE_LIBCARLA_SERVER = ARGV.libcarla_server or ARGV.all ENABLE_LIBCARLA = any([ ENABLE_CARLA_UE, ENABLE_PYTHON_API, ENABLE_LIBCARLA_CLIENT, ENABLE_LIBCARLA_SERVER, ]) ENABLE_CARSIM = ARGV.carsim ENABLE_CHRONO = ARGV.chrono ENABLE_ROS2 = ARGV.ros2 ENABLE_UNITY_BUILD = ARGV.unity_build ENABLE_NVIDIA_OMNIVERSE = ARGV.nv_omniverse ENABLE_RSS = ARGV.rss UPDATE_DEPENDENCIES = ARGV.update_deps or ARGV.all BUILD_DEPENDENCIES = ARGV.build_deps or ARGV.all UPDATE_CARLA_UE_ASSETS = ARGV.update_ue_assets or ARGV.all PARALLELISM = ARGV.parallelism # Root paths: CARLA_VERSION_STRING = ARGV.version BUILD_PATH = Path(ARGV.build_path) BUILD_TEMP_PATH = BUILD_PATH / 'Temp' LOG_PATH = BUILD_PATH / 'BuildLogs' DEPENDENCIES_PATH = BUILD_PATH / 'Dependencies' LIBCARLA_BUILD_PATH = BUILD_PATH / 'LibCarla' LIBCARLA_INSTALL_PATH = WORKSPACE_PATH / 'Install' / 'LibCarla' # Language options C_COMPILER = FindExecutable([ARGV.c_compiler]) if not C_COMPILER: sys.exit(f"Error C Compiler not found: {ARGV.c_compiler}") CPP_COMPILER = FindExecutable([ARGV.cpp_compiler]) if not CPP_COMPILER: sys.exit(f"Error CPP Compiler not found: {ARGV.cpp_compiler}") LINKER = FindExecutable([ARGV.linker]) if not LINKER: sys.exit(f"Error LINKER not found: {ARGV.linker}") LIB = FindExecutable([ARGV.ar]) if not LIB: sys.exit(f"Error AR not found {ARGV.ar}") C_STANDARD = ARGV.c_standard CPP_STANDARD = ARGV.cpp_standard C_COMPILER_CLI_TYPE = 'msvc' if ('cl' in C_COMPILER and os.name == 'nt') else 'gnu' CPP_COMPILER_CLI_TYPE = 'msvc' if ('cl' in CPP_COMPILER and os.name == 'nt') else 'gnu' C_COMPILER_IS_CLANG = 'clang' in C_COMPILER CPP_COMPILER_IS_CLANG = 'clang' in CPP_COMPILER C_ENABLE_MARCH_NATIVE = ARGV.march_native and C_COMPILER_CLI_TYPE == 'gnu' CPP_ENABLE_MARCH_NATIVE = ARGV.march_native and CPP_COMPILER_CLI_TYPE == 'gnu' LIB_IS_AR = 'ar' in LIB # Unreal Engine UNREAL_ENGINE_PATH = Path(ARGV.ue_path) # Dependencies: # Boost BOOST_USE_SUPERPROJECT = True BOOST_VERSION = (1, 84, 0) BOOST_VERSION_MAJOR, BOOST_VERSION_MINOR, BOOST_VERSION_PATCH = BOOST_VERSION BOOST_VERSION_STRING = f'{BOOST_VERSION_MAJOR}.{BOOST_VERSION_MINOR}.{BOOST_VERSION_PATCH}' BOOST_TOOLSET = ARGV.boost_toolset BOOST_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-source' BOOST_BUILD_PATH = DEPENDENCIES_PATH / 'boost-build' BOOST_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-install' BOOST_INCLUDE_PATH = BOOST_INSTALL_PATH / 'include' BOOST_LIBRARY_PATH = BOOST_INSTALL_PATH / 'lib' BOOST_B2_PATH = BOOST_SOURCE_PATH / f'b2{EXE_EXT}' BOOST_ALGORITHM_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-algorithm-source' BOOST_ALGORITHM_BUILD_PATH = DEPENDENCIES_PATH / 'boost-algorithm-build' BOOST_ALGORITHM_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-algorithm-install' BOOST_ALGORITHM_INCLUDE_PATH = BOOST_ALGORITHM_INSTALL_PATH / 'include' BOOST_ALGORITHM_LIB_PATH = BOOST_ALGORITHM_INSTALL_PATH / 'lib' BOOST_ASIO_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-asio-source' BOOST_ASIO_BUILD_PATH = DEPENDENCIES_PATH / 'boost-asio-build' BOOST_ASIO_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-asio-install' BOOST_ASIO_INCLUDE_PATH = BOOST_ASIO_INSTALL_PATH / 'include' BOOST_ASIO_LIB_PATH = BOOST_ASIO_INSTALL_PATH / 'lib' BOOST_DATE_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-date-time-source' BOOST_DATE_BUILD_PATH = DEPENDENCIES_PATH / 'boost-date-time-build' BOOST_DATE_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-date-install' BOOST_DATE_INCLUDE_PATH = BOOST_DATE_INSTALL_PATH / 'include' BOOST_DATE_LIB_PATH = BOOST_DATE_INSTALL_PATH / 'lib' BOOST_GEOMETRY_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-geometry-source' BOOST_GEOMETRY_BUILD_PATH = DEPENDENCIES_PATH / 'boost-geometry-build' BOOST_GEOMETRY_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-geometry-install' BOOST_GEOMETRY_INCLUDE_PATH = BOOST_GEOMETRY_INSTALL_PATH / 'include' BOOST_GEOMETRY_LIB_PATH = BOOST_GEOMETRY_INSTALL_PATH / 'lib' BOOST_GIL_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-gil-source' BOOST_GIL_BUILD_PATH = DEPENDENCIES_PATH / 'boost-gil-build' BOOST_GIL_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-gil-install' BOOST_GIL_INCLUDE_PATH = BOOST_GIL_INSTALL_PATH / 'include' BOOST_GIL_LIB_PATH = BOOST_GIL_INSTALL_PATH / 'lib' BOOST_ITERATOR_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-iterator-source' BOOST_ITERATOR_BUILD_PATH = DEPENDENCIES_PATH / 'boost-iterator-build' BOOST_ITERATOR_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-iterator-install' BOOST_ITERATOR_INCLUDE_PATH = BOOST_ITERATOR_INSTALL_PATH / 'include' BOOST_ITERATOR_LIB_PATH = BOOST_ITERATOR_INSTALL_PATH / 'lib' BOOST_PYTHON_SOURCE_PATH = DEPENDENCIES_PATH / 'boost-python-source' BOOST_PYTHON_BUILD_PATH = DEPENDENCIES_PATH / 'boost-python-build' BOOST_PYTHON_INSTALL_PATH = DEPENDENCIES_PATH / 'boost-python-install' BOOST_PYTHON_INCLUDE_PATH = BOOST_PYTHON_INSTALL_PATH / 'include' BOOST_PYTHON_LIB_PATH = BOOST_PYTHON_INSTALL_PATH / 'lib' # Eigen EIGEN_SOURCE_PATH = DEPENDENCIES_PATH / 'eigen-source' EIGEN_BUILD_PATH = DEPENDENCIES_PATH / 'eigen-build' EIGEN_INSTALL_PATH = DEPENDENCIES_PATH / 'eigen-install' EIGEN_INCLUDE_PATH = EIGEN_INSTALL_PATH / 'include' # Chrono CHRONO_SOURCE_PATH = DEPENDENCIES_PATH / 'chrono-source' CHRONO_BUILD_PATH = DEPENDENCIES_PATH / 'chrono-build' CHRONO_INSTALL_PATH = DEPENDENCIES_PATH / 'chrono-install' CHRONO_INCLUDE_PATH = CHRONO_INSTALL_PATH / 'include' CHRONO_LIBRARY_PATH = CHRONO_INSTALL_PATH / 'lib' # Google Test GTEST_SOURCE_PATH = DEPENDENCIES_PATH / 'gtest-source' GTEST_BUILD_PATH = DEPENDENCIES_PATH / 'gtest-build' GTEST_INSTALL_PATH = DEPENDENCIES_PATH / 'gtest-install' GTEST_INCLUDE_PATH = GTEST_INSTALL_PATH / 'include' GTEST_LIBRARY_PATH = GTEST_INSTALL_PATH / 'lib' # ZLib ZLIB_SOURCE_PATH = DEPENDENCIES_PATH / 'zlib-source' ZLIB_BUILD_PATH = DEPENDENCIES_PATH / 'zlib-build' ZLIB_INSTALL_PATH = DEPENDENCIES_PATH / 'zlib-install' ZLIB_INCLUDE_PATH = ZLIB_INSTALL_PATH / 'include' ZLIB_LIBRARY_PATH = ZLIB_INSTALL_PATH / 'lib' ZLIB_LIB_PATH = ZLIB_LIBRARY_PATH / (f'zlibstatic{LIB_EXT}' if os.name == 'nt' else f'libz{LIB_EXT}') # LibPNG LIBPNG_SOURCE_PATH = DEPENDENCIES_PATH / 'libpng-source' LIBPNG_BUILD_PATH = DEPENDENCIES_PATH / 'libpng-build' LIBPNG_INSTALL_PATH = DEPENDENCIES_PATH / 'libpng-install' LIBPNG_INCLUDE_PATH = LIBPNG_INSTALL_PATH / 'include' LIBPNG_LIBRARY_PATH = LIBPNG_INSTALL_PATH / 'lib' # SQLite SQLITE_SOURCE_PATH = DEPENDENCIES_PATH / 'sqlite-source' SQLITE_BUILD_PATH = DEPENDENCIES_PATH / 'sqlite-build' SQLITE_LIBRARY_PATH = SQLITE_BUILD_PATH SQLITE_INCLUDE_PATH = SQLITE_SOURCE_PATH SQLITE_LIB_PATH = SQLITE_BUILD_PATH / f'{LIB_PREFIX}sqlite{LIB_EXT}' SQLITE_EXE_PATH = SQLITE_BUILD_PATH / f'sqlite{EXE_EXT}' # Proj PROJ_SOURCE_PATH = DEPENDENCIES_PATH / 'proj-source' PROJ_BUILD_PATH = DEPENDENCIES_PATH / 'proj-build' PROJ_INSTALL_PATH = DEPENDENCIES_PATH / 'proj-install' PROJ_INCLUDE_PATH = PROJ_INSTALL_PATH / 'include' PROJ_LIBRARY_PATH = PROJ_INSTALL_PATH / 'lib' # Recast & Detour RECAST_SOURCE_PATH = DEPENDENCIES_PATH / 'recast-source' RECAST_BUILD_PATH = DEPENDENCIES_PATH / 'recast-build' RECAST_INSTALL_PATH = DEPENDENCIES_PATH / 'recast-install' RECAST_INCLUDE_PATH = RECAST_INSTALL_PATH / 'include' RECAST_LIBRARY_PATH = RECAST_INSTALL_PATH / 'lib' # RPCLib RPCLIB_SOURCE_PATH = DEPENDENCIES_PATH / 'rpclib-source' RPCLIB_BUILD_PATH = DEPENDENCIES_PATH / 'rpclib-build' RPCLIB_INSTALL_PATH = DEPENDENCIES_PATH / 'rpclib-install' RPCLIB_INCLUDE_PATH = RPCLIB_INSTALL_PATH / 'include' RPCLIB_LIBRARY_PATH = RPCLIB_INSTALL_PATH / 'lib' # Xerces-C XERCESC_SOURCE_PATH = DEPENDENCIES_PATH / 'xercesc-source' XERCESC_BUILD_PATH = DEPENDENCIES_PATH / 'xercesc-build' XERCESC_INSTALL_PATH = DEPENDENCIES_PATH / 'xercesc-install' XERCESC_LIBRARY_PATH = XERCESC_INSTALL_PATH / 'lib' XERCESC_INCLUDE_PATH = XERCESC_INSTALL_PATH / 'include' # LibOSMScout LIBOSMSCOUT_SOURCE_PATH = DEPENDENCIES_PATH / 'libosmscout-source' LIBOSMSCOUT_BUILD_PATH = DEPENDENCIES_PATH / 'libosmscout-build' LIBOSMSCOUT_INSTALL_PATH = DEPENDENCIES_PATH / 'libosmscout-install' LIBOSMSCOUT_INCLUDE_PATH = LIBOSMSCOUT_INSTALL_PATH / 'include' LIBOSMSCOUT_LIBRARY_PATH = LIBOSMSCOUT_INSTALL_PATH / 'lib' # LunaSVG LUNASVG_SOURCE_PATH = DEPENDENCIES_PATH / 'lunasvg-source' LUNASVG_BUILD_PATH = DEPENDENCIES_PATH / 'lunasvg-build' LUNASVG_INSTALL_PATH = DEPENDENCIES_PATH / 'lunasvg-install' LUNASVG_INCLUDE_PATH = LUNASVG_INSTALL_PATH / 'include' LUNASVG_LIBRARY_PATH = LUNASVG_INSTALL_PATH / 'lib' # SUMO SUMO_SOURCE_PATH = DEPENDENCIES_PATH / 'sumo-source' SUMO_BUILD_PATH = DEPENDENCIES_PATH / 'sumo-build' SUMO_INSTALL_PATH = DEPENDENCIES_PATH / 'sumo-install' SUMO_INCLUDE_PATH = SUMO_INSTALL_PATH / 'include' SUMO_LIBRARY_PATH = SUMO_INSTALL_PATH / 'lib' # Nvidia Omniverse NV_OMNIVERSE_PLUGIN_PATH = UNREAL_ENGINE_PATH / 'Engine' / 'Plugins' / 'Marketplace' / 'NVIDIA' / 'Omniverse' NV_OMNIVERSE_PATCH_PATH = PATCHES_PATH / 'omniverse_4.26' # Basic IO functions: def Log(message): message = str(message) message += '\n' print(message, end='') def LaunchSubprocess( cmd : list, working_directory : Path = None, log = None): return subprocess.run( cmd, stdout = log, stderr = log, cwd = working_directory) def LaunchSubprocessImmediate( cmd : list, working_directory : Path = None, log_name : str = None): sp = None if not ARGV.no_log and log_name != None: with open(LOG_PATH / f'{log_name}.log', 'w') as file: sp = subprocess.run( cmd, cwd = working_directory, stdout = file, stderr = file) else: sp = subprocess.run( cmd, cwd = working_directory, stdout = subprocess.PIPE, stderr = subprocess.PIPE) sp.check_returncode() # Convenience classes for listing dependencies: class Download: def __init__(self, url : str): self.url = url class GitRepository: def __init__(self, url : str, tag_or_branch : str = None, commit : str = None): self.url = url self.tag_or_branch = tag_or_branch self.commit = commit class Dependency: def __init__(self, name : str, *sources): self.name = name self.sources = [ *sources ] assert all( type(e) is Download or type(e) is GitRepository for e in self.sources) class DependencyUEPlugin(Dependency): def __init__(self, name: str, *sources): super().__init__(name, *sources) DEFAULT_DEPENDENCIES = [ Dependency( 'eigen', GitRepository('https://gitlab.com/libeigen/eigen.git', tag_or_branch = '3.4.0')), Dependency( 'libpng', GitRepository('https://github.com/glennrp/libpng.git', tag_or_branch = 'v1.6.40')), Dependency( 'zlib', GitRepository('https://github.com/madler/zlib.git'),), Dependency( 'sqlite', Download('https://www.sqlite.org/2021/sqlite-amalgamation-3340100.zip')), Dependency( 'rpclib', GitRepository('https://github.com/carla-simulator/rpclib.git', tag_or_branch = 'carla')), Dependency( 'recast', GitRepository('https://github.com/carla-simulator/recastnavigation.git', tag_or_branch = 'carla')), ] + [ Dependency( 'boost', Download(f'https://github.com/boostorg/boost/releases/download/boost-{BOOST_VERSION_STRING}/boost-{BOOST_VERSION_STRING}.zip')), ] if BOOST_USE_SUPERPROJECT else [ Dependency( 'boost-algorithm', GitRepository('https://github.com/boostorg/algorithm.git')), Dependency( 'boost-asio', GitRepository('https://github.com/boostorg/asio.git')), Dependency( 'boost-iterator', GitRepository('https://github.com/boostorg/iterator.git')), Dependency( 'boost-python', GitRepository('https://github.com/boostorg/python.git')), Dependency( 'boost-geometry', GitRepository('https://github.com/boostorg/geometry.git')), Dependency( 'boost-date-time', GitRepository('https://github.com/boostorg/date_time.git')), Dependency( 'boost-gil', GitRepository('https://github.com/boostorg/gil.git')), ] if ENABLE_GTEST: DEFAULT_DEPENDENCIES.append(Dependency( 'gtest', GitRepository('https://github.com/google/googletest.git', tag_or_branch = 'v1.14.0'))) CHRONO_DEPENDENCIES = [ Dependency( 'chrono', GitRepository('https://github.com/projectchrono/chrono.git', tag_or_branch = '8.0.0')), ] OSM_WORLD_RENDERER_DEPENDENCIES = [ Dependency( 'libosmscout', GitRepository('https://github.com/Framstag/libosmscout.git')), Dependency( 'lunasvg', GitRepository('https://github.com/sammycage/lunasvg.git')), ] OSM2ODR_DEPENDENCIES = [ Dependency( 'proj', GitRepository('https://github.com/OSGeo/PROJ.git', tag_or_branch = '7.2.1')), Dependency( 'xercesc', GitRepository('https://github.com/apache/xerces-c.git', tag_or_branch = 'v3.2.4')), Dependency( 'sumo', GitRepository('https://github.com/carla-simulator/sumo.git', tag_or_branch = 'carla_osm2odr')), ] CARLA_UE_DEPENDENCIES = [ DependencyUEPlugin( 'StreetMap', GitRepository( 'https://github.com/carla-simulator/StreetMap.git', tag_or_branch = 'UE5Native')), ] CARLA_UE_ASSETS_DEPENDENCIES = [ Dependency( 'Carla', GitRepository( 'https://bitbucket.org/carla-simulator/carla-content.git', tag_or_branch = 'marcel/5.3' # @CARLAUE5 This branch should be removed once merged. )) ] class Task: def __init__(self, name : str, in_edges : list, body, *args): assert all([ type(e) == Task for e in in_edges ]) self.name = name self.body = body self.args = args self.in_edge_done_count = 0 self.in_edges = in_edges self.out_edges = [] # Filled right before task graph execution. self.done = False def CreateSubprocessTask(name : str, in_edges : list, command : list): return Task(name, in_edges, LaunchSubprocessImmediate, command, None, name) def CreateCMakeConfigureDefaultCommandLine(source_path : Path, build_path : Path) -> list: cpp_flags_release = '' if os.name == 'nt': cpp_flags_release = '/MD' if CPP_ENABLE_MARCH_NATIVE: cpp_flags_release += ' -march=native' cmd = [ 'cmake', '-G', ARGV.generator, '-S', source_path, '-B', build_path, '-DCMAKE_C_COMPILER=' + C_COMPILER, '-DCMAKE_CXX_COMPILER=' + CPP_COMPILER, '-DCMAKE_BUILD_TYPE=Release', f'-DCMAKE_CXX_FLAGS_RELEASE={cpp_flags_release}', ] if os.name != 'nt': cmd.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON') CPP_ABI_VERSIONS = f'-fuse-ld={LINKER} -nostdinc++ -nostdlib++' CPP_LIB= f'-isystem {UNREAL_ENGINE_PATH}/Engine/Source/ThirdParty/Unix/LibCxx/include/c++/v1 -L {UNREAL_ENGINE_PATH}/Engine/Source/ThirdParty/Unix/LibCxx/lib/Unix/x86_64-unknown-linux-gnu/ -lc++' cmd.append(f'-DCMAKE_CXX_FLAGS={CPP_ABI_VERSIONS} {CPP_LIB}') return cmd def CreateCMakeConfigureDefault(name : str, in_edges : list, source_path : Path, build_path : Path, *args, install_path : Path = None): cmd = Task.CreateCMakeConfigureDefaultCommandLine(source_path, build_path) if install_path != None: cmd.append('-DCMAKE_INSTALL_PREFIX=' + str(install_path)) cmd.extend([ *args, source_path ]) return Task.CreateSubprocessTask(name, in_edges, cmd) def CreateCMakeBuildDefault(name : str, in_edges : list, build_path : Path, *args): cmd = [ 'cmake', '--build', build_path ] cmd.extend([ *args ]) return Task.CreateSubprocessTask(name, in_edges, cmd) def CreateCMakeInstallDefault(name : str, in_edges : list, build_path : Path, install_path : Path, *args): cmd = [ 'cmake', '--install', build_path, '--prefix', install_path ] cmd.extend([ *args ]) return Task.CreateSubprocessTask(name, in_edges, cmd) def Run(self): self.body(*self.args) def ToString(self): return f'{[ e.name for e in self.in_edges ]} -> {self.name}' class TaskGraph: def __init__(self, parallelism : int = PARALLELISM): self.sequential = SEQUENTIAL self.parallelism = parallelism self.tasks = [] self.sources = [] self.task_map = {} def Reset(self): self.tasks = [] self.sources = [] self.task_map = {} def Add(self, task : Task): self.tasks.append(task) if len(task.in_edges) == 0: self.sources.append(task) self.task_map[task.name] = self.tasks[-1] return task def Validate(self): return True def ToString(self): return '\n'.join([ e.ToString() for e in self.tasks ]) def Print(self): Log(self.ToString()) def Execute(self, sequential : bool = False): if len(self.tasks) == 0: return Log('-- Running task graph --') self.Print() assert self.Validate() prior_sequential = self.sequential self.sequential = sequential try: for task in self.tasks: for in_edge in task.in_edges: assert in_edge != None in_edge.out_edges.append(task) task_queue = deque() active_count = 0 done_count = 0 def UpdateOutEdges(task): nonlocal task_queue if len(task.out_edges) == 0: return for out in task.out_edges: assert out.in_edge_done_count < len(out.in_edges) out.in_edge_done_count += 1 if out.in_edge_done_count == len(out.in_edges): task_queue.append(out) def Flush(): nonlocal futures nonlocal future_map nonlocal done_count nonlocal active_count if active_count != 0: done = [ e for e in as_completed(futures) ] done_tasks = [ future_map[e] for e in done ] for e in done_tasks: e.done = True Log(f'> {task.name} - DONE') UpdateOutEdges(e) assert active_count == len(done_tasks) done_count += len(done_tasks) active_count = 0 future_map = {} futures = [] assert len(set(self.sources)) == len(self.sources) task_queue.extend(self.sources) with ProcessPoolExecutor(self.parallelism) as pool: futures = [] future_map = {} while len(task_queue) != 0: while len(task_queue) != 0 and active_count < self.parallelism: task = task_queue.popleft() Log(f'> {task.name} - STARTED') if not self.sequential: active_count += 1 future = pool.submit(task.Run) future_map[future] = task futures.append(future) else: task.Run() Log(f'> {task.name} - DONE') task.done = True done_count += 1 UpdateOutEdges(task) Flush() if done_count != len(self.tasks): pending_tasks = [] for e in self.tasks: if not e.done: pending_tasks.append(e) Log(f'> {len(self.tasks) - done_count} did not complete: {pending_tasks}.') assert False finally: Log('-- Done --') self.sequential = prior_sequential self.Reset() def UpdateGitRepository(path : Path, url : str, branch : str = None, commit : str = None): if path.exists(): LaunchSubprocessImmediate([ 'git', '-C', str(path), 'pull' ]) else: cmd = [ 'git', '-C', str(path.parent), 'clone', '--depth', '1', '--single-branch' ] if branch != None: cmd.extend([ '-b', branch ]) cmd.extend([ url, path.stem ]) LaunchSubprocessImmediate(cmd) if commit != None: LaunchSubprocessImmediate([ 'git', '-C', str(path), 'fetch' ]) LaunchSubprocessImmediate([ 'git', '-C', str(path), 'checkout', commit ]) def DownloadDependency(name : str, path : Path, url : str): # Download: try: temp_path = Path(str(path) + '.tmp') with requests.Session() as session: with session.get(url, stream = True) as result: result.raise_for_status() with open(temp_path, 'wb') as file: shutil.copyfileobj(result.raw, file) except Exception as err: Log(f'Failed to download dependency "{name}": {err}') # Extract: try: extract_path = path.with_name(path.name + '-temp') if url.endswith('.tar.gz'): archive_path = temp_path.with_suffix('.tar.gz') temp_path.rename(archive_path) with tarfile.open(archive_path) as file: file.extractall(extract_path) elif url.endswith('.zip'): archive_path = temp_path.with_suffix('.zip') temp_path.rename(archive_path) zipfile.ZipFile(archive_path).extractall(extract_path) # Peel unnecessary outer directory: entries = [ file for file in extract_path.iterdir() ] if len(entries) == 1 and entries[0].is_dir(): Path(entries[0]).rename(path) if extract_path.exists(): extract_path.rmdir() else: extract_path.rename(path) except Exception as err: Log(f'Failed to extract dependency "{name}": {err}') def UpdateDependency( dep : Dependency, download_path : Path = None): name = dep.name if download_path == None: download_path = DEPENDENCIES_PATH / f'{name}-source' if type(dep) is DependencyUEPlugin: # Override download path if we're dealing with an Unreal Engine Plugin. download_path = CARLA_UE_PLUGIN_ROOT_PATH / name for source in dep.sources: try: if type(source) is GitRepository: UpdateGitRepository(download_path, source.url, source.tag_or_branch, source.commit) elif type(source) is Download: if download_path.exists(): Log(f'Dependency "{name}" already present. Delete "{download_path}" if you wish for it to be downloaded again.') else: DownloadDependency(name, download_path, source.url) return finally: pass Log(f'Failed to update dependency "{name}".') assert False def UpdateDependencies(task_graph : TaskGraph): DEPENDENCIES_PATH.mkdir(exist_ok = True) unique_deps = set(DEFAULT_DEPENDENCIES) if ENABLE_CARLA_UE: unique_deps.update(CARLA_UE_DEPENDENCIES) if ENABLE_OSM_WORLD_RENDERER: unique_deps.update(OSM_WORLD_RENDERER_DEPENDENCIES) if ENABLE_OSM2ODR: unique_deps.update(OSM2ODR_DEPENDENCIES) if ENABLE_CHRONO: unique_deps.update(CHRONO_DEPENDENCIES) return [ task_graph.Add(Task(f'{dep.name}-update', [], UpdateDependency, dep)) for dep in unique_deps ] def CleanDownloadsMain(): for ext in [ '*.tmp', '*.zip', '*.tar.gz' ]: for e in DEPENDENCIES_PATH.glob(ext): e.unlink(missing_ok = True) def CleanDownloads(task_graph : TaskGraph): return task_graph.Add(Task('clean-downloads', [], CleanDownloadsMain)) def ConfigureBoost(): if BOOST_B2_PATH.exists(): return bootstrap_file_path = BOOST_SOURCE_PATH / f'bootstrap{SHELL_EXT}' log_name = 'boost-configure' if os.name != 'nt': try: os.chmod(bootstrap_file_path, stat.S_IEXEC | stat.S_IREAD) os.chmod(BOOST_SOURCE_PATH / 'tools/build/src/engine/build.sh', stat.S_IEXEC | stat.S_IREAD) except Exception as e: with open(LOG_PATH / f'{log_name}.log', 'w') as log_file: log_file.write(f"Cannot give permisions: {str(e)}") LaunchSubprocessImmediate( [ bootstrap_file_path ], working_directory = BOOST_SOURCE_PATH, log_name = log_name) def BuildAndInstallBoost(): LaunchSubprocessImmediate([ BOOST_B2_PATH, f'-j{PARALLELISM}', 'architecture=x86', 'address-model=64', f'toolset={BOOST_TOOLSET}', 'variant=release', 'link=static', 'runtime-link=shared', 'threading=multi', '--layout=versioned', '--with-system', '--with-python', '--with-date_time', f'--build-dir={BOOST_BUILD_PATH}', f'--prefix={BOOST_INSTALL_PATH}', f'--libdir={BOOST_LIBRARY_PATH}', f'--includedir={BOOST_INCLUDE_PATH}', 'install' ], working_directory = BOOST_SOURCE_PATH, log_name = 'boost-build') if (BOOST_INCLUDE_PATH / 'boost').exists(): return candidates = glob.glob(f'{BOOST_INCLUDE_PATH}/**/boost', recursive = True) candidates.sort() boost_path = Path(candidates[0]) shutil.move(boost_path, BOOST_INCLUDE_PATH / 'boost') boost_path.parent.rmdir() def BuildSQLite(): SQLITE_BUILD_PATH.mkdir(exist_ok = True) sqlite_sources = glob.glob(f'{SQLITE_SOURCE_PATH}/**/*.c', recursive = True) sqlite_sources = [ Path(e) for e in sqlite_sources ] if not SQLITE_EXE_PATH.exists(): cmd = [ C_COMPILER ] cmd.extend([ f'-std=c89', f'-fuse-ld={LINKER}', '-O2', '-Wno-error=int-conversion', ] if C_COMPILER_CLI_TYPE == 'gnu' else [ f'/std:c{C_STANDARD}', '/O2', '/Qvec', '/MD', '/EHsc', ]) if C_ENABLE_MARCH_NATIVE: cmd.append('-march=native') cmd.extend(sqlite_sources) if C_COMPILER_CLI_TYPE == 'msvc': cmd.append(f'/Fe{SQLITE_EXE_PATH}') else: cmd.extend([ '-o', SQLITE_EXE_PATH ]) LaunchSubprocessImmediate(cmd, log_name = 'sqlite-exe-build') if not SQLITE_LIB_PATH.exists(): objs = [] BUILD_TEMP_PATH.mkdir(exist_ok = True) for e in sqlite_sources: cmd = [ C_COMPILER, '-c' if C_COMPILER_CLI_TYPE == 'gnu' else '/c', ] cmd.extend([ f'-std=c89', '-march=native', '-O2', '-Wno-error=int-conversion', '-fPIC', '-nostdinc++', '-nostdlib++', '-isystem', f'{UNREAL_ENGINE_PATH}/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v22_clang-16.0.6-centos7/x86_64-unknown-linux-gnu/usr/include/', '-L', f'{UNREAL_ENGINE_PATH}/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v22_clang-16.0.6-centos7/x86_64-unknown-linux-gnu/usr/lib64', '-lc', ] if C_COMPILER_CLI_TYPE == 'gnu' else [ f'/std:c{C_STANDARD}', '/O2', '/Qvec', '/MD', '/EHsc', ]) obj_path = BUILD_TEMP_PATH / f'{e.name}{OBJ_EXT}' if C_COMPILER_CLI_TYPE != 'gnu': cmd.extend([ e, f'/Fo{obj_path}' ]) else: cmd.extend([ e, '-o', obj_path ]) LaunchSubprocessImmediate(cmd, log_name = f'sqlite-{e.stem}-build') objs.append(obj_path) cmd = [ LIB ] if os.name == 'nt': cmd.append(f'/OUT:{SQLITE_LIB_PATH}') else: cmd.extend(['rsc', SQLITE_LIB_PATH]) cmd.extend(objs) LaunchSubprocessImmediate(cmd, log_name = 'sqlite-lib-build') def ConfigureSUMO(): xercesc_path_candidates = glob.glob(f'{XERCESC_INSTALL_PATH}/**/{LIB_PREFIX}xerces-c*{LIB_EXT}', recursive=True) if len(xercesc_path_candidates) == 0: raise Exception('Could not configure SUMO since xerces-c could not be found.') if len(xercesc_path_candidates) > 1: Log('Warning: multiple xerces-c libraries found.') xercesc_path_candidates.sort() XERCESC_LIB_PATH = xercesc_path_candidates[0] cmd = Task.CreateCMakeConfigureDefaultCommandLine( SUMO_SOURCE_PATH, SUMO_BUILD_PATH) proj_candidates = glob.glob(str(PROJ_INSTALL_PATH / 'lib' / '**' / f'*{LIB_PREFIX}proj{LIB_EXT}'), recursive = True) if len(proj_candidates) == 0: raise Exception('Could not configure SUMO since PROJ could not be found.') PROJ_LIB_PATH = proj_candidates[0] cmd.extend([ f'-DZLIB_INCLUDE_DIR={ZLIB_INCLUDE_PATH}', f'-DZLIB_LIBRARY={ZLIB_LIB_PATH}', f'-DPROJ_INCLUDE_DIR={PROJ_INSTALL_PATH}/include', f'-DPROJ_LIBRARY={PROJ_LIB_PATH}', f'-DXercesC_INCLUDE_DIR={XERCESC_INSTALL_PATH}/include', f'-DXercesC_LIBRARY={XERCESC_LIB_PATH}', # '-DSUMO_LIBRARIES=OFF', # '-DPROFILING=OFF', # '-DPPROF=OFF', # '-DCOVERAGE=OFF', # '-DSUMO_UTILS=OFF', # '-DFMI=OFF', # '-DNETEDIT=OFF', # '-DENABLE_JAVA_BINDINGS=OFF', # '-DENABLE_CS_BINDINGS=OFF', # '-DCCACHE_SUPPORT=OFF', ]) LaunchSubprocessImmediate(cmd, log_name='sumo-configure') def BuildDependencies(task_graph : TaskGraph): # There are some dependencies that need sqlite to be built before configuring. build_sqlite = task_graph.Add(Task('sqlite-build', [], BuildSQLite)) configure_zlib = task_graph.Add(Task.CreateCMakeConfigureDefault( 'zlib-configure', [], ZLIB_SOURCE_PATH, ZLIB_BUILD_PATH, install_path = ZLIB_INSTALL_PATH)) build_zlib = task_graph.Add(Task.CreateCMakeBuildDefault( 'zlib-build', [ configure_zlib ], ZLIB_BUILD_PATH)) # Configure step: if BOOST_USE_SUPERPROJECT: task_graph.Add(Task('boost-configure', [], ConfigureBoost)) else: task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-algorithm-configure', [], BOOST_ALGORITHM_SOURCE_PATH, BOOST_ALGORITHM_BUILD_PATH, install_path = BOOST_ALGORITHM_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-asio-configure', [], BOOST_ASIO_SOURCE_PATH, BOOST_ASIO_BUILD_PATH, install_path = BOOST_ASIO_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-date-time-configure', [], BOOST_DATE_SOURCE_PATH, BOOST_DATE_BUILD_PATH, install_path = BOOST_DATE_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-geometry-configure', [], BOOST_GEOMETRY_SOURCE_PATH, BOOST_GEOMETRY_BUILD_PATH, install_path = BOOST_GEOMETRY_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-gil-configure', [], BOOST_GIL_SOURCE_PATH, BOOST_GIL_BUILD_PATH, install_path = BOOST_GIL_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-iterator-configure', [], BOOST_ITERATOR_SOURCE_PATH, BOOST_ITERATOR_BUILD_PATH, install_path = BOOST_ITERATOR_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'boost-python-configure', [], BOOST_PYTHON_SOURCE_PATH, BOOST_PYTHON_BUILD_PATH, install_path = BOOST_PYTHON_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault( 'zlib-install', [ build_zlib ], ZLIB_BUILD_PATH, ZLIB_INSTALL_PATH)) if ENABLE_GTEST: task_graph.Add(Task.CreateCMakeConfigureDefault( 'gtest-configure', [], GTEST_SOURCE_PATH, GTEST_BUILD_PATH)) task_graph.Add(Task.CreateCMakeConfigureDefault( 'libpng-configure', [], LIBPNG_SOURCE_PATH, LIBPNG_BUILD_PATH, '-DPNG_TESTS=OFF', '-DPNG_SHARED=OFF', '-DPNG_TOOLS=OFF', '-DPNG_BUILD_ZLIB=ON', f'-DZLIB_INCLUDE_DIRS={ZLIB_INCLUDE_PATH}', f'-DZLIB_LIBRARIES={ZLIB_LIB_PATH}')) task_graph.Add(Task.CreateCMakeConfigureDefault( 'recast-configure', [], RECAST_SOURCE_PATH, RECAST_BUILD_PATH, '-DRECASTNAVIGATION_DEMO=OFF', '-DRECASTNAVIGATION_TESTS=OFF', '-DRECASTNAVIGATION_EXAMPLES=OFF')) task_graph.Add(Task.CreateCMakeConfigureDefault( 'rpclib-configure', [], RPCLIB_SOURCE_PATH, RPCLIB_BUILD_PATH, '-DRPCLIB_BUILD_TESTS=OFF', '-DRPCLIB_GENERATE_COMPDB=OFF', '-DRPCLIB_BUILD_EXAMPLES=OFF', '-DRPCLIB_ENABLE_LOGGING=OFF', '-DRPCLIB_ENABLE_COVERAGE=OFF', '-DRPCLIB_MSVC_STATIC_RUNTIME=OFF', '-DCMAKE_POSITION_INDEPENDENT_CODE=ON')) if ENABLE_OSM_WORLD_RENDERER: task_graph.Add(Task.CreateCMakeConfigureDefault( 'libosmscout-configure', [], LIBOSMSCOUT_SOURCE_PATH, LIBOSMSCOUT_BUILD_PATH, '-DOSMSCOUT_BUILD_TOOL_STYLEEDITOR=OFF', '-DOSMSCOUT_BUILD_TOOL_OSMSCOUT2=OFF', '-DOSMSCOUT_BUILD_TESTS=OFF', '-DOSMSCOUT_BUILD_CLIENT_QT=OFF', '-DOSMSCOUT_BUILD_DEMOS=OFF')) task_graph.Add(Task.CreateCMakeConfigureDefault( 'lunasvg-configure', [], LUNASVG_SOURCE_PATH, LUNASVG_BUILD_PATH)) if ENABLE_CHRONO: task_graph.Add(Task.CreateCMakeConfigureDefault( 'chrono-configure', [], CHRONO_SOURCE_PATH, CHRONO_BUILD_PATH, f'-DEIGEN3_INCLUDE_DIR={EIGEN_SOURCE_PATH}', '-DENABLE_MODULE_VEHICLE=ON')) if ENABLE_OSM2ODR: task_graph.Add(Task.CreateCMakeConfigureDefault( 'proj-configure', [ build_sqlite ], PROJ_SOURCE_PATH, PROJ_BUILD_PATH, f'-DSQLITE3_INCLUDE_DIR={SQLITE_SOURCE_PATH}', f'-DSQLITE3_LIBRARY={SQLITE_LIB_PATH}', f'-DEXE_SQLITE3={SQLITE_EXE_PATH}', '-DWIN32_LEAN_AND_MEAN=1', '-DVC_EXTRALEAN=1', '-DNOMINMAX=1', '-DENABLE_TIFF=OFF', '-DENABLE_CURL=OFF', '-DBUILD_SHARED_LIBS=OFF', '-DBUILD_PROJSYNC=OFF', '-DBUILD_PROJINFO=OFF', '-DBUILD_CCT=OFF', '-DBUILD_CS2CS=OFF', '-DBUILD_GEOD=OFF', '-DBUILD_GIE=OFF', '-DBUILD_PROJ=OFF', '-DBUILD_TESTING=OFF', install_path = PROJ_INSTALL_PATH)) configure_xercesc = task_graph.Add(Task.CreateCMakeConfigureDefault( 'xercesc-configure', [], XERCESC_SOURCE_PATH, XERCESC_BUILD_PATH, '-DBUILD_SHARED_LIBS=OFF')) # SUMO requires that Proj and Xerces be built and installed before its configure step: build_xercesc = task_graph.Add(Task.CreateCMakeBuildDefault( 'xercesc-build', [ configure_xercesc ], XERCESC_BUILD_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault( 'xercesc-install', [ build_xercesc ], XERCESC_BUILD_PATH, XERCESC_INSTALL_PATH)) # We wait for all pending tasks to finish here, then we'll switch to sequential task execution for the build step. task_graph.Execute() # Build: if BOOST_USE_SUPERPROJECT: task_graph.Add(Task('boost-build', [], BuildAndInstallBoost)) else: task_graph.Add(Task.CreateCMakeBuildDefault('boost-algorithm-build', [], BOOST_ALGORITHM_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-asio-build', [], BOOST_ASIO_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-date-build', [], BOOST_DATE_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-geometry-build', [], BOOST_GEOMETRY_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-gil-build', [], BOOST_GIL_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-iterator-build', [], BOOST_ITERATOR_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-python-build', [], BOOST_PYTHON_BUILD_PATH)) if ENABLE_GTEST: task_graph.Add(Task.CreateCMakeBuildDefault('gtest-build', [], GTEST_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('libpng-build', [], LIBPNG_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('recast-build', [], RECAST_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('rpclib-build', [], RPCLIB_BUILD_PATH)) if ENABLE_OSM2ODR: build_proj = task_graph.Add(Task.CreateCMakeBuildDefault('proj-build', [], PROJ_BUILD_PATH)) install_proj = task_graph.Add(Task.CreateCMakeInstallDefault('proj-install', [ build_proj ], PROJ_BUILD_PATH, PROJ_INSTALL_PATH)) configure_sumo = task_graph.Add(Task('sumo-configure', [ install_proj ], ConfigureSUMO)) task_graph.Add(Task.CreateCMakeBuildDefault('sumo-build', [ configure_sumo ], SUMO_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('xercesc-build', [], XERCESC_BUILD_PATH)) if ENABLE_OSM_WORLD_RENDERER: task_graph.Add(Task.CreateCMakeBuildDefault('lunasvg-build', [], LUNASVG_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('libosmscout-build', [], LIBOSMSCOUT_BUILD_PATH)) if ENABLE_CHRONO: task_graph.Add(Task.CreateCMakeBuildDefault('chrono-build', [], CHRONO_BUILD_PATH)) task_graph.Execute(sequential = True) # The underlying build system should already parallelize. # Install: if not BOOST_USE_SUPERPROJECT: task_graph.Add(Task.CreateCMakeBuildDefault('boost-algorithm-install', [], BOOST_ALGORITHM_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-asio-install', [], BOOST_ASIO_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-date-install', [], BOOST_DATE_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-geometry-install', [], BOOST_GEOMETRY_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-gil-install', [], BOOST_GIL_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-iterator-install', [], BOOST_ITERATOR_BUILD_PATH)) task_graph.Add(Task.CreateCMakeBuildDefault('boost-python-install', [], BOOST_PYTHON_BUILD_PATH)) if ENABLE_GTEST: task_graph.Add(Task.CreateCMakeInstallDefault('gtest-install', [], GTEST_BUILD_PATH, GTEST_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('libpng-install', [], LIBPNG_BUILD_PATH, LIBPNG_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('recast-install', [], RECAST_BUILD_PATH, RECAST_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('rpclib-install', [], RPCLIB_BUILD_PATH, RPCLIB_INSTALL_PATH)) if ENABLE_OSM_WORLD_RENDERER: task_graph.Add(Task.CreateCMakeInstallDefault('lunasvg-install', [], LUNASVG_BUILD_PATH, LUNASVG_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('libosmscout-install', [], LIBOSMSCOUT_BUILD_PATH, LIBOSMSCOUT_INSTALL_PATH)) if ENABLE_OSM2ODR: task_graph.Add(Task.CreateCMakeInstallDefault('sumo-install', [], SUMO_BUILD_PATH, SUMO_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('proj-install', [], PROJ_BUILD_PATH, PROJ_INSTALL_PATH)) task_graph.Add(Task.CreateCMakeInstallDefault('xercesc-install', [], XERCESC_BUILD_PATH, XERCESC_INSTALL_PATH)) if ENABLE_CHRONO: task_graph.Add(Task.CreateCMakeInstallDefault('chrono-install', [], CHRONO_BUILD_PATH, CHRONO_INSTALL_PATH)) task_graph.Execute() def BuildLibCarlaMain(task_graph : TaskGraph): configure_libcarla = task_graph.Add(Task.CreateCMakeConfigureDefault( 'libcarla-configure', [], WORKSPACE_PATH, LIBCARLA_BUILD_PATH, f'-DCARLA_DEPENDENCIES_PATH={DEPENDENCIES_PATH}', f'-DBUILD_LIBCARLA_SERVER={"ON" if ARGV.libcarla_server else "OFF"}', f'-DBUILD_LIBCARLA_CLIENT={"ON" if ARGV.libcarla_client else "OFF"}', f'-DBUILD_OSM_WORLD_RENDERER={"ON" if ENABLE_OSM_WORLD_RENDERER else "OFF"}', f'-DLIBCARLA_PYTORCH={"ON" if ARGV.pytorch else "OFF"}')) build_libcarla = task_graph.Add(Task.CreateCMakeBuildDefault( 'libcarla-build', [ configure_libcarla ], LIBCARLA_BUILD_PATH)) return task_graph.Add(Task.CreateCMakeInstallDefault( 'libcarla-install', [ build_libcarla ], LIBCARLA_BUILD_PATH, LIBCARLA_INSTALL_PATH)) def BuildPythonAPIMain(): content = '' with open(PYTHON_API_PATH / 'setup.py.in', 'r') as file: content = file.read() content = content.format_map(globals()) if os.name == 'nt': content = content.replace(os.sep, '\\\\') with open(PYTHON_API_PATH / 'setup.py', 'w') as file: file.write(content) LaunchSubprocessImmediate( [ sys.executable, 'setup.py', 'bdist_egg', 'bdist_wheel' ], working_directory = PYTHON_API_PATH, log_name = 'python-api-build') def BuildPythonAPI(task_graph : TaskGraph): install_libcarla = task_graph.task_map.get('libcarla-install') task_graph.Add(Task('python-api-build', [ install_libcarla ], BuildPythonAPIMain)) def SetupUnrealEngine(task_graph : TaskGraph): pass def UpdateCarlaUEAssets(task_graph : TaskGraph): CARLA_UE_CONTENT_PATH.mkdir(parents = True, exist_ok = True) Log('Cloning CARLA UE content...') for e in CARLA_UE_ASSETS_DEPENDENCIES: UpdateDependency(e, CARLA_UE_CONTENT_CARLA_PATH) def BuildCarlaUEMain(): assert UNREAL_ENGINE_PATH.exists() unreal_build_tool_args = [] if ENABLE_CARSIM: unreal_build_tool_args.append('-carsim') if ENABLE_CHRONO: unreal_build_tool_args.append('-chrono') if ENABLE_ROS2: unreal_build_tool_args.append('-ros2') if ENABLE_UNITY_BUILD: unreal_build_tool_args.append('-unity-build') if ENABLE_NVIDIA_OMNIVERSE: unreal_build_tool_args.append('-nv-omniverse') if os.name == 'nt': LaunchSubprocessImmediate([ UNREAL_ENGINE_PATH / 'Engine' / 'Build' / 'BatchFiles' / 'Build.bat', 'CarlaUE4Editor', 'Win64', 'Development', '-WaitMutex', '-FromMsBuild', CARLA_UE_PATH / 'CarlaUE4.uproject', ], log_name = 'carla-ue-editor-build') else: pass def BuildCarlaUE(task_graph : TaskGraph): if ENABLE_NVIDIA_OMNIVERSE: task_graph.Add(Task('nv-omniverse-install', [], InstallNVIDIAOmniverse)) dependencies = [] if ENABLE_LIBCARLA: dependencies.append(task_graph.task_map.get('libcarla-install')) if ENABLE_PYTHON_API: dependencies.append(task_graph.task_map.get('python-api-build')) task_graph.Add(Task('carla-ue-build', dependencies, BuildCarlaUEMain)) def InstallNVIDIAOmniverse(): filename = 'USDCarlaInterface' header = f'{filename}.h' source = f'{filename}.cpp' omniverse_usd_path = NV_OMNIVERSE_PLUGIN_PATH / 'Source' / 'OmniverseUSD' files = [ [ omniverse_usd_path / 'Public' / header, NV_OMNIVERSE_PATCH_PATH / header ], [ omniverse_usd_path / 'Private' / source, NV_OMNIVERSE_PATCH_PATH / source ], ] for src, dst in files: shutil.copyfile(src, dst) def Clean(): if not BUILD_PATH.exists(): return try: shutil.rmtree(BUILD_PATH) finally: Log(f'Failed to remove {BUILD_PATH}.') exit(-1) if __name__ == '__main__': try: task_graph = TaskGraph(PARALLELISM) if ARGV.clean or ARGV.rebuild: Clean() BUILD_PATH.mkdir(exist_ok = True) if not ARGV.no_log: LOG_PATH.mkdir(exist_ok = True) if UPDATE_DEPENDENCIES: UpdateDependencies(task_graph) CleanDownloads(task_graph) task_graph.Execute() if BUILD_DEPENDENCIES: BuildDependencies(task_graph) if ENABLE_LIBCARLA: BuildLibCarlaMain(task_graph) if ENABLE_PYTHON_API: BuildPythonAPI(task_graph) if UPDATE_CARLA_UE_ASSETS: UpdateCarlaUEAssets(task_graph) if ENABLE_CARLA_UE: BuildCarlaUE(task_graph) task_graph.Execute() except Exception as err: Log(err) Log(DEFAULT_ERROR_MESSAGE) exit(-1) finally: try: ARGS_SYNC_PATH.unlink(missing_ok = True) finally: pass exit(0)