From 05a09b1007d28d6dfbedd63c732d235bf06d1c83 Mon Sep 17 00:00:00 2001 From: Alejandro Fraga Cimadevila Date: Tue, 15 Jun 2021 10:48:42 +0200 Subject: [PATCH] First iteration of the required files functionality --- LibCarla/source/carla/client/Client.h | 12 +++ LibCarla/source/carla/client/FileTransfer.h | 86 +++++++++++++++++++ .../source/carla/client/detail/Client.cpp | 41 ++++++++- LibCarla/source/carla/client/detail/Client.h | 8 ++ .../source/carla/client/detail/Simulator.cpp | 21 +++++ .../source/carla/client/detail/Simulator.h | 14 +++ .../carla/client/detail/WalkerNavigation.cpp | 3 +- PythonAPI/carla/source/libcarla/Client.cpp | 19 ++++ .../Carla/Source/Carla/Game/CarlaEpisode.cpp | 6 +- .../Source/Carla/Game/CarlaGameInstance.cpp | 2 + .../Source/Carla/Game/CarlaGameInstance.h | 36 ++++++++ .../Carla/Source/Carla/Server/CarlaServer.cpp | 47 ++++++++++ 12 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 LibCarla/source/carla/client/FileTransfer.h diff --git a/LibCarla/source/carla/client/Client.h b/LibCarla/source/carla/client/Client.h index 2309a7c45..4b540a5dc 100644 --- a/LibCarla/source/carla/client/Client.h +++ b/LibCarla/source/carla/client/Client.h @@ -54,6 +54,18 @@ namespace client { return _simulator->GetAvailableMaps(); } + bool SetFilesBaseFolder(const std::string &path) { + return _simulator->SetFilesBaseFolder(path); + } + + std::vector GetRequiredFiles(const std::string &folder = "", const bool download = true) const { + return _simulator->GetRequiredFiles(folder, download); + } + + void RequestFile(const std::string &name) const { + _simulator->RequestFile(name); + } + World ReloadWorld(bool reset_settings = true) const { return World{_simulator->ReloadEpisode(reset_settings)}; } diff --git a/LibCarla/source/carla/client/FileTransfer.h b/LibCarla/source/carla/client/FileTransfer.h new file mode 100644 index 000000000..9ec3a9948 --- /dev/null +++ b/LibCarla/source/carla/client/FileTransfer.h @@ -0,0 +1,86 @@ + +// Copyright (c) 2021 Computer Vision Center (CVC) at the Universitat Autonoma +// de Barcelona (UAB). +// +// This work is licensed under the terms of the MIT license. +// For a copy, see . + +#pragma once + +#include "carla/FileSystem.h" + +#include +#include +#include +#include + +namespace carla { +namespace client { + + class FileTransfer { + + public: + + FileTransfer() = delete; + + static bool SetFilesBaseFolder(const std::string &path) { + if (path.empty()) return false; + + // Check that the path ends in a slash, add it otherwise + if (path[path.size() - 1] != '/' && path[path.size() - 1] != '\\') { + _filesBaseFolder = path + "/"; + } + + return true; + } + + static const std::string& GetFilesBaseFolder() { + return _filesBaseFolder; + } + + static bool FileExists(std::string file) { + // Check if the file exists or not + struct stat buffer; + return (stat((_filesBaseFolder + file).c_str(), &buffer) == 0); + } + + static bool WriteFile(std::string path, std::vector content) { + std::string writePath = _filesBaseFolder + path; + + // Validate and create the file path + carla::FileSystem::ValidateFilePath(writePath); + + // Open the file to truncate it in binary mode + std::ofstream out(writePath, std::ios::trunc | std::ios::binary); + if(!out.good()) return false; + + // Write the content on and close it + for(auto file : content) { + out << file; + } + out.close(); + + return true; + } + + static std::vector ReadFile(std::string path) { + // Read the binary file from the base folder + std::ifstream file(_filesBaseFolder + path, std::ios::binary); + std::vector content(std::istreambuf_iterator(file), {}); + return content; + } + + private: + + static std::string _filesBaseFolder; + + }; + + #ifdef _WIN32 + std::string FileTransfer::_filesBaseFolder = std::string(getenv("USER")) + "/carlaCache/"; + #else + std::string FileTransfer::_filesBaseFolder = std::string(getenv("HOME")) + "/carlaCache/"; + #endif + +} // namespace client +} // namespace carla \ No newline at end of file diff --git a/LibCarla/source/carla/client/detail/Client.cpp b/LibCarla/source/carla/client/detail/Client.cpp index 408742689..549e61d5d 100644 --- a/LibCarla/source/carla/client/detail/Client.cpp +++ b/LibCarla/source/carla/client/detail/Client.cpp @@ -8,6 +8,7 @@ #include "carla/Exception.h" #include "carla/Version.h" +#include "carla/client/FileTransfer.h" #include "carla/client/TimeoutException.h" #include "carla/rpc/ActorDescription.h" #include "carla/rpc/BoneTransformData.h" @@ -172,6 +173,44 @@ namespace detail { return _pimpl->CallAndWait>("get_navigation_mesh"); } + bool Client::SetFilesBaseFolder(const std::string &path) { + return FileTransfer::SetFilesBaseFolder(path); + } + + std::vector Client::GetRequiredFiles(const std::string &folder, const bool download) const { + // Get the list of required files + auto requiredFiles = _pimpl->CallAndWait>("get_required_files", folder); + + if (download) { + + // For each required file, check if it exists and request it otherwise + for(auto requiredFile : requiredFiles) { + if(!FileTransfer::FileExists(requiredFile)) { + RequestFile(requiredFile); + } + } + } + return requiredFiles; + } + + void Client::RequestFile(const std::string &name) const { + // Download the binary content of the file from the server and write it on the client + auto content = _pimpl->CallAndWait>("request_file", name); + FileTransfer::WriteFile(name, content); + } + + std::vector Client::GetCacheFile(const std::string &name, const bool request_otherwise) const { + // Get the file from the cache in the file transfer + std::vector file = FileTransfer::ReadFile(name); + + // If it isn't in the cache, download it if request otherwise is true + if (file.empty() && request_otherwise) { + RequestFile(name); + file = FileTransfer::ReadFile(name); + } + return file; + } + std::vector Client::GetAvailableMaps() { return _pimpl->CallAndWait>("get_available_maps"); } @@ -231,7 +270,7 @@ namespace detail { void Client::SetWheelSteerDirection( rpc::ActorId vehicle, rpc::VehicleWheelLocation vehicle_wheel, - float angle_in_deg){ + float angle_in_deg) { return _pimpl->AsyncCall("set_wheel_steer_direction", vehicle, vehicle_wheel, angle_in_deg); } diff --git a/LibCarla/source/carla/client/detail/Client.h b/LibCarla/source/carla/client/detail/Client.h index 2822d40f4..216f4ca77 100644 --- a/LibCarla/source/carla/client/detail/Client.h +++ b/LibCarla/source/carla/client/detail/Client.h @@ -107,6 +107,14 @@ namespace detail { std::vector GetNavigationMesh() const; + bool SetFilesBaseFolder(const std::string &path); + + std::vector GetRequiredFiles(const std::string &folder = "", const bool download = true) const; + + void RequestFile(const std::string &name) const; + + std::vector GetCacheFile(const std::string &name, const bool request_otherwise = true) const; + std::vector GetAvailableMaps(); std::vector GetActorDefinitions(); diff --git a/LibCarla/source/carla/client/detail/Simulator.cpp b/LibCarla/source/carla/client/detail/Simulator.cpp index 9aed9cc1c..38d2c4127 100644 --- a/LibCarla/source/carla/client/detail/Simulator.cpp +++ b/LibCarla/source/carla/client/detail/Simulator.cpp @@ -158,6 +158,27 @@ namespace detail { return _cached_map; } + // =========================================================================== + // -- Required files --------------------------------------------------------- + // =========================================================================== + + + bool Simulator::SetFilesBaseFolder(const std::string &path) { + return _client.SetFilesBaseFolder(path); + } + + std::vector Simulator::GetRequiredFiles(const std::string &folder, const bool download) const { + return _client.GetRequiredFiles(folder, download); + } + + void Simulator::RequestFile(const std::string &name) const { + _client.RequestFile(name); + } + + std::vector Simulator::GetCacheFile(const std::string &name, const bool request_otherwise) const { + return _client.GetCacheFile(name, request_otherwise); + } + // =========================================================================== // -- Tick ------------------------------------------------------------------- // =========================================================================== diff --git a/LibCarla/source/carla/client/detail/Simulator.h b/LibCarla/source/carla/client/detail/Simulator.h index 93a64a2af..0052978b8 100644 --- a/LibCarla/source/carla/client/detail/Simulator.h +++ b/LibCarla/source/carla/client/detail/Simulator.h @@ -121,6 +121,20 @@ namespace detail { return _client.GetAvailableMaps(); } + /// @} + // ========================================================================= + /// @name Required files related methods + // ========================================================================= + /// @{ + + bool SetFilesBaseFolder(const std::string &path); + + std::vector GetRequiredFiles(const std::string &folder = "", const bool download = true) const; + + void RequestFile(const std::string &name) const; + + std::vector GetCacheFile(const std::string &name, const bool request_otherwise) const; + /// @} // ========================================================================= /// @name Garbage collection policy diff --git a/LibCarla/source/carla/client/detail/WalkerNavigation.cpp b/LibCarla/source/carla/client/detail/WalkerNavigation.cpp index 19c4eaae0..e17fc4881 100644 --- a/LibCarla/source/carla/client/detail/WalkerNavigation.cpp +++ b/LibCarla/source/carla/client/detail/WalkerNavigation.cpp @@ -22,7 +22,8 @@ namespace detail { WalkerNavigation::WalkerNavigation(Client &client) : _client(client), _next_check_index(0) { // Here call the server to retrieve the navmesh data. - _nav.Load(_client.GetNavigationMesh()); + auto files = _client.GetRequiredFiles("Nav"); + _nav.Load(_client.GetCacheFile(files[0])); } void WalkerNavigation::Tick(std::shared_ptr episode) { diff --git a/PythonAPI/carla/source/libcarla/Client.cpp b/PythonAPI/carla/source/libcarla/Client.cpp index d3e2a7177..58aa1756e 100644 --- a/PythonAPI/carla/source/libcarla/Client.cpp +++ b/PythonAPI/carla/source/libcarla/Client.cpp @@ -30,6 +30,22 @@ static auto GetAvailableMaps(const carla::client::Client &self) { return result; } +static auto SetFilesBaseFolder(carla::client::Client &self, const std::string &path) { + return self.SetFilesBaseFolder(path); +} + +static auto GetRequiredFiles(const carla::client::Client &self, const std::string &folder, const bool download) { + boost::python::list result; + for (const auto &str : self.GetRequiredFiles(folder, download)) { + result.append(str); + } + return result; +} + +static void RequestFile(const carla::client::Client &self, const std::string &name) { + self.RequestFile(name); +} + static void ApplyBatchCommands( const carla::client::Client &self, const boost::python::object &commands, @@ -180,6 +196,9 @@ void export_client() { .def("get_server_version", CONST_CALL_WITHOUT_GIL(cc::Client, GetServerVersion)) .def("get_world", &cc::Client::GetWorld) .def("get_available_maps", &GetAvailableMaps) + .def("set_files_base_folder", &SetFilesBaseFolder, (arg("path"))) + .def("get_required_files", &GetRequiredFiles, (arg("folder")="", arg("download")=true)) + .def("request_file", &RequestFile, (arg("name"))) .def("reload_world", CONST_CALL_WITHOUT_GIL_1(cc::Client, ReloadWorld, bool), (arg("reset_settings")=true)) .def("load_world", CONST_CALL_WITHOUT_GIL_3(cc::Client, LoadWorld, std::string, bool, rpc::MapLayer), (arg("map_name"), arg("reset_settings")=true, arg("map_layers")=rpc::MapLayer::All)) .def("generate_opendrive_world", CONST_CALL_WITHOUT_GIL_3(cc::Client, GenerateOpenDriveWorld, std::string, diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaEpisode.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaEpisode.cpp index 9ad18c223..7c9c506ba 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaEpisode.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaEpisode.cpp @@ -69,8 +69,10 @@ bool UCarlaEpisode::LoadNewEpisode(const FString &MapString, bool ResetSettings) } // Some conversions... FinalPath = FinalPath.Replace(TEXT("/Game/"), *FPaths::ProjectContentDir()); - if (FPaths::FileExists(IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FinalPath))) - { + FinalPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FinalPath); + + if (FPaths::FileExists(FinalPath)) { + UCarlaStatics::GetGameInstance(GetWorld())->SetMapPath(FinalPath); bIsFileFound = true; FinalPath = MapString; } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.cpp index 80a1bb45f..b3ecd09ad 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.cpp @@ -17,6 +17,8 @@ UCarlaGameInstance::UCarlaGameInstance() { check(CarlaSettings != nullptr); CarlaSettings->LoadSettings(); CarlaSettings->LogSettings(); + + SetDefaultMapPath(); } UCarlaGameInstance::~UCarlaGameInstance() = default; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.h index d4e24e861..1642f037f 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.h @@ -103,6 +103,39 @@ public: return CurrentMapLayer; } + UFUNCTION(BlueprintCallable) + void SetDefaultMapPath() { + // Read the config file + FConfigFile ConfigFile = FConfigFile(); + FString configFStr = FPaths::ProjectDir(); + configFStr += "Config/DefaultEngine.ini"; + ConfigFile.Read(configFStr); + + // Depending on where we are, set the editor or game default map +#ifdef UE_EDITOR + ConfigFile.GetString(TEXT("/Script/EngineSettings.GameMapsSettings"), TEXT("EditorStartupMap"), _MapPath); +#else + ConfigFile.GetString(TEXT("/Script/EngineSettings.GameMapsSettings"), TEXT("GameDefaultMap"), _MapPath); +#endif + + // Format and convert the path to absolute + _MapPath = _MapPath.Replace(TEXT("/Game/"), *FPaths::ProjectContentDir()); + _MapPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*_MapPath); + _MapPath = FPaths::GetBaseFilename(_MapPath, false); + } + + UFUNCTION(BlueprintCallable) + void SetMapPath(const FString &MapPath) + { + _MapPath = MapPath; + } + + UFUNCTION(BlueprintCallable) + const FString &GetMapPath() const + { + return _MapPath; + } + private: UPROPERTY(Category = "CARLA Settings", EditAnywhere) @@ -118,4 +151,7 @@ private: UPROPERTY(Category = "CARLA Game Instance", EditAnywhere) int32 CurrentMapLayer = static_cast(carla::rpc::MapLayer::All); + UPROPERTY() + FString _MapPath; + }; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp index cfcd9b2c7..c2c6b03f9 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp @@ -24,6 +24,7 @@ #include "Carla/Actor/ActorData.h" #include "CarlaServerResponse.h" #include "Carla/Util/BoundingBoxCalculator.h" +#include "Misc/FileHelper.h" #include #include @@ -358,6 +359,52 @@ void FCarlaServer::FPimpl::BindActions() return Result; }; + BIND_SYNC(get_required_files) << [this](std::string folder = "") -> R> + { + REQUIRE_CARLA_EPISODE(); + + // Check that the path ends in a slash, add it otherwise + if (folder[folder.size() - 1] != '/' && folder[folder.size() - 1] != '\\') { + folder += "/"; + } + + // Get the map's folder absolute path and check if it's in its own folder + auto mapDir = FPaths::GetPath(UCarlaStatics::GetGameInstance(Episode->GetWorld())->GetMapPath()) + "/" + folder.c_str(); + auto fileName = mapDir.EndsWith(Episode->GetMapName() + "/") ? "*" : Episode->GetMapName(); + + // Find all the xodr and bin files from the map + TArray Files; + auto &FileManager = IFileManager::Get(); + FileManager.FindFilesRecursive(Files, *mapDir, *FString(fileName + ".xodr"), true, false, false); + FileManager.FindFilesRecursive(Files, *mapDir, *FString(fileName + ".bin"), true, false, false); + + // Remove the start of the path until the content folder and put each file in the result + std::vector result; + for (auto File : Files) { + File.RemoveFromStart(FPaths::ConvertRelativePathToFull(FPaths::ProjectContentDir())); + result.emplace_back(TCHAR_TO_UTF8(*File)); + } + + return result; + }; + + BIND_SYNC(request_file) << [this](std::string name) -> R> + { + REQUIRE_CARLA_EPISODE(); + + // Get the absolute path of the file + FString path(FPaths::ConvertRelativePathToFull(FPaths::ProjectContentDir())); + path.Append(name.c_str()); + + // Copy the binary data of the file into the result and return it + TArray Content; + FFileHelper::LoadFileToArray(Content, *path, 0); + std::vector Result(Content.Num()); + memcpy(&Result[0], Content.GetData(), Content.Num()); + + return Result; + }; + BIND_SYNC(get_episode_settings) << [this]() -> R { REQUIRE_CARLA_EPISODE();