First iteration of the required files functionality

LibCarla/source/carla/client/Client.h

@@ -54,6 +54,18 @@ namespace client {
       return _simulator->GetAvailableMaps();
     }
 
+    bool SetFilesBaseFolder(const std::string &path) {
+      return _simulator->SetFilesBaseFolder(path);
+    }
+
+    std::vector<std::string> GetRequiredFiles(const std::string &folder = "", const bool download = true) const {
+      return _simulator->GetRequiredFiles(folder, download);
+    }
+
+    void RequestFile(const std::string &name) const {
+      _simulator->RequestFile(name);
+    }
+
     World ReloadWorld(bool reset_settings = true) const {
       return World{_simulator->ReloadEpisode(reset_settings)};
     }

LibCarla/source/carla/client/FileTransfer.h

@@ -0,0 +1,86 @@
+
+// Copyright (c) 2021 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/FileSystem.h"
+
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <sys/stat.h>
+
+namespace carla {
+namespace client {
+
+  class FileTransfer {
+
+  public:
+
+    FileTransfer() = delete;
+
+    static bool SetFilesBaseFolder(const std::string &path) {
+      if (path.empty()) return false;
+
+      // Check that the path ends in a slash, add it otherwise
+      if (path[path.size() - 1] != '/' && path[path.size() - 1] != '\\') {
+        _filesBaseFolder = path + "/";
+      }
+
+      return true;
+    }
+
+    static const std::string& GetFilesBaseFolder() {
+      return _filesBaseFolder;
+    }
+
+    static bool FileExists(std::string file) {
+      // Check if the file exists or not
+      struct stat buffer;
+      return (stat((_filesBaseFolder + file).c_str(), &buffer) == 0);
+    }
+
+    static bool WriteFile(std::string path, std::vector<uint8_t> content) {
+      std::string writePath = _filesBaseFolder + path;
+
+      // Validate and create the file path
+      carla::FileSystem::ValidateFilePath(writePath);
+
+      // Open the file to truncate it in binary mode
+      std::ofstream out(writePath, std::ios::trunc | std::ios::binary);
+      if(!out.good()) return false;
+
+      // Write the content on and close it
+      for(auto file : content) {
+        out << file;
+      }
+      out.close();
+
+      return true;
+    }
+
+    static std::vector<uint8_t> ReadFile(std::string path) {
+      // Read the binary file from the base folder
+      std::ifstream file(_filesBaseFolder + path, std::ios::binary);
+      std::vector<uint8_t> content(std::istreambuf_iterator<char>(file), {});
+      return content;
+    }
+
+  private:
+
+    static std::string _filesBaseFolder;
+
+  };
+
+    #ifdef _WIN32
+      std::string FileTransfer::_filesBaseFolder = std::string(getenv("USER")) + "/carlaCache/";
+    #else
+      std::string FileTransfer::_filesBaseFolder = std::string(getenv("HOME")) + "/carlaCache/";
+    #endif
+
+} // namespace client
+} // namespace carla

LibCarla/source/carla/client/detail/Client.cpp

@@ -8,6 +8,7 @@
 
 #include "carla/Exception.h"
 #include "carla/Version.h"
+#include "carla/client/FileTransfer.h"
 #include "carla/client/TimeoutException.h"
 #include "carla/rpc/ActorDescription.h"
 #include "carla/rpc/BoneTransformData.h"
@@ -172,6 +173,44 @@ namespace detail {
     return _pimpl->CallAndWait<std::vector<uint8_t>>("get_navigation_mesh");
   }
 
+  bool Client::SetFilesBaseFolder(const std::string &path) {
+    return FileTransfer::SetFilesBaseFolder(path);
+  }
+
+  std::vector<std::string> Client::GetRequiredFiles(const std::string &folder, const bool download) const {
+    // Get the list of required files
+    auto requiredFiles = _pimpl->CallAndWait<std::vector<std::string>>("get_required_files", folder);
+
+    if (download) {
+
+      // For each required file, check if it exists and request it otherwise
+      for(auto requiredFile : requiredFiles) {
+        if(!FileTransfer::FileExists(requiredFile)) {
+          RequestFile(requiredFile);
+        }
+      }
+    }
+    return requiredFiles;
+  }
+
+  void Client::RequestFile(const std::string &name) const {
+    // Download the binary content of the file from the server and write it on the client
+    auto content = _pimpl->CallAndWait<std::vector<uint8_t>>("request_file", name);
+    FileTransfer::WriteFile(name, content);
+  }
+
+  std::vector<uint8_t> Client::GetCacheFile(const std::string &name, const bool request_otherwise) const {
+    // Get the file from the cache in the file transfer
+    std::vector<uint8_t> file = FileTransfer::ReadFile(name);
+
+    // If it isn't in the cache, download it if request otherwise is true
+    if (file.empty() && request_otherwise) {
+      RequestFile(name);
+      file = FileTransfer::ReadFile(name);
+    }
+    return file;
+  }
+
   std::vector<std::string> Client::GetAvailableMaps() {
     return _pimpl->CallAndWait<std::vector<std::string>>("get_available_maps");
   }
@@ -231,7 +270,7 @@ namespace detail {
   void Client::SetWheelSteerDirection(
         rpc::ActorId vehicle,
         rpc::VehicleWheelLocation vehicle_wheel,
-        float angle_in_deg){
+        float angle_in_deg) {
     return _pimpl->AsyncCall("set_wheel_steer_direction", vehicle, vehicle_wheel, angle_in_deg);
   }
 

LibCarla/source/carla/client/detail/Client.h

@@ -107,6 +107,14 @@ namespace detail {
 
     std::vector<uint8_t> GetNavigationMesh() const;
 
+    bool SetFilesBaseFolder(const std::string &path);
+
+    std::vector<std::string> GetRequiredFiles(const std::string &folder = "", const bool download = true) const;
+
+    void RequestFile(const std::string &name) const;
+
+    std::vector<uint8_t> GetCacheFile(const std::string &name, const bool request_otherwise = true) const;
+
     std::vector<std::string> GetAvailableMaps();
 
     std::vector<rpc::ActorDefinition> GetActorDefinitions();

LibCarla/source/carla/client/detail/Simulator.cpp

@@ -158,6 +158,27 @@ namespace detail {
     return _cached_map;
   }
 
+  // ===========================================================================
+  // -- Required files ---------------------------------------------------------
+  // ===========================================================================
+
+
+    bool Simulator::SetFilesBaseFolder(const std::string &path) {
+      return _client.SetFilesBaseFolder(path);
+    }
+
+    std::vector<std::string> Simulator::GetRequiredFiles(const std::string &folder, const bool download) const {
+      return _client.GetRequiredFiles(folder, download);
+    }
+
+    void Simulator::RequestFile(const std::string &name) const {
+      _client.RequestFile(name);
+    }
+
+    std::vector<uint8_t> Simulator::GetCacheFile(const std::string &name, const bool request_otherwise) const {
+      return _client.GetCacheFile(name, request_otherwise);
+    }
+
   // ===========================================================================
   // -- Tick -------------------------------------------------------------------
   // ===========================================================================

LibCarla/source/carla/client/detail/Simulator.h

@@ -121,6 +121,20 @@ namespace detail {
       return _client.GetAvailableMaps();
     }
 
+    /// @}
+    // =========================================================================
+    /// @name Required files related methods
+    // =========================================================================
+    /// @{
+
+    bool SetFilesBaseFolder(const std::string &path);
+
+    std::vector<std::string> GetRequiredFiles(const std::string &folder = "", const bool download = true) const;
+
+    void RequestFile(const std::string &name) const;
+
+    std::vector<uint8_t> GetCacheFile(const std::string &name, const bool request_otherwise) const;
+
     /// @}
     // =========================================================================
     /// @name Garbage collection policy

LibCarla/source/carla/client/detail/WalkerNavigation.cpp

@@ -22,7 +22,8 @@ namespace detail {
 
   WalkerNavigation::WalkerNavigation(Client &client) : _client(client), _next_check_index(0) {
     // Here call the server to retrieve the navmesh data.
-    _nav.Load(_client.GetNavigationMesh());
+    auto files = _client.GetRequiredFiles("Nav");
+    _nav.Load(_client.GetCacheFile(files[0]));
   }
 
   void WalkerNavigation::Tick(std::shared_ptr<Episode> episode) {

PythonAPI/carla/source/libcarla/Client.cpp

@@ -30,6 +30,22 @@ static auto GetAvailableMaps(const carla::client::Client &self) {
   return result;
 }
 
+static auto SetFilesBaseFolder(carla::client::Client &self, const std::string &path) {
+  return self.SetFilesBaseFolder(path);
+}
+
+static auto GetRequiredFiles(const carla::client::Client &self, const std::string &folder, const bool download) {
+  boost::python::list result;
+  for (const auto &str : self.GetRequiredFiles(folder, download)) {
+    result.append(str);
+  }
+  return result;
+}
+
+static void RequestFile(const carla::client::Client &self, const std::string &name) {
+  self.RequestFile(name);
+}
+
 static void ApplyBatchCommands(
     const carla::client::Client &self,
     const boost::python::object &commands,
@@ -180,6 +196,9 @@ void export_client() {
     .def("get_server_version", CONST_CALL_WITHOUT_GIL(cc::Client, GetServerVersion))
     .def("get_world", &cc::Client::GetWorld)
     .def("get_available_maps", &GetAvailableMaps)
+    .def("set_files_base_folder", &SetFilesBaseFolder, (arg("path")))
+    .def("get_required_files", &GetRequiredFiles, (arg("folder")="", arg("download")=true))
+    .def("request_file", &RequestFile, (arg("name")))
     .def("reload_world", CONST_CALL_WITHOUT_GIL_1(cc::Client, ReloadWorld, bool), (arg("reset_settings")=true))
     .def("load_world", CONST_CALL_WITHOUT_GIL_3(cc::Client, LoadWorld, std::string, bool, rpc::MapLayer), (arg("map_name"), arg("reset_settings")=true, arg("map_layers")=rpc::MapLayer::All))
     .def("generate_opendrive_world", CONST_CALL_WITHOUT_GIL_3(cc::Client, GenerateOpenDriveWorld, std::string,

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaEpisode.cpp

@@ -69,8 +69,10 @@ bool UCarlaEpisode::LoadNewEpisode(const FString &MapString, bool ResetSettings)
     }
     // Some conversions...
     FinalPath = FinalPath.Replace(TEXT("/Game/"), *FPaths::ProjectContentDir());
-    if (FPaths::FileExists(IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FinalPath)))
-    {
+    FinalPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FinalPath);
+
+    if (FPaths::FileExists(FinalPath)) {
+      UCarlaStatics::GetGameInstance(GetWorld())->SetMapPath(FinalPath);
       bIsFileFound = true;
       FinalPath = MapString;
     }

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.cpp

@@ -17,6 +17,8 @@ UCarlaGameInstance::UCarlaGameInstance() {
   check(CarlaSettings != nullptr);
   CarlaSettings->LoadSettings();
   CarlaSettings->LogSettings();
+
+  SetDefaultMapPath();
 }
 
 UCarlaGameInstance::~UCarlaGameInstance() = default;

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameInstance.h

@@ -103,6 +103,39 @@ public:
     return CurrentMapLayer;
   }
 
+  UFUNCTION(BlueprintCallable)
+  void SetDefaultMapPath() {
+    // Read the config file
+    FConfigFile ConfigFile = FConfigFile();
+    FString configFStr = FPaths::ProjectDir();
+    configFStr += "Config/DefaultEngine.ini";
+    ConfigFile.Read(configFStr);
+
+    // Depending on where we are, set the editor or game default map
+#ifdef UE_EDITOR
+    ConfigFile.GetString(TEXT("/Script/EngineSettings.GameMapsSettings"), TEXT("EditorStartupMap"), _MapPath);
+#else
+    ConfigFile.GetString(TEXT("/Script/EngineSettings.GameMapsSettings"), TEXT("GameDefaultMap"), _MapPath);
+#endif
+
+    // Format and convert the path to absolute
+    _MapPath = _MapPath.Replace(TEXT("/Game/"), *FPaths::ProjectContentDir());
+    _MapPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*_MapPath);
+    _MapPath = FPaths::GetBaseFilename(_MapPath, false);
+  }
+
+  UFUNCTION(BlueprintCallable)
+  void SetMapPath(const FString &MapPath)
+  {
+    _MapPath = MapPath;
+  }
+
+  UFUNCTION(BlueprintCallable)
+  const FString &GetMapPath() const
+  {
+    return _MapPath;
+  }
+
 private:
 
   UPROPERTY(Category = "CARLA Settings", EditAnywhere)
@@ -118,4 +151,7 @@ private:
   UPROPERTY(Category = "CARLA Game Instance", EditAnywhere)
   int32 CurrentMapLayer = static_cast<int32>(carla::rpc::MapLayer::All);
 
+  UPROPERTY()
+  FString _MapPath;
+
 };

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp

@@ -24,6 +24,7 @@
 #include "Carla/Actor/ActorData.h"
 #include "CarlaServerResponse.h"
 #include "Carla/Util/BoundingBoxCalculator.h"
+#include "Misc/FileHelper.h"
 
 #include <compiler/disable-ue4-macros.h>
 #include <carla/Functional.h>
@@ -358,6 +359,52 @@ void FCarlaServer::FPimpl::BindActions()
     return Result;
   };
 
+  BIND_SYNC(get_required_files) << [this](std::string folder = "") -> R<std::vector<std::string>>
+  {
+    REQUIRE_CARLA_EPISODE();
+
+    // Check that the path ends in a slash, add it otherwise
+    if (folder[folder.size() - 1] != '/' && folder[folder.size() - 1] != '\\') {
+      folder += "/";
+    }
+    
+    // Get the map's folder absolute path and check if it's in its own folder
+    auto mapDir = FPaths::GetPath(UCarlaStatics::GetGameInstance(Episode->GetWorld())->GetMapPath()) + "/" + folder.c_str();
+    auto fileName = mapDir.EndsWith(Episode->GetMapName() + "/") ? "*" : Episode->GetMapName();
+
+    // Find all the xodr and bin files from the map
+    TArray<FString> Files;
+    auto &FileManager = IFileManager::Get();
+    FileManager.FindFilesRecursive(Files, *mapDir, *FString(fileName + ".xodr"), true, false, false);
+    FileManager.FindFilesRecursive(Files, *mapDir, *FString(fileName + ".bin"), true, false, false);
+
+    // Remove the start of the path until the content folder and put each file in the result
+    std::vector<std::string> result;
+    for (auto File : Files) {
+      File.RemoveFromStart(FPaths::ConvertRelativePathToFull(FPaths::ProjectContentDir()));
+      result.emplace_back(TCHAR_TO_UTF8(*File));
+    }
+
+    return result;
+  };
+  
+  BIND_SYNC(request_file) << [this](std::string name) -> R<std::vector<uint8_t>>
+  {
+    REQUIRE_CARLA_EPISODE();
+
+    // Get the absolute path of the file
+    FString path(FPaths::ConvertRelativePathToFull(FPaths::ProjectContentDir()));
+    path.Append(name.c_str());
+
+    // Copy the binary data of the file into the result and return it
+    TArray<uint8_t> Content;
+    FFileHelper::LoadFileToArray(Content, *path, 0);
+    std::vector<uint8_t> Result(Content.Num());
+    memcpy(&Result[0], Content.GetData(), Content.Num());
+
+    return Result;
+  };
+
   BIND_SYNC(get_episode_settings) << [this]() -> R<cr::EpisodeSettings>
   {
     REQUIRE_CARLA_EPISODE();