From 89e329b73802d39095f5051a24ed72ea5650984b Mon Sep 17 00:00:00 2001 From: manishthani Date: Tue, 17 Sep 2019 14:57:22 +0200 Subject: [PATCH] Fixed bugs of importing assets (#2068) * Refactor Commandlet + made small fix * fixed scale of assets when importing them * Fixed bug of adding assets automatically, but semantic segmentation to be fixed * small fix + hiding movemeshes call until its stable * Meshes are moved to semantic segmentation folders * Retagging semantic segmentation * Redefined tags, refactor and added comments * Updated Changelog * created a move assets commandlet * Removing RoadRunnerFiles folder * readded flag of only prepare maps in import script * Removing ContentBrowser module * Added Import folder * updated readme * Apply zero rotation * updated doc link * updated readme * refactoring * Adding more comments and refactoring * Removed unnecesary include header files * Remove unnecessary includes in source files --- .gitignore | 3 +- CHANGELOG.md | 1 + Import/README.md | 4 + RoadRunnerFiles/README.txt | 34 --- .../Carla/Commandlet/MoveAssetsCommandlet.cpp | 170 ++++++++++++ .../Carla/Commandlet/MoveAssetsCommandlet.h | 78 ++++++ .../PrepareAssetsForCookingCommandlet.cpp | 247 ++++++++++-------- .../PrepareAssetsForCookingCommandlet.h | 43 +-- Util/BuildTools/Import.py | 70 +++-- 9 files changed, 459 insertions(+), 191 deletions(-) create mode 100644 Import/README.md delete mode 100644 RoadRunnerFiles/README.txt create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.cpp create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.h diff --git a/.gitignore b/.gitignore index 478671ee1..96c188b38 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,7 @@ Util/Build Install /ExportedMaps -/Import -/RoadRunnerFiles/*/ +/Import/*/ *.VC.db *.VC.opendb diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a33902c..e9f3aeda3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Added Doxygen documentation online with automatic updates through Jenkins pipeline * Fixed client_bounding_boxes.py example script * Exposed in the API: camera, exposure, depth of field, tone mapper and color attributes for the RGB sensor + * Fixed materials and semantic segmentation issues regarding importing assets ## CARLA 0.9.6 diff --git a/Import/README.md b/Import/README.md new file mode 100644 index 000000000..881c68578 --- /dev/null +++ b/Import/README.md @@ -0,0 +1,4 @@ +CARLA Simulator +=============== + +Please, place all the packages containing props and maps tom import in this folder. For more information, see topic [Creating standalone asset packages for distribution](https://carla.readthedocs.io/en/latest/asset_packages_for_dist/) diff --git a/RoadRunnerFiles/README.txt b/RoadRunnerFiles/README.txt deleted file mode 100644 index a98d806e1..000000000 --- a/RoadRunnerFiles/README.txt +++ /dev/null @@ -1,34 +0,0 @@ -This is the place where the generate_map.py will look for the maps. - -The structure must be one folder per map, with the same name as the .fbx and .xodr files that will be inside said folder. - -P.e.: -RoadRunnerFiles/ -├── AwesomeMap01 -│   ├── Asphalt1_Diff.png -│   ├── Asphalt1_Norm.png -│   ├── Asphalt1_Spec.png -│   ├── Grass1_Diff.png -│   ├── Grass1_Norm.png -│   ├── Grass1_Spec.png -│   ├── LaneMarking1_Diff.png -│   ├── LaneMarking1_Norm.png -│   ├── LaneMarking1_Spec.png -│   ├── AwesomeMap01.fbx -│   └── AwesomeMap01.xodr -└── AwesomeMap02 - ├── Asphalt1_Diff.png - ├── Asphalt1_Norm.png - ├── Asphalt1_Spec.png - ├── Concrete1_Diff.png - ├── Concrete1_Norm.png - ├── Concrete1_Spec.png - ├── Grass1_Diff.png - ├── Grass1_Norm.png - ├── Grass1_Spec.png - ├── LaneMarking1_Diff.png - ├── LaneMarking1_Norm.png - ├── LaneMarking1_Spec.png - ├── AwesomeMap02.fbx - └── AwesomeMap02.xodr - diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.cpp new file mode 100644 index 000000000..112f7a542 --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.cpp @@ -0,0 +1,170 @@ +// Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma +// de Barcelona (UAB). +// +// This work is licensed under the terms of the MIT license. +// For a copy, see . + +#include "MoveAssetsCommandlet.h" + +UMoveAssetsCommandlet::UMoveAssetsCommandlet() +{ + IsClient = false; + IsEditor = true; + IsServer = false; + LogToConsole = true; +} +#if WITH_EDITORONLY_DATA + +// NOTE: Assets imported from a map FBX will be classified for semantic +// segmentation as ROAD, ROADLINES AND TERRAIN based on the asset name +// defined in RoadRunner. These tags will be used for moving the meshes +// and for specifying the path to these meshes when spawning them in a world. +namespace SSTags { + // Carla Semantic Segmentation Folder Tags + static const FString ROAD = TEXT("Roads"); + static const FString ROADLINES = TEXT("RoadLines"); + static const FString TERRAIN = TEXT("Terrain"); + + // RoadRunner Tags + static const FString R_ROAD = TEXT("RoadNode"); + static const FString R_TERRAIN = TEXT("Terrain"); + static const FString R_MARKING = TEXT("MarkingNode"); +} + +FMovePackageParams UMoveAssetsCommandlet::ParseParams(const FString &InParams) const +{ + TArray Tokens; + TArray Params; + + ParseCommandLine(*InParams, Tokens, Params); + + // Parse and store package name + FMovePackageParams PackageParams; + FParse::Value(*InParams, TEXT("PackageName="), PackageParams.Name); + + // Parse and store maps name in an array + FString Maps; + FParse::Value(*InParams, TEXT("Maps="), Maps); + + TArray MapNames; + Maps.ParseIntoArray(MapNames, TEXT(" "), true); + + PackageParams.MapNames = MapNames; + + return PackageParams; +} + +void UMoveAssetsCommandlet::MoveAssets(const FMovePackageParams &PackageParams) +{ + // Create a library instance for loading all the assets + AssetsObjectLibrary = UObjectLibrary::CreateLibrary(UStaticMesh::StaticClass(), false, GIsEditor); + AssetsObjectLibrary->AddToRoot(); + + // Start loading all the assets in the library and classify them for semantic + // segmentation + for (const auto &Map : PackageParams.MapNames) + { + MoveAssetsFromMapForSemanticSegmentation(PackageParams.Name, Map); + } +} + +void MoveFiles(const TArray &Assets, const FString &DestPath) +{ + check(DestPath.Len() > 0); + + FAssetToolsModule &AssetToolsModule = FModuleManager::LoadModuleChecked("AssetTools"); + TArray AssetsAndNames; + for (auto AssetIt = Assets.CreateConstIterator(); AssetIt; ++AssetIt) + { + UObject *Asset = *AssetIt; + + if (!ensure(Asset)) + { + continue; + } + + new(AssetsAndNames) FAssetRenameData(Asset, DestPath, Asset->GetName()); + } + + if (AssetsAndNames.Num() > 0) + { + AssetToolsModule.Get().RenameAssetsWithDialog(AssetsAndNames); + } +} + +void UMoveAssetsCommandlet::MoveAssetsFromMapForSemanticSegmentation( + const FString &PackageName, + const FString &MapName) +{ + // Prepare a UObjectLibrary for moving assets + const FString SrcPath = TEXT("/Game/") + PackageName + TEXT("/Maps/") + MapName; + AssetsObjectLibrary->LoadAssetDataFromPath(*SrcPath); + AssetsObjectLibrary->LoadAssetsFromAssetData(); + + // Load Assets to move + MapContents.Empty(); + AssetsObjectLibrary->GetAssetDataList(MapContents); + AssetsObjectLibrary->ClearLoaded(); + + TArray DestinationPaths = {SSTags::ROAD, SSTags::ROADLINES, SSTags::TERRAIN}; + + // Init Map with keys + TMap> AssetDataMap; + for (const auto &Paths : DestinationPaths) + { + AssetDataMap.Add(Paths, {}); + } + + for (const auto &MapAsset : MapContents) + { + // Get AssetName + UStaticMesh *MeshAsset = CastChecked(MapAsset.GetAsset()); + FString ObjectName = MeshAsset->GetName(); + + FString AssetName; + MapAsset.AssetName.ToString(AssetName); + + if (SrcPath.Len()) + { + + const FString CurrentPackageName = MeshAsset->GetOutermost()->GetName(); + + if (!ensure(CurrentPackageName.StartsWith(SrcPath))) + { + continue; + } + + // Bind between tags and classify assets according to semantic + // segmentation + if (AssetName.Contains(SSTags::R_ROAD)) + { + AssetDataMap[SSTags::ROAD].Add(MeshAsset); + } + else if (AssetName.Contains(SSTags::R_MARKING)) + { + AssetDataMap[SSTags::ROADLINES].Add(MeshAsset); + } + else if (AssetName.Contains(SSTags::R_TERRAIN)) + { + AssetDataMap[SSTags::TERRAIN].Add(MeshAsset); + } + } + } + + // Move assets to correspoding semantic segmentation folders + for (const auto &Elem : AssetDataMap) + { + FString DestPath = TEXT("/Game/") + PackageName + TEXT("/Static/") + Elem.Key + "/" + MapName; + MoveFiles(Elem.Value, DestPath); + } +} + +int32 UMoveAssetsCommandlet::Main(const FString &Params) +{ + FMovePackageParams PackageParams = ParseParams(Params); + + MoveAssets(PackageParams); + + return 0; +} +#endif diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.h new file mode 100644 index 000000000..5988c66b3 --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/MoveAssetsCommandlet.h @@ -0,0 +1,78 @@ +// Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma +// de Barcelona (UAB). +// +// This work is licensed under the terms of the MIT license. +// For a copy, see . + +#pragma once + +#include "Carla/OpenDrive/OpenDriveActor.h" +#include "Commandlets/Commandlet.h" +#include "Runtime/Engine/Classes/Engine/ObjectLibrary.h" + +#if WITH_EDITORONLY_DATA +#include "AssetRegistry/Public/AssetRegistryModule.h" +#include "Developer/AssetTools/Public/AssetToolsModule.h" +#endif // WITH_EDITORONLY_DATA +#include "MoveAssetsCommandlet.generated.h" + +/// Struct containing Package Params, used for storing the parsed arguments when +/// invoking this commandlet +USTRUCT() +struct CARLA_API FMovePackageParams +{ + GENERATED_USTRUCT_BODY() + + FString Name; + + TArray MapNames; +}; + +UCLASS() +class UMoveAssetsCommandlet : public UCommandlet +{ + GENERATED_BODY() + +public: + + /// Default constructor. + UMoveAssetsCommandlet(); +#if WITH_EDITORONLY_DATA + + /// Parses the command line parameters provided through @a InParams The + /// arguments to parse are the package name and a list of map names + /// concatenated in a string. + FMovePackageParams ParseParams(const FString &InParams) const; + + /// Moves all the assets contained in a map from @a SrcPath to @a DestPath + void MoveAssetsFromMapForSemanticSegmentation(const FString &PackageName, const FString &MapName); + + /// Moves the meshes of all maps listed in a @PackageParams + void MoveAssets(const FMovePackageParams &PackageParams); + +public: + + /// Main method and entry of the commandlet, taking as input parameters @a + /// Params. + virtual int32 Main(const FString &Params) override; + +#endif // WITH_EDITORONLY_DATA + +private: + + /// The following data structures are declared as class members and with + /// UPROPERTY macro to avoid UE4 to garbage collect them. + + /// Loaded assets from any object library + UPROPERTY() + TArray AssetDatas; + + /// Loaded maps from any object library + UPROPERTY() + TArray MapContents; + + /// Used for loading assets in object library. Loaded Data is stored in + /// AssetDatas. + UPROPERTY() + UObjectLibrary *AssetsObjectLibrary; +}; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.cpp index 0a1a3ffbd..5eb935cae 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.cpp @@ -5,20 +5,20 @@ // For a copy, see . #include "PrepareAssetsForCookingCommandlet.h" -#include "GameFramework/WorldSettings.h" -#include "HAL/PlatformFilemanager.h" -#include "HAL/PlatformFile.h" -#include "UObject/MetaData.h" +#include "HAL/PlatformFilemanager.h" UPrepareAssetsForCookingCommandlet::UPrepareAssetsForCookingCommandlet() { + // Set necessary flags to run commandlet IsClient = false; IsEditor = true; IsServer = false; LogToConsole = true; #if WITH_EDITORONLY_DATA + // Get Carla Default materials, these will be used for maps that need to use + // Carla materials static ConstructorHelpers::FObjectFinder MarkingNode(TEXT( "Material'/Game/Carla/Static/GenericMaterials/LaneMarking/M_MarkingLane_W.M_MarkingLane_W'")); static ConstructorHelpers::FObjectFinder RoadNode(TEXT( @@ -44,7 +44,11 @@ FPackageParams UPrepareAssetsForCookingCommandlet::ParseParams(const FString &In ParseCommandLine(*InParams, Tokens, Params); FPackageParams PackageParams; + + // Parse and store Package name FParse::Value(*InParams, TEXT("PackageName="), PackageParams.Name); + + // Parse and store flag for only preparing maps FParse::Bool(*InParams, TEXT("OnlyPrepareMaps="), PackageParams.bOnlyPrepareMaps); return PackageParams; } @@ -70,40 +74,22 @@ void UPrepareAssetsForCookingCommandlet::LoadWorld(FAssetData &AssetData) TArray UPrepareAssetsForCookingCommandlet::SpawnMeshesToWorld( const TArray &AssetsPaths, - bool bUseCarlaMaterials, - bool bIsPropsMap) + bool bUseCarlaMaterials) { TArray SpawnedMeshes; - // Remove the meshes names from the original path for props, so we can load - // props inside folder - TArray AssetsPathsDirectories = AssetsPaths; - if (bIsPropsMap) - { - for (auto &AssetPath : AssetsPathsDirectories) - { - AssetPath.Split( - TEXT("/"), - &AssetPath, - nullptr, - ESearchCase::Type::IgnoreCase, - ESearchDir::Type::FromEnd); - } - } - - // Load assets specified in AssetsPathsDirectories by using an object library + // Load assets specified in AssetsPaths by using an object library // for building map world AssetsObjectLibrary = UObjectLibrary::CreateLibrary(UStaticMesh::StaticClass(), false, GIsEditor); AssetsObjectLibrary->AddToRoot(); - AssetsObjectLibrary->LoadAssetDataFromPaths(AssetsPathsDirectories); + + AssetsObjectLibrary->LoadAssetDataFromPaths(AssetsPaths); AssetsObjectLibrary->LoadAssetsFromAssetData(); MapContents.Empty(); AssetsObjectLibrary->GetAssetDataList(MapContents); // Create default Transform for all assets to spawn const FTransform zeroTransform = FTransform(); - FVector initialVector = FVector(0, 0, 0); - FRotator initialRotator = FRotator(0, 180, 0); UStaticMesh *MeshAsset; AStaticMeshActor *MeshActor; @@ -111,29 +97,32 @@ TArray UPrepareAssetsForCookingCommandlet::SpawnMeshesToWorl for (auto MapAsset : MapContents) { // Spawn Static Mesh - MeshAsset = CastChecked(MapAsset.GetAsset()); - MeshActor = World->SpawnActor(AStaticMeshActor::StaticClass(), - initialVector, - initialRotator); - MeshActor->GetStaticMeshComponent()->SetStaticMesh(CastChecked(MeshAsset)); - SpawnedMeshes.Add(MeshActor); - if (bUseCarlaMaterials) + MeshAsset = Cast(MapAsset.GetAsset()); + if (MeshAsset) { - // Set Carla Materials - FString AssetName; - MapAsset.AssetName.ToString(AssetName); - if (AssetName.Contains("MarkingNode")) + MeshActor = World->SpawnActor(AStaticMeshActor::StaticClass(), zeroTransform); + MeshActor->GetStaticMeshComponent()->SetStaticMesh(CastChecked(MeshAsset)); + + SpawnedMeshes.Add(MeshActor); + if (bUseCarlaMaterials) { - MeshActor->GetStaticMeshComponent()->SetMaterial(0, MarkingNodeMaterial); - MeshActor->GetStaticMeshComponent()->SetMaterial(1, MarkingNodeMaterialAux); - } - else if (AssetName.Contains("RoadNode")) - { - MeshActor->GetStaticMeshComponent()->SetMaterial(0, RoadNodeMaterial); - } - else if (AssetName.Contains("Terrain")) - { - MeshActor->GetStaticMeshComponent()->SetMaterial(0, TerrainNodeMaterial); + // Set Carla Materials depending on RoadRunner's Semantic Segmentation + // tag + FString AssetName; + MapAsset.AssetName.ToString(AssetName); + if (AssetName.Contains(SSTags::R_MARKING)) + { + MeshActor->GetStaticMeshComponent()->SetMaterial(0, MarkingNodeMaterial); + MeshActor->GetStaticMeshComponent()->SetMaterial(1, MarkingNodeMaterialAux); + } + else if (AssetName.Contains(SSTags::R_ROAD)) + { + MeshActor->GetStaticMeshComponent()->SetMaterial(0, RoadNodeMaterial); + } + else if (AssetName.Contains(SSTags::R_TERRAIN)) + { + MeshActor->GetStaticMeshComponent()->SetMaterial(0, TerrainNodeMaterial); + } } } } @@ -162,9 +151,9 @@ void UPrepareAssetsForCookingCommandlet::DestroySpawnedActorsInWorld( bool UPrepareAssetsForCookingCommandlet::SaveWorld( FAssetData &AssetData, - FString &PackageName, - FString &DestPath, - FString &WorldName) + const FString &PackageName, + const FString &DestPath, + const FString &WorldName) { // Create Package to save UPackage *Package = AssetData.GetPackage(); @@ -263,7 +252,7 @@ FAssetsPaths UPrepareAssetsForCookingCommandlet::GetAssetsPathFromPackage(const FString PropAssetPath = PropJsonObject->GetStringField(TEXT("path")); - AssetsPaths.PropsPaths.Add(PropAssetPath); + AssetsPaths.PropsPaths.Add(std::move(PropAssetPath)); } } } @@ -309,6 +298,103 @@ bool UPrepareAssetsForCookingCommandlet::SavePackage(const FString &PackagePath, *PackageFileName, GError, nullptr, true, true, SAVE_NoError); } +void UPrepareAssetsForCookingCommandlet::GenerateMapPathsFile( + const FAssetsPaths &AssetsPaths, + const FString &PropsMapPath) +{ + FString MapPathData; + for (const auto &Map : AssetsPaths.MapsPaths) + { + MapPathData.Append(Map.Path + TEXT("/") + Map.Name + TEXT("+")); + } + + if (PropsMapPath.IsEmpty()) + { + MapPathData.Append(PropsMapPath); + } + else + { + if (!MapPathData.IsEmpty()) + { + MapPathData.RemoveFromEnd(TEXT("+")); + } + } + + FString SaveDirectory = FPaths::ProjectContentDir(); + FString FileName = FString("MapPaths.txt"); + SaveStringTextToFile(SaveDirectory, FileName, MapPathData, true); +} + +void UPrepareAssetsForCookingCommandlet::GeneratePackagePathFile(const FString &PackageName) +{ + FString SaveDirectory = FPaths::ProjectContentDir(); + FString FileName = FString("PackagePath.txt"); + FString PackageJsonFilePath = GetFirstPackagePath(PackageName); + SaveStringTextToFile(SaveDirectory, FileName, PackageJsonFilePath, true); +} + +void UPrepareAssetsForCookingCommandlet::PrepareMapsForCooking( + const FString &PackageName, + const TArray &MapsPaths) +{ + // Load World + FAssetData AssetData; + LoadWorld(AssetData); + World = CastChecked(AssetData.GetAsset()); + + FString BasePath = TEXT("/Game/") + PackageName + TEXT("/Static/"); + + for (const auto &Map : MapsPaths) + { + FString MapPath = TEXT("/") + Map.Name; + + FString DefaultPath = TEXT("/Game/") + PackageName + TEXT("/Maps/") + Map.Name; + FString RoadsPath = BasePath + SSTags::ROAD + MapPath; + FString RoadLinesPath = BasePath + SSTags::ROADLINES + MapPath; + FString TerrainPath = BasePath + SSTags::TERRAIN + MapPath; + + // Spawn assets located in semantic segmentation fodlers + TArray DataPath = {DefaultPath, RoadsPath, RoadLinesPath, TerrainPath}; + + TArray SpawnedActors = SpawnMeshesToWorld(DataPath, Map.bUseCarlaMapMaterials); + + // Save the World in specified path + SaveWorld(AssetData, PackageName, Map.Path, Map.Name); + + // Remove spawned actors from world to keep equal as BaseMap + DestroySpawnedActorsInWorld(SpawnedActors); + } +} + +void UPrepareAssetsForCookingCommandlet::PreparePropsForCooking( + FString &PackageName, + const TArray &PropsPaths, + FString &MapDestPath) +{ + // Load World + FAssetData AssetData; + LoadWorld(AssetData); + World = CastChecked(AssetData.GetAsset()); + + // Remove the meshes names from the original path for props, so we can load + // props inside folder + TArray PropPathDirs = PropsPaths; + + for (auto &PropPath : PropPathDirs) + { + PropPath.Split(TEXT("/"), &PropPath, nullptr, ESearchCase::Type::IgnoreCase, ESearchDir::Type::FromEnd); + } + + // Add props in a single Base Map + TArray SpawnedActors = SpawnMeshesToWorld(PropPathDirs, false); + + FString MapName("PropsMap"); + SaveWorld(AssetData, PackageName, MapDestPath, MapName); + + DestroySpawnedActorsInWorld(SpawnedActors); + MapObjectLibrary->ClearLoaded(); +} + int32 UPrepareAssetsForCookingCommandlet::Main(const FString &Params) { FPackageParams PackageParams = ParseParams(Params); @@ -316,71 +402,26 @@ int32 UPrepareAssetsForCookingCommandlet::Main(const FString &Params) // Get Props and Maps Path FAssetsPaths AssetsPaths = GetAssetsPathFromPackage(PackageParams.Name); - // Load World - FAssetData AssetData; - LoadWorld(AssetData); - World = CastChecked(AssetData.GetAsset()); - if (PackageParams.bOnlyPrepareMaps) { - for (auto Map : AssetsPaths.MapsPaths) - { - FString RoadsPath = TEXT("/Game/") + PackageParams.Name + TEXT("/Static/RoadNode/") + Map.Name; - FString MarkingLinePath = TEXT("/Game/") + PackageParams.Name + TEXT("/Static/MarkingNode/") + Map.Name; - FString TerrainPath = TEXT("/Game/") + PackageParams.Name + TEXT("/Static/TerrainNode/") + Map.Name; - - TArray DataPath = {RoadsPath, MarkingLinePath, TerrainPath}; - - // Add Meshes to inside the loaded World - TArray SpawnedActors = SpawnMeshesToWorld(DataPath, Map.bUseCarlaMapMaterials); - - // Save the World in specified path - SaveWorld(AssetData, PackageParams.Name, Map.Path, Map.Name); - - // Remove spawned actors from world to keep equal as BaseMap - DestroySpawnedActorsInWorld(SpawnedActors); - } + PrepareMapsForCooking(PackageParams.Name, AssetsPaths.MapsPaths); } else { - FString MapPathData; - for (auto Map : AssetsPaths.MapsPaths) - { - MapPathData.Append(Map.Path + TEXT("/") + Map.Name + TEXT("+")); - } + FString PropsMapPath(""); if (AssetsPaths.PropsPaths.Num() > 0) { FString MapName("PropsMap"); - FString WorldDestPath = TEXT("/Game/") + PackageParams.Name + - TEXT("/Maps/") + MapName; - - MapPathData.Append(WorldDestPath + TEXT("/") + MapName); - - // Add props in a single Base Map - TArray SpawnedActors = SpawnMeshesToWorld(AssetsPaths.PropsPaths, false, true); - - SaveWorld(AssetData, PackageParams.Name, WorldDestPath, MapName); - - DestroySpawnedActorsInWorld(SpawnedActors); - MapObjectLibrary->ClearLoaded(); - } - else - { - if (MapPathData.Len() >= 0) - { - MapPathData.RemoveFromEnd(TEXT("+")); - } + PropsMapPath = TEXT("/Game/") + PackageParams.Name + TEXT("/Maps/") + MapName; + PreparePropsForCooking(PackageParams.Name, AssetsPaths.PropsPaths, MapName); } // Save Map Path File for further use - FString SaveDirectory = FPaths::ProjectContentDir(); - FString FileName = FString("MapPaths.txt"); - SaveStringTextToFile(SaveDirectory, FileName, MapPathData, true); + GenerateMapPathsFile(AssetsPaths, PropsMapPath); - FileName = FString("PackagePath.txt"); - FString PackageJsonFilePath = GetFirstPackagePath(PackageParams.Name); - SaveStringTextToFile(SaveDirectory, FileName, PackageJsonFilePath, true); + // Saves Package path for further use + GeneratePackagePathFile(PackageParams.Name); } return 0; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.h index 9e3831436..115dd477a 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Commandlet/PrepareAssetsForCookingCommandlet.h @@ -8,22 +8,15 @@ #include "Carla/OpenDrive/OpenDriveActor.h" #include "Commandlets/Commandlet.h" -#include "Containers/Map.h" -#include "CoreMinimal.h" -#include "Engine/World.h" -#include "Misc/PackageName.h" #include "Runtime/Engine/Classes/Engine/ObjectLibrary.h" -#include "UObject/Package.h" -#if WITH_EDITORONLY_DATA -#include "AssetRegistry/Public/AssetRegistryModule.h" -#include "Developer/AssetTools/Public/AssetToolsModule.h" -#include "Developer/AssetTools/Public/IAssetTools.h" -#endif // WITH_EDITORONLY_DATA #include "Runtime/Engine/Classes/Engine/StaticMeshActor.h" #include "PrepareAssetsForCookingCommandlet.generated.h" -/// Struct containing Package Params +/// Struct containing Package with @a Name and @a bOnlyPrepareMaps flag used to +/// separate the cooking of maps and props across the different stages (Maps +/// will be imported during make import command and Props will be imported +/// during make package command) USTRUCT() struct CARLA_API FPackageParams { @@ -85,12 +78,15 @@ public: /// @pre World is expected to be previously loaded TArray SpawnMeshesToWorld( const TArray &AssetsPaths, - bool bUseCarlaMaterials, - bool bIsPropsMap = false); + bool bUseCarlaMaterials); /// Saves the current World, contained in @a AssetData, into @a DestPath /// composed of @a PackageName and with @a WorldName. - bool SaveWorld(FAssetData &AssetData, FString &PackageName, FString &DestPath, FString &WorldName); + bool SaveWorld( + FAssetData &AssetData, + const FString &PackageName, + const FString &DestPath, + const FString &WorldName); /// Destroys all the previously spawned actors stored in @a SpawnedActors void DestroySpawnedActorsInWorld(TArray &SpawnedActors); @@ -99,6 +95,23 @@ public: /// @a PackageName FAssetsPaths GetAssetsPathFromPackage(const FString &PackageName) const; + /// Generates the MapPaths file provided @a AssetsPaths and @a PropsMapPath + void GenerateMapPathsFile(const FAssetsPaths &AssetsPaths, const FString &PropsMapPath); + + /// Generates the PackagePat file that contains the path of a package with @a + /// PackageName + void GeneratePackagePathFile(const FString &PackageName); + + /// For each Map data contained in @MapsPaths, it creates a World, spawn its + /// actors inside the world and saves it in .umap format + /// in a destination path built from @a PackageName. + void PrepareMapsForCooking(const FString &PackageName, const TArray &MapsPaths); + + /// For all the props inside @a PropsPaths, it creates a single World, spawn + /// all the props inside the world and saves it in .umap format + /// in a destination path built from @a PackageName and @a MapDestPath. + void PreparePropsForCooking(FString &PackageName, const TArray &PropsPaths, FString &MapDestPath); + public: /// Main method and entry of the commandlet, taking as input parameters @a @@ -113,7 +126,7 @@ private: UPROPERTY() TArray AssetDatas; - /// Loaded maps from any object library + /// Loaded map content from any object library UPROPERTY() TArray MapContents; diff --git a/Util/BuildTools/Import.py b/Util/BuildTools/Import.py index 53a565e10..259c0b656 100755 --- a/Util/BuildTools/Import.py +++ b/Util/BuildTools/Import.py @@ -63,8 +63,7 @@ def generate_import_setting_file(package_name, json_dirname, props, maps): with open(importfile, "w+") as fh: import_groups = [] file_names = [] - import_settings = [] - import_settings.append({ + import_settings = { "bImportMesh": 1, "bConvertSceneUnit": 1, "bConvertScene": 1, @@ -78,9 +77,10 @@ def generate_import_setting_file(package_name, json_dirname, props, maps): "StaticMeshImportData": { "bRemoveDegenerates": 1, "bAutoGenerateCollision": 0, - "bCombineMeshes": 0 + "bCombineMeshes": 0, + "bConvertSceneUnit": 1 } - }) + } for prop in props: props_dest = "/" + "/".join(["Game", package_name, "Static", prop["tag"], prop["name"]]) @@ -207,52 +207,48 @@ def import_assets_from_json_list(json_list): data = json.load(json_file) # Take all the fbx registered in the provided json files # and place it inside unreal in the provided path (by the json file) - maps = data["maps"] - props = data["props"] + maps = [] + props = [] + if "maps" in data: + maps = data["maps"] + if "props" in data: + props = data["props"] + package_name = filename.replace(".json", "") import_assets(package_name, dirname, props, maps) - move_uassets(package_name, maps) if not package_name: print("No Packages JSONs found, nothing to import. Skipping package.") continue - prepare_maps_commandlet_for_cooking(package_name) + + # First we only move the meshes to the tagged folders for semantic + # segmentation + move_assets_commandlet(package_name, maps) + + # We prepare only the maps for cooking after moving them. Props cooking will be done from Package.sh script. + prepare_maps_commandlet_for_cooking(package_name, only_prepare_maps=True) -def move_uassets(package_name, maps): - for umap in maps: - origin_path = os.path.join(CARLA_ROOT_PATH, "Unreal", "CarlaUE4", "Content", package_name, "Maps", umap["name"]) - dest_base_path = os.path.join(CARLA_ROOT_PATH, "Unreal", "CarlaUE4", "Content", package_name, "Static") - - # Create the 3 posible destination folder path - marking_dir = os.path.join(dest_base_path, "MarkingNode", umap["name"]) - road_dir = os.path.join(dest_base_path, "RoadNode", umap["name"]) - terrain_dir = os.path.join(dest_base_path, "TerrainNode", umap["name"]) - - # Create folders if they do not exist - if not os.path.exists(marking_dir): - os.makedirs(marking_dir) - if not os.path.exists(road_dir): - os.makedirs(road_dir) - if not os.path.exists(terrain_dir): - os.makedirs(terrain_dir) - - # Move uassets to corresponding folder - for filename in os.listdir(origin_path): - if "MarkingNode" in filename: - shutil.move(os.path.join(origin_path, filename), os.path.join(marking_dir, filename)) - if "RoadNode" in filename: - shutil.move(os.path.join(origin_path, filename), os.path.join(road_dir, filename)) - if "TerrainNode" in filename: - shutil.move(os.path.join(origin_path, filename), os.path.join(terrain_dir, filename)) - -def prepare_maps_commandlet_for_cooking(package_name): +def prepare_maps_commandlet_for_cooking(package_name, only_prepare_maps): commandlet_name = "PrepareAssetsForCooking" commandlet_arguments = "-PackageName=%s" % package_name - commandlet_arguments += " -OnlyPrepareMaps=true" + commandlet_arguments += " -OnlyPrepareMaps=%d" % only_prepare_maps invoke_commandlet(commandlet_name, commandlet_arguments) + +def move_assets_commandlet(package_name, maps): + commandlet_name = "MoveAssets" + commandlet_arguments = "-PackageName=%s" % package_name + + umap_names = "" + for umap in maps: + umap_names += umap["name"] + " " + commandlet_arguments += " -Maps=%s" % umap_names + + invoke_commandlet(commandlet_name, commandlet_arguments) + + def main(): import_folder = os.path.join(CARLA_ROOT_PATH, "Import") json_list = get_packages_json_list(import_folder)