From c498ff357a39deae43c45a555b1760003ae7d950 Mon Sep 17 00:00:00 2001 From: nsubiron Date: Thu, 6 Apr 2017 15:56:43 +0200 Subject: [PATCH] Make adding capture cameras fully customizable in settings file --- Resources/Example.CarlaSettings.ini | 56 +++++---- Source/Carla/Carla.h | 2 +- Source/Carla/Game/CameraDescription.h | 32 ++++++ Source/Carla/Game/CapturedImage.h | 27 +++++ Source/Carla/Game/CarlaGameController.cpp | 28 +---- Source/Carla/Game/CarlaGameMode.cpp | 85 ++------------ Source/Carla/Game/CarlaGameMode.h | 2 - Source/Carla/Game/CarlaPlayerState.cpp | 5 +- Source/Carla/Game/CarlaPlayerState.h | 45 +++++--- Source/Carla/Game/CarlaSettings.cpp | 115 +++++++++---------- Source/Carla/Game/CarlaSettings.h | 67 +---------- Source/Carla/Game/CarlaVehicleController.cpp | 65 +++++------ Source/Carla/Game/CarlaVehicleController.h | 14 +-- Source/Carla/Game/PostProcessEffect.cpp | 13 +++ Source/Carla/Game/PostProcessEffect.h | 23 ++++ Source/Carla/SceneCaptureCamera.cpp | 10 +- Source/Carla/SceneCaptureCamera.h | 22 +--- 17 files changed, 276 insertions(+), 335 deletions(-) create mode 100644 Source/Carla/Game/CameraDescription.h create mode 100644 Source/Carla/Game/CapturedImage.h create mode 100644 Source/Carla/Game/PostProcessEffect.cpp create mode 100644 Source/Carla/Game/PostProcessEffect.h diff --git a/Resources/Example.CarlaSettings.ini b/Resources/Example.CarlaSettings.ini index 768976905..ac00aa882 100644 --- a/Resources/Example.CarlaSettings.ini +++ b/Resources/Example.CarlaSettings.ini @@ -15,15 +15,22 @@ WritePort=2001 ReadPort=2002 [CARLA/SceneCapture] -; Valid values: Mono, Stereo, NoCapture. -; The mode is overriden by the client if networking is enabled. -Mode=Mono -; Size of the captured image in pixels. +; Names of the scene capture cameras to attach to the player, each of them +; should be defined in its own subsection. +Cameras=CameraStereoLeft/RGB,CameraStereoLeft/Depth,CameraStereoRight/RGB,CameraStereoRight/Depth +; Cameras=CameraMono + +; Defaults for each camera can be set here, e.g. ImageSizeX=720 ImageSizeY=512 -[CARLA/SceneCapture/Mono] -; Image size can also be changed by mode. +[CARLA/SceneCapture/CameraMono] +; Post-processing effect to be applied. Valid values: +; * None No effects applied. +; * SceneFinal Post-processing present at scene (bloom, fog, etc). +; * Depth Depth map only. +PostProcessing=SceneFinal +; Size of the captured image in pixels. ImageSizeX=720 ImageSizeY=512 ; Position of the camera relative to the car in centimeters. @@ -35,18 +42,25 @@ CameraRotationPitch=0 CameraRotationRoll=0 CameraRotationYaw=0 -[CARLA/SceneCapture/Stereo] -; Position of the cameras relative to the car in centimeters. -Camera0PositionX=170 -Camera0PositionY=30 -Camera0PositionZ=150 -Camera1PositionX=170 -Camera1PositionY=-30 -Camera1PositionZ=150 -; Rotation of the cameras relative to the car in degrees. -Camera0RotationPitch=0 -Camera0RotationRoll=0 -Camera0RotationYaw=0 -Camera1RotationPitch=0 -Camera1RotationRoll=0 -Camera1RotationYaw=0 +[CARLA/SceneCapture/CameraStereoLeft] +CameraPositionX=170 +CameraPositionY=-30 +CameraPositionZ=150 +CameraRotationPitch=0 +CameraRotationRoll=0 +CameraRotationYaw=0 +[CARLA/SceneCapture/CameraStereoLeft/RGB] +PostProcessing=SceneFinal +[CARLA/SceneCapture/CameraStereoLeft/Depth] +PostProcessing=Depth +[CARLA/SceneCapture/CameraStereoRight] +CameraPositionX=170 +CameraPositionY=30 +CameraPositionZ=150 +CameraRotationPitch=0 +CameraRotationRoll=0 +CameraRotationYaw=0 +[CARLA/SceneCapture/CameraStereoRight/RGB] +PostProcessing=SceneFinal +[CARLA/SceneCapture/CameraStereoRight/Depth] +PostProcessing=Depth diff --git a/Source/Carla/Carla.h b/Source/Carla/Carla.h index b73343c0d..21296ef50 100644 --- a/Source/Carla/Carla.h +++ b/Source/Carla/Carla.h @@ -14,7 +14,7 @@ DECLARE_LOG_CATEGORY_EXTERN(LogCarlaServer, Log, All); // Options to compile with extra debug options. #ifdef WITH_EDITOR // #define CARLA_ROAD_GENERATOR_PRINT_OUT /// @todo #1 Crashes in Linux. -// #define CARLA_SERVER_CHECK_IMAGES +// #define CARLA_SERVER_CHECK_IMAGES /// @todo #13 Uncomment and compile ;) #endif // WITH_EDITOR class FCarlaModule : public IModuleInterface diff --git a/Source/Carla/Game/CameraDescription.h b/Source/Carla/Game/CameraDescription.h new file mode 100644 index 000000000..1271cdf61 --- /dev/null +++ b/Source/Carla/Game/CameraDescription.h @@ -0,0 +1,32 @@ +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) + +#pragma once + +#include "PostProcessEffect.h" +#include "CameraDescription.generated.h" + +USTRUCT() +struct FCameraDescription +{ + GENERATED_BODY() + + /** X size in pixels of the captured image. */ + UPROPERTY(Category = "Camera Description", EditDefaultsOnly) + uint32 ImageSizeX = 720u; + + /** Y size in pixels of the captured image. */ + UPROPERTY(Category = "Camera Description", EditDefaultsOnly) + uint32 ImageSizeY = 512u; + + /** Position relative to the player. */ + UPROPERTY(Category = "Camera Description", EditDefaultsOnly) + FVector Position = {170.0f, 0.0f, 150.0f}; + + /** Rotation relative to the player. */ + UPROPERTY(Category = "Camera Description", EditDefaultsOnly) + FRotator Rotation = {0.0f, 0.0f, 0.0f}; + + /** Post-process effect to be applied to the captured image. */ + UPROPERTY(Category = "Camera Description", EditDefaultsOnly) + EPostProcessEffect PostProcessEffect = EPostProcessEffect::SceneFinal; +}; diff --git a/Source/Carla/Game/CapturedImage.h b/Source/Carla/Game/CapturedImage.h new file mode 100644 index 000000000..5d6023d7e --- /dev/null +++ b/Source/Carla/Game/CapturedImage.h @@ -0,0 +1,27 @@ +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) + +#pragma once + +#include "PostProcessEffect.h" +#include "CapturedImage.generated.h" + +/// Bitmap and meta info of a scene capture. +/// +/// The bitmap may be empty if the capture failed. +USTRUCT() +struct FCapturedImage +{ + GENERATED_BODY() + + UPROPERTY(VisibleAnywhere) + uint32 SizeX = 0u; + + UPROPERTY(VisibleAnywhere) + uint32 SizeY = 0u; + + UPROPERTY(VisibleAnywhere) + EPostProcessEffect PostProcessEffect = EPostProcessEffect::INVALID; + + UPROPERTY(VisibleAnywhere) + TArray BitMap; +}; diff --git a/Source/Carla/Game/CarlaGameController.cpp b/Source/Carla/Game/CarlaGameController.cpp index 41e419f77..a37d65ee7 100644 --- a/Source/Carla/Game/CarlaGameController.cpp +++ b/Source/Carla/Game/CarlaGameController.cpp @@ -53,7 +53,7 @@ static void Set(std::vector &cImage, const TArray &BitMap) static bool CheckImage( const FString &Tag, - const ACarlaPlayerState::Image &Image, + const FCapturedImage &Image, const std::vector &ImageReward, uint32 SizeX, uint32 SizeY) @@ -82,7 +82,7 @@ static bool CheckImageValidity(const ACarlaPlayerState &Player, const carla::Rew // Wait for the scene init to be sent, return false if we need to restart the // server. /// @todo At the moment we just ignored what it is sent. -static bool ReadSceneInit(carla::CarlaServer &Server, ESceneCaptureMode &CaptureMode) +static bool ReadSceneInit(carla::CarlaServer &Server) { carla::Mode Mode; uint32 Scene; @@ -92,13 +92,6 @@ static bool ReadSceneInit(carla::CarlaServer &Server, ESceneCaptureMode &Capture if (!Server.tryReadSceneInit(Mode, Scene, Success)) return false; } - if (Mode == carla::Mode::MONO) { - CaptureMode = ESceneCaptureMode::Mono; - } else if (Mode == carla::Mode::STEREO) { - CaptureMode = ESceneCaptureMode::Stereo; - } else { - CaptureMode = ESceneCaptureMode::NoCapture; - } return true; } @@ -160,19 +153,8 @@ static bool SendReward( Set(reward->collision_general, PlayerState.GetCollisionIntensityOther()); Set(reward->intersect_other_lane, PlayerState.GetOtherLaneIntersectionFactor()); Set(reward->intersect_offroad, PlayerState.GetOffRoadIntersectionFactor()); - { // Add images. - using CPS = ACarlaPlayerState; - auto &ImageRGB0 = PlayerState.GetImage(CPS::ImageRGB0); - if (ImageRGB0.BitMap.Num() > 0) { - // Do not add any camera if first is invalid, also assume all the images - // have the same size. - reward->image_width = ImageRGB0.SizeX; - reward->image_height = ImageRGB0.SizeY; - Set(reward->image_rgb_0, ImageRGB0.BitMap); - Set(reward->image_rgb_1, PlayerState.GetImage(CPS::ImageRGB1).BitMap); - Set(reward->image_depth_0, PlayerState.GetImage(CPS::ImageDepth0).BitMap); - Set(reward->image_depth_1, PlayerState.GetImage(CPS::ImageDepth1).BitMap); - } + for (const auto &Image : PlayerState.GetImages()) { + /// @todo #13 Add images to reward. } #ifdef CARLA_SERVER_CHECK_IMAGES check(CheckImageValidity(PlayerState, *reward)); @@ -212,7 +194,7 @@ void CarlaGameController::Initialize(UCarlaSettings &CarlaSettings) { if (bServerNeedsRestart) { UE_LOG(LogCarlaServer, Log, TEXT("Initializing CarlaServer")); - if (Server->init(1u) && ReadSceneInit(*Server, CarlaSettings.SceneCaptureMode)) { + if (Server->init(1u) && ReadSceneInit(*Server)) { bServerNeedsRestart = false; } else { UE_LOG(LogCarlaServer, Warning, TEXT("Failed to initialize, server needs restart")); diff --git a/Source/Carla/Game/CarlaGameMode.cpp b/Source/Carla/Game/CarlaGameMode.cpp index bb4070785..3ec488e0a 100644 --- a/Source/Carla/Game/CarlaGameMode.cpp +++ b/Source/Carla/Game/CarlaGameMode.cpp @@ -7,8 +7,6 @@ #include "EngineUtils.h" #include "GameFramework/PlayerStart.h" -#include "SceneCaptureCamera.h" - #include "CarlaGameInstance.h" #include "CarlaGameState.h" #include "CarlaHUD.h" @@ -89,85 +87,16 @@ void ACarlaGameMode::RegisterPlayer(AController &NewPlayer) AttachCaptureCamerasToPlayer(*PlayerController); } -void ACarlaGameMode::RegisterCaptureCamera( - ASceneCaptureCamera &CaptureCamera, - AController &Player) -{ - AddTickPrerequisiteActor(&CaptureCamera); - ACarlaVehicleController *Vehicle = Cast(&Player); - if (Vehicle != nullptr) { - Vehicle->RegisterCaptureCamera(CaptureCamera); - } else { - UE_LOG(LogCarla, Warning, TEXT("Trying to register camera but player is not a ACarlaVehicleController")); - } -} - -static ASceneCaptureCamera *SpawnAndAttachCamera( - AController &Player, - const FVector &RelativeLocation, - const FRotator &RelativeRotation, - uint32 ImageSizeX, - uint32 ImageSizeY, - EPostProcessEffect PostProcessEffect) -{ - auto Camera = Player.GetWorld()->SpawnActor(RelativeLocation, RelativeRotation); - Camera->SetImageSize(ImageSizeX, ImageSizeY); - Camera->SetPostProcessEffect(PostProcessEffect); - Camera->AttachToActor(Player.GetPawn(), FAttachmentTransformRules::KeepRelativeTransform); - Camera->SetOwner(Player.GetPawn()); - return Camera; -} - void ACarlaGameMode::AttachCaptureCamerasToPlayer(AController &Player) { + ACarlaVehicleController *Vehicle = Cast(&Player); + if (Vehicle == nullptr) { + UE_LOG(LogCarla, Warning, TEXT("Trying to add capture cameras but player is not a ACarlaVehicleController")); + return; + } auto &Settings = GameInstance->GetCarlaSettings(); - if (Settings.SceneCaptureMode == ESceneCaptureMode::Mono) { - auto TheCamera = - SpawnAndAttachCamera( - Player, - Settings.Mono_CameraPosition, - Settings.Mono_CameraRotation, - Settings.Mono_ImageSizeX, - Settings.Mono_ImageSizeY, - EPostProcessEffect::SceneFinal); - RegisterCaptureCamera(*TheCamera, Player); - } else if (Settings.SceneCaptureMode == ESceneCaptureMode::Stereo) { - auto RGBCamera0 = - SpawnAndAttachCamera( - Player, - Settings.Stereo_Camera0Position, - Settings.Stereo_Camera0Rotation, - Settings.Stereo_ImageSizeX, - Settings.Stereo_ImageSizeY, - EPostProcessEffect::SceneFinal); - auto DepthCamera0 = - SpawnAndAttachCamera( - Player, - Settings.Stereo_Camera0Position, - Settings.Stereo_Camera0Rotation, - Settings.Stereo_ImageSizeX, - Settings.Stereo_ImageSizeY, - EPostProcessEffect::Depth); - auto RGBCamera1 = - SpawnAndAttachCamera( - Player, - Settings.Stereo_Camera1Position, - Settings.Stereo_Camera1Rotation, - Settings.Stereo_ImageSizeX, - Settings.Stereo_ImageSizeY, - EPostProcessEffect::SceneFinal); - auto DepthCamera1 = - SpawnAndAttachCamera( - Player, - Settings.Stereo_Camera1Position, - Settings.Stereo_Camera1Rotation, - Settings.Stereo_ImageSizeX, - Settings.Stereo_ImageSizeY, - EPostProcessEffect::Depth); - RegisterCaptureCamera(*RGBCamera0, Player); - RegisterCaptureCamera(*DepthCamera0, Player); - RegisterCaptureCamera(*RGBCamera1, Player); - RegisterCaptureCamera(*DepthCamera1, Player); + for (const auto &Item : Settings.CameraDescriptions) { + Vehicle->AddSceneCaptureCamera(Item.Value); } } diff --git a/Source/Carla/Game/CarlaGameMode.h b/Source/Carla/Game/CarlaGameMode.h index 71c374f19..4e3adc28a 100644 --- a/Source/Carla/Game/CarlaGameMode.h +++ b/Source/Carla/Game/CarlaGameMode.h @@ -33,8 +33,6 @@ private: void RegisterPlayer(AController &NewPlayer); - void RegisterCaptureCamera(ASceneCaptureCamera &CaptureCamera, AController &Player); - void AttachCaptureCamerasToPlayer(AController &Player); /// Iterate all the APlayerStart present in the world and add the ones with diff --git a/Source/Carla/Game/CarlaPlayerState.cpp b/Source/Carla/Game/CarlaPlayerState.cpp index c15bfd2f8..604657925 100644 --- a/Source/Carla/Game/CarlaPlayerState.cpp +++ b/Source/Carla/Game/CarlaPlayerState.cpp @@ -10,10 +10,7 @@ void ACarlaPlayerState::Reset() CollisionIntensityCars = 0.0f; CollisionIntensityPedestrians = 0.0f; CollisionIntensityOther = 0.0f; - // Invalidate images. - for (auto &image : Images) { - image = Image(); // Reset. - } + Images.Empty(); } void ACarlaPlayerState::CopyProperties(APlayerState *PlayerState) diff --git a/Source/Carla/Game/CarlaPlayerState.h b/Source/Carla/Game/CarlaPlayerState.h index 1849f7192..af18c6400 100644 --- a/Source/Carla/Game/CarlaPlayerState.h +++ b/Source/Carla/Game/CarlaPlayerState.h @@ -2,8 +2,8 @@ #pragma once -#include #include "GameFramework/PlayerState.h" +#include "CapturedImage.h" #include "CarlaPlayerState.generated.h" /// Current state of the player, updated every frame by ACarlaVehicleController. @@ -20,19 +20,6 @@ class CARLA_API ACarlaPlayerState : public APlayerState // =========================================================================== public: - enum ImageTag { - ImageRGB0, - ImageRGB1, - ImageDepth0, - ImageDepth1, - NUMBER_OF_IMAGES - }; - - struct Image { - uint32 SizeX = 0u; - uint32 SizeY = 0u; - TArray BitMap; - }; // =========================================================================== // -- APlayerState ----------------------------------------------------------- @@ -48,64 +35,76 @@ public: // =========================================================================== public: + UFUNCTION() int32 GetTimeStamp() const { return TimeStamp; } + UFUNCTION() const FVector &GetLocation() const { return Location; } + UFUNCTION() const FVector &GetOrientation() const { return Orientation; } + UFUNCTION() const FVector &GetAcceleration() const { return Acceleration; } + UFUNCTION() float GetForwardSpeed() const { return ForwardSpeed; } + UFUNCTION() int32 GetCurrentGear() const { return CurrentGear; } + UFUNCTION() float GetCollisionIntensityCars() const { return CollisionIntensityCars; } + UFUNCTION() float GetCollisionIntensityPedestrians() const { return CollisionIntensityPedestrians; } + UFUNCTION() float GetCollisionIntensityOther() const { return CollisionIntensityOther; } + UFUNCTION() float GetOtherLaneIntersectionFactor() const { return OtherLaneIntersectionFactor; } + UFUNCTION() float GetOffRoadIntersectionFactor() const { return OffRoadIntersectionFactor; } - const Image &GetImage(ImageTag Tag) const + UFUNCTION() + const TArray &GetImages() const { - return Images[Tag]; + return Images; } // =========================================================================== @@ -127,27 +126,39 @@ private: // If you add another variable here, don't forget to copy it inside // CopyProperties if necessary. + UPROPERTY(VisibleAnywhere) int32 TimeStamp; + UPROPERTY(VisibleAnywhere) FVector Location; + UPROPERTY(VisibleAnywhere) FVector Orientation; + UPROPERTY(VisibleAnywhere) FVector Acceleration; + UPROPERTY(VisibleAnywhere) int32 CurrentGear; + UPROPERTY(VisibleAnywhere) float ForwardSpeed = 0.0f; + UPROPERTY(VisibleAnywhere) float CollisionIntensityCars = 0.0f; + UPROPERTY(VisibleAnywhere) float CollisionIntensityPedestrians = 0.0f; + UPROPERTY(VisibleAnywhere) float CollisionIntensityOther = 0.0f; + UPROPERTY(VisibleAnywhere) float OtherLaneIntersectionFactor = 0.0f; + UPROPERTY(VisibleAnywhere) float OffRoadIntersectionFactor = 0.0f; - std::array Images; + UPROPERTY(VisibleAnywhere) + TArray Images; }; diff --git a/Source/Carla/Game/CarlaSettings.cpp b/Source/Carla/Game/CarlaSettings.cpp index f190b0d04..753c06888 100644 --- a/Source/Carla/Game/CarlaSettings.cpp +++ b/Source/Carla/Game/CarlaSettings.cpp @@ -11,8 +11,6 @@ // INI file sections. #define S_CARLA_SERVER TEXT("CARLA/Server") #define S_CARLA_SCENECAPTURE TEXT("CARLA/SceneCapture") -#define S_CARLA_SCENECAPTURE_MONO TEXT("CARLA/SceneCapture/Mono") -#define S_CARLA_SCENECAPTURE_STEREO TEXT("CARLA/SceneCapture/Stereo") template static void SafeCastTo(SOURCE source, TARGET &target) @@ -62,16 +60,19 @@ public: } } - void GetSceneCaptureMode(const TCHAR* Section, const TCHAR* Key, ESceneCaptureMode &Target) const + void GetPostProcessEffect(const TCHAR* Section, const TCHAR* Key, EPostProcessEffect &Target) const { - FString ModeString; - if (ConfigFile.GetString(Section, Key, ModeString)) { - if (ModeString == "Stereo") { - Target = ESceneCaptureMode::Stereo; - } else if (ModeString == "Mono") { - Target = ESceneCaptureMode::Mono; - } else if (ModeString == "NoCapture") { - Target = ESceneCaptureMode::NoCapture; + FString ValueString; + if (ConfigFile.GetString(Section, Key, ValueString)) { + if (ValueString == "None") { + Target = EPostProcessEffect::None; + } else if (ValueString == "SceneFinal") { + Target = EPostProcessEffect::SceneFinal; + } else if (ValueString == "Depth") { + Target = EPostProcessEffect::Depth; + } else { + UE_LOG(LogCarla, Error, TEXT("Invalid post-processing \"%s\" in INI file"), *ValueString); + Target = EPostProcessEffect::INVALID; } } } @@ -85,6 +86,22 @@ private: // -- Other static methods ----------------------------------------------------- // ============================================================================= +static void GetCameraDescription( + const INIFile &ConfigFile, + const TCHAR* Section, + FCameraDescription &Camera) +{ + ConfigFile.GetInt(Section, TEXT("ImageSizeX"), Camera.ImageSizeX); + ConfigFile.GetInt(Section, TEXT("ImageSizeY"), Camera.ImageSizeY); + ConfigFile.GetInt(Section, TEXT("CameraPositionX"), Camera.Position.X); + ConfigFile.GetInt(Section, TEXT("CameraPositionY"), Camera.Position.Y); + ConfigFile.GetInt(Section, TEXT("CameraPositionZ"), Camera.Position.Z); + ConfigFile.GetInt(Section, TEXT("CameraRotationPitch"), Camera.Rotation.Pitch); + ConfigFile.GetInt(Section, TEXT("CameraRotationRoll"), Camera.Rotation.Roll); + ConfigFile.GetInt(Section, TEXT("CameraRotationYaw"), Camera.Rotation.Yaw); + ConfigFile.GetPostProcessEffect(Section, TEXT("PostProcessing"), Camera.PostProcessEffect); +} + static void LoadSettingsFromFile(const FString &FileName, UCarlaSettings &Settings) { UE_LOG(LogCarla, Log, TEXT("Loading settings from \"%s\""), *FileName); @@ -95,35 +112,24 @@ static void LoadSettingsFromFile(const FString &FileName, UCarlaSettings &Settin ConfigFile.GetInt(S_CARLA_SERVER, TEXT("WritePort"), Settings.WritePort); ConfigFile.GetInt(S_CARLA_SERVER, TEXT("ReadPort"), Settings.ReadPort); // SceneCapture. - ConfigFile.GetSceneCaptureMode(S_CARLA_SCENECAPTURE, TEXT("Mode"), Settings.SceneCaptureMode); - // SceneCapture - Mono. - ConfigFile.GetInt(S_CARLA_SCENECAPTURE, TEXT("ImageSizeX"), Settings.Mono_ImageSizeX); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE, TEXT("ImageSizeY"), Settings.Mono_ImageSizeY); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("ImageSizeX"), Settings.Mono_ImageSizeX); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("ImageSizeY"), Settings.Mono_ImageSizeY); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraPositionX"), Settings.Mono_CameraPosition.X); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraPositionY"), Settings.Mono_CameraPosition.Y); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraPositionZ"), Settings.Mono_CameraPosition.Z); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraRotationPitch"), Settings.Mono_CameraRotation.Pitch); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraRotationRoll"), Settings.Mono_CameraRotation.Roll); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_MONO, TEXT("CameraRotationYaw"), Settings.Mono_CameraRotation.Yaw); - // SceneCapture - Stereo. - ConfigFile.GetInt(S_CARLA_SCENECAPTURE, TEXT("ImageSizeX"), Settings.Stereo_ImageSizeX); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE, TEXT("ImageSizeY"), Settings.Stereo_ImageSizeY); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("ImageSizeX"), Settings.Stereo_ImageSizeX); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("ImageSizeY"), Settings.Stereo_ImageSizeY); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0PositionX"), Settings.Stereo_Camera0Position.X); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0PositionY"), Settings.Stereo_Camera0Position.Y); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0PositionZ"), Settings.Stereo_Camera0Position.Z); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1PositionX"), Settings.Stereo_Camera1Position.X); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1PositionY"), Settings.Stereo_Camera1Position.Y); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1PositionZ"), Settings.Stereo_Camera1Position.Z); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0RotationPitch"), Settings.Stereo_Camera0Rotation.Pitch); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0RotationRoll"), Settings.Stereo_Camera0Rotation.Roll); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera0RotationYaw"), Settings.Stereo_Camera0Rotation.Yaw); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1RotationPitch"), Settings.Stereo_Camera1Rotation.Pitch); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1RotationRoll"), Settings.Stereo_Camera1Rotation.Roll); - ConfigFile.GetInt(S_CARLA_SCENECAPTURE_STEREO, TEXT("Camera1RotationYaw"), Settings.Stereo_Camera1Rotation.Yaw); + FString Cameras; + ConfigFile.GetString(S_CARLA_SCENECAPTURE, TEXT("Cameras"), Cameras); + TArray CameraNames; + Cameras.ParseIntoArray(CameraNames, TEXT(","), true); + for (FString &Name : CameraNames) { + FCameraDescription &Camera = Settings.CameraDescriptions.FindOrAdd(Name); + GetCameraDescription(ConfigFile, S_CARLA_SCENECAPTURE, Camera); + + TArray SubSections; + Name.ParseIntoArray(SubSections, TEXT("/"), true); + check(SubSections.Num() > 0); + FString Section = S_CARLA_SCENECAPTURE; + for (FString &SubSection : SubSections) { + Section += TEXT("/"); + Section += SubSection; + GetCameraDescription(ConfigFile, *Section, Camera); + } + } } static bool GetSettingsFileName(FString &Value) @@ -167,14 +173,6 @@ void UCarlaSettings::LoadSettings() void UCarlaSettings::LogSettings() { - auto ModeAsString = [](ESceneCaptureMode Mode) { - switch (Mode) { - case ESceneCaptureMode::Mono: return TEXT("Mono"); - case ESceneCaptureMode::Stereo: return TEXT("Stereo"); - case ESceneCaptureMode::NoCapture: return TEXT("NoCapture"); - default: return TEXT("INVALID"); - }; - }; UE_LOG(LogCarla, Log, TEXT("== CARLA Settings ==============================================================")); UE_LOG(LogCarla, Log, TEXT("Settings file: %s"), *CurrentFileName); UE_LOG(LogCarla, Log, TEXT("[%s]"), S_CARLA_SERVER); @@ -183,21 +181,16 @@ void UCarlaSettings::LogSettings() UE_LOG(LogCarla, Log, TEXT("Write Port = %d"), WritePort); UE_LOG(LogCarla, Log, TEXT("Read Port = %d"), ReadPort); UE_LOG(LogCarla, Log, TEXT("[%s]"), S_CARLA_SCENECAPTURE); - UE_LOG(LogCarla, Log, TEXT("Mode = %s"), ModeAsString(SceneCaptureMode)); - UE_LOG(LogCarla, Log, TEXT("[%s]"), S_CARLA_SCENECAPTURE_MONO); - UE_LOG(LogCarla, Log, TEXT("Image Size = %dx%d"), Mono_ImageSizeX, Mono_ImageSizeY); - UE_LOG(LogCarla, Log, TEXT("Camera Position = (%s)"), *Mono_CameraPosition.ToString()); - UE_LOG(LogCarla, Log, TEXT("Camera Rotation = (%s)"), *Mono_CameraRotation.ToString()); - UE_LOG(LogCarla, Log, TEXT("[%s]"), S_CARLA_SCENECAPTURE_STEREO); - UE_LOG(LogCarla, Log, TEXT("ImageSize = %dx%d"), Stereo_ImageSizeX, Stereo_ImageSizeY); - UE_LOG(LogCarla, Log, TEXT("Camera0 Position = (%s)"), *Stereo_Camera0Position.ToString()); - UE_LOG(LogCarla, Log, TEXT("Camera0 Rotation = (%s)"), *Stereo_Camera0Rotation.ToString()); - UE_LOG(LogCarla, Log, TEXT("Camera1 Position = (%s)"), *Stereo_Camera1Position.ToString()); - UE_LOG(LogCarla, Log, TEXT("Camera1 Rotation = (%s)"), *Stereo_Camera1Rotation.ToString()); + UE_LOG(LogCarla, Log, TEXT("Added %d cameras."), CameraDescriptions.Num()); + for (auto &Item : CameraDescriptions) { + UE_LOG(LogCarla, Log, TEXT("[%s/%s]"), S_CARLA_SCENECAPTURE, *Item.Key); + UE_LOG(LogCarla, Log, TEXT("Image Size = %dx%d"), Item.Value.ImageSizeX, Item.Value.ImageSizeY); + UE_LOG(LogCarla, Log, TEXT("Camera Position = (%s)"), *Item.Value.Position.ToString()); + UE_LOG(LogCarla, Log, TEXT("Camera Rotation = (%s)"), *Item.Value.Rotation.ToString()); + UE_LOG(LogCarla, Log, TEXT("Post-Processing = %s"), *PostProcessEffect::ToString(Item.Value.PostProcessEffect)); + } UE_LOG(LogCarla, Log, TEXT("================================================================================")); } #undef S_CARLA_SERVER #undef S_CARLA_SCENECAPTURE -#undef S_CARLA_SCENECAPTURE_MONO -#undef S_CARLA_SCENECAPTURE_STEREO diff --git a/Source/Carla/Game/CarlaSettings.h b/Source/Carla/Game/CarlaSettings.h index 89098b755..0b56688fc 100644 --- a/Source/Carla/Game/CarlaSettings.h +++ b/Source/Carla/Game/CarlaSettings.h @@ -3,16 +3,9 @@ #pragma once #include "UObject/NoExportTypes.h" +#include "CameraDescription.h" #include "CarlaSettings.generated.h" -UENUM(BlueprintType) -enum class ESceneCaptureMode : uint8 -{ - Mono UMETA(DisplayName = "Mono: Single RGB capture"), - Stereo UMETA(DisplayName = "Stereo: Stereo RGB captures and Depth maps"), - NoCapture UMETA(DisplayName = "No capture") -}; - /// Global settings for CARLA. UCLASS() class CARLA_API UCarlaSettings : public UObject @@ -62,63 +55,9 @@ public: /// @{ public: - /** Controls the number and type of scene capture cameras that are added to the player. */ + /** Descriptions of the cameras to be attached to the player. */ UPROPERTY(Category = "Scene Capture", EditDefaultsOnly) - ESceneCaptureMode SceneCaptureMode = ESceneCaptureMode::Mono; - - /// @} - // =========================================================================== - /// @name Scene Capture - Mono - // =========================================================================== - /// @{ -public: - - /** X size in pixels of the captured image. */ - UPROPERTY(Category = "Scene Capture|Mono", EditDefaultsOnly) - uint32 Mono_ImageSizeX = 720u; - - /** Y size in pixels of the captured image. */ - UPROPERTY(Category = "Scene Capture|Mono", EditDefaultsOnly) - uint32 Mono_ImageSizeY = 512u; - - /** Camera position relative to the car. */ - UPROPERTY(Category = "Scene Capture|Mono", EditDefaultsOnly) - FVector Mono_CameraPosition = {170.0f, 0.0f, 150.0f}; - - /** Camera rotation relative to the car. */ - UPROPERTY(Category = "Scene Capture|Mono", EditDefaultsOnly) - FRotator Mono_CameraRotation = {0.0f, 0.0f, 0.0f}; - - /// @} - // =========================================================================== - /// @name Scene Capture - Stereo - // =========================================================================== - /// @{ -public: - - /** X size in pixels of the captured image. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - uint32 Stereo_ImageSizeX = 720u; - - /** Y size in pixels of the captured image. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - uint32 Stereo_ImageSizeY = 512u; - - /** Camera0 position relative to the car. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - FVector Stereo_Camera0Position = {170.0f, 30.0f, 150.0f}; - - /** Camera0 rotation relative to the car. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - FRotator Stereo_Camera0Rotation = {0.0f, 0.0f, 0.0f}; - - /** Camera1 position relative to the car. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - FVector Stereo_Camera1Position = {170.0f, -30.0f, 150.0f}; - - /** Camera1 rotation relative to the car. */ - UPROPERTY(Category = "Scene Capture|Stereo", EditDefaultsOnly) - FRotator Stereo_Camera1Rotation = {0.0f, 0.0f, 0.0f}; + TMap CameraDescriptions; /// @} }; diff --git a/Source/Carla/Game/CarlaVehicleController.cpp b/Source/Carla/Game/CarlaVehicleController.cpp index 16903bb56..3b3bc719f 100644 --- a/Source/Carla/Game/CarlaVehicleController.cpp +++ b/Source/Carla/Game/CarlaVehicleController.cpp @@ -6,6 +6,7 @@ #include "Camera/CameraComponent.h" #include "GameFramework/Pawn.h" #include "GameFramework/SpringArmComponent.h" +#include "SceneCaptureCamera.h" #include "WheeledVehicle.h" #include "WheeledVehicleMovementComponent.h" @@ -93,16 +94,19 @@ void ACarlaVehicleController::Possess(APawn *aPawn) } } -static void ReadCameraPixels( - const ASceneCaptureCamera *Camera, - ACarlaPlayerState::Image &Image) +void ACarlaVehicleController::BeginPlay() { - if (Camera != nullptr) { - if (Camera->ReadPixels(Image.BitMap)) { + CarlaPlayerState->Images.Empty(); + const auto NumberOfCameras = SceneCaptureCameras.Num(); + if (NumberOfCameras > 0) { + CarlaPlayerState->Images.AddDefaulted(NumberOfCameras); + for (auto i = 0; i < NumberOfCameras; ++i) { + auto *Camera = SceneCaptureCameras[i]; + check(Camera != nullptr); + auto &Image = CarlaPlayerState->Images[i]; Image.SizeX = Camera->GetImageSizeX(); Image.SizeY = Camera->GetImageSizeY(); - } else { - Image.BitMap.Empty(); // Clears the array. + Image.PostProcessEffect = Camera->GetPostProcessEffect(); } } } @@ -120,12 +124,15 @@ void ACarlaVehicleController::Tick(float DeltaTime) const FVector CurrentSpeed = CarlaPlayerState->ForwardSpeed * CarlaPlayerState->Orientation; CarlaPlayerState->Acceleration = (CurrentSpeed - PreviousSpeed) / DeltaTime; CarlaPlayerState->CurrentGear = GetVehicleCurrentGear(); - /// @todo Set intersection factors. - using CPS = ACarlaPlayerState; - ReadCameraPixels(RGBCameras[0u], CarlaPlayerState->Images[CPS::ImageRGB0]); - ReadCameraPixels(RGBCameras[1u], CarlaPlayerState->Images[CPS::ImageRGB1]); - ReadCameraPixels(DepthCameras[0u], CarlaPlayerState->Images[CPS::ImageDepth0]); - ReadCameraPixels(DepthCameras[1u], CarlaPlayerState->Images[CPS::ImageDepth1]); + /// @todo #15 Set intersection factors. + const auto NumberOfCameras = SceneCaptureCameras.Num(); + check(NumberOfCameras == CarlaPlayerState->Images.Num()); + for (auto i = 0; i < NumberOfCameras; ++i) { + auto &Image = CarlaPlayerState->Images[i]; + if (!SceneCaptureCameras[i]->ReadPixels(Image.BitMap)) { + Image.BitMap.Empty(); + } + } } } @@ -166,30 +173,20 @@ int32 ACarlaVehicleController::GetVehicleCurrentGear() const // -- Scene capture ------------------------------------------------------------ // ============================================================================= -void ACarlaVehicleController::RegisterCaptureCamera(ASceneCaptureCamera &CaptureCamera) +void ACarlaVehicleController::AddSceneCaptureCamera(const FCameraDescription &Description) { - AddTickPrerequisiteActor(&CaptureCamera); - const auto Effect = CaptureCamera.GetPostProcessEffect(); - auto &Cameras = (Effect == EPostProcessEffect::Depth ? DepthCameras : RGBCameras); - - for (auto i = 0u; i < Cameras.size(); ++i) { - if (Cameras[i] == nullptr) { - Cameras[i] = &CaptureCamera; - UE_LOG( - LogCarla, - Log, - TEXT("Registered capture camera %d with postprocess \"%s\""), - i, - *CaptureCamera.GetPostProcessEffectAsString()); - return; - } - } + auto Camera = GetWorld()->SpawnActor(Description.Position, Description.Rotation); + Camera->Set(Description); + Camera->AttachToActor(GetPawn(), FAttachmentTransformRules::KeepRelativeTransform); + Camera->SetOwner(GetPawn()); + AddTickPrerequisiteActor(Camera); + SceneCaptureCameras.Add(Camera); UE_LOG( LogCarla, - Warning, - TEXT("Attempting to register a capture camera of type \"%d\" but already have %d, captures from this camera won't be sent"), - *CaptureCamera.GetPostProcessEffectAsString(), - Cameras.size()); + Log, + TEXT("Created capture camera %d with postprocess \"%s\""), + SceneCaptureCameras.Num() - 1, + *PostProcessEffect::ToString(Camera->GetPostProcessEffect())); } // ============================================================================= diff --git a/Source/Carla/Game/CarlaVehicleController.h b/Source/Carla/Game/CarlaVehicleController.h index f3c0624e8..fdcfd1f4d 100644 --- a/Source/Carla/Game/CarlaVehicleController.h +++ b/Source/Carla/Game/CarlaVehicleController.h @@ -11,6 +11,7 @@ class ASceneCaptureCamera; class UCameraComponent; class USpringArmComponent; class UWheeledVehicleMovementComponent; +struct FCameraDescription; /** * @@ -41,6 +42,8 @@ public: virtual void Possess(APawn *aPawn) override; + virtual void BeginPlay() override; + virtual void Tick(float DeltaTime) override; virtual void CalcCamera(float DeltaTime, FMinimalViewInfo& OutResult) override; @@ -80,7 +83,7 @@ public: /// @{ public: - void RegisterCaptureCamera(ASceneCaptureCamera &CaptureCamera); + void AddSceneCaptureCamera(const FCameraDescription &CameraDescription); /// @} // =========================================================================== @@ -175,6 +178,9 @@ private: UPROPERTY() UWheeledVehicleMovementComponent *MovementComponent; + UPROPERTY() + TArray SceneCaptureCameras; + // Cast for quick access to the custom player state. UPROPERTY() ACarlaPlayerState *CarlaPlayerState; @@ -182,10 +188,4 @@ private: // Cast for quick access to the custom HUD. UPROPERTY() ACarlaHUD *CarlaHUD; - - using CaptureCameraArray = std::array; - - CaptureCameraArray RGBCameras; - - CaptureCameraArray DepthCameras; }; diff --git a/Source/Carla/Game/PostProcessEffect.cpp b/Source/Carla/Game/PostProcessEffect.cpp new file mode 100644 index 000000000..c137fe91f --- /dev/null +++ b/Source/Carla/Game/PostProcessEffect.cpp @@ -0,0 +1,13 @@ +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) + +#include "Carla.h" +#include "PostProcessEffect.h" +#include "Package.h" + +FString PostProcessEffect::ToString(EPostProcessEffect PostProcessEffect) +{ + const UEnum* ptr = FindObject(ANY_PACKAGE, TEXT("EPostProcessEffect"), true); + if(!ptr) + return FString("Invalid"); + return ptr->GetEnumName(static_cast(PostProcessEffect)); +} diff --git a/Source/Carla/Game/PostProcessEffect.h b/Source/Carla/Game/PostProcessEffect.h new file mode 100644 index 000000000..1a72da45e --- /dev/null +++ b/Source/Carla/Game/PostProcessEffect.h @@ -0,0 +1,23 @@ +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) + +#pragma once + +#include "PostProcessEffect.generated.h" + +UENUM(BlueprintType) +enum class EPostProcessEffect : uint8 +{ + None UMETA(DisplayName = "RGB without any post-processing"), + SceneFinal UMETA(DisplayName = "RGB with post-processing present at the scene"), + Depth UMETA(DisplayName = "Depth Map"), + + SIZE UMETA(Hidden), + INVALID UMETA(Hidden), +}; + +/// Helper class for working with EPostProcessEffect. +class CARLA_API PostProcessEffect { +public: + + static FString ToString(EPostProcessEffect PostProcessEffect); +}; diff --git a/Source/Carla/SceneCaptureCamera.cpp b/Source/Carla/SceneCaptureCamera.cpp index 39cd9b9fd..3f3acca26 100644 --- a/Source/Carla/SceneCaptureCamera.cpp +++ b/Source/Carla/SceneCaptureCamera.cpp @@ -1,4 +1,4 @@ -// Fill out your copyright notice in the Description page of Project Settings. +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) #include "Carla.h" #include "SceneCaptureCamera.h" @@ -100,12 +100,10 @@ void ASceneCaptureCamera::SetPostProcessEffect(EPostProcessEffect otherPostProce PostProcessEffect = otherPostProcessEffect; } -FString ASceneCaptureCamera::GetPostProcessEffectAsString() const +void ASceneCaptureCamera::Set(const FCameraDescription &CameraDescription) { - const UEnum* ptr = FindObject(ANY_PACKAGE, TEXT("EPostProcessEffect"), true); - if(!ptr) - return FString("Invalid"); - return ptr->GetEnumName(static_cast(PostProcessEffect)); + SetImageSize(CameraDescription.ImageSizeX, CameraDescription.ImageSizeY); + SetPostProcessEffect(CameraDescription.PostProcessEffect); } bool ASceneCaptureCamera::ReadPixels(TArray &BitMap) const diff --git a/Source/Carla/SceneCaptureCamera.h b/Source/Carla/SceneCaptureCamera.h index 22539da9d..43ea32922 100644 --- a/Source/Carla/SceneCaptureCamera.h +++ b/Source/Carla/SceneCaptureCamera.h @@ -1,14 +1,10 @@ -// Fill out your copyright notice in the Description page of Project Settings. - -/** - * Own SceneCapture, re-implementing some of the methods since ASceneCapture - * cannot be subclassed. - */ +// CARLA, Copyright (C) 2017 Computer Vision Center (CVC) #pragma once #include "GameFramework/Actor.h" #include "StaticMeshResources.h" +#include "Game/CameraDescription.h" #include "SceneCaptureCamera.generated.h" class UDrawFrustumComponent; @@ -16,16 +12,8 @@ class USceneCaptureComponent2D; class UStaticMeshComponent; class UTextureRenderTarget2D; -UENUM(BlueprintType) -enum class EPostProcessEffect : uint8 -{ - None UMETA(DisplayName = "RGB without any post-processing"), - SceneFinal UMETA(DisplayName = "RGB with post-processing present at the scene"), - Depth UMETA(DisplayName = "Depth Map"), - - SIZE UMETA(Hidden) -}; - +/// Own SceneCapture, re-implementing some of the methods since ASceneCapture +/// cannot be subclassed. UCLASS(hidecategories=(Collision, Attachment, Actor)) class CARLA_API ASceneCaptureCamera : public AActor { @@ -62,7 +50,7 @@ public: void SetPostProcessEffect(EPostProcessEffect PostProcessEffect); - FString GetPostProcessEffectAsString() const; + void Set(const FCameraDescription &CameraDescription); bool ReadPixels(TArray &BitMap) const;