Reorganize where images are stored

Source/Carla/Game/CarlaGameController.cpp

@@ -32,11 +32,11 @@ static inline void Set(carla::Vector2D &cVector, const FVector &uVector)
   cVector = {uVector.X, uVector.Y};
 }
 
-static void Set(std::vector<carla::Color> &cImage, const ASceneCaptureCamera *Camera)
+static void Set(std::vector<carla::Color> &cImage, const TArray<FColor> &BitMap)
 {
-  if (Camera != nullptr) {
+  if (BitMap.Num() > 0) {
-    cImage.reserve(Camera->GetImage().Num());
+    cImage.reserve(BitMap.Num());
-    for (const auto &color : Camera->GetImage()) {
+    for (const auto &color : BitMap) {
       cImage.emplace_back();
       cImage.back().R = color.R;
       cImage.back().G = color.G;
@@ -98,12 +98,10 @@ static bool SendAndReadSceneValues(
 
 static void SendReward(
     carla::CarlaServer &Server,
-    const ACarlaPlayerState &PlayerState,
+    const ACarlaPlayerState &PlayerState)
-    const std::array<const ASceneCaptureCamera *, 2u> &RGBCameras,
-    const std::array<const ASceneCaptureCamera *, 2u> &DepthCameras)
 {
   carla::Reward_Values reward;
-  reward.timestamp = FMath::RoundHalfToZero(1000.0 * FPlatformTime::Seconds());
+  reward.timestamp = PlayerState.GetTimeStamp();
   Set(reward.player_location, PlayerState.GetLocation());
   Set(reward.player_orientation, PlayerState.GetOrientation());
   Set(reward.player_acceleration, PlayerState.GetAcceleration());
@@ -111,15 +109,21 @@ static void SendReward(
   Set(reward.collision_car, PlayerState.GetCollisionIntensityCars());
   Set(reward.collision_pedestrian, PlayerState.GetCollisionIntensityPedestrians());
   Set(reward.collision_general, PlayerState.GetCollisionIntensityOther());
-  // Set(reward.intersect_other_lane, );
+  Set(reward.intersect_other_lane, PlayerState.GetOtherLaneIntersectionFactor());
-  // Set(reward.intersect_offroad, );
+  Set(reward.intersect_offroad, PlayerState.GetOffRoadIntersectionFactor());
-  if (RGBCameras[0u] != nullptr) { // Do not add any camera if first is invalid.
+  { // Add images.
-    reward.image_width = RGBCameras[0u]->GetImageSizeX();
+    using CPS = ACarlaPlayerState;
-    reward.image_height = RGBCameras[0u]->GetImageSizeY();
+    auto &ImageRGB0 = PlayerState.GetImage(CPS::ImageRGB0);
-    Set(reward.image_rgb_0, RGBCameras[0u]);
+    if (ImageRGB0.BitMap.Num() > 0) {
-    Set(reward.image_rgb_1, RGBCameras[1u]);
+      // Do not add any camera if first is invalid, also assume all the images
-    Set(reward.image_depth_0, DepthCameras[0u]);
+      // have the same size.
-    Set(reward.image_depth_1, DepthCameras[1u]);
+      reward.image_width = ImageRGB0.SizeX;
+      reward.image_height = ImageRGB0.SizeY;
+      Set(reward.image_rgb_0, ImageRGB0.BitMap);
+      Set(reward.image_rgb_1, PlayerState.GetImage(CPS::ImageRGB1).BitMap);
+      Set(reward.image_depth_0, PlayerState.GetImage(CPS::ImageDepth0).BitMap);
+      Set(reward.image_depth_1, PlayerState.GetImage(CPS::ImageDepth1).BitMap);
+    }
   }
   UE_LOG(LogCarla, Log, CSTEXT("Sending reward"));
   Server.sendReward(reward);
@@ -142,9 +146,7 @@ static void TryReadControl(carla::CarlaServer &Server, ACarlaVehicleController &
 
 CarlaGameController::CarlaGameController() :
   Server(MakeUnique<carla::CarlaServer>(2001u, 2002u, 2000u)),
-  Player(nullptr),
+  Player(nullptr) {}
-  RGBCameras({{nullptr}}),
-  DepthCameras({{nullptr}}) {}
 
 CarlaGameController::~CarlaGameController()
 {
@@ -162,8 +164,6 @@ void CarlaGameController::Initialize()
       UE_LOG(LogCarla, Warning, CSTEXT("Read scene init failed, server needs restart"));
     }
   }
-  RGBCameras = {{nullptr}};
-  DepthCameras = {{nullptr}};
 }
 
 APlayerStart *CarlaGameController::ChoosePlayerStart(
@@ -183,32 +183,6 @@ void CarlaGameController::RegisterPlayer(AController &NewPlayer)
   check(Player != nullptr);
 }
 
-void CarlaGameController::RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera)
-{
-  const auto Effect = CaptureCamera.GetPostProcessEffect();
-  check((Effect == EPostProcessEffect::None) || (Effect == EPostProcessEffect::Depth))
-  auto &Cameras = (Effect == EPostProcessEffect::None ? RGBCameras : DepthCameras);
-
-  for (auto i = 0u; i < Cameras.size(); ++i) {
-    if (Cameras[i] == nullptr) {
-      Cameras[i] = &CaptureCamera;
-      UE_LOG(
-          LogCarla,
-          Log,
-          CSTEXT("Registered capture camera %d with postprocess \"%s\""),
-          i,
-          *CaptureCamera.GetPostProcessEffectAsString());
-      return;
-    }
-  }
-  UE_LOG(
-      LogCarla,
-      Warning,
-      CSTEXT("Attempting to register a capture camera of type \"%d\" but already have %d, captures from this camera won't be sent"),
-      *CaptureCamera.GetPostProcessEffectAsString(),
-      Cameras.size());
-}
-
 void CarlaGameController::BeginPlay()
 {
   Server->sendEndReset();
@@ -228,7 +202,7 @@ void CarlaGameController::Tick(float DeltaSeconds)
     return;
   }
 
-  SendReward(*Server, Player->GetPlayerState(), RGBCameras, DepthCameras);
+  SendReward(*Server, Player->GetPlayerState());
   TryReadControl(*Server, *Player);
 }
 

Source/Carla/Game/CarlaGameController.h

@@ -25,8 +25,6 @@ public:
 
   virtual void RegisterPlayer(AController &NewPlayer) override;
 
-  virtual void RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera) override;
-
   virtual void BeginPlay() override;
 
   virtual void Tick(float DeltaSeconds) override;
@@ -39,9 +37,5 @@ private:
 
   ACarlaVehicleController *Player;
 
-  std::array<const ASceneCaptureCamera *, 2u> RGBCameras;
-
-  std::array<const ASceneCaptureCamera *, 2u> DepthCameras;
-
   bool bServerNeedsRestart = true;
 };

Source/Carla/Game/CarlaGameControllerBase.h

@@ -6,7 +6,6 @@
 
 class AController;
 class APlayerStart;
-class ASceneCaptureCamera;
 
 /// Base class for a CARLA game controller.
 class CARLA_API CarlaGameControllerBase
@@ -21,8 +20,6 @@ public:
 
   virtual void RegisterPlayer(AController &NewPlayer) = 0;
 
-  virtual void RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera) = 0;
-
   virtual void BeginPlay() = 0;
 
   virtual void Tick(float DeltaSeconds) = 0;

Source/Carla/Game/CarlaGameMode.cpp

@@ -16,7 +16,8 @@
 
 ACarlaGameMode::ACarlaGameMode() :
   Super(),
-  GameController(nullptr)
+  GameController(nullptr),
+  PlayerController(nullptr)
 {
   PrimaryActorTick.bCanEverTick = true;
   PrimaryActorTick.TickGroup = TG_PrePhysics;
@@ -79,7 +80,10 @@ void ACarlaGameMode::RegisterCaptureCamera(ASceneCaptureCamera &CaptureCamera)
 {
   check(GameController != nullptr);
   AddTickPrerequisiteActor(&CaptureCamera);
-  GameController->RegisterCaptureCamera(CaptureCamera);
+  ACarlaVehicleController *Player = Cast<ACarlaVehicleController>(PlayerController);
+  if (Player != nullptr) {
+    Player->RegisterCaptureCamera(CaptureCamera);
+  }
 }
 
 void ACarlaGameMode::RegisterPlayer(AController &NewPlayer)
@@ -87,6 +91,7 @@ void ACarlaGameMode::RegisterPlayer(AController &NewPlayer)
   check(GameController != nullptr);
   AddTickPrerequisiteActor(&NewPlayer);
   GameController->RegisterPlayer(NewPlayer);
+  PlayerController = &NewPlayer;
 }
 
 APlayerStart *ACarlaGameMode::FindUnOccupiedStartPoints(

Source/Carla/Game/CarlaGameMode.h

@@ -48,4 +48,6 @@ private:
       TArray<APlayerStart *> &UnOccupiedStartPoints);
 
   CarlaGameControllerBase *GameController;
+
+  AController *PlayerController;
 };

Source/Carla/Game/CarlaPlayerState.cpp

@@ -6,9 +6,14 @@
 void ACarlaPlayerState::Reset()
 {
   Super::Reset();
+  // Reset incremental values.
   CollisionIntensityCars = 0.0f;
   CollisionIntensityPedestrians = 0.0f;
   CollisionIntensityOther = 0.0f;
+  // Invalidate images.
+  for (auto &image : Images) {
+    image = Image(); // Reset.
+  }
 }
 
 void ACarlaPlayerState::CopyProperties(APlayerState *PlayerState)
@@ -19,6 +24,7 @@ void ACarlaPlayerState::CopyProperties(APlayerState *PlayerState)
     ACarlaPlayerState *Other = Cast<ACarlaPlayerState>(PlayerState);
     if (Other != nullptr)
     {
+      TimeStamp = Other->TimeStamp;
       Location = Other->Location;
       Orientation = Other->Orientation;
       Acceleration = Other->Acceleration;
@@ -26,6 +32,10 @@ void ACarlaPlayerState::CopyProperties(APlayerState *PlayerState)
       CollisionIntensityCars = Other->CollisionIntensityCars;
       CollisionIntensityPedestrians = Other->CollisionIntensityPedestrians;
       CollisionIntensityOther = Other->CollisionIntensityOther;
+      OtherLaneIntersectionFactor = Other->OtherLaneIntersectionFactor;
+      OffRoadIntersectionFactor = Other->OffRoadIntersectionFactor;
+      Images = Other->Images;
+      UE_LOG(LogCarla, Log, TEXT("Copied properties of ACarlaPlayerState"));
     }
   }
 }
@@ -34,3 +44,8 @@ void ACarlaPlayerState::RegisterCollision(AActor */*Actor*/, FVector NormalImpul
 {
   CollisionIntensityOther += NormalImpulse.Size();
 }
+
+void ACarlaPlayerState::UpdateTimeStamp()
+{
+  TimeStamp = FMath::RoundHalfToZero(1000.0 * FPlatformTime::Seconds());
+}

Source/Carla/Game/CarlaPlayerState.h

@@ -2,24 +2,56 @@
 
 #pragma once
 
+#include <array>
 #include "GameFramework/PlayerState.h"
 #include "CarlaPlayerState.generated.h"
 
-/**
+/// Current state of the player, updated every frame by ACarlaVehicleController.
- *
+///
- */
+/// This class matches the reward that it is sent to the client over the
+/// network.
 UCLASS()
 class CARLA_API ACarlaPlayerState : public APlayerState
 {
   GENERATED_BODY()
 
+  // ===========================================================================
+  // -- Types ------------------------------------------------------------------
+  // ===========================================================================
+public:
+
+  enum ImageTag {
+    ImageRGB0,
+    ImageRGB1,
+    ImageDepth0,
+    ImageDepth1,
+    NUMBER_OF_IMAGES
+  };
+
+  struct Image {
+    uint32 SizeX = 0u;
+    uint32 SizeY = 0u;
+    TArray<FColor> BitMap;
+  };
+
+  // ===========================================================================
+  // -- APlayerState -----------------------------------------------------------
+  // ===========================================================================
 public:
 
   virtual void Reset() override;
 
   virtual void CopyProperties(APlayerState *PlayerState) override;
 
-  void RegisterCollision(AActor *Actor, FVector NormalImpulse);
+  // ===========================================================================
+  // -- Getters ----------------------------------------------------------------
+  // ===========================================================================
+public:
+
+  int32 GetTimeStamp() const
+  {
+    return TimeStamp;
+  }
 
   const FVector &GetLocation() const
   {
@@ -56,21 +88,59 @@ public:
     return CollisionIntensityOther;
   }
 
+  float GetOtherLaneIntersectionFactor() const
+  {
+    return OtherLaneIntersectionFactor;
+  }
+
+  float GetOffRoadIntersectionFactor() const
+  {
+    return OffRoadIntersectionFactor;
+  }
+
+  const Image &GetImage(ImageTag Tag) const
+  {
+    return Images[Tag];
+  }
+
+  // ===========================================================================
+  // -- Modifiers --------------------------------------------------------------
+  // ===========================================================================
+private:
+
+  void RegisterCollision(AActor *Actor, FVector NormalImpulse);
+
+  void UpdateTimeStamp();
+
+  // ===========================================================================
+  // -- Private members --------------------------------------------------------
+  // ===========================================================================
 private:
 
   friend class ACarlaVehicleController;
 
+  // If you add another variable here, don't forget to copy it inside
+  // CopyProperties if necessary.
+
+  int32 TimeStamp;
+
   FVector Location;
 
   FVector Orientation;
 
   FVector Acceleration;
 
-  float ForwardSpeed;
+  float ForwardSpeed = 0.0f;
 
   float CollisionIntensityCars = 0.0f;
 
   float CollisionIntensityPedestrians = 0.0f;
 
   float CollisionIntensityOther = 0.0f;
+
+  float OtherLaneIntersectionFactor = 0.0f;
+
+  float OffRoadIntersectionFactor = 0.0f;
+
+  std::array<Image, NUMBER_OF_IMAGES> Images;
 };

Source/Carla/Game/CarlaVehicleController.cpp

@@ -86,17 +86,36 @@ void ACarlaVehicleController::Possess(APawn *aPawn)
   }
 }
 
+static void ReadCameraPixels(const ASceneCaptureCamera *Camera, ACarlaPlayerState::Image &Image)
+{
+  if (Camera != nullptr) {
+    if (Camera->ReadPixels(Image.BitMap)) {
+      Image.SizeX = Camera->GetImageSizeX();
+      Image.SizeY = Camera->GetImageSizeY();
+    } else {
+      Image.BitMap.Empty(); // Clears the array.
+    }
+  }
+}
+
 void ACarlaVehicleController::Tick(float DeltaTime)
 {
   Super::PlayerTick(DeltaTime);
 
   if (IsPossessingAVehicle()) {
+    CarlaPlayerState->UpdateTimeStamp();
     CarlaPlayerState->Location = GetVehicleLocation();
     const FVector PreviousSpeed = CarlaPlayerState->ForwardSpeed * CarlaPlayerState->Orientation;
     CarlaPlayerState->Orientation = GetVehicleOrientation();
     CarlaPlayerState->ForwardSpeed = GetVehicleForwardSpeed();
     const FVector CurrentSpeed = CarlaPlayerState->ForwardSpeed * CarlaPlayerState->Orientation;
     CarlaPlayerState->Acceleration = (CurrentSpeed - PreviousSpeed) / DeltaTime;
+    /// @todo Set intersection factors.
+    using CPS = ACarlaPlayerState;
+    ReadCameraPixels(RGBCameras[0u], CarlaPlayerState->Images[CPS::ImageRGB0]);
+    ReadCameraPixels(RGBCameras[1u], CarlaPlayerState->Images[CPS::ImageRGB1]);
+    ReadCameraPixels(DepthCameras[0u], CarlaPlayerState->Images[CPS::ImageDepth0]);
+    ReadCameraPixels(DepthCameras[1u], CarlaPlayerState->Images[CPS::ImageDepth1]);
   }
 }
 
@@ -127,10 +146,34 @@ FVector ACarlaVehicleController::GetVehicleOrientation() const
   return GetPawn()->GetTransform().GetRotation().GetForwardVector();
 }
 
-void ACarlaVehicleController::ResetPlayerState()
+// =============================================================================
+// -- Scene capture ------------------------------------------------------------
+// =============================================================================
+
+void ACarlaVehicleController::RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera)
 {
-  check(CarlaPlayerState != nullptr);
+  const auto Effect = CaptureCamera.GetPostProcessEffect();
-  CarlaPlayerState->Reset();
+  check((Effect == EPostProcessEffect::None) || (Effect == EPostProcessEffect::Depth))
+  auto &Cameras = (Effect == EPostProcessEffect::None ? RGBCameras : DepthCameras);
+
+  for (auto i = 0u; i < Cameras.size(); ++i) {
+    if (Cameras[i] == nullptr) {
+      Cameras[i] = &CaptureCamera;
+      UE_LOG(
+          LogCarla,
+          Log,
+          TEXT("Registered capture camera %d with postprocess \"%s\""),
+          i,
+          *CaptureCamera.GetPostProcessEffectAsString());
+      return;
+    }
+  }
+  UE_LOG(
+      LogCarla,
+      Warning,
+      TEXT("Attempting to register a capture camera of type \"%d\" but already have %d, captures from this camera won't be sent"),
+      *CaptureCamera.GetPostProcessEffectAsString(),
+      Cameras.size());
 }
 
 // =============================================================================

Source/Carla/Game/CarlaVehicleController.h

@@ -6,6 +6,7 @@
 #include "CarlaVehicleController.generated.h"
 
 class ACarlaPlayerState;
+class ASceneCaptureCamera;
 class UCameraComponent;
 class USpringArmComponent;
 class UWheeledVehicleMovementComponent;
@@ -48,6 +49,7 @@ public:
   /// @name Vehicle pawn info
   // ===========================================================================
   /// @{
+public:
 
   bool IsPossessingAVehicle() const
   {
@@ -68,7 +70,14 @@ public:
     return *CarlaPlayerState;
   }
 
-  void ResetPlayerState();
+  /// @}
+  // ===========================================================================
+  /// @name Scene Capture
+  // ===========================================================================
+  /// @{
+public:
+
+  void RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera);
 
   /// @}
   // ===========================================================================
@@ -166,4 +175,10 @@ private:
   // Cast for quick access to the custom player state.
   UPROPERTY()
   ACarlaPlayerState *CarlaPlayerState;
+
+  using CaptureCameraArray = std::array<const ASceneCaptureCamera *, 2u>;
+
+  CaptureCameraArray RGBCameras;
+
+  CaptureCameraArray DepthCameras;
 };

Source/Carla/Game/MockGameController.cpp

@@ -24,11 +24,6 @@ void MockGameController::RegisterPlayer(AController &NewPlayer)
   }
 }
 
-void MockGameController::RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera)
-{
-
-}
-
 void MockGameController::BeginPlay()
 {
 

Source/Carla/Game/MockGameController.h

@@ -15,8 +15,6 @@ public:
 
   virtual void RegisterPlayer(AController &NewPlayer) override;
 
-  virtual void RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera) override;
-
   virtual void BeginPlay() override;
 
   virtual void Tick(float DeltaSeconds) override;

Source/Carla/SceneCaptureCamera.cpp

@@ -20,7 +20,7 @@ ASceneCaptureCamera::ASceneCaptureCamera(const FObjectInitializer& ObjectInitial
   SizeY(200u),
   PostProcessEffect(EPostProcessEffect::None)
 {
-  PrimaryActorTick.bCanEverTick = true;
+  PrimaryActorTick.bCanEverTick = true; /// @todo Does it need to tick?
   PrimaryActorTick.TickGroup = TG_PrePhysics;
 
   MeshComp = CreateDefaultSubobject<UStaticMeshComponent>(TEXT("CamMesh0"));
@@ -83,16 +83,6 @@ void ASceneCaptureCamera::BeginPlay()
     GameMode->RegisterCaptureCamera(*this);
 }
 
-void ASceneCaptureCamera::Tick(float Delta)
-{
-  Super::Tick(Delta);
-  // Update the image bitmap.
-  FTextureRenderTargetResource* RTResource = CaptureRenderTarget->GameThread_GetRenderTargetResource();
-  FReadSurfaceDataFlags ReadPixelFlags(RCM_UNorm);
-  ReadPixelFlags.SetLinearToGamma(true);
-  RTResource->ReadPixels(ImageBitMap, ReadPixelFlags);
-}
-
 FString ASceneCaptureCamera::GetPostProcessEffectAsString() const
 {
   const UEnum* ptr = FindObject<UEnum>(ANY_PACKAGE, TEXT("EPostProcessEffect"), true);
@@ -101,6 +91,14 @@ FString ASceneCaptureCamera::GetPostProcessEffectAsString() const
   return ptr->GetEnumName(static_cast<int32>(PostProcessEffect));
 }
 
+bool ASceneCaptureCamera::ReadPixels(TArray<FColor> &BitMap) const
+{
+  FTextureRenderTargetResource* RTResource = CaptureRenderTarget->GameThread_GetRenderTargetResource();
+  FReadSurfaceDataFlags ReadPixelFlags(RCM_UNorm);
+  ReadPixelFlags.SetLinearToGamma(true);
+  return RTResource->ReadPixels(BitMap, ReadPixelFlags);
+}
+
 void ASceneCaptureCamera::UpdateDrawFrustum()
 {
   if(DrawFrustum && CaptureComponent2D)

Source/Carla/SceneCaptureCamera.h

@@ -41,8 +41,6 @@ public:
 
   virtual void BeginPlay() override;
 
-  virtual void Tick(float Delta) override;
-
   uint32 GetImageSizeX() const
   {
     return SizeX;
@@ -60,10 +58,7 @@ public:
 
   FString GetPostProcessEffectAsString() const;
 
-  const TArray<FColor> &GetImage() const
+  bool ReadPixels(TArray<FColor> &BitMap) const;
-  {
-    return ImageBitMap;
-  }
 
 private:
 
@@ -95,7 +90,4 @@ private:
   /** Scene capture component. */
   UPROPERTY(EditAnywhere)
   class USceneCaptureComponent2D* CaptureComponent2D;
-
-  UPROPERTY()
-  TArray<FColor> ImageBitMap;
 };