Improving visual quality of the rgb scene capture (#1549)

* Adding TemporalAA
* Setting alpha values of pixels in buffer to 255 to account for TemporalAA
* Alpha now removed from pixels on the client side in the image deserialize function
* Adding motion blur

CHANGELOG.md

@@ -1,4 +1,7 @@
 ## Latest
+  * Improved visual quality of the screen capture for the rgb sensor
+    - Enabled Temporal AA for screen captures with no post-processing to prevent jaggies during motion
+    - Reduced the target gamma of render target to 1.4 to minimize brightness differences with main camera 
   * Upgraded to Unreal Engine 4.22
   * Recorder fixes:
     - Fixed a possible crash if an actor is respawned before the episode is ready when a new map is loaded automatically.

LibCarla/source/carla/sensor/s11n/ImageSerializer.cpp

@@ -13,7 +13,12 @@ namespace sensor {
 namespace s11n {
 
   SharedPtr<SensorData> ImageSerializer::Deserialize(RawData data) {
-    return SharedPtr<data::Image>(new data::Image{std::move(data)});
+    auto image = SharedPtr<data::Image>(new data::Image{std::move(data)});
+    // Set alpha of each pixel in the buffer to max to make it 100% opaque
+    for (auto &pixel : *image) {
+      pixel.a = 255u;
+    }
+    return image;
   }
 
 } // namespace s11n

Unreal/CarlaUE4/Config/DefaultEngine.ini

@@ -19,7 +19,7 @@ TransitionMap=/Game/Carla/Maps/Town03.Town03
 GlobalDefaultServerGameMode=/Game/Carla/Blueprints/Game/CarlaGameMode.CarlaGameMode_C
 
 [/Script/Engine.RendererSettings]
-r.DefaultFeature.MotionBlur=False
+r.DefaultFeature.MotionBlur=True
 r.AllowStaticLighting=False
 r.DiscardUnusedQuality=True
 r.DefaultFeature.Bloom=False

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.cpp

@@ -22,13 +22,15 @@
 // -- Local variables and types ------------------------------------------------
 // =============================================================================
 
-struct LockTexture {
+struct LockTexture
+{
   LockTexture(FRHITexture2D *InTexture, uint32 &Stride)
     : Texture(InTexture),
       Source(reinterpret_cast<const uint8 *>(
-          RHILockTexture2D(Texture, 0, RLM_ReadOnly, Stride, false))) {}
+            RHILockTexture2D(Texture, 0, RLM_ReadOnly, Stride, false))) {}
 
-  ~LockTexture() {
+  ~LockTexture()
+  {
     RHIUnlockTexture2D(Texture, 0, false);
   }
 
@@ -98,7 +100,8 @@ TUniquePtr<TImagePixelData<FColor>> FPixelReader::DumpPixels(
 {
   const FIntPoint DestSize(RenderTarget.GetSurfaceWidth(), RenderTarget.GetSurfaceHeight());
   TUniquePtr<TImagePixelData<FColor>> PixelData = MakeUnique<TImagePixelData<FColor>>(DestSize);
-  if (!WritePixelsToArray(RenderTarget, PixelData->Pixels)) {
+  if (!WritePixelsToArray(RenderTarget, PixelData->Pixels))
+  {
     return nullptr;
   }
   return PixelData;
@@ -112,14 +115,14 @@ TFuture<bool> FPixelReader::SavePixelsToDisk(
 }
 
 TFuture<bool> FPixelReader::SavePixelsToDisk(
-      TUniquePtr<TImagePixelData<FColor>> PixelData,
-      const FString &FilePath)
+    TUniquePtr<TImagePixelData<FColor>> PixelData,
+    const FString &FilePath)
 {
   TUniquePtr<FImageWriteTask> ImageTask = MakeUnique<FImageWriteTask>();
   ImageTask->PixelData = MoveTemp(PixelData);
   ImageTask->Filename = FilePath;
   ImageTask->Format = EImageFormat::PNG;
-  ImageTask->CompressionQuality = (int32)EImageCompressionQuality::Default;
+  ImageTask->CompressionQuality = (int32) EImageCompressionQuality::Default;
   ImageTask->bOverwriteFile = true;
   ImageTask->PixelPreProcessors.Add(TAsyncAlphaWrite<FColor>(255));
 
@@ -133,9 +136,9 @@ void FPixelReader::WritePixelsToBuffer(
     uint32 Offset,
     FRHICommandListImmediate &
 #if CARLA_WITH_VULKAN_SUPPORT == 1
-      InRHICmdList
+    InRHICmdList
 #endif // CARLA_WITH_VULKAN_SUPPORT
-  )
+    )
 {
   check(IsInRenderingThread());
 

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h

@@ -74,6 +74,7 @@ private:
       carla::Buffer &Buffer,
       uint32 Offset,
       FRHICommandListImmediate &InRHICmdList);
+
 };
 
 // =============================================================================

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp

@@ -26,7 +26,7 @@ namespace SceneCaptureSensor_local_ns {
 
   static void SetCameraDefaultOverrides(USceneCaptureComponent2D &CaptureComponent2D);
 
-  static void RemoveShowFlags(FEngineShowFlags &ShowFlags);
+  static void ConfigureShowFlags(FEngineShowFlags &ShowFlags, bool bPostProcessing = true);
 
   static auto GetQualitySettings(UWorld *World)
   {
@@ -71,6 +71,7 @@ ASceneCaptureSensor::ASceneCaptureSensor(const FObjectInitializer& ObjectInitial
   CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(
       FName(*FString::Printf(TEXT("SceneCaptureComponent2D_%d"), SCENE_CAPTURE_COUNTER)));
   CaptureComponent2D->SetupAttachment(MeshComp);
+
   SceneCaptureSensor_local_ns::SetCameraDefaultOverrides(*CaptureComponent2D);
 
   ++SCENE_CAPTURE_COUNTER;
@@ -133,12 +134,17 @@ void ASceneCaptureSensor::BeginPlay()
   using namespace SceneCaptureSensor_local_ns;
 
   // Setup render target.
+
+  // Determine the gamma of the player.
+
   const bool bInForceLinearGamma = !bEnablePostProcessingEffects;
-  if (bEnablePostProcessingEffects) {
-    CaptureRenderTarget->TargetGamma = 2.4f;
-  }
+
   CaptureRenderTarget->InitCustomFormat(ImageWidth, ImageHeight, PF_B8G8R8A8, bInForceLinearGamma);
 
+  if (bEnablePostProcessingEffects) {
+    CaptureRenderTarget->TargetGamma = 2.2f;
+  }
+
   check(IsValid(CaptureComponent2D) && !CaptureComponent2D->IsPendingKill());
 
   CaptureComponent2D->Deactivate();
@@ -158,11 +164,6 @@ void ASceneCaptureSensor::BeginPlay()
     CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR;
   }
 
-  if (!bEnablePostProcessingEffects)
-  {
-    SceneCaptureSensor_local_ns::RemoveShowFlags(CaptureComponent2D->ShowFlags);
-  }
-
   CaptureComponent2D->UpdateContent();
   CaptureComponent2D->Activate();
 
@@ -171,6 +172,8 @@ void ASceneCaptureSensor::BeginPlay()
       GetWorld(),
       FString("g.TimeoutForBlockOnRenderFence 300000"));
 
+  SceneCaptureSensor_local_ns::ConfigureShowFlags(CaptureComponent2D->ShowFlags, bEnablePostProcessingEffects);
+
   Super::BeginPlay();
 }
 
@@ -205,20 +208,27 @@ namespace SceneCaptureSensor_local_ns {
   static void SetCameraDefaultOverrides(USceneCaptureComponent2D &CaptureComponent2D)
   {
     auto &PostProcessSettings = CaptureComponent2D.PostProcessSettings;
-    PostProcessSettings.bOverride_AutoExposureMethod = true;
-    PostProcessSettings.AutoExposureMethod = AEM_Histogram;
-    PostProcessSettings.bOverride_AutoExposureMinBrightness = true;
-    PostProcessSettings.AutoExposureMinBrightness = 0.27f;
-    PostProcessSettings.bOverride_AutoExposureMaxBrightness = true;
-    PostProcessSettings.AutoExposureMaxBrightness = 5.0f;
-    PostProcessSettings.bOverride_AutoExposureBias = true;
-    PostProcessSettings.AutoExposureBias = -3.5f;
+
+    // Set motion Blur settings
+    PostProcessSettings.bOverride_MotionBlurAmount = true;
+    PostProcessSettings.MotionBlurAmount = 0.5f;
+    PostProcessSettings.bOverride_MotionBlurMax = true;
+    PostProcessSettings.MotionBlurMax = 10.0f;
+    PostProcessSettings.bOverride_MotionBlurPerObjectSize = true;
+    PostProcessSettings.MotionBlurPerObjectSize = 0.5f;
   }
 
   // Remove the show flags that might interfere with post-processing effects like
   // depth and semantic segmentation.
-  static void RemoveShowFlags(FEngineShowFlags &ShowFlags)
+  static void ConfigureShowFlags(FEngineShowFlags &ShowFlags, bool bPostProcessing)
   {
+    if (bPostProcessing)
+    {
+      ShowFlags.EnableAdvancedFeatures();
+      ShowFlags.SetMotionBlur(true);
+      return;
+    }
+
     ShowFlags.SetAmbientOcclusion(false);
     ShowFlags.SetAntiAliasing(false);
     ShowFlags.SetAtmosphericFog(false);