p71924506
/
carla
mirror of https://github.com/carla-simulator/carla.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removing the PixelReader2

This commit is contained in:
bernatx 2022-07-08 09:58:24 +02:00 committed by bernat
parent a9a8e88ed9
commit 2ff36dbfda
6 changed files with 135 additions and 429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.cpp
View File

@ -8,6 +8,7 @@
#include "Carla/Sensor/PixelReader.h" #include "Carla/Sensor/PixelReader.h"
#include "Engine/TextureRenderTarget2D.h" #include "Engine/TextureRenderTarget2D.h"
#include "Async/Async.h"
#include "HighResScreenshot.h" #include "HighResScreenshot.h"
#include "Runtime/ImageWriteQueue/Public/ImageWriteQueue.h" #include "Runtime/ImageWriteQueue/Public/ImageWriteQueue.h"
@ -36,18 +37,15 @@ struct LockTexture
// -- Static local functions --------------------------------------------------- // -- Static local functions ---------------------------------------------------
// ============================================================================= // =============================================================================
// Temporal; this avoid allocating the array each time and also avoids checking
// for a bigger texture, ReadSurfaceData will allocate the space needed.
TArray<FColor> gPixels;
static void WritePixelsToBuffer_Vulkan( static void WritePixelsToBuffer_Vulkan(
const UTextureRenderTarget2D &RenderTarget, const UTextureRenderTarget2D &RenderTarget,
carla::Buffer &Buffer,
uint32 Offset, uint32 Offset,
FRHICommandListImmediate &InRHICmdList) FRHICommandListImmediate &RHICmdList,
FPixelReader::Payload FuncForSending)
{ {
TRACE_CPUPROFILER_EVENT_SCOPE_STR("WritePixelsToBuffer_Vulkan"); TRACE_CPUPROFILER_EVENT_SCOPE_STR("WritePixelsToBuffer_Vulkan");
check(IsInRenderingThread()); check(IsInRenderingThread());
auto RenderResource = auto RenderResource =
static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource); static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource);
FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture(); FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture();
@ -56,21 +54,46 @@ static void WritePixelsToBuffer_Vulkan(
return; return;
} }
FIntPoint Rect = RenderResource->GetSizeXY(); auto BackBufferReadback = std::make_unique<FRHIGPUTextureReadback>(TEXT("CameraBufferReadback"));
FIntPoint BackBufferSize = Texture->GetSizeXY();
EPixelFormat BackBufferPixelFormat = Texture->GetFormat();
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("EnqueueCopy");
BackBufferReadback->EnqueueCopy(RHICmdList, Texture, FResolveRect(0, 0, BackBufferSize.X, BackBufferSize.Y));
}
// NS: Extra copy here, don't know how to avoid it. // workaround to force RHI with Vulkan to refresh the fences state in the middle of frame
{ {
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Read Surface"); FRenderQueryRHIRef Query = RHICreateRenderQuery(RQT_AbsoluteTime);
InRHICmdList.ReadSurfaceData( TRACE_CPUPROFILER_EVENT_SCOPE_STR("create query");
Texture, RHICmdList.EndRenderQuery(Query);
FIntRect(0, 0, Rect.X, Rect.Y), TRACE_CPUPROFILER_EVENT_SCOPE_STR("Flush");
gPixels, RHICmdList.ImmediateFlush(EImmediateFlushType::FlushRHIThread);
FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX)); TRACE_CPUPROFILER_EVENT_SCOPE_STR("query result");
} uint64 OldAbsTime = 0;
{ RHICmdList.GetRenderQueryResult(Query, OldAbsTime, true);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Buffer Copy");
Buffer.copy_from(Offset, gPixels);
} }
AsyncTask(ENamedThreads::AnyNormalThreadNormalTask, [=, Readback=std::move(BackBufferReadback)]() mutable {
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Wait GPU transfer");
while (!Readback->IsReady())
{
std::this_thread::yield();
}
}
FPixelFormatInfo PixelFormat = GPixelFormats[BackBufferPixelFormat];
int32 Count = (BackBufferSize.Y * (PixelFormat.BlockBytes * BackBufferSize.X));
void* LockedData = Readback->Lock(Count);
if (LockedData)
{
FuncForSending(LockedData, Count, Offset);
}
Readback->Unlock();
Readback.reset();
});
} }
// Temporal; this avoid allocating the array each time // Temporal; this avoid allocating the array each time
@ -78,12 +101,13 @@ TArray<FFloat16Color> gFloatPixels;
static void WriteFloatPixelsToBuffer_Vulkan( static void WriteFloatPixelsToBuffer_Vulkan(
const UTextureRenderTarget2D &RenderTarget, const UTextureRenderTarget2D &RenderTarget,
carla::Buffer &Buffer,
uint32 Offset, uint32 Offset,
FRHICommandListImmediate &InRHICmdList) FRHICommandListImmediate &RHICmdList,
FPixelReader::Payload FuncForSending)
{ {
TRACE_CPUPROFILER_EVENT_SCOPE_STR("WritePixelsToBuffer_Vulkan");
check(IsInRenderingThread()); check(IsInRenderingThread());
gFloatPixels.Empty();
auto RenderResource = auto RenderResource =
static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource); static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource);
FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture(); FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture();
@ -92,25 +116,58 @@ static void WriteFloatPixelsToBuffer_Vulkan(
return; return;
} }
FIntPoint Rect = RenderResource->GetSizeXY(); auto BackBufferReadback = std::make_unique<FRHIGPUTextureReadback>(TEXT("CameraBufferReadback"));
FIntPoint BackBufferSize = Texture->GetSizeXY();
// NS: Extra copy here, don't know how to avoid it. EPixelFormat BackBufferPixelFormat = Texture->GetFormat();
InRHICmdList.ReadSurfaceFloatData( {
Texture, TRACE_CPUPROFILER_EVENT_SCOPE_STR("EnqueueCopy");
FIntRect(0, 0, Rect.X, Rect.Y), BackBufferReadback->EnqueueCopy(RHICmdList, Texture, FResolveRect(0, 0, BackBufferSize.X, BackBufferSize.Y));
gFloatPixels,
CubeFace_PosX,0,0);
TArray<float> IntermediateBuffer;
IntermediateBuffer.Reserve(gFloatPixels.Num() * 2);
for (FFloat16Color& color : gFloatPixels) {
float x = (color.R.GetFloat() - 0.5f)*4.f;
float y = (color.G.GetFloat() - 0.5f)*4.f;
IntermediateBuffer.Add(x);
IntermediateBuffer.Add(y);
} }
Buffer.copy_from(Offset, IntermediateBuffer);
} // workaround to force RHI with Vulkan to refresh the fences state in the middle of frame
{
FRenderQueryRHIRef Query = RHICreateRenderQuery(RQT_AbsoluteTime);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("create query");
RHICmdList.EndRenderQuery(Query);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Flush");
RHICmdList.ImmediateFlush(EImmediateFlushType::FlushRHIThread);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("query result");
uint64 OldAbsTime = 0;
RHICmdList.GetRenderQueryResult(Query, OldAbsTime, true);
}
AsyncTask(ENamedThreads::AnyNormalThreadNormalTask, [=, Readback=std::move(BackBufferReadback)]() mutable {
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Wait GPU transfer");
while (!Readback->IsReady())
{
std::this_thread::yield();
}
}
FPixelFormatInfo PixelFormat = GPixelFormats[BackBufferPixelFormat];
int32 Count = (BackBufferSize.Y * (PixelFormat.BlockBytes * BackBufferSize.X));
int32 TotalPixels = (BackBufferSize.Y * BackBufferSize.X);
void* LockedData = Readback->Lock(Count);
if (LockedData)
{
TArray<float> IntermediateBuffer;
FFloat16Color *Data = reinterpret_cast<FFloat16Color *>(LockedData);
IntermediateBuffer.Reserve(TotalPixels * 2);
for (int i=0; i<TotalPixels; ++i)
{
float x = (Data->R.GetFloat() - 0.5f) * 4.f;
float y = (Data->G.GetFloat() - 0.5f) * 4.f;
IntermediateBuffer.Add(x);
IntermediateBuffer.Add(y);
++Data;
}
FuncForSending(reinterpret_cast<void *>(IntermediateBuffer.GetData()), TotalPixels * sizeof(float) * 2 , Offset);
}
Readback->Unlock();
Readback.reset();
});
}
// ============================================================================= // =============================================================================
// -- FPixelReader ------------------------------------------------------------- // -- FPixelReader -------------------------------------------------------------
@ -171,11 +228,10 @@ TFuture<bool> FPixelReader::SavePixelsToDisk(
void FPixelReader::WritePixelsToBuffer( void FPixelReader::WritePixelsToBuffer(
UTextureRenderTarget2D &RenderTarget, UTextureRenderTarget2D &RenderTarget,
carla::Buffer &Buffer,
uint32 Offset, uint32 Offset,
FRHICommandListImmediate &InRHICmdList, FRHICommandListImmediate &InRHICmdList,
bool use16BitFormat FPixelReader::Payload FuncForSending,
) bool use16BitFormat)
{ {
TRACE_CPUPROFILER_EVENT_SCOPE_STR("WritePixelsToBuffer"); TRACE_CPUPROFILER_EVENT_SCOPE_STR("WritePixelsToBuffer");
check(IsInRenderingThread()); check(IsInRenderingThread());
@ -184,15 +240,15 @@ void FPixelReader::WritePixelsToBuffer(
{ {
if (use16BitFormat) if (use16BitFormat)
{ {
WriteFloatPixelsToBuffer_Vulkan(RenderTarget, Buffer, Offset, InRHICmdList); WriteFloatPixelsToBuffer_Vulkan(RenderTarget, Offset, InRHICmdList, std::move(FuncForSending));
} }
else else
{ {
WritePixelsToBuffer_Vulkan(RenderTarget, Buffer, Offset, InRHICmdList); WritePixelsToBuffer_Vulkan(RenderTarget, Offset, InRHICmdList, std::move(FuncForSending));
} }
return; return;
} }
/*
FTextureRenderTargetResource* RenderTargetResource = RenderTarget.GetRenderTargetResource(); FTextureRenderTargetResource* RenderTargetResource = RenderTarget.GetRenderTargetResource();
if(!RenderTargetResource) if(!RenderTargetResource)
{ {
@ -235,4 +291,5 @@ void FPixelReader::WritePixelsToBuffer(
Buffer.copy_from(Offset, Source, ExpectedStride * Height); Buffer.copy_from(Offset, Source, ExpectedStride * Height);
} }
} }
*/
} }

44
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h
View File

@ -11,6 +11,7 @@
#include "Runtime/ImageWriteQueue/Public/ImagePixelData.h" #include "Runtime/ImageWriteQueue/Public/ImagePixelData.h"
#include <compiler/disable-ue4-macros.h> #include <compiler/disable-ue4-macros.h>
#include <carla/Logging.h>
#include <carla/Buffer.h> #include <carla/Buffer.h>
#include <carla/sensor/SensorRegistry.h> #include <carla/sensor/SensorRegistry.h>
#include <compiler/enable-ue4-macros.h> #include <compiler/enable-ue4-macros.h>
@ -26,6 +27,8 @@ class FPixelReader
{ {
public: public:
using Payload = std::function<void(void *, uint32, uint32)>;
/// Copy the pixels in @a RenderTarget into @a BitMap. /// Copy the pixels in @a RenderTarget into @a BitMap.
/// ///
/// @pre To be called from game-thread. /// @pre To be called from game-thread.
@ -71,9 +74,9 @@ private:
/// @pre To be called from render-thread. /// @pre To be called from render-thread.
static void WritePixelsToBuffer( static void WritePixelsToBuffer(
UTextureRenderTarget2D &RenderTarget, UTextureRenderTarget2D &RenderTarget,
carla::Buffer &Buffer,
uint32 Offset, uint32 Offset,
FRHICommandListImmediate &InRHICmdList, FRHICommandListImmediate &InRHICmdList,
FPixelReader::Payload FuncForSending,
bool use16BitFormat = false); bool use16BitFormat = false);
}; };
@ -101,26 +104,43 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat
// game-thread. // game-thread.
ENQUEUE_RENDER_COMMAND(FWritePixels_SendPixelsInRenderThread) ENQUEUE_RENDER_COMMAND(FWritePixels_SendPixelsInRenderThread)
( (
[&Sensor, Stream=Sensor.GetDataStream(Sensor), use16BitFormat](auto &InRHICmdList) mutable [&Sensor, use16BitFormat](auto &InRHICmdList) mutable
{ {
TRACE_CPUPROFILER_EVENT_SCOPE_STR("FWritePixels_SendPixelsInRenderThread"); TRACE_CPUPROFILER_EVENT_SCOPE_STR("FWritePixels_SendPixelsInRenderThread");
/// @todo Can we make sure the sensor is not going to be destroyed? /// @todo Can we make sure the sensor is not going to be destroyed?
if (!Sensor.IsPendingKill()) if (!Sensor.IsPendingKill())
{ {
auto Buffer = Stream.PopBufferFromPool(); FPixelReader::Payload FuncForSending = [&Sensor, Frame = FCarlaEngine::GetFrameCounter()](void *LockedData, uint32 Count, uint32 Offset)
{
if (Sensor.IsPendingKill()) return;
auto Stream = Sensor.GetDataStream(Sensor);
// Stream.SetFrameNumber(Frame);
auto Buffer = Stream.PopBufferFromPool();
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Buffer Copy");
Buffer.copy_from(Offset, boost::asio::buffer(LockedData, Count));
}
{
// send
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Sending buffer");
if(Buffer.data())
{
SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send");
Stream.Send(Sensor, std::move(Buffer));
}
}
};
WritePixelsToBuffer( WritePixelsToBuffer(
*Sensor.CaptureRenderTarget, *Sensor.CaptureRenderTarget,
Buffer,
carla::sensor::SensorRegistry::get<TSensor *>::type::header_offset, carla::sensor::SensorRegistry::get<TSensor *>::type::header_offset,
InRHICmdList, use16BitFormat); InRHICmdList,
std::move(FuncForSending),
if(Buffer.data()) use16BitFormat);
{
SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send");
Stream.Send(Sensor, std::move(Buffer));
}
} }
} }
); );

220
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader2.cpp
View File

@ -1,220 +0,0 @@
// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
// de Barcelona (UAB).
//
// This work is licensed under the terms of the MIT license.
// For a copy, see <https://opensource.org/licenses/MIT>.
#include "Carla.h"
#include "Carla/Sensor/PixelReader2.h"
#include "Engine/TextureRenderTarget2D.h"
#include "Async/Async.h"
#include "HighResScreenshot.h"
#include "Runtime/ImageWriteQueue/Public/ImageWriteQueue.h"
// =============================================================================
// -- Local variables and types ------------------------------------------------
// =============================================================================
struct LockTexture2
{
LockTexture2(FRHITexture2D *InTexture, uint32 &Stride)
: Texture(InTexture),
Source(reinterpret_cast<const uint8 *>(
RHILockTexture2D(Texture, 0, RLM_ReadOnly, Stride, false))) {}
~LockTexture2()
{
RHIUnlockTexture2D(Texture, 0, false);
}
FRHITexture2D *Texture;
const uint8 *Source;
};
// =============================================================================
// -- Static local functions ---------------------------------------------------
// =============================================================================
// Temporal; this avoid allocating the array each time and also avoids checking
// for a bigger texture, ReadSurfaceData will allocate the space needed.
TArray<FColor> gPixels2;
static void WritePixelsToBuffer_Vulkan2(
const UTextureRenderTarget2D &RenderTarget,
uint32 Offset,
FRHICommandListImmediate &RHICmdList,
std::function<void(void *, uint32, uint32)> FuncForSending)
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR(__FUNCTION__);
check(IsInRenderingThread());
auto RenderResource = static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource);
FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture();
if (!Texture)
{
return;
}
auto BackBufferReadback = std::make_unique<FRHIGPUTextureReadback>(TEXT("CameraBufferReadback"));
FIntPoint BackBufferSize = Texture->GetSizeXY();
EPixelFormat BackBufferPixelFormat = Texture->GetFormat();
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("EnqueueCopy");
BackBufferReadback->EnqueueCopy(RHICmdList, Texture, FResolveRect(0, 0, BackBufferSize.X, BackBufferSize.Y));
}
// workaround to force RHI with Vulkan to refresh the fences state in the middle of frame
{
FRenderQueryRHIRef Query = RHICreateRenderQuery(RQT_AbsoluteTime);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("create query");
RHICmdList.EndRenderQuery(Query);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Flush");
RHICmdList.ImmediateFlush(EImmediateFlushType::FlushRHIThread);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("query result");
uint64 OldAbsTime = 0;
RHICmdList.GetRenderQueryResult(Query, OldAbsTime, true);
}
AsyncTask(ENamedThreads::AnyNormalThreadNormalTask, [=, Readback=std::move(BackBufferReadback)]() mutable {
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Wait GPU transfer");
while (!Readback->IsReady())
{
std::this_thread::yield();
}
}
FPixelFormatInfo PixelFormat = GPixelFormats[BackBufferPixelFormat];
int32 Count = (BackBufferSize.Y * (PixelFormat.BlockBytes * BackBufferSize.X));
void* LockedData = Readback->Lock(Count);
if (LockedData)
{
FuncForSending(LockedData, Count, Offset);
}
Readback->Unlock();
Readback.reset();
});
}
// Temporal; this avoid allocating the array each time
TArray<FFloat16Color> gFloatPixels2;
static void WriteFloatPixelsToBuffer_Vulkan2(
const UTextureRenderTarget2D &RenderTarget,
carla::Buffer &Buffer,
uint32 Offset,
FRHICommandListImmediate &InRHICmdList)
{
check(IsInRenderingThread());
gFloatPixels2.Empty();
auto RenderResource =
static_cast<const FTextureRenderTarget2DResource *>(RenderTarget.Resource);
FTexture2DRHIRef Texture = RenderResource->GetRenderTargetTexture();
if (!Texture)
{
return;
}
FIntPoint Rect = RenderResource->GetSizeXY();
// NS: Extra copy here, don't know how to avoid it.
InRHICmdList.ReadSurfaceFloatData(
Texture,
FIntRect(0, 0, Rect.X, Rect.Y),
gFloatPixels2,
CubeFace_PosX,0,0);
TArray<float> IntermediateBuffer;
IntermediateBuffer.Reserve(gFloatPixels2.Num() * 2);
for (FFloat16Color& color : gFloatPixels2) {
float x = (color.R.GetFloat() - 0.5f)*4.f;
float y = (color.G.GetFloat() - 0.5f)*4.f;
IntermediateBuffer.Add(x);
IntermediateBuffer.Add(y);
}
Buffer.copy_from(Offset, IntermediateBuffer);
}
// =============================================================================
// -- FPixelReader -------------------------------------------------------------
// =============================================================================
bool FPixelReader2::WritePixelsToArray(
UTextureRenderTarget2D &RenderTarget,
TArray<FColor> &BitMap)
{
check(IsInGameThread());
FTextureRenderTargetResource *RTResource =
RenderTarget.GameThread_GetRenderTargetResource();
if (RTResource == nullptr)
{
UE_LOG(LogCarla, Error, TEXT("FPixelReader2: UTextureRenderTarget2D missing render target"));
return false;
}
FReadSurfaceDataFlags ReadPixelFlags(RCM_UNorm);
ReadPixelFlags.SetLinearToGamma(true);
return RTResource->ReadPixels(BitMap, ReadPixelFlags);
}
TUniquePtr<TImagePixelData<FColor>> FPixelReader2::DumpPixels(
UTextureRenderTarget2D &RenderTarget)
{
const FIntPoint DestSize(RenderTarget.GetSurfaceWidth(), RenderTarget.GetSurfaceHeight());
TUniquePtr<TImagePixelData<FColor>> PixelData = MakeUnique<TImagePixelData<FColor>>(DestSize);
TArray<FColor> Pixels(PixelData->Pixels.GetData(), PixelData->Pixels.Num());
if (!WritePixelsToArray(RenderTarget, Pixels))
{
return nullptr;
}
return PixelData;
}
TFuture<bool> FPixelReader2::SavePixelsToDisk(
UTextureRenderTarget2D &RenderTarget,
const FString &FilePath)
{
return SavePixelsToDisk(DumpPixels(RenderTarget), FilePath);
}
TFuture<bool> FPixelReader2::SavePixelsToDisk(
TUniquePtr<TImagePixelData<FColor>> PixelData,
const FString &FilePath)
{
TUniquePtr<FImageWriteTask> ImageTask = MakeUnique<FImageWriteTask>();
ImageTask->PixelData = MoveTemp(PixelData);
ImageTask->Filename = FilePath;
ImageTask->Format = EImageFormat::PNG;
ImageTask->CompressionQuality = (int32) EImageCompressionQuality::Default;
ImageTask->bOverwriteFile = true;
ImageTask->PixelPreProcessors.Add(TAsyncAlphaWrite<FColor>(255));
FHighResScreenshotConfig &HighResScreenshotConfig = GetHighResScreenshotConfig();
return HighResScreenshotConfig.ImageWriteQueue->Enqueue(MoveTemp(ImageTask));
}
void FPixelReader2::WritePixelsToBuffer(
UTextureRenderTarget2D &RenderTarget,
uint32 Offset,
FRHICommandListImmediate &InRHICmdList,
std::function<void(void *, uint32, uint32)> FuncForSending,
bool use16BitFormat)
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR(__FUNCTION__);
check(IsInRenderingThread());
if (IsVulkanPlatform(GMaxRHIShaderPlatform) || IsD3DPlatform(GMaxRHIShaderPlatform, false))
{
if (use16BitFormat)
{
// WriteFloatPixelsToBuffer_Vulkan2(RenderTarget, Buffer, Offset, InRHICmdList);
}
else
{
WritePixelsToBuffer_Vulkan2(RenderTarget, Offset, InRHICmdList, std::move(FuncForSending));
}
}
}

150
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader2.h
View File

@ -1,150 +0,0 @@
// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
// de Barcelona (UAB).
//
// This work is licensed under the terms of the MIT license.
// For a copy, see <https://opensource.org/licenses/MIT>.
#pragma once
#include "CoreGlobals.h"
#include "Engine/TextureRenderTarget2D.h"
#include "Runtime/ImageWriteQueue/Public/ImagePixelData.h"
#include <compiler/disable-ue4-macros.h>
#include <carla/Logging.h>
#include <carla/Buffer.h>
#include <carla/sensor/SensorRegistry.h>
#include <compiler/enable-ue4-macros.h>
// =============================================================================
// -- FPixelReader -------------------------------------------------------------
// =============================================================================
/// Utils for reading pixels from UTextureRenderTarget2D.
///
/// @todo This class only supports PF_R8G8B8A8 format.
class FPixelReader2
{
public:
/// Copy the pixels in @a RenderTarget into @a BitMap.
///
/// @pre To be called from game-thread.
static bool WritePixelsToArray(
UTextureRenderTarget2D &RenderTarget,
TArray<FColor> &BitMap);
/// Dump the pixels in @a RenderTarget.
///
/// @pre To be called from game-thread.
static TUniquePtr<TImagePixelData<FColor>> DumpPixels(
UTextureRenderTarget2D &RenderTarget);
/// Asynchronously save the pixels in @a RenderTarget to disk.
///
/// @pre To be called from game-thread.
static TFuture<bool> SavePixelsToDisk(
UTextureRenderTarget2D &RenderTarget,
const FString &FilePath);
/// Asynchronously save the pixels in @a PixelData to disk.
///
/// @pre To be called from game-thread.
static TFuture<bool> SavePixelsToDisk(
TUniquePtr<TImagePixelData<FColor>> PixelData,
const FString &FilePath);
/// Convenience function to enqueue a render command that sends the pixels
/// down the @a Sensor's data stream. It expects a sensor derived from
/// ASceneCaptureSensor or compatible.
///
/// Note that the serializer needs to define a "header_offset" that it's
/// allocated in front of the buffer.
///
/// @pre To be called from game-thread.
template <typename TSensor>
static void SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat = false);
private:
/// Copy the pixels in @a RenderTarget into @a Buffer.
///
/// @pre To be called from render-thread.
static void WritePixelsToBuffer(
UTextureRenderTarget2D &RenderTarget,
uint32 Offset,
FRHICommandListImmediate &InRHICmdList,
std::function<void(void *, uint32, uint32)> FuncForSending,
bool use16BitFormat = false);
};
// =============================================================================
// -- FPixelReader::SendPixelsInRenderThread -----------------------------------
// =============================================================================
template <typename TSensor>
void FPixelReader2::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat)
{
TRACE_CPUPROFILER_EVENT_SCOPE(FPixelReader2::SendPixelsInRenderThread);
check(Sensor.CaptureRenderTarget != nullptr);
if (!Sensor.HasActorBegunPlay() || Sensor.IsPendingKill())
{
return;
}
/// Blocks until the render thread has finished all it's tasks.
Sensor.EnqueueRenderSceneImmediate();
// Enqueue a command in the render-thread that will write the image buffer to
// the data stream. The stream is created in the capture thus executed in the
// game-thread.
ENQUEUE_RENDER_COMMAND(FWritePixels2_SendPixelsInRenderThread)
(
[&Sensor, use16BitFormat](auto &InRHICmdList) mutable
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("FWritePixels2_SendPixelsInRenderThread");
/// @todo Can we make sure the sensor is not going to be destroyed?
if (!Sensor.IsPendingKill())
{
std::function<void(void *, uint32, uint32)> FuncForSending = [&Sensor, Frame = FCarlaEngine::GetFrameCounter()](void *LockedData, uint32 Count, uint32 Offset)
{
if (Sensor.IsPendingKill()) return;
auto Stream = Sensor.GetDataStream(Sensor);
// Stream.SetFrameNumber(Frame);
auto Buffer = Stream.PopBufferFromPool();
{
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Buffer Copy");
Buffer.copy_from(Offset, boost::asio::buffer(LockedData, Count));
}
{
// send
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Sending buffer");
if(Buffer.data())
{
SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend);
TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send");
Stream.Send(Sensor, std::move(Buffer));
}
}
};
WritePixelsToBuffer(
*Sensor.CaptureRenderTarget,
carla::sensor::SensorRegistry::get<TSensor *>::type::header_offset,
InRHICmdList,
std::move(FuncForSending),
use16BitFormat);
}
}
);
// Blocks until the render thread has finished all it's tasks
Sensor.WaitForRenderThreadToFinish();
}

2
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp
View File

@ -60,5 +60,5 @@ void ASceneCaptureCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float
TRACE_CPUPROFILER_EVENT_SCOPE_TEXT(*ProfilerText); TRACE_CPUPROFILER_EVENT_SCOPE_TEXT(*ProfilerText);
} }
); );
FPixelReader2::SendPixelsInRenderThread(*this); FPixelReader::SendPixelsInRenderThread(*this);
} }

1
Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h
View File

@ -32,7 +32,6 @@ class CARLA_API ASceneCaptureSensor : public ASensor
friend class ACarlaGameModeBase; friend class ACarlaGameModeBase;
friend class FPixelReader; friend class FPixelReader;
friend class FPixelReader2;
public: public: