From a2b6f8abf65117e56ae2649ac29d2c3738ac1331 Mon Sep 17 00:00:00 2001 From: doterop Date: Tue, 28 Jul 2020 11:36:31 +0200 Subject: [PATCH] Added fences and separeted each step --- .../Source/Carla/Game/CarlaGameModeBase.cpp | 138 +++++++++++++++--- .../Source/Carla/Game/CarlaGameModeBase.h | 92 +++++++++++- .../Carla/Source/Carla/Sensor/DepthCamera.cpp | 7 +- .../Carla/Sensor/SceneCaptureCamera.cpp | 8 +- .../Carla/Sensor/SceneCaptureSensor.cpp | 32 ++++ .../Source/Carla/Sensor/SceneCaptureSensor.h | 38 ++--- .../Sensor/SemanticSegmentationCamera.cpp | 8 +- 7 files changed, 264 insertions(+), 59 deletions(-) diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.cpp index 818f14bae..d3f8a36d7 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.cpp @@ -192,7 +192,8 @@ void ACarlaGameModeBase::BeginPlay() Recorder->GetReplayer()->CheckPlayAfterMapLoaded(); } - CaptureAtlasDelegate = FCoreDelegates::OnEndFrameRT.AddUObject(this, &ACarlaGameModeBase::CaptureAtlas); + SendAtlasDelegate = FCoreDelegates::OnEndFrame.AddUObject(this, &ACarlaGameModeBase::SendAtlas); + CaptureAtlasDelegate = FCoreDelegates::OnBeginFrame.AddUObject(this, &ACarlaGameModeBase::CaptureAtlas); } @@ -206,10 +207,6 @@ void ACarlaGameModeBase::Tick(float DeltaSeconds) Recorder->Tick(DeltaSeconds); } - if(!IsAtlasTextureValid) - { - CreateAtlasTextures(); - } } void ACarlaGameModeBase::EndPlay(const EEndPlayReason::Type EndPlayReason) @@ -224,7 +221,22 @@ void ACarlaGameModeBase::EndPlay(const EEndPlayReason::Type EndPlayReason) CarlaSettingsDelegate->Reset(); } - FCoreDelegates::OnEndFrameRT.Remove(CaptureAtlasDelegate); + // TODO: improve this delete processs; not multithread friendly + while(!AtlasCopyRequestQueue.IsEmpty()) + { + delete AtlasCopyRequestQueue.Peek(); + AtlasCopyRequestQueue.Pop(); + } + + while(!AtlasCopyRequestsQueuePool.IsEmpty()) + { + delete AtlasCopyRequestsQueuePool.Peek(); + AtlasCopyRequestsQueuePool.Pop(); + } + + + FCoreDelegates::OnEndFrame.Remove(CaptureAtlasDelegate); + FCoreDelegates::OnBeginFrame.Remove(SendAtlasDelegate); } void ACarlaGameModeBase::SpawnActorFactories() @@ -380,6 +392,17 @@ void ACarlaGameModeBase::DebugShowSignals(bool enable) } +FAtlasCopyRequest* ACarlaGameModeBase::GetAtlasCopyRequest() +{ + if(AtlasCopyRequestsQueuePool.IsEmpty()) { + UE_LOG(LogCarla, Warning, TEXT("ACarlaGameModeBase::GetAtlasCopyRequest new FAtlasCopyRequest")); + return new FAtlasCopyRequest(); + } + FAtlasCopyRequest* AtlasCopyRequest = nullptr; + AtlasCopyRequestsQueuePool.Dequeue(AtlasCopyRequest); + return AtlasCopyRequest; +} + void ACarlaGameModeBase::CreateAtlasTextures() { if(AtlasTextureWidth > 0 && AtlasTextureHeight > 0) @@ -390,7 +413,8 @@ void ACarlaGameModeBase::CreateAtlasTextures() for(int i = 0; i < kMaxNumTextures; i++) { CamerasAtlasTexture[i] = RHICreateTexture2D(AtlasTextureWidth, AtlasTextureHeight, PF_B8G8R8A8, 1, 1, TexCreate_CPUReadback, CreateInfo); - AtlasPixels[i].Init(FColor(), AtlasTextureWidth * AtlasTextureHeight); + // Prepare some AtlasCopyRequests + AtlasCopyRequestQueue.Enqueue(new FAtlasCopyRequest()); } IsAtlasTextureValid = true; } @@ -400,20 +424,32 @@ extern FDynamicRHI* GDynamicRHI; void ACarlaGameModeBase::CaptureAtlas() { + ACarlaGameModeBase* This = this; - if(!IsAtlasTextureValid || !SceneCaptureSensors.Num()) return; + if(!SceneCaptureSensors.Num()) return; -#if !UE_BUILD_SHIPPING - if (!ReadSurface) return; -#endif + if(!IsAtlasTextureValid) + { + CreateAtlasTextures(); + } - ENQUEUE_RENDER_COMMAND(ASceneCaptureSensor_CaptureAtlas) + //if (!AtlasCopyRequestQueue.IsEmpty()) + //{ + // UE_LOG(LogCarla, Error, TEXT("ACarlaGameModeBase::CaptureAtlas: request full")); + // return; + //} + //UE_LOG(LogCarla, Warning, TEXT("ACarlaGameModeBase::CaptureAtlas: creating request")); + + FAtlasCopyRequest* AtlasCopyRequest = GetAtlasCopyRequest(); + AtlasCopyRequest->ResizeBuffer(AtlasTextureWidth, AtlasTextureHeight); + + ENQUEUE_RENDER_COMMAND(ACarlaGameModeBase_CaptureAtlas) ( - [This](FRHICommandListImmediate& RHICmdList) mutable + [This, &CurrentAtlasPixels = AtlasCopyRequest->AtlasImage](FRHICommandListImmediate& RHICmdList) mutable { - FTexture2DRHIRef AtlasTexture = This->CamerasAtlasTexture[This->CurrentAtlas]; - TArray& CurrentAtlasPixels = This->AtlasPixels[This->CurrentAtlas]; + FTexture2DRHIRef AtlasTexture = This->CamerasAtlasTexture[This->PreviousAtlas]; + //TArray& CurrentAtlasPixels = This->AtlasPixels[This->CurrentAtlas]; if (!AtlasTexture) { @@ -422,21 +458,22 @@ void ACarlaGameModeBase::CaptureAtlas() } // Download Atlas texture - SCOPE_CYCLE_COUNTER(STAT_CarlaSensorReadRT); FIntRect Rect = FIntRect(0, 0, This->AtlasTextureWidth, This->AtlasTextureHeight); + #if !UE_BUILD_SHIPPING - if(This->ReadSurface == 2) { - Rect = FIntRect(0, 0, This->SurfaceW, This->SurfaceH); - } + if(This->ReadSurfaceMode == 2) { + Rect = FIntRect(0, 0, This->SurfaceW, This->SurfaceH); + } #endif // GDynamicRHI->RHIReadSurfaceData(AtlasTexture, Rect, CurrentAtlasPixels, FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX)); - // FRHICommandBeginRenderPass RHICommandBeginRenderPass; - // FRHICommandBeginParallelRenderPass RHICommandBeginParallelRenderPass; - +#if !UE_BUILD_SHIPPING + if (This->ReadSurfaceMode == 0) return; +#endif + SCOPE_CYCLE_COUNTER(STAT_CarlaSensorReadRT); RHICmdList.ReadSurfaceData( AtlasTexture, Rect, @@ -444,8 +481,61 @@ void ACarlaGameModeBase::CaptureAtlas() FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX)); - // PreviousTexture = CurrentTexture; - // CurrentTexture = (CurrentTexture + 1) & ~MaxNumTextures; + This->PreviousAtlas = This->CurrentAtlas; + This->CurrentAtlas = (This->CurrentAtlas + 1) & ~kMaxNumTextures; } ); + + // AtlasCopyRequest->Start(); + // AtlasCopyRequest->Wait(); + AtlasCopyRequestQueue.Enqueue(AtlasCopyRequest); + +} + +void ACarlaGameModeBase::SendAtlas() +{ + if(!AtlasCopyRequestQueue.IsEmpty()) + { + FAtlasCopyRequest* AtlasCopyRequest = nullptr; + AtlasCopyRequestQueue.Peek(AtlasCopyRequest); + + // Be sure that the request has finished + + if(AtlasCopyRequest /* && AtlasCopyRequest->IsComplete() */ ) + { + //UE_LOG(LogCarla, Warning, TEXT("ACarlaGameModeBase::SendAtlas: DONE")); +#if !UE_BUILD_SHIPPING + if(!AtlasCopyToCamera) + { + AtlasCopyRequestQueue.Pop(); + AtlasCopyRequestsQueuePool.Enqueue(AtlasCopyRequest); + return; + } +#endif + + ACarlaGameModeBase* This = this; + // We need to make a copy of the atlas to avoid possible threads issues later + // since the AtlasCopyRequest could be used later + //TArray AtlasImage(AtlasCopyRequest->AtlasImage); + //ParallelFor(SceneCaptureSensors.Num(), + // [This, &AtlasImage, AtlasTextureWidth = AtlasCopyRequest->AtlasTextureWidth](int32 Index) + for(int32 Index = 0; Index < SceneCaptureSensors.Num(); Index++) + { + ASceneCaptureSensor* Sensor = SceneCaptureSensors[Index]; + Sensor->CopyTextureFromAtlas(AtlasCopyRequest->AtlasImage, AtlasTextureWidth); + } + //); + + // Remove from pending queue ... + AtlasCopyRequestQueue.Pop(); + // ... add to pool + AtlasCopyRequestsQueuePool.Enqueue(AtlasCopyRequest); + } + else + { + //UE_LOG(LogCarla, Error, TEXT("ACarlaGameModeBase::SendAtlas: request didn't finish")); + } + + } + } \ No newline at end of file diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.h index 116be53ed..0de743aff 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameModeBase.h @@ -26,6 +26,45 @@ #include "CarlaGameModeBase.generated.h" + +USTRUCT() +struct FAtlasCopyRequest +{ + GENERATED_BODY() + + void ResizeBuffer(uint32 Width, uint32 Height) + { + if(AtlasTextureWidth != Width || AtlasTextureHeight != Height) + { + UE_LOG(LogCarla, Warning, TEXT("FAtlasCopyRequest::ResizeBuffer %dx%d"), Width, Height); + AtlasTextureWidth = Width; + AtlasTextureHeight = Height; + AtlasImage.Init(FColor(), Width * Height); + } + } + + void Start() + { + RenderFence.BeginFence(true); + } + + void Wait() + { + RenderFence.Wait(false); + } + + bool IsComplete() + { + return RenderFence.IsFenceComplete(); + } + + TArray AtlasImage; + FRenderCommandFence RenderFence; + uint32 AtlasTextureWidth = 0u; + uint32 AtlasTextureHeight = 0u; +}; + + /// Base class for the CARLA Game Mode. UCLASS(HideCategories=(ActorTick)) class CARLA_API ACarlaGameModeBase : public AGameModeBase @@ -78,9 +117,9 @@ public: } UFUNCTION(Exec) - void SwitchReadSurface(uint32 Mode) { + void SwitchReadSurfaceMode(uint32 Mode) { #if !UE_BUILD_SHIPPING - ReadSurface = Mode; + ReadSurfaceMode = Mode; #endif } @@ -92,6 +131,39 @@ public: #endif } + UFUNCTION(Exec) + void EnableCameraCopyToAtlas(bool Enable) { +#if !UE_BUILD_SHIPPING + CameraCopyToAtlasEnable = Enable; +#endif + } + + UFUNCTION(Exec) + void EnableAtlasCopyToCamera(bool Enable) { +#if !UE_BUILD_SHIPPING + AtlasCopyToCamera = Enable; +#endif + } + + UFUNCTION(Exec) + void EnableCameraStream(bool Enable) { +#if !UE_BUILD_SHIPPING + CameraStreamEnable = Enable; +#endif + } + +#if !UE_BUILD_SHIPPING + + bool IsCameraCopyToAtlasEnabled() const { + return CameraCopyToAtlasEnable; + } + + bool IsCameraStreamEnabled() const { + return CameraStreamEnable; + } + +#endif + protected: void InitGame(const FString &MapName, const FString &Options, FString &ErrorMessage) override; @@ -110,10 +182,14 @@ private: void ParseOpenDrive(const FString &MapName); + FAtlasCopyRequest* GetAtlasCopyRequest(); + void CreateAtlasTextures(); void CaptureAtlas(); + void SendAtlas(); + UPROPERTY() UCarlaGameInstance *GameInstance = nullptr; @@ -147,9 +223,14 @@ private: boost::optional Map; FDelegateHandle CaptureAtlasDelegate; + FDelegateHandle SendAtlasDelegate; + + TQueue AtlasCopyRequestQueue; + TQueue AtlasCopyRequestsQueuePool; static const uint32 kMaxNumTextures = 2u; // This has to be POT - TArray AtlasPixels[kMaxNumTextures]; + // FRenderCommandFence RenderFence[kMaxNumTextures]; + TArray AtlasPixels[kMaxNumTextures]; // TODO: remove TArray SceneCaptureSensors; FTexture2DRHIRef CamerasAtlasTexture[kMaxNumTextures]; uint32 AtlasTextureWidth = 0u; @@ -160,9 +241,12 @@ private: bool IsAtlasTextureValid = false; #if !UE_BUILD_SHIPPING - uint32 ReadSurface = 1; + uint32 ReadSurfaceMode = 1; uint32 SurfaceW = 0; uint32 SurfaceH = 0; + bool CameraCopyToAtlasEnable = true; + bool AtlasCopyToCamera = true; + bool CameraStreamEnable = true; #endif }; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp index 694a3a244..4e44cc3be 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp @@ -34,7 +34,10 @@ void ADepthCamera::Tick(float DeltaTime) { Super::Tick(DeltaTime); +#if !UE_BUILD_SHIPPING ACarlaGameModeBase* GameMode = Cast(GetWorld()->GetAuthGameMode()); - SendPixelsInStream(*this, GameMode->GetCurrentAtlasPixels(), GameMode->GetAtlasTextureWidth()); -} + if(!GameMode->IsCameraStreamEnabled()) return; +#endif + SendPixelsInStream(*this); +} diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp index b2cac7c08..26eace3b4 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp @@ -28,6 +28,10 @@ void ASceneCaptureCamera::Tick(float DeltaTime) { Super::Tick(DeltaTime); +#if !UE_BUILD_SHIPPING ACarlaGameModeBase* GameMode = Cast(GetWorld()->GetAuthGameMode()); - SendPixelsInStream(*this, GameMode->GetCurrentAtlasPixels(), GameMode->GetAtlasTextureWidth()); -} \ No newline at end of file + if(!GameMode->IsCameraStreamEnabled()) return; +#endif + + SendPixelsInStream(*this); +} diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp index 83f305dbf..2f8087b26 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp @@ -18,6 +18,7 @@ #include "ContentStreaming.h" #include "Async/Async.h" #include "RHICommandList.h" +#include "HAL/UnrealMemory.h" static auto SCENE_CAPTURE_COUNTER = 0u; @@ -488,6 +489,8 @@ void ASceneCaptureSensor::BeginPlay() ACarlaGameModeBase* GameMode = Cast(GetWorld()->GetAuthGameMode()); GameMode->AddSceneCaptureSensor(this); + ImageToSend.Init(FColor(), ImageWidth * ImageHeight); + } void ASceneCaptureSensor::Tick(float DeltaTime) @@ -515,6 +518,31 @@ void ASceneCaptureSensor::EndPlay(const EEndPlayReason::Type EndPlayReason) GameMode->RemoveSceneCaptureSensor(this); } +void ASceneCaptureSensor::CopyTextureFromAtlas( + const TArray& AtlasImage, + uint32 AtlasTextureWidth) +{ + + if(AtlasImage.Num() > 0 && ImageToSend.Num() > 0) + { + SCOPE_CYCLE_COUNTER(STAT_CarlaSensorBufferCopy); + + const FColor* Source = &AtlasImage[0] + PositionInAtlas.X; + FColor* Dest = ImageToSend.GetData(); + + check(ImageToSend.Num() > 0); + check(Source != nullptr); + check(Dest != nullptr); + check(ImageWidth > 0); + + for (uint32 Row = 0u; Row < ImageHeight; Row++) + { + FMemory::Memcpy(Dest, Source, sizeof(FColor)*ImageWidth); + Source += AtlasTextureWidth; + Dest += ImageWidth; + } + } +} void ASceneCaptureSensor::CopyTextureToAtlas() { @@ -526,6 +554,10 @@ void ASceneCaptureSensor::CopyTextureToAtlas() ACarlaGameModeBase* GameMode = Cast(GetWorld()->GetAuthGameMode()); if(!GameMode->IsCameraAtlasTextureValid()) return; +#if !UE_BUILD_SHIPPING + if(!GameMode->IsCameraCopyToAtlasEnabled()) return; +#endif + ENQUEUE_RENDER_COMMAND(ASceneCaptureSensor_CopyTextureToAtlas) ( [This, GameMode](FRHICommandListImmediate& RHICmdList) mutable diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h index 5e6e83b99..33a78523f 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h @@ -270,38 +270,24 @@ public: FPixelReader::SavePixelsToDisk(*CaptureRenderTarget, FilePath); } + void CopyTextureFromAtlas( + const TArray& AtlasImage, + uint32 AtlasTextureWidth ); + template - void SendPixelsInStream(TSensor &Sensor, const TArray& AtlasPixels, uint32 AtlasTextureWidth) + void SendPixelsInStream(TSensor &Sensor) { - if(AtlasPixels.Num() > 0) + if(ImageToSend.Num() > 0) { - auto Stream = GetDataStream(Sensor); - auto Buffer = Stream.PopBufferFromPool(); - - { - SCOPE_CYCLE_COUNTER(STAT_CarlaSensorBufferCopy); - - constexpr uint32 BytesPerPixel = sizeof(FColor); - const uint32 ExpectedStride = ImageWidth * BytesPerPixel; - - const FColor* Source = &AtlasPixels[0]; - uint8 *SrcRow = (uint8*)Source + PositionInAtlas.X * BytesPerPixel; - uint32 DstOffset = Offset; - - Buffer.reset(Offset + ImageWidth * ImageHeight * BytesPerPixel); - - for (uint32 Row = 0u; Row < ImageHeight; Row++) - { - Buffer.copy_from(DstOffset, SrcRow, ExpectedStride); - DstOffset += ExpectedStride; - SrcRow += AtlasTextureWidth * BytesPerPixel; - } - } - { SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend); + auto Stream = GetDataStream(Sensor); + auto Buffer = Stream.PopBufferFromPool(); + Buffer.copy_from(Offset, ImageToSend); Stream.Send(Sensor, std::move(Buffer)); } + // TODO: remove + ImageToSend.Init(FColor(), ImageWidth * ImageHeight); } } @@ -319,6 +305,8 @@ protected: FDelegateHandle CopyTextureDelegate; + TArray ImageToSend; + /// Render target necessary for scene capture. UPROPERTY(EditAnywhere) UTextureRenderTarget2D *CaptureRenderTarget = nullptr; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp index dc218b24d..a160ff825 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp @@ -30,6 +30,10 @@ void ASemanticSegmentationCamera::Tick(float DeltaTime) { Super::Tick(DeltaTime); +#if !UE_BUILD_SHIPPING ACarlaGameModeBase* GameMode = Cast(GetWorld()->GetAuthGameMode()); - SendPixelsInStream(*this, GameMode->GetCurrentAtlasPixels(), GameMode->GetAtlasTextureWidth()); -} \ No newline at end of file + if(!GameMode->IsCameraStreamEnabled()) return; +#endif + + SendPixelsInStream(*this); +}