From ed332ecbcd541ff716065e83f12bc359c7ea25b3 Mon Sep 17 00:00:00 2001 From: bernatx Date: Tue, 6 Jun 2023 11:30:48 +0200 Subject: [PATCH] Check for stream ready, and also disable g-buffers --- .../Carla/Source/Carla/Sensor/DataStream.h | 12 +++++-- .../Carla/Source/Carla/Sensor/PixelReader.h | 18 +++++----- .../Carla/Sensor/SceneCaptureSensor.cpp | 9 +++-- .../Source/Carla/Sensor/SceneCaptureSensor.h | 14 ++++++-- .../Carla/Source/Carla/Sensor/Sensor.h | 12 ++++++- .../Source/Carla/Sensor/SensorFactory.cpp | 36 +++++++++---------- .../Source/Carla/Sensor/WorldObserver.cpp | 4 +++ 7 files changed, 69 insertions(+), 36 deletions(-) diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DataStream.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DataStream.h index 2053ac249..5a56b0c4b 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DataStream.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DataStream.h @@ -37,22 +37,28 @@ public: /// /// @pre This functions needs to be called in the game-thread. template - auto MakeAsyncDataStream(const SensorT &Sensor, double Timestamp) + FAsyncDataStreamTmpl MakeAsyncDataStream(const SensorT &Sensor, double Timestamp) { check(Stream.has_value()); return FAsyncDataStreamTmpl{Sensor, Timestamp, *Stream}; } + bool IsStreamReady() + { + return Stream.has_value(); + } + /// Return the token that allows subscribing to this stream. auto GetToken() const { check(Stream.has_value()); - return (*Stream).token(); + return Stream->token(); } bool AreClientsListening() { - return Stream ? Stream->AreClientsListening() : false; + check(Stream.has_value()); + return Stream->AreClientsListening(); } private: diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h index 9cd559cf5..d40711317 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/PixelReader.h @@ -95,7 +95,7 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat TRACE_CPUPROFILER_EVENT_SCOPE(FPixelReader::SendPixelsInRenderThread); check(Sensor.CaptureRenderTarget != nullptr); - if (!Sensor.HasActorBegunPlay() || Sensor.IsPendingKill()) + if (!Sensor.HasActorBegunPlay() || Sensor.IsPendingKill() || !Sensor.IsStreamReady()) { return; } @@ -115,7 +115,7 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat /// @todo Can we make sure the sensor is not going to be destroyed? if (!Sensor.IsPendingKill()) { - FPixelReader::Payload FuncForSending = + FPixelReader::Payload FuncForSending = [&Sensor, Frame = FCarlaEngine::GetFrameCounter(), Conversor = std::move(Conversor)](void *LockedData, uint32 Size, uint32 Offset, uint32 ExpectedRowBytes) { if (Sensor.IsPendingKill()) return; @@ -130,15 +130,15 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat LockedData = reinterpret_cast(Converted.GetData()); Size = Converted.Num() * Converted.GetTypeSize(); } - + auto Stream = Sensor.GetDataStream(Sensor); Stream.SetFrameNumber(Frame); auto Buffer = Stream.PopBufferFromPool(); - + uint32 CurrentRowBytes = ExpectedRowBytes; -#ifdef _WIN32 - // DirectX uses additional bytes to align each row to 256 boundry, + #ifdef _WIN32 + // DirectX uses additional bytes to align each row to 256 boundry, // so we need to remove that extra data if (IsD3DPlatform(GMaxRHIShaderPlatform, false)) { @@ -159,7 +159,7 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat } } } -#endif // _WIN32 + #endif // _WIN32 if (ExpectedRowBytes == CurrentRowBytes) { @@ -179,11 +179,11 @@ void FPixelReader::SendPixelsInRenderThread(TSensor &Sensor, bool use16BitFormat } } }; - + WritePixelsToBuffer( *Sensor.CaptureRenderTarget, carla::sensor::SensorRegistry::get::type::header_offset, - InRHICmdList, + InRHICmdList, std::move(FuncForSending)); } } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp index 31e098b6b..6489308d4 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp @@ -449,8 +449,11 @@ float ASceneCaptureSensor::GetChromAberrOffset() const void ASceneCaptureSensor::EnqueueRenderSceneImmediate() { TRACE_CPUPROFILER_EVENT_SCOPE(ASceneCaptureSensor::EnqueueRenderSceneImmediate); - // Equivalent to "CaptureComponent2D->CaptureScene" + (optional) GBuffer extraction. - CaptureSceneExtended(); + // Creates an snapshot of the scene, requieres bCaptureEveryFrame = false. + GetCaptureComponent2D()->CaptureScene(); + + // // Equivalent to "CaptureComponent2D->CaptureScene" + (optional) GBuffer extraction. + // CaptureSceneExtended(); } constexpr const TCHAR* GBufferNames[] = @@ -508,7 +511,7 @@ void ASceneCaptureSensor::CaptureSceneExtended() if (Prior != GBufferPtr->DesiredTexturesMask) UE_LOG(LogCarla, Verbose, TEXT("GBuffer selection changed (%llu)."), GBufferPtr->DesiredTexturesMask); - + Prior = GBufferPtr->DesiredTexturesMask; GBufferPtr->OwningActor = CaptureComponent2D->GetViewOwner(); diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h index 22584ad69..412136cff 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h @@ -60,6 +60,11 @@ struct FCameraGBufferUint8 template FAsyncDataStream GetDataStream(const SensorT &Self) { + while (!Stream.IsStreamReady()) + { + std::this_thread::yield(); + } + return Stream.MakeAsyncDataStream(Self, Self.GetEpisode().GetElapsedGameTime()); } @@ -102,9 +107,14 @@ struct FCameraGBufferFloat template FAsyncDataStream GetDataStream(const SensorT &Self) { + while (!Stream.IsStreamReady()) + { + std::this_thread::yield(); + } + return Stream.MakeAsyncDataStream(Self, Self.GetEpisode().GetElapsedGameTime()); } - + mutable bool bIsUsed = false; FDataStream Stream; }; @@ -420,7 +430,7 @@ public: } CameraGBuffers; protected: - + void CaptureSceneExtended(); virtual void SendGBufferTextures(FGBufferRequest& GBuffer); diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/Sensor.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/Sensor.h index a78a15978..afca2e685 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/Sensor.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/Sensor.h @@ -56,6 +56,11 @@ public: return Stream.GetToken(); } + bool IsStreamReady() + { + return Stream.IsStreamReady(); + } + void Tick(const float DeltaTime) final; virtual void PrePhysTick(float DeltaSeconds) {} @@ -65,7 +70,7 @@ public: // Small interface to notify sensors when no clients are listening virtual void OnLastClientDisconnected() {}; - + void PostPhysTickInternal(UWorld *World, ELevelTick TickType, float DeltaSeconds); UFUNCTION(BlueprintCallable) @@ -102,6 +107,11 @@ protected: template FAsyncDataStream GetDataStream(const SensorT &Self) { + while (!Stream.IsStreamReady()) + { + std::this_thread::yield(); + } + return Stream.MakeAsyncDataStream(Self, GetEpisode().GetElapsedGameTime()); } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SensorFactory.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SensorFactory.cpp index 8c6ecf5c6..6b8a9c2e7 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SensorFactory.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SensorFactory.cpp @@ -134,27 +134,27 @@ FActorSpawnResult ASensorFactory::SpawnActor( { auto *Episode = GameInstance->GetCarlaEpisode(); check(Episode != nullptr); - + Sensor->SetEpisode(*Episode); Sensor->Set(Description); Sensor->SetDataStream(GameInstance->GetServer().OpenStream()); - ASceneCaptureSensor * SceneCaptureSensor = Cast(Sensor); - if(SceneCaptureSensor) - { - SceneCaptureSensor->CameraGBuffers.SceneColor.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.SceneDepth.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.SceneStencil.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferA.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferB.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferC.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferD.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferE.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.GBufferF.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.Velocity.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.SSAO.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.CustomDepth.SetDataStream(GameInstance->GetServer().OpenStream()); - SceneCaptureSensor->CameraGBuffers.CustomStencil.SetDataStream(GameInstance->GetServer().OpenStream()); - } + // ASceneCaptureSensor * SceneCaptureSensor = Cast(Sensor); + // if(SceneCaptureSensor) + // { + // SceneCaptureSensor->CameraGBuffers.SceneColor.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.SceneDepth.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.SceneStencil.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferA.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferB.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferC.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferD.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferE.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.GBufferF.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.Velocity.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.SSAO.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.CustomDepth.SetDataStream(GameInstance->GetServer().OpenStream()); + // SceneCaptureSensor->CameraGBuffers.CustomStencil.SetDataStream(GameInstance->GetServer().OpenStream()); + // } } UGameplayStatics::FinishSpawningActor(Sensor, Transform); return FActorSpawnResult{Sensor}; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/WorldObserver.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/WorldObserver.cpp index 7937b4e23..768102a11 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/WorldObserver.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/WorldObserver.cpp @@ -371,6 +371,10 @@ void FWorldObserver::BroadcastTick( bool PendingLightUpdates) { TRACE_CPUPROFILER_EVENT_SCOPE_STR(__FUNCTION__); + + if (!Stream.IsStreamReady()) + return; + auto AsyncStream = Stream.MakeAsyncDataStream(*this, Episode.GetElapsedGameTime()); carla::Buffer buffer = FWorldObserver_Serialize(