From d215264cb67c17a4824e0e01631a220e6f68050f Mon Sep 17 00:00:00 2001 From: Marcel Pi <25649656+MarcelPiNacy@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:26:02 +0100 Subject: [PATCH] Fix image sending issue + move EnqueueRenderSceneImmediate to PostPhysTick. --- .../carla/sensor/s11n/ImageSerializer.h | 17 +++++++++-- PythonAPI/examples/prop_control.py | 6 ++-- Unreal/CMakeLists.txt | 11 ++++++- .../Carla/Source/Carla/Sensor/DepthCamera.cpp | 8 +++-- .../Sensor/InstanceSegmentationCamera.cpp | 8 +++-- .../Source/Carla/Sensor/NormalsCamera.cpp | 8 +++-- .../Carla/Sensor/SceneCaptureCamera.cpp | 9 ++++-- .../Carla/Sensor/SceneCaptureSensor.cpp | 1 + .../Sensor/SemanticSegmentationCamera.cpp | 8 +++-- .../Carla/Source/Carla/Sensor/Sensor.h | 30 +++++++++++-------- 10 files changed, 72 insertions(+), 34 deletions(-) diff --git a/LibCarla/source/carla/sensor/s11n/ImageSerializer.h b/LibCarla/source/carla/sensor/s11n/ImageSerializer.h index e69b9a534..22b54b168 100644 --- a/LibCarla/source/carla/sensor/s11n/ImageSerializer.h +++ b/LibCarla/source/carla/sensor/s11n/ImageSerializer.h @@ -31,6 +31,8 @@ namespace s11n { }; #pragma pack(pop) + static_assert (sizeof(ImageHeader) == sizeof(uint32_t) * 3); + constexpr static auto header_offset = sizeof(ImageHeader); static const ImageHeader &DeserializeHeader(const RawData &data) { @@ -38,20 +40,29 @@ namespace s11n { } template - static Buffer Serialize(const Sensor &sensor, Buffer &&bitmap); + static Buffer Serialize(Sensor&& sensor, Buffer&& bitmap); static SharedPtr Deserialize(RawData &&data); }; template - inline Buffer ImageSerializer::Serialize(const Sensor &sensor, Buffer &&bitmap) { + inline Buffer ImageSerializer::Serialize( + Sensor&& sensor, + Buffer&& bitmap) + { DEBUG_ASSERT(bitmap.size() > sizeof(ImageHeader)); + ImageHeader header = { sensor.GetImageWidth(), sensor.GetImageHeight(), sensor.GetFOVAngle() }; - std::memcpy(bitmap.data(), reinterpret_cast(&header), sizeof(header)); + + std::memcpy( + bitmap.data(), + reinterpret_cast(&header), + sizeof(header)); + return std::move(bitmap); } diff --git a/PythonAPI/examples/prop_control.py b/PythonAPI/examples/prop_control.py index d9f94c94d..a97b2bce2 100644 --- a/PythonAPI/examples/prop_control.py +++ b/PythonAPI/examples/prop_control.py @@ -955,9 +955,9 @@ class CameraManager(object): image.convert(self.sensors[self.index][1]) array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8")) # TODO: Remove the length - length = image.height * image.width * 4 - array.size - if length > 0: - array = np.concatenate((array, np.zeros(length))) + # length = image.height * image.width * 4 - array.size + # if length > 0: + # array = np.concatenate((array, np.zeros(length))) array = np.reshape(array, (image.height, image.width, 4)) array = array[:, :, :3] array = array[:, :, ::-1] diff --git a/Unreal/CMakeLists.txt b/Unreal/CMakeLists.txt index a62eafb3a..89115ef26 100644 --- a/Unreal/CMakeLists.txt +++ b/Unreal/CMakeLists.txt @@ -264,6 +264,11 @@ add_custom_target ( VERBATIM ) +add_dependencies ( + CarlaUnreal + ${UE_DEPENDENCIES_ORDER_ONLY} +) + add_custom_target ( CarlaUnrealEditor COMMAND @@ -280,6 +285,11 @@ add_custom_target ( VERBATIM ) +add_dependencies ( + CarlaUnrealEditor + ${UE_DEPENDENCIES_ORDER_ONLY} +) + set ( @@ -303,7 +313,6 @@ add_custom_target ( add_dependencies ( launch CarlaUnrealEditor - ${UE_DEPENDENCIES_ORDER_ONLY} ) add_custom_target ( diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp index 01b07880c..1d04c25c4 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp @@ -33,13 +33,15 @@ ADepthCamera::ADepthCamera(const FObjectInitializer &ObjectInitializer) void ADepthCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) { TRACE_CPUPROFILER_EVENT_SCOPE(ADepthCamera::PostPhysTick); + Super::PostPhysTick(World, TickType, DeltaSeconds); -#if 0 - ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this]( +#if 1 + auto FrameIndex = FCarlaEngine::GetFrameCounter(); + ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex]( TArrayView Pixels, FIntPoint Size) -> bool { - SendImageDataToClient(*this, Pixels); + SendImageDataToClient(*this, Pixels, FrameIndex); return true; }); #else diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp index 21b6bfbac..304e16281 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp @@ -52,6 +52,7 @@ void AInstanceSegmentationCamera::SetUpSceneCaptureComponent(USceneCaptureCompon void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) { TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationCamera::PostPhysTick); + Super::PostPhysTick(World, TickType, DeltaSeconds); USceneCaptureComponent2D* SceneCapture = GetCaptureComponent2D(); TArray TaggedComponents; @@ -63,12 +64,13 @@ void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickTyp SceneCapture->ShowOnlyComponents.Emplace(Component); } -#if 0 - ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this]( +#if 1 + auto FrameIndex = FCarlaEngine::GetFrameCounter(); + ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex]( TArrayView Pixels, FIntPoint Size) -> bool { - SendImageDataToClient(*this, Pixels); + SendImageDataToClient(*this, Pixels, FrameIndex); return true; }); #else diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/NormalsCamera.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/NormalsCamera.cpp index cd3ead64c..7e688d1e0 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/NormalsCamera.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/NormalsCamera.cpp @@ -27,13 +27,15 @@ ANormalsCamera::ANormalsCamera(const FObjectInitializer &ObjectInitializer) void ANormalsCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) { TRACE_CPUPROFILER_EVENT_SCOPE(ANormalsCamera::PostPhysTick); + Super::PostPhysTick(World, TickType, DeltaSeconds); -#if 0 - ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this]( +#if 1 + auto FrameIndex = FCarlaEngine::GetFrameCounter(); + ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex]( TArrayView Pixels, FIntPoint Size) -> bool { - SendImageDataToClient(*this, Pixels); + SendImageDataToClient(*this, Pixels, FrameIndex); return true; }); #else diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp index f3dd04d64..d0d74a533 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp @@ -49,6 +49,8 @@ void ASceneCaptureCamera::EndPlay(const EEndPlayReason::Type EndPlayReason) void ASceneCaptureCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) { TRACE_CPUPROFILER_EVENT_SCOPE(ASceneCaptureCamera::PostPhysTick); + Super::PostPhysTick(World, TickType, DeltaSeconds); + ENQUEUE_RENDER_COMMAND(MeasureTime) ( [](auto &InRHICmdList) @@ -63,12 +65,13 @@ void ASceneCaptureCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float } ); -#if 0 - ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this]( +#if 1 + auto FrameIndex = FCarlaEngine::GetFrameCounter(); + ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex]( TArrayView Pixels, FIntPoint Size) -> bool { - SendImageDataToClient(*this, Pixels); + SendImageDataToClient(*this, Pixels, FrameIndex); return true; }); #else diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp index 45751ddc9..0b4a44bf5 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp @@ -515,6 +515,7 @@ void ASceneCaptureSensor::PrePhysTick(float DeltaSeconds) void ASceneCaptureSensor::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaTime) { Super::PostPhysTick(World, TickType, DeltaTime); + EnqueueRenderSceneImmediate(); } void ASceneCaptureSensor::EndPlay(const EEndPlayReason::Type EndPlayReason) diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp index 6d6052e57..6adabd750 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp @@ -27,13 +27,15 @@ ASemanticSegmentationCamera::ASemanticSegmentationCamera( void ASemanticSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) { TRACE_CPUPROFILER_EVENT_SCOPE(ASemanticSegmentationCamera::PostPhysTick); + Super::PostPhysTick(World, TickType, DeltaSeconds); -#if 0 - ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this]( +#if 1 + auto FrameIndex = FCarlaEngine::GetFrameCounter(); + ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex]( TArrayView Pixels, FIntPoint Size) -> bool { - SendImageDataToClient(*this, Pixels); + SendImageDataToClient(*this, Pixels, FrameIndex); return true; }); #else diff --git a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/Sensor.h b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/Sensor.h index bf0897b5f..ce2f5afaf 100644 --- a/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/Sensor.h +++ b/Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/Sensor.h @@ -133,22 +133,27 @@ protected: typename PixelType> static void SendImageDataToClient( SensorType&& Sensor, - TArrayView Pixels) + TArrayView Pixels, + uint32 FrameIndex) { + using carla::sensor::SensorRegistry; + using SensorT = std::remove_const_t>; + auto Stream = Sensor.GetDataStream(Sensor); - Stream.SetFrameNumber(FCarlaEngine::GetFrameCounter()); + Stream.SetFrameNumber(FrameIndex); auto Buffer = Stream.PopBufferFromPool(); - Buffer.copy_from(0, boost::asio::buffer(Pixels.GetData(), Pixels.Num() * sizeof(FColor))); + Buffer.copy_from( + carla::sensor::SensorRegistry::get::type::header_offset, + boost::asio::buffer( + Pixels.GetData(), + Pixels.Num() * sizeof(FColor))); if (!Buffer.data()) return; - carla::Buffer BufferReady( - std::move( - carla::sensor::SensorRegistry::Serialize( - Sensor, - std::move(Buffer)))); - auto BufferView = - carla::BufferView::CreateFrom( - std::move(BufferReady)); + auto Serialized = SensorRegistry::Serialize( + Sensor, + std::move(Buffer)); + auto SerializedBuffer = carla::Buffer(std::move(Serialized)); + auto BufferView = carla::BufferView::CreateFrom(std::move(SerializedBuffer)); #if defined(WITH_ROS2) auto ROS2 = carla::ros2::ROS2::GetInstance(); if (ROS2->IsEnabled()) @@ -186,7 +191,8 @@ protected: }); } #endif - Stream.Send(Sensor, BufferView); + if (Sensor.AreClientsListening()) + Stream.Send(Sensor, BufferView); } /// Seed of the pseudo-random engine.