Fix image sending issue + move EnqueueRenderSceneImmediate to PostPhysTick.

LibCarla/source/carla/sensor/s11n/ImageSerializer.h

@@ -31,6 +31,8 @@ namespace s11n {
     };
 #pragma pack(pop)
 
+    static_assert (sizeof(ImageHeader) == sizeof(uint32_t) * 3);
+
     constexpr static auto header_offset = sizeof(ImageHeader);
 
     static const ImageHeader &DeserializeHeader(const RawData &data) {
@@ -38,20 +40,29 @@ namespace s11n {
     }
 
     template <typename Sensor>
-    static Buffer Serialize(const Sensor &sensor, Buffer &&bitmap);
+    static Buffer Serialize(Sensor&& sensor, Buffer&& bitmap);
 
     static SharedPtr<SensorData> Deserialize(RawData &&data);
   };
 
   template <typename Sensor>
-  inline Buffer ImageSerializer::Serialize(const Sensor &sensor, Buffer &&bitmap) {
+  inline Buffer ImageSerializer::Serialize(
+    Sensor&& sensor,
+    Buffer&& bitmap) 
+  {
     DEBUG_ASSERT(bitmap.size() > sizeof(ImageHeader));
+
     ImageHeader header = {
       sensor.GetImageWidth(),
       sensor.GetImageHeight(),
       sensor.GetFOVAngle()
     };
-    std::memcpy(bitmap.data(), reinterpret_cast<const void *>(&header), sizeof(header));
+
+    std::memcpy(
+      bitmap.data(),
+      reinterpret_cast<const void *>(&header),
+      sizeof(header));
+    
     return std::move(bitmap);
   }
 

PythonAPI/examples/prop_control.py

@@ -955,9 +955,9 @@ class CameraManager(object):
             image.convert(self.sensors[self.index][1])
             array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
             # TODO: Remove the length
-            length = image.height * image.width * 4 - array.size
-            if length > 0:
-                array = np.concatenate((array, np.zeros(length)))
+            # length = image.height * image.width * 4 - array.size
+            # if length > 0:
+            #     array = np.concatenate((array, np.zeros(length)))
             array = np.reshape(array, (image.height, image.width, 4))
             array = array[:, :, :3]
             array = array[:, :, ::-1]

Unreal/CMakeLists.txt

@@ -264,6 +264,11 @@ add_custom_target (
   VERBATIM
 )
 
+add_dependencies (
+  CarlaUnreal
+  ${UE_DEPENDENCIES_ORDER_ONLY}
+)
+
 add_custom_target (
   CarlaUnrealEditor
   COMMAND
@@ -280,6 +285,11 @@ add_custom_target (
   VERBATIM
 )
 
+add_dependencies (
+  CarlaUnrealEditor
+  ${UE_DEPENDENCIES_ORDER_ONLY}
+)
+
 
 
 set (
@@ -303,7 +313,6 @@ add_custom_target (
 add_dependencies (
   launch
   CarlaUnrealEditor
-  ${UE_DEPENDENCIES_ORDER_ONLY}
 )
 
 add_custom_target (

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp

@@ -33,13 +33,15 @@ ADepthCamera::ADepthCamera(const FObjectInitializer &ObjectInitializer)
 void ADepthCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ADepthCamera::PostPhysTick);
+  Super::PostPhysTick(World, TickType, DeltaSeconds);
 
-#if 0
-  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this](
+#if 1
+  auto FrameIndex = FCarlaEngine::GetFrameCounter();
+  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex](
     TArrayView<const FColor> Pixels,
     FIntPoint Size) -> bool
   {
-    SendImageDataToClient(*this, Pixels);
+    SendImageDataToClient(*this, Pixels, FrameIndex);
     return true;
   });
 #else

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp

@@ -52,6 +52,7 @@ void AInstanceSegmentationCamera::SetUpSceneCaptureComponent(USceneCaptureCompon
 void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationCamera::PostPhysTick);
+  Super::PostPhysTick(World, TickType, DeltaSeconds);
 
   USceneCaptureComponent2D* SceneCapture = GetCaptureComponent2D();
   TArray<UObject *> TaggedComponents;
@@ -63,12 +64,13 @@ void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickTyp
     SceneCapture->ShowOnlyComponents.Emplace(Component);
   }
 
-#if 0
-  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this](
+#if 1
+  auto FrameIndex = FCarlaEngine::GetFrameCounter();
+  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex](
     TArrayView<const FColor> Pixels,
     FIntPoint Size) -> bool
   {
-    SendImageDataToClient(*this, Pixels);
+    SendImageDataToClient(*this, Pixels, FrameIndex);
     return true;
   });
 #else

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/NormalsCamera.cpp

@@ -27,13 +27,15 @@ ANormalsCamera::ANormalsCamera(const FObjectInitializer &ObjectInitializer)
 void ANormalsCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ANormalsCamera::PostPhysTick);
+  Super::PostPhysTick(World, TickType, DeltaSeconds);
 
-#if 0
-  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this](
+#if 1
+  auto FrameIndex = FCarlaEngine::GetFrameCounter();
+  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex](
     TArrayView<const FColor> Pixels,
     FIntPoint Size) -> bool
   {
-    SendImageDataToClient(*this, Pixels);
+    SendImageDataToClient(*this, Pixels, FrameIndex);
     return true;
   });
 #else

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureCamera.cpp

@@ -49,6 +49,8 @@ void ASceneCaptureCamera::EndPlay(const EEndPlayReason::Type EndPlayReason)
 void ASceneCaptureCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ASceneCaptureCamera::PostPhysTick);
+  Super::PostPhysTick(World, TickType, DeltaSeconds);
+  
   ENQUEUE_RENDER_COMMAND(MeasureTime)
   (
     [](auto &InRHICmdList)
@@ -63,12 +65,13 @@ void ASceneCaptureCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float
     }
   );
 
-#if 0
-  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this](
+#if 1
+  auto FrameIndex = FCarlaEngine::GetFrameCounter();
+  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex](
     TArrayView<const FColor> Pixels,
     FIntPoint Size) -> bool
   {
-    SendImageDataToClient(*this, Pixels);
+    SendImageDataToClient(*this, Pixels, FrameIndex);
     return true;
   });
 #else

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.cpp

@@ -515,6 +515,7 @@ void ASceneCaptureSensor::PrePhysTick(float DeltaSeconds)
 void ASceneCaptureSensor::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaTime)
 {
   Super::PostPhysTick(World, TickType, DeltaTime);
+  EnqueueRenderSceneImmediate();
 }
 
 void ASceneCaptureSensor::EndPlay(const EEndPlayReason::Type EndPlayReason)

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp

@@ -27,13 +27,15 @@ ASemanticSegmentationCamera::ASemanticSegmentationCamera(
 void ASemanticSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ASemanticSegmentationCamera::PostPhysTick);
+  Super::PostPhysTick(World, TickType, DeltaSeconds);
 
-#if 0
-  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this](
+#if 1
+  auto FrameIndex = FCarlaEngine::GetFrameCounter();
+  ImageUtil::ReadSensorImageDataAsyncFColor(*this, [this, FrameIndex](
     TArrayView<const FColor> Pixels,
     FIntPoint Size) -> bool
   {
-    SendImageDataToClient(*this, Pixels);
+    SendImageDataToClient(*this, Pixels, FrameIndex);
     return true;
   });
 #else

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/Sensor.h

@@ -133,22 +133,27 @@ protected:
     typename PixelType>
   static void SendImageDataToClient(
     SensorType&& Sensor,
-    TArrayView<PixelType> Pixels)
+    TArrayView<PixelType> Pixels,
+    uint32 FrameIndex)
   {
+    using carla::sensor::SensorRegistry;
+    using SensorT = std::remove_const_t<std::remove_reference_t<SensorType>>;
+
     auto Stream = Sensor.GetDataStream(Sensor);
-    Stream.SetFrameNumber(FCarlaEngine::GetFrameCounter());
+    Stream.SetFrameNumber(FrameIndex);
     auto Buffer = Stream.PopBufferFromPool();
-    Buffer.copy_from(0, boost::asio::buffer(Pixels.GetData(), Pixels.Num() * sizeof(FColor)));
+    Buffer.copy_from(
+      carla::sensor::SensorRegistry::get<SensorT*>::type::header_offset,
+      boost::asio::buffer(
+        Pixels.GetData(),
+        Pixels.Num() * sizeof(FColor)));
     if (!Buffer.data())
       return;
-    carla::Buffer BufferReady(
-      std::move(
-        carla::sensor::SensorRegistry::Serialize(
-          Sensor,
-          std::move(Buffer))));
-    auto BufferView =
-      carla::BufferView::CreateFrom(
-        std::move(BufferReady));
+    auto Serialized = SensorRegistry::Serialize(
+      Sensor,
+      std::move(Buffer));
+    auto SerializedBuffer = carla::Buffer(std::move(Serialized));
+    auto BufferView = carla::BufferView::CreateFrom(std::move(SerializedBuffer));
 #if defined(WITH_ROS2)
     auto ROS2 = carla::ros2::ROS2::GetInstance();
     if (ROS2->IsEnabled())
@@ -186,7 +191,8 @@ protected:
       });
     }
 #endif
-    Stream.Send(Sensor, BufferView);
+    if (Sensor.AreClientsListening()) 
+      Stream.Send(Sensor, BufferView);
   }
 
   /// Seed of the pseudo-random engine.