Marcel/empty gbuffer fix (#6013)

* Change handling of empty gbuffers.

Now, instead of sending an empty image, a black one with the size of the viewport is sent if the target gbuffer is unavailable.

* Add more GBufferID checks + improve empty gbuffer handling.

* Fix manual_control_gbuffer.py error on repeated listen_to_gbuffer.

* Expose is_listening and is_listening_gbuffer to the PythonAPI.

* Fix data race + autoremove unused gbuffers in manual_control_gbuffer.

* Update PythonAPI docs.

* Remove magic number.

* Switch from error to warning on invalid sensor type when requesting gbuffers.

Docs/python_api.md

@@ -2161,6 +2161,12 @@ Sensors compound a specific family of actors quite diverse and unique. They are
 When <b>True</b> the sensor will be waiting for data.  
 
 ### Methods
+- <a name="carla.Sensor.is_listening"></a>**<font color="#7fb800">is_listening</font>**(<font color="#00a6ed">**self**</font>)  
+Returns whether the sensor is in a listening state.  
+- <a name="carla.Sensor.is_listening_gbuffer"></a>**<font color="#7fb800">is_listening_gbuffer</font>**(<font color="#00a6ed">**self**</font>, <font color="#00a6ed">**gbuffer_id**</font>)  
+Returns whether the sensor is in a listening state for a specific GBuffer texture.  
+    - **Parameters:**
+        - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the target Unreal Engine GBuffer texture.  
 - <a name="carla.Sensor.listen"></a>**<font color="#7fb800">listen</font>**(<font color="#00a6ed">**self**</font>, <font color="#00a6ed">**callback**</font>)<button class="SnipetButton" id="carla.Sensor.listen-snipet_button">snippet &rarr;</button>  
 The function the sensor will be calling to every time a new measurement is received. This function needs for an argument containing an object type [carla.SensorData](#carla.SensorData) to work with.  
     - **Parameters:**
@@ -2168,7 +2174,7 @@ The function the sensor will be calling to every time a new measurement is recei
 - <a name="carla.Sensor.listen_to_gbuffer"></a>**<font color="#7fb800">listen_to_gbuffer</font>**(<font color="#00a6ed">**self**</font>, <font color="#00a6ed">**gbuffer_id**</font>, <font color="#00a6ed">**callback**</font>)  
 The function the sensor will be calling to every time the desired GBuffer texture is received.<br> This function needs for an argument containing an object type [carla.SensorData](#carla.SensorData) to work with.  
     - **Parameters:**
-        - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the desired Unreal Engine GBuffer texture.  
+        - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the target Unreal Engine GBuffer texture.  
         - `callback` (_function_) - The called function with one argument containing the received GBuffer texture.  
 - <a name="carla.Sensor.stop"></a>**<font color="#7fb800">stop</font>**(<font color="#00a6ed">**self**</font>)  
 Commands the sensor to stop listening for data.  

LibCarla/source/carla/client/ServerSideSensor.cpp

@@ -11,6 +11,8 @@
 
 #include <exception>
 
+constexpr size_t GBufferTextureCount = 13;
+
 namespace carla {
 namespace client {
 
@@ -23,11 +25,11 @@ namespace client {
     }
     if (IsListening() && GetEpisode().IsValid()) {
       try {
-        Stop();
-        for (uint32_t i = 1; i != 16; ++i) {
+        for (uint32_t i = 1; i != GBufferTextureCount + 1; ++i) {
           if (listening_mask.test(i))
-            StopGBuffer(i);
+            StopGBuffer(i - 1);
         }
+        Stop();
       } catch (const std::exception &e) {
         log_error("exception trying to stop sensor:", GetDisplayId(), ':', e.what());
       }
@@ -55,9 +57,10 @@ namespace client {
 
   void ServerSideSensor::ListenToGBuffer(uint32_t GBufferId, CallbackFunctionType callback) {
     log_debug(GetDisplayId(), ": subscribing to gbuffer stream");
+    RELEASE_ASSERT(GBufferId < GBufferTextureCount);
     if (GetActorDescription().description.id != "sensor.camera.rgb")
     {
-      log_error("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb).");
+      log_warning("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb).");
       return;
     }
     GetEpisode().Lock()->SubscribeToGBuffer(*this, GBufferId, std::move(callback));
@@ -67,29 +70,26 @@ namespace client {
 
   void ServerSideSensor::StopGBuffer(uint32_t GBufferId) {
     log_debug(GetDisplayId(), ": unsubscribing from gbuffer stream");
+    RELEASE_ASSERT(GBufferId < GBufferTextureCount);
     if (GetActorDescription().description.id != "sensor.camera.rgb")
     {
-      log_error("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb).");
+      log_warning("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb).");
       return;
     }
     GetEpisode().Lock()->UnSubscribeFromGBuffer(*this, GBufferId);
     listening_mask.reset(GBufferId + 1);
-    if (listening_mask.count() == 1) {
-      listening_mask.reset(0);
-    }
   }
 
   bool ServerSideSensor::Destroy() {
     log_debug("calling sensor Destroy() ", GetDisplayId());
     if (IsListening()) {
-      for (uint32_t i = 1; i != 16; ++i) {
+      for (uint32_t i = 1; i != GBufferTextureCount + 1; ++i) {
         if (listening_mask.test(i)) {
-          StopGBuffer(i);
+          StopGBuffer(i - 1);
         }
       }
       Stop();
     }
-    listening_mask = {};
     return Actor::Destroy();
   }
 

LibCarla/source/carla/client/ServerSideSensor.h

@@ -54,7 +54,7 @@ namespace client {
 
   private:
 
-    std::bitset<32> listening_mask;
+    std::bitset<16> listening_mask;
 
   };
 

PythonAPI/carla/source/libcarla/Sensor.cpp

@@ -28,6 +28,7 @@ void export_sensor() {
   class_<cc::Sensor, bases<cc::Actor>, boost::noncopyable, boost::shared_ptr<cc::Sensor>>("Sensor", no_init)
     .add_property("is_listening", &cc::Sensor::IsListening)
     .def("listen", &SubscribeToStream, (arg("callback")))
+    .def("is_listening", &cc::Sensor::IsListening)
     .def("stop", &cc::Sensor::Stop)
     .def(self_ns::str(self_ns::self))
   ;
@@ -35,6 +36,7 @@ void export_sensor() {
   class_<cc::ServerSideSensor, bases<cc::Sensor>, boost::noncopyable, boost::shared_ptr<cc::ServerSideSensor>>
       ("ServerSideSensor", no_init)
     .def("listen_to_gbuffer", &SubscribeToGBuffer, (arg("gbuffer_id"), arg("callback")))
+    .def("is_listening_gbuffer", &cc::ServerSideSensor::IsListeningGBuffer, (arg("gbuffer_id")))
     .def("stop_gbuffer", &cc::ServerSideSensor::StopGBuffer, (arg("gbuffer_id")))
     .def(self_ns::str(self_ns::self))
   ;

PythonAPI/docs/sensor.yml

@@ -41,6 +41,10 @@
       doc: >
         The function the sensor will be calling to every time a new measurement is received. This function needs for an argument containing an object type carla.SensorData to work with.
     # --------------------------------------
+    - def_name: is_listening
+      doc: >
+        Returns whether the sensor is in a listening state.
+    # --------------------------------------
     - def_name: stop
       doc: >
         Commands the sensor to stop listening for data.
@@ -50,7 +54,7 @@
       - param_name: gbuffer_id
         type: carla.GBufferTextureID
         doc: >
-          The ID of the desired Unreal Engine GBuffer texture.
+          The ID of the target Unreal Engine GBuffer texture.
       - param_name: callback
         type: function
         doc: >
@@ -59,6 +63,15 @@
         The function the sensor will be calling to every time the desired GBuffer texture is received.<br>
         This function needs for an argument containing an object type carla.SensorData to work with.
     # --------------------------------------
+    - def_name: is_listening_gbuffer
+      params:
+      - param_name: gbuffer_id
+        type: carla.GBufferTextureID
+        doc: >
+          The ID of the target Unreal Engine GBuffer texture.
+      doc: >
+        Returns whether the sensor is in a listening state for a specific GBuffer texture.
+    # --------------------------------------
     - def_name: stop_gbuffer
       params:
       - param_name: gbuffer_id

PythonAPI/examples/manual_control_gbuffer.py

@@ -1212,14 +1212,16 @@ class CameraManager(object):
             self.hud.notification('ERROR: Unsupported operation, see log for more info.')
             print('ERROR: GBuffer methods are not available for the current sensor type"%s". Only "sensor.camera.rgb" is currently supported.' % name)
             return False
-        self.output_texture_id = index % len(gbuffer_names)
         if self.output_texture_id != 0:
             self.sensor.stop_gbuffer(self.output_texture_id - 1)
+        self.output_texture_id = index % len(gbuffer_names)
+        adjusted_index = self.output_texture_id - 1
         if self.output_texture_id != 0:
-            self.sensor.listen_to_gbuffer(
-                self.output_texture_id - 1,
-                lambda image, index = self.output_texture_id: # Need to capture the output_texture_id by value.
-                    CameraManager._parse_image(weak_self, image, index))
+            if not self.sensor.is_listening_gbuffer(adjusted_index):
+                self.sensor.listen_to_gbuffer(
+                    adjusted_index,
+                    lambda image, index = self.output_texture_id:
+                        CameraManager._parse_image(weak_self, image, index))
         return True
 
     def next_gbuffer(self):
@@ -1234,10 +1236,8 @@ class CameraManager(object):
             display.blit(self.surface, (0, 0))
 
     @staticmethod
-    def _parse_image(weak_self, image, output_texture_id = 0):
+    def _parse_image(weak_self, image, index = 0):
         self = weak_self()
-        if self.output_texture_id != output_texture_id:
-            return
         if not self:
             return
         if self.sensors[self.index][0].startswith('sensor.lidar'):
@@ -1270,6 +1270,8 @@ class CameraManager(object):
             array = array[:, :, ::-1]
             self.surface = pygame.surfarray.make_surface(array.swapaxes(0, 1))
         else:
+            if self.output_texture_id != index:
+                return
             image.convert(self.sensors[self.index][1])
             array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
             array = np.reshape(array, (image.height, image.width, 4))

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h

@@ -476,7 +476,7 @@ private:
         std::is_same<std::remove_reference_t<CameraGBufferT>, FCameraGBufferUint8>::value,
         FColor,
         FLinearColor>::type;
-      FIntPoint ViewSize = {};
+      FIntPoint ViewSize;
       TArray<PixelType> Pixels;
       if (GBufferData.WaitForTextureTransfer(TextureID))
       {
@@ -484,26 +484,48 @@ private:
         void* PixelData;
         int32 SourcePitch;
         FIntPoint SourceExtent;
-        GBufferData.MapTextureData(TextureID, PixelData, SourcePitch, SourceExtent);
-        ViewSize = GBufferData.ViewRect.Size();
+        GBufferData.MapTextureData(
+          TextureID,
+          PixelData,
+          SourcePitch,
+          SourceExtent);
         auto Format = GBufferData.Readbacks[(size_t)TextureID]->GetFormat();
+        ViewSize = GBufferData.ViewRect.Size();
         Pixels.AddUninitialized(ViewSize.X * ViewSize.Y);
         FReadSurfaceDataFlags Flags = {};
         Flags.SetLinearToGamma(true);
-        ImageUtil::DecodePixelsByFormat(PixelData, SourcePitch, SourceExtent, ViewSize, Format, Flags, Pixels);
+        ImageUtil::DecodePixelsByFormat(
+          PixelData,
+          SourcePitch,
+          SourceExtent,
+          ViewSize,
+          Format,
+          Flags,
+          Pixels);
         GBufferData.UnmapTextureData(TextureID);
       }
+      else
+      {
+        ViewSize = GBufferData.ViewRect.Size();
+        Pixels.SetNum(ViewSize.X * ViewSize.Y);
+        for (auto& Pixel : Pixels)
+          Pixel = PixelType::Black;
+      }
       auto GBufferStream = CameraGBuffer.GetDataStream(Self);
       auto Buffer = GBufferStream.PopBufferFromPool();
-      Buffer.copy_from(carla::sensor::SensorRegistry::get<CameraGBufferT*>::type::header_offset, Pixels);
+      Buffer.copy_from(
+        carla::sensor::SensorRegistry::get<CameraGBufferT*>::type::header_offset,
+        Pixels);
       if (Buffer.empty()) {
         return;
       }
       SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend);
       TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send");
       GBufferStream.Send(
-        CameraGBuffer, std::move(Buffer),
-        ViewSize.X, ViewSize.Y,
+        CameraGBuffer,
+        std::move(Buffer),
+        ViewSize.X,
+        ViewSize.Y,
         Self.GetFOVAngle());
   }