From 182a48a2f962ab3d7c905b30a6b04298632cbda2 Mon Sep 17 00:00:00 2001 From: Marcel Pi Date: Wed, 7 Dec 2022 17:51:17 +0000 Subject: [PATCH] Marcel/empty gbuffer fix (#6013) * Change handling of empty gbuffers. Now, instead of sending an empty image, a black one with the size of the viewport is sent if the target gbuffer is unavailable. * Add more GBufferID checks + improve empty gbuffer handling. * Fix manual_control_gbuffer.py error on repeated listen_to_gbuffer. * Expose is_listening and is_listening_gbuffer to the PythonAPI. * Fix data race + autoremove unused gbuffers in manual_control_gbuffer. * Update PythonAPI docs. * Remove magic number. * Switch from error to warning on invalid sensor type when requesting gbuffers. --- Docs/python_api.md | 8 ++++- .../source/carla/client/ServerSideSensor.cpp | 22 ++++++------ .../source/carla/client/ServerSideSensor.h | 2 +- PythonAPI/carla/source/libcarla/Sensor.cpp | 2 ++ PythonAPI/docs/sensor.yml | 15 +++++++- PythonAPI/examples/manual_control_gbuffer.py | 20 ++++++----- .../Source/Carla/Sensor/SceneCaptureSensor.h | 36 +++++++++++++++---- 7 files changed, 75 insertions(+), 30 deletions(-) diff --git a/Docs/python_api.md b/Docs/python_api.md index 4a30bb08a..f628fbe07 100644 --- a/Docs/python_api.md +++ b/Docs/python_api.md @@ -2161,6 +2161,12 @@ Sensors compound a specific family of actors quite diverse and unique. They are When True the sensor will be waiting for data. ### Methods +- **is_listening**(**self**) +Returns whether the sensor is in a listening state. +- **is_listening_gbuffer**(**self**, **gbuffer_id**) +Returns whether the sensor is in a listening state for a specific GBuffer texture. + - **Parameters:** + - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the target Unreal Engine GBuffer texture. - **listen**(**self**, **callback**) The function the sensor will be calling to every time a new measurement is received. This function needs for an argument containing an object type [carla.SensorData](#carla.SensorData) to work with. - **Parameters:** @@ -2168,7 +2174,7 @@ The function the sensor will be calling to every time a new measurement is recei - **listen_to_gbuffer**(**self**, **gbuffer_id**, **callback**) The function the sensor will be calling to every time the desired GBuffer texture is received.
This function needs for an argument containing an object type [carla.SensorData](#carla.SensorData) to work with. - **Parameters:** - - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the desired Unreal Engine GBuffer texture. + - `gbuffer_id` (_[carla.GBufferTextureID](#carla.GBufferTextureID)_) - The ID of the target Unreal Engine GBuffer texture. - `callback` (_function_) - The called function with one argument containing the received GBuffer texture. - **stop**(**self**) Commands the sensor to stop listening for data. diff --git a/LibCarla/source/carla/client/ServerSideSensor.cpp b/LibCarla/source/carla/client/ServerSideSensor.cpp index 2caba3ce7..38c09b7af 100644 --- a/LibCarla/source/carla/client/ServerSideSensor.cpp +++ b/LibCarla/source/carla/client/ServerSideSensor.cpp @@ -11,6 +11,8 @@ #include +constexpr size_t GBufferTextureCount = 13; + namespace carla { namespace client { @@ -23,11 +25,11 @@ namespace client { } if (IsListening() && GetEpisode().IsValid()) { try { - Stop(); - for (uint32_t i = 1; i != 16; ++i) { + for (uint32_t i = 1; i != GBufferTextureCount + 1; ++i) { if (listening_mask.test(i)) - StopGBuffer(i); + StopGBuffer(i - 1); } + Stop(); } catch (const std::exception &e) { log_error("exception trying to stop sensor:", GetDisplayId(), ':', e.what()); } @@ -55,9 +57,10 @@ namespace client { void ServerSideSensor::ListenToGBuffer(uint32_t GBufferId, CallbackFunctionType callback) { log_debug(GetDisplayId(), ": subscribing to gbuffer stream"); + RELEASE_ASSERT(GBufferId < GBufferTextureCount); if (GetActorDescription().description.id != "sensor.camera.rgb") { - log_error("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb)."); + log_warning("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb)."); return; } GetEpisode().Lock()->SubscribeToGBuffer(*this, GBufferId, std::move(callback)); @@ -67,29 +70,26 @@ namespace client { void ServerSideSensor::StopGBuffer(uint32_t GBufferId) { log_debug(GetDisplayId(), ": unsubscribing from gbuffer stream"); + RELEASE_ASSERT(GBufferId < GBufferTextureCount); if (GetActorDescription().description.id != "sensor.camera.rgb") { - log_error("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb)."); + log_warning("GBuffer methods are not supported on non-RGB sensors (sensor.camera.rgb)."); return; } GetEpisode().Lock()->UnSubscribeFromGBuffer(*this, GBufferId); listening_mask.reset(GBufferId + 1); - if (listening_mask.count() == 1) { - listening_mask.reset(0); - } } bool ServerSideSensor::Destroy() { log_debug("calling sensor Destroy() ", GetDisplayId()); if (IsListening()) { - for (uint32_t i = 1; i != 16; ++i) { + for (uint32_t i = 1; i != GBufferTextureCount + 1; ++i) { if (listening_mask.test(i)) { - StopGBuffer(i); + StopGBuffer(i - 1); } } Stop(); } - listening_mask = {}; return Actor::Destroy(); } diff --git a/LibCarla/source/carla/client/ServerSideSensor.h b/LibCarla/source/carla/client/ServerSideSensor.h index d55a777b8..fb1f5091b 100644 --- a/LibCarla/source/carla/client/ServerSideSensor.h +++ b/LibCarla/source/carla/client/ServerSideSensor.h @@ -54,7 +54,7 @@ namespace client { private: - std::bitset<32> listening_mask; + std::bitset<16> listening_mask; }; diff --git a/PythonAPI/carla/source/libcarla/Sensor.cpp b/PythonAPI/carla/source/libcarla/Sensor.cpp index 5b66743a1..6d3813a97 100644 --- a/PythonAPI/carla/source/libcarla/Sensor.cpp +++ b/PythonAPI/carla/source/libcarla/Sensor.cpp @@ -28,6 +28,7 @@ void export_sensor() { class_, boost::noncopyable, boost::shared_ptr>("Sensor", no_init) .add_property("is_listening", &cc::Sensor::IsListening) .def("listen", &SubscribeToStream, (arg("callback"))) + .def("is_listening", &cc::Sensor::IsListening) .def("stop", &cc::Sensor::Stop) .def(self_ns::str(self_ns::self)) ; @@ -35,6 +36,7 @@ void export_sensor() { class_, boost::noncopyable, boost::shared_ptr> ("ServerSideSensor", no_init) .def("listen_to_gbuffer", &SubscribeToGBuffer, (arg("gbuffer_id"), arg("callback"))) + .def("is_listening_gbuffer", &cc::ServerSideSensor::IsListeningGBuffer, (arg("gbuffer_id"))) .def("stop_gbuffer", &cc::ServerSideSensor::StopGBuffer, (arg("gbuffer_id"))) .def(self_ns::str(self_ns::self)) ; diff --git a/PythonAPI/docs/sensor.yml b/PythonAPI/docs/sensor.yml index 9a393e183..6401c928a 100644 --- a/PythonAPI/docs/sensor.yml +++ b/PythonAPI/docs/sensor.yml @@ -41,6 +41,10 @@ doc: > The function the sensor will be calling to every time a new measurement is received. This function needs for an argument containing an object type carla.SensorData to work with. # -------------------------------------- + - def_name: is_listening + doc: > + Returns whether the sensor is in a listening state. + # -------------------------------------- - def_name: stop doc: > Commands the sensor to stop listening for data. @@ -50,7 +54,7 @@ - param_name: gbuffer_id type: carla.GBufferTextureID doc: > - The ID of the desired Unreal Engine GBuffer texture. + The ID of the target Unreal Engine GBuffer texture. - param_name: callback type: function doc: > @@ -59,6 +63,15 @@ The function the sensor will be calling to every time the desired GBuffer texture is received.
This function needs for an argument containing an object type carla.SensorData to work with. # -------------------------------------- + - def_name: is_listening_gbuffer + params: + - param_name: gbuffer_id + type: carla.GBufferTextureID + doc: > + The ID of the target Unreal Engine GBuffer texture. + doc: > + Returns whether the sensor is in a listening state for a specific GBuffer texture. + # -------------------------------------- - def_name: stop_gbuffer params: - param_name: gbuffer_id diff --git a/PythonAPI/examples/manual_control_gbuffer.py b/PythonAPI/examples/manual_control_gbuffer.py index 050e7be40..87a51e0b8 100644 --- a/PythonAPI/examples/manual_control_gbuffer.py +++ b/PythonAPI/examples/manual_control_gbuffer.py @@ -1204,7 +1204,7 @@ class CameraManager(object): def next_sensor(self): self.set_sensor(self.index + 1) - + def set_gbuffer(self, index): weak_self = weakref.ref(self) name = self.sensors[self.index][0] @@ -1212,14 +1212,16 @@ class CameraManager(object): self.hud.notification('ERROR: Unsupported operation, see log for more info.') print('ERROR: GBuffer methods are not available for the current sensor type"%s". Only "sensor.camera.rgb" is currently supported.' % name) return False - self.output_texture_id = index % len(gbuffer_names) if self.output_texture_id != 0: self.sensor.stop_gbuffer(self.output_texture_id - 1) + self.output_texture_id = index % len(gbuffer_names) + adjusted_index = self.output_texture_id - 1 if self.output_texture_id != 0: - self.sensor.listen_to_gbuffer( - self.output_texture_id - 1, - lambda image, index = self.output_texture_id: # Need to capture the output_texture_id by value. - CameraManager._parse_image(weak_self, image, index)) + if not self.sensor.is_listening_gbuffer(adjusted_index): + self.sensor.listen_to_gbuffer( + adjusted_index, + lambda image, index = self.output_texture_id: + CameraManager._parse_image(weak_self, image, index)) return True def next_gbuffer(self): @@ -1234,10 +1236,8 @@ class CameraManager(object): display.blit(self.surface, (0, 0)) @staticmethod - def _parse_image(weak_self, image, output_texture_id = 0): + def _parse_image(weak_self, image, index = 0): self = weak_self() - if self.output_texture_id != output_texture_id: - return if not self: return if self.sensors[self.index][0].startswith('sensor.lidar'): @@ -1270,6 +1270,8 @@ class CameraManager(object): array = array[:, :, ::-1] self.surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) else: + if self.output_texture_id != index: + return image.convert(self.sensors[self.index][1]) array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8")) array = np.reshape(array, (image.height, image.width, 4)) diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h index c263310b2..22584ad69 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h @@ -476,7 +476,7 @@ private: std::is_same, FCameraGBufferUint8>::value, FColor, FLinearColor>::type; - FIntPoint ViewSize = {}; + FIntPoint ViewSize; TArray Pixels; if (GBufferData.WaitForTextureTransfer(TextureID)) { @@ -484,26 +484,48 @@ private: void* PixelData; int32 SourcePitch; FIntPoint SourceExtent; - GBufferData.MapTextureData(TextureID, PixelData, SourcePitch, SourceExtent); - ViewSize = GBufferData.ViewRect.Size(); + GBufferData.MapTextureData( + TextureID, + PixelData, + SourcePitch, + SourceExtent); auto Format = GBufferData.Readbacks[(size_t)TextureID]->GetFormat(); + ViewSize = GBufferData.ViewRect.Size(); Pixels.AddUninitialized(ViewSize.X * ViewSize.Y); FReadSurfaceDataFlags Flags = {}; Flags.SetLinearToGamma(true); - ImageUtil::DecodePixelsByFormat(PixelData, SourcePitch, SourceExtent, ViewSize, Format, Flags, Pixels); + ImageUtil::DecodePixelsByFormat( + PixelData, + SourcePitch, + SourceExtent, + ViewSize, + Format, + Flags, + Pixels); GBufferData.UnmapTextureData(TextureID); } + else + { + ViewSize = GBufferData.ViewRect.Size(); + Pixels.SetNum(ViewSize.X * ViewSize.Y); + for (auto& Pixel : Pixels) + Pixel = PixelType::Black; + } auto GBufferStream = CameraGBuffer.GetDataStream(Self); auto Buffer = GBufferStream.PopBufferFromPool(); - Buffer.copy_from(carla::sensor::SensorRegistry::get::type::header_offset, Pixels); + Buffer.copy_from( + carla::sensor::SensorRegistry::get::type::header_offset, + Pixels); if (Buffer.empty()) { return; } SCOPE_CYCLE_COUNTER(STAT_CarlaSensorStreamSend); TRACE_CPUPROFILER_EVENT_SCOPE_STR("Stream Send"); GBufferStream.Send( - CameraGBuffer, std::move(Buffer), - ViewSize.X, ViewSize.Y, + CameraGBuffer, + std::move(Buffer), + ViewSize.X, + ViewSize.Y, Self.GetFOVAngle()); }