Add temporary workaround for the memory blowup issue (#8098)

* Add temporary workaround for the memory blowup issue during save_to_disk.

* Add more fixes on the server side. Update gitignore to remove the _out folder in PythonAPI/examples.

* Enable clangd support. (#8104)

* Enable clangd support.

* Disable CMAKE_EXPORT_COMPILE_COMMANDS by default.

* Revert unwanted target rename.

* Solves the crash on server side

We still have an unhandled exception on client side, but it doesn't affect nominal behaviour.

* Revert "Merge branch 'marcel/leak-workaround' of https://github.com/carla-simulator/carla into marcel/leak-workaround"

This reverts commit 89c211e780, reversing
changes made to 908c203fca.

* Reapply "Merge branch 'marcel/leak-workaround' of https://github.com/carla-simulator/carla into marcel/leak-workaround"

This reverts commit fc2402efdb.

* Fixed crash on client side

Now it's both client and server crashes fixed. By removing posts but keeping async read/write, we make sure saving to disk is sequential but there's no error if it gets interrupted.

* Cleanup of Changes -- Preparation for PR

---------

Co-authored-by: Jorge Virgos <jorgevirgos.dev@gmail.com>

LibCarla/source/carla/streaming/detail/tcp/Client.cpp

@@ -15,7 +15,6 @@
 #include <boost/asio/connect.hpp>
 #include <boost/asio/read.hpp>
 #include <boost/asio/write.hpp>
-#include <boost/asio/post.hpp>
 #include <boost/asio/bind_executor.hpp>
 
 #include <exception>
@@ -86,7 +85,6 @@ namespace tcp {
 
   void Client::Connect() {
     auto self = shared_from_this();
-    boost::asio::post(_strand, [this, self]() {
       if (_done) {
         return;
       }
@@ -139,18 +137,15 @@ namespace tcp {
 
       log_debug("streaming client: connecting to", ep);
       _socket.async_connect(ep, boost::asio::bind_executor(_strand, handle_connect));
-    });
   }
 
   void Client::Stop() {
     _connection_timer.cancel();
     auto self = shared_from_this();
-    boost::asio::post(_strand, [this, self]() {
       _done = true;
       if (_socket.is_open()) {
         _socket.close();
       }
-    });
   }
 
   void Client::Reconnect() {
@@ -165,7 +160,6 @@ namespace tcp {
 
   void Client::ReadData() {
     auto self = shared_from_this();
-    boost::asio::post(_strand, [this, self]() {
       if (_done) {
         return;
       }
@@ -182,7 +176,7 @@ namespace tcp {
           // Move the buffer to the callback function and start reading the next
           // piece of data.
           // log_debug("streaming client: success reading data, calling the callback");
-          boost::asio::post(_strand, [self, message]() { self->_callback(message->pop()); });
+          _callback(message->pop());
           ReadData();
         } else {
           // As usual, if anything fails start over from the very top.
@@ -219,7 +213,6 @@ namespace tcp {
           _socket,
           message->size_as_buffer(),
           boost::asio::bind_executor(_strand, handle_read_header));
-    });
   }
 
 } // namespace tcp

LibCarla/source/carla/streaming/detail/tcp/ServerSession.cpp

@@ -79,7 +79,6 @@ namespace tcp {
     DEBUG_ASSERT(message != nullptr);
     DEBUG_ASSERT(!message->empty());
     auto self = shared_from_this();
-    boost::asio::post(_strand, [=]() {
     if (!_socket.is_open()) {
       return;
     }
@@ -111,11 +110,8 @@ namespace tcp {
     log_debug("session", _session_id, ": sending message of", message->size(), "bytes");
 
     _deadline.expires_from_now(_timeout);
-      boost::asio::async_write(
+    boost::asio::async_write(_socket, message->GetBufferSequence(), 
-          _socket,
+      boost::asio::bind_executor(_strand, handle_sent));
-          message->GetBufferSequence(),
-          handle_sent);
-    });
   }
 
   void ServerSession::Close() {

PythonAPI/examples/.gitignore

@@ -1,2 +1 @@
-carla.so
+_out/
-carla.pyd

Unreal/CarlaUnreal/Plugins/Carla/Source/Carla/Sensor/ImageUtil.cpp

@@ -226,7 +226,16 @@ namespace ImageUtil
 
 
 
-  static void ReadImageDataAsyncCommand(
+  struct ReadImageDataContext
+  {
+    EPixelFormat Format;
+    FIntPoint Size;
+    ReadImageDataAsyncCallback Callback;
+    TUniquePtr<FRHIGPUTextureReadback> Readback;
+  };
+
+  static void ReadImageDataBegin(
+    ReadImageDataContext& Self,
     UTextureRenderTarget2D& RenderTarget,
     ReadImageDataAsyncCallback&& Callback)
   {
@@ -239,12 +248,13 @@ namespace ImageUtil
     auto Texture = Resource->GetRenderTargetTexture();
     if (Texture == nullptr)
       return;
-    auto Readback = MakeUnique<FRHIGPUTextureReadback>(
+    Self.Callback = std::move(Callback);
+    Self.Readback = MakeUnique<FRHIGPUTextureReadback>(
       TEXT("ReadImageData-Readback"));
-    auto Size = Texture->GetSizeXY();
+    Self.Size = Texture->GetSizeXY();
-    auto Format = Texture->GetFormat();
+    Self.Format = Texture->GetFormat();
     auto ResolveRect = FResolveRect();
-    Readback->EnqueueCopy(CmdList, Texture, ResolveRect);
+    Self.Readback->EnqueueCopy(CmdList, Texture, ResolveRect);
 
     auto Query = RenderQueryPool->AllocateQuery();
     CmdList.EndRenderQuery(Query.GetQuery());
@@ -252,26 +262,42 @@ namespace ImageUtil
     uint64 DeltaTime;
     RHIGetRenderQueryResult(Query.GetQuery(), DeltaTime, true);
     Query.ReleaseQuery();
+  }
 
-    AsyncTask(
+  static void ReadImageDataEnd(
-      ENamedThreads::HighTaskPriority, [
+    ReadImageDataContext& Self)
-      Readback = MoveTemp(Readback),
-      Callback = std::move(Callback),
-      Size,
-      Format]
   {
-      while (!Readback->IsReady())
-        std::this_thread::yield();
     int32 RowPitch, BufferHeight;
-      auto MappedPtr = Readback->Lock(RowPitch, &BufferHeight);
+    auto MappedPtr = Self.Readback->Lock(RowPitch, &BufferHeight);
     if (MappedPtr != nullptr)
     {
-        ScopedCallback Unlock = [&] { Readback->Unlock(); };
+      ScopedCallback Unlock = [&] { Self.Readback->Unlock(); };
-        Callback(MappedPtr, RowPitch, BufferHeight, Format, Size);
+      Self.Callback(MappedPtr, RowPitch, BufferHeight, Self.Format, Self.Size);
     }
+  }
+
+  static void ReadImageDataEndAsync(
+    ReadImageDataContext&& Self)
+  {
+    AsyncTask(
+      ENamedThreads::HighTaskPriority, [
+      Self = std::move(Self)]() mutable
+    {
+      while (!Self.Readback->IsReady())
+        std::this_thread::yield();
+      ReadImageDataEnd(Self);
     });
   }
 
+  static void ReadImageDataAsyncCommand(
+    UTextureRenderTarget2D& RenderTarget,
+    ReadImageDataAsyncCallback&& Callback)
+  {
+    ReadImageDataContext Context = { };
+    ReadImageDataBegin(Context, RenderTarget, std::move(Callback));
+    ReadImageDataEndAsync(std::move(Context));
+  }
+
 
 
   bool ReadImageDataAsync(
@@ -287,14 +313,14 @@ namespace ImageUtil
     else
     {
       ENQUEUE_RENDER_COMMAND(ReadImageDataAsyncCmd)([
-        &RenderTarget,
+        &RenderTarget, Callback = std::move(Callback)
-        Callback = std::move(Callback)
       ](auto& CmdList) mutable
       {
-        ReadImageDataAsyncCommand(
+        ReadImageDataContext Context = { };
-          RenderTarget,
+        ReadImageDataBegin(Context, RenderTarget, std::move(Callback));
-          std::move(Callback));
+        ReadImageDataEnd(Context);
       });
+      FlushRenderingCommands();
     }
     return true;
   }