Image serialization complete pipeline

LibCarla/cmake/client/CMakeLists.txt

@@ -63,6 +63,21 @@ file(GLOB libcarla_carla_rpc_headers
     "${libcarla_source_path}/carla/rpc/*.h")
 install(FILES ${libcarla_carla_rpc_headers} DESTINATION include/carla/rpc)
 
+file(GLOB libcarla_carla_sensor_headers
+    "${libcarla_source_path}/carla/sensor/*.cpp"
+    "${libcarla_source_path}/carla/sensor/*.h")
+install(FILES ${libcarla_carla_sensor_headers} DESTINATION include/carla/sensor)
+
+file(GLOB libcarla_carla_sensor_data_headers
+    "${libcarla_source_path}/carla/sensor/data/*.cpp"
+    "${libcarla_source_path}/carla/sensor/data/*.h")
+install(FILES ${libcarla_carla_sensor_data_headers} DESTINATION include/carla/sensor/data)
+
+file(GLOB libcarla_carla_sensor_s11n_headers
+    "${libcarla_source_path}/carla/sensor/s11n/*.cpp"
+    "${libcarla_source_path}/carla/sensor/s11n/*.h")
+install(FILES ${libcarla_carla_sensor_s11n_headers} DESTINATION include/carla/sensor/s11n)
+
 file(GLOB libcarla_carla_streaming_headers
     "${libcarla_source_path}/carla/streaming/*.cpp"
     "${libcarla_source_path}/carla/streaming/*.h")

LibCarla/cmake/server/CMakeLists.txt

@@ -23,6 +23,9 @@ install(FILES ${libcarla_carla_rpc_headers} DESTINATION include/carla/rpc)
 file(GLOB libcarla_carla_sensor_headers "${libcarla_source_path}/carla/sensor/*.h")
 install(FILES ${libcarla_carla_sensor_headers} DESTINATION include/carla/sensor)
 
+file(GLOB libcarla_carla_sensor_data_headers "${libcarla_source_path}/carla/sensor/data/*.h")
+install(FILES ${libcarla_carla_sensor_data_headers} DESTINATION include/carla/sensor/data)
+
 file(GLOB libcarla_carla_sensor_s11n_headers "${libcarla_source_path}/carla/sensor/s11n/*.h")
 install(FILES ${libcarla_carla_sensor_s11n_headers} DESTINATION include/carla/sensor/s11n)
 

LibCarla/source/carla/Memory.h

@@ -6,26 +6,24 @@
 
 #pragma once
 
-// In this namespace, we use boost::shared_ptr for now to make it compatible
-// with boost::python, but it would be nice to make an adaptor for
-// std::shared_ptr.
 #include <boost/enable_shared_from_this.hpp>
 #include <boost/make_shared.hpp>
 #include <boost/shared_ptr.hpp>
 
 namespace carla {
-namespace client {
+
+  // Use this SharedPtr (boost::shared_ptr) to keep compatibility with
+  // boost::python, but it would be nice if in the future we can make a Python
+  // adaptor for std::shared_ptr.
+  template <typename T>
+  using SharedPtr = boost::shared_ptr<T>;
 
   template <typename T>
   using EnableSharedFromThis = boost::enable_shared_from_this<T>;
 
-  template <typename T>
-  using SharedPtr = boost::shared_ptr<T>;
-
   template <typename T, typename ... Args>
   static inline auto MakeShared(Args && ... args) {
     return boost::make_shared<T>(std::forward<Args>(args) ...);
   }
 
-} // namespace client
 } // namespace carla

LibCarla/source/carla/client/Actor.h

@@ -7,8 +7,8 @@
 #pragma once
 
 #include "carla/Debug.h"
+#include "carla/Memory.h"
 #include "carla/NonCopyable.h"
-#include "carla/client/Memory.h"
 #include "carla/client/World.h"
 #include "carla/rpc/Actor.h"
 

LibCarla/source/carla/client/BlueprintLibrary.h

@@ -8,9 +8,9 @@
 
 #include "carla/Debug.h"
 #include "carla/Iterator.h"
+#include "carla/Memory.h"
 #include "carla/NonCopyable.h"
 #include "carla/client/ActorBlueprint.h"
-#include "carla/client/Memory.h"
 
 #include <type_traits>
 #include <unordered_map>

LibCarla/source/carla/client/Client.h

@@ -6,13 +6,14 @@
 
 #pragma once
 
+#include "carla/Memory.h"
 #include "carla/NonCopyable.h"
 #include "carla/Time.h"
 #include "carla/Version.h"
 #include "carla/client/Control.h"
-#include "carla/client/Memory.h"
 #include "carla/client/Transform.h"
 #include "carla/rpc/Client.h"
+#include "carla/sensor/Deserializer.h"
 #include "carla/streaming/Client.h"
 
 #include <string>
@@ -51,8 +52,10 @@ namespace client {
     }
 
     template <typename Functor>
-    void SubscribeToStream(const streaming::Token &token, Functor &&callback) {
+    void SubscribeToStream(const streaming::Token &token, Functor callback) {
-      _streaming_client.Subscribe(token, std::forward<Functor>(callback));
+      _streaming_client.Subscribe(token, [callback](auto buffer) {
+        callback(sensor::Deserializer::Deserialize(std::move(buffer)));
+      });
     }
 
     std::string GetClientVersion() const {

LibCarla/source/carla/client/Image.cpp

@@ -1,86 +0,0 @@
-// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
-// de Barcelona (UAB).
-//
-// This work is licensed under the terms of the MIT license.
-// For a copy, see <https://opensource.org/licenses/MIT>.
-
-#include "carla/Debug.h"
-#include "carla/client/Image.h"
-
-#include <cstdint>
-#include <cstring>
-
-namespace carla {
-namespace client {
-
-  /// @todo This object should be shared and packed.
-  struct FImageHeaderData
-  {
-    uint64_t FrameNumber;
-    uint32_t Width;
-    uint32_t Height;
-    uint32_t Type;
-    float FOV;
-  };
-
-  static std::string GetTypeString(uint32_t type) {
-    switch (type) {
-      case 0u: return "None";
-      case 1u: return "SceneFinal";
-      case 2u: return "Depth";
-      case 3u: return "SemanticSegmentation";
-      default: return "Invalid";
-    }
-  }
-
-  SharedPtr<Image> Image::FromBuffer(Buffer buffer) {
-    /// @todo We can avoid making another copy of the buffer here.
-    if (buffer.size() < sizeof(FImageHeaderData)) {
-      throw std::invalid_argument("buffer too small to be an image");
-    }
-    auto begin = reinterpret_cast<const byte_type *>(buffer.data());
-    auto image = MakeShared<Image>();
-    FImageHeaderData data;
-    std::memcpy(&data, begin, sizeof(data));
-    image->_frame_number = data.FrameNumber;
-    image->_width = data.Width;
-    image->_height = data.Height;
-    image->_type = GetTypeString(data.Type);
-    image->_fov = data.FOV;
-    const auto size = image->GetSize();
-    DEBUG_ASSERT((size + sizeof(FImageHeaderData)) == buffer.size());
-    auto raw_data = std::make_unique<byte_type[]>(size);
-    std::memcpy(raw_data.get(), begin + sizeof(FImageHeaderData), size);
-    image->_raw_data = std::move(raw_data);
-    return image;
-  }
-
-  Image::Image() {
-    Clear();
-  }
-
-  Image::Image(Image &&rhs) {
-    (*this) = std::move(rhs);
-  }
-
-  Image &Image::operator=(Image &&rhs) {
-    _width = rhs._width;
-    _height = rhs._height;
-    _type = rhs._type;
-    _fov = rhs._fov;
-    _raw_data = std::move(rhs._raw_data);
-    rhs.Clear();
-    return *this;
-  }
-
-  void Image::Clear() {
-    _frame_number = 0u;
-    _width = 0u;
-    _height = 0u;
-    _type = "Invalid";
-    _fov = 0.0f;
-    _raw_data = nullptr;
-  }
-
-} // namespace client
-} // namespace carla

LibCarla/source/carla/client/Image.h

@@ -1,82 +0,0 @@
-// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
-// de Barcelona (UAB).
-//
-// This work is licensed under the terms of the MIT license.
-// For a copy, see <https://opensource.org/licenses/MIT>.
-
-#pragma once
-
-#include "carla/Buffer.h"
-#include "carla/NonCopyable.h"
-#include "carla/client/Memory.h"
-
-#include <memory>
-
-namespace carla {
-namespace client {
-
-  class Image
-    : public EnableSharedFromThis<Image>,
-      private NonCopyable  {
-  public:
-
-    using byte_type = unsigned char;
-
-    static SharedPtr<Image> FromBuffer(Buffer buffer);
-
-    Image();
-
-    Image(Image &&rhs);
-    Image &operator=(Image &&rhs);
-
-    void Clear();
-
-    size_t GetFrameNumber() const {
-      return _frame_number;
-    }
-
-    size_t GetWidth() const {
-      return _width;
-    }
-
-    size_t GetHeight() const {
-      return _height;
-    }
-
-    const std::string &GetType() const {
-      return _type;
-    }
-
-    float GetFOV() const {
-      return _fov;
-    }
-
-    byte_type *GetData() {
-      return _raw_data.get();
-    }
-
-    const byte_type *GetData() const {
-      return _raw_data.get();
-    }
-
-    size_t GetSize() const {
-      return 4u * _width * _height;
-    }
-
-  private:
-
-    size_t _frame_number;
-
-    size_t _width;
-
-    size_t _height;
-
-    std::string _type;
-
-    float _fov;
-
-    std::unique_ptr<byte_type[]> _raw_data;
-  };
-
-} // namespace client
-} // namespace carla

LibCarla/source/carla/client/World.h

@@ -7,9 +7,9 @@
 #pragma once
 
 #include "carla/Debug.h"
+#include "carla/Memory.h"
 #include "carla/NonCopyable.h"
 #include "carla/client/Client.h"
-#include "carla/client/Memory.h"
 
 namespace carla {
 namespace client {

LibCarla/source/carla/sensor/CompositeSerializer.h

@@ -6,34 +6,40 @@
 
 #pragma once
 
+#include "carla/Buffer.h"
+#include "carla/Memory.h"
 #include "carla/sensor/CompileTimeTypeMap.h"
 
 namespace carla {
 namespace sensor {
 
+  class SensorData;
+
   template <typename... Items>
   class CompositeSerializer : public CompileTimeTypeMap<Items...> {
     using Super = CompileTimeTypeMap<Items...>;
 
   public:
 
-    using interpreted_type = size_t; /// @todo
+    using interpreted_type = SharedPtr<SensorData>;
 
     template <typename Sensor, typename... Args>
-    static auto Serialize(Sensor &sensor, Args &&... args) {
+    static Buffer Serialize(Sensor &sensor, Args &&... args) {
       using TheSensor = typename std::remove_const<Sensor>::type;
       using Serializer = typename Super::template get<TheSensor*>::type;
       return Serializer::Serialize(sensor, std::forward<Args>(args)...);
     }
 
+    static interpreted_type Deserialize(Buffer data);
+
+  private:
+
     template <size_t Index, typename Data>
-    static interpreted_type Deserialize(Data &&data) {
+    static interpreted_type Deserialize_impl(Data &&data) {
       using Serializer = typename Super::template get_by_index<Index>::type;
       return Serializer::Deserialize(std::forward<Data>(data));
     }
 
-  private:
-
     template <typename Data, size_t... Is>
     static interpreted_type Deserialize_impl(size_t i, Data &&data, std::index_sequence<Is...>) {
       // This function is equivalent to creating a switch statement with a case
@@ -41,13 +47,11 @@ namespace sensor {
       // into a jump table. See https://stackoverflow.com/a/46282159/5308925.
       interpreted_type result;
       std::initializer_list<int> ({
-          (i == Is ? (result = Deserialize<Is>(std::forward<Data>(data))), 0 : 0)...
+          (i == Is ? (result = Deserialize_impl<Is>(std::forward<Data>(data))), 0 : 0)...
       });
       return result;
     }
 
-  public:
-
     template <typename Data>
     static interpreted_type Deserialize(size_t index, Data &&data) {
       return Deserialize_impl(
@@ -59,3 +63,19 @@ namespace sensor {
 
 } // namespace sensor
 } // namespace carla
+
+#include "carla/sensor/DataMessage.h"
+
+namespace carla {
+namespace sensor {
+
+  template <typename... Items>
+  inline typename CompositeSerializer<Items...>::interpreted_type
+  CompositeSerializer<Items...>::Deserialize(Buffer data) {
+    DataMessage message{std::move(data)};
+    size_t index = message.GetSensorTypeId();
+    return Deserialize(index, std::move(message));
+  }
+
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/DataMessage.h

@@ -0,0 +1,71 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/Buffer.h"
+#include "carla/sensor/s11n/SensorHeaderSerializer.h"
+
+#include <cstdint>
+#include <iterator>
+
+namespace carla {
+namespace sensor {
+
+  class DataMessage {
+   using HeaderSerializer = s11n::SensorHeaderSerializer;
+  private:
+
+    const auto &GetHeader() const {
+     return HeaderSerializer::Deserialize(_buffer);
+    }
+
+  public:
+
+    uint64_t GetSensorTypeId() const {
+     return GetHeader().sensor_type;
+    }
+
+    uint64_t GetFrameNumber() const {
+     return GetHeader().frame_number;
+    }
+
+    const rpc::Transform &GetSensorTransform() const {
+     return GetHeader().sensor_transform;
+    }
+
+    auto begin() {
+     return _buffer.begin() + HeaderSerializer::header_offset;
+    }
+
+    auto begin() const {
+     return _buffer.begin() + HeaderSerializer::header_offset;
+    }
+
+    auto end() {
+     return _buffer.end();
+    }
+
+    auto end() const {
+     return _buffer.end();
+    }
+
+    size_t size() const {
+     return std::distance(begin(), end());
+    }
+
+  private:
+
+    template <typename... Items>
+    friend class CompositeSerializer;
+
+    DataMessage(Buffer buffer) : _buffer(std::move(buffer)) {}
+
+    Buffer _buffer;
+  };
+
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/Deserializer.cpp

@@ -0,0 +1,19 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#include "carla/sensor/Deserializer.h"
+
+#include "carla/sensor/SensorRegistry.h"
+
+namespace carla {
+namespace sensor {
+
+  SharedPtr<SensorData> Deserializer::Deserialize(Buffer buffer) {
+    return SensorRegistry::Deserialize(std::move(buffer));
+  }
+
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/Deserializer.h

@@ -0,0 +1,24 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/Buffer.h"
+#include "carla/Memory.h"
+
+namespace carla {
+namespace sensor {
+
+  class SensorData;
+
+  class Deserializer {
+  public:
+
+    static SharedPtr<SensorData> Deserialize(Buffer buffer);
+  };
+
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/SensorData.h

@@ -0,0 +1,46 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/Memory.h"
+#include "carla/NonCopyable.h"
+#include "carla/sensor/DataMessage.h"
+
+namespace carla {
+namespace sensor {
+
+  /// Base class for all the objects containing data generated by a sensor.
+  class SensorData
+    : public EnableSharedFromThis<SensorData>,
+      private NonCopyable  {
+  protected:
+
+    explicit SensorData(const DataMessage &message)
+      : _frame_number(message.GetFrameNumber()),
+        _sensor_transform(message.GetSensorTransform()) {}
+
+  public:
+
+    virtual ~SensorData() = default;
+
+    size_t GetFrameNumber() const {
+      return _frame_number;
+    }
+
+    const rpc::Transform &GetSensorTransform() const {
+      return _sensor_transform;
+    }
+
+  private:
+
+    const size_t _frame_number;
+
+    const rpc::Transform _sensor_transform;
+  };
+
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/data/Array.h

@@ -0,0 +1,98 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/Debug.h"
+#include "carla/sensor/SensorData.h"
+
+#include <type_traits>
+#include <iterator>
+
+namespace carla {
+namespace sensor {
+namespace data {
+
+  template <size_t Offset, typename T>
+  class Array : public SensorData {
+  public:
+
+    using value_type = T;
+    using iterator = value_type *;
+    using const_iterator = typename std::add_const<value_type>::type *;
+    using size_type = typename std::iterator_traits<iterator>::difference_type;
+    using pointer = typename std::iterator_traits<iterator>::pointer;
+    using reference = typename std::iterator_traits<iterator>::reference;
+
+    iterator begin() {
+     return reinterpret_cast<iterator>(_message.begin() + Offset);
+    }
+
+    const_iterator cbegin() const {
+     return reinterpret_cast<const_iterator>(_message.begin() + Offset);
+    }
+
+    const_iterator begin() const {
+     return cbegin();
+    }
+
+    iterator end() {
+     return reinterpret_cast<iterator>(_message.end());
+    }
+
+    const_iterator cend() const {
+     return reinterpret_cast<const_iterator>(_message.end());
+    }
+
+    const_iterator end() const {
+     return cend();
+    }
+
+    bool empty() const {
+      return begin() == end();
+    }
+
+    size_type size() const {
+     return std::distance(begin(), end());
+    }
+
+    value_type *data() {
+      return begin();
+    }
+
+    const value_type *data() const {
+      return begin();
+    }
+
+    reference operator[](size_type i) {
+      return data()[i];
+    }
+
+    const reference operator[](size_type i) const {
+      return data()[i];
+    }
+
+  protected:
+
+    explicit Array(DataMessage message)
+      : SensorData(message),
+        _message(std::move(message)) {
+      DEBUG_ASSERT(_message.size() >= Offset);
+      DEBUG_ASSERT((_message.size() - Offset) % sizeof(T) == 0u);
+    }
+
+    const DataMessage &GetMessage() const {
+      return _message;
+    }
+
+  private:
+
+    DataMessage _message;
+  };
+
+} // namespace data
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/data/Color.h

@@ -0,0 +1,28 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include <cstdint>
+
+namespace carla {
+namespace sensor {
+namespace data {
+
+#pragma pack(push, 1)
+    struct Color {
+      uint8_t b;
+      uint8_t g;
+      uint8_t r;
+      uint8_t a;
+    };
+#pragma pack(pop)
+
+    static_assert(sizeof(Color) == sizeof(uint32_t), "Invalid color size!");
+
+} // namespace data
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/data/Image.h

@@ -0,0 +1,20 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/sensor/data/Color.h"
+#include "carla/sensor/data/ImageTmpl.h"
+
+namespace carla {
+namespace sensor {
+namespace data {
+
+  using Image = ImageTmpl<Color>;
+
+} // namespace data
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/data/ImageTmpl.h

@@ -0,0 +1,54 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#pragma once
+
+#include "carla/Debug.h"
+#include "carla/sensor/data/Array.h"
+#include "carla/sensor/s11n/ImageSerializer.h"
+
+namespace carla {
+namespace sensor {
+namespace data {
+
+  template <typename T>
+  class ImageTmpl : public Array<s11n::ImageSerializer::header_offset, T>  {
+    using Super = Array<s11n::ImageSerializer::header_offset, T>;
+  protected:
+
+    using Serializer = s11n::ImageSerializer;
+
+    friend Serializer;
+
+    explicit ImageTmpl(DataMessage message)
+      : Super(std::move(message)) {
+      DEBUG_ASSERT(GetWidth() * GetHeight() == Super::size());
+    }
+
+  private:
+
+   const auto &GetHeader() const {
+     return Serializer::DeserializeHeader(Super::GetMessage());
+   }
+
+  public:
+
+    auto GetWidth() const {
+     return GetHeader().width;
+    }
+
+    auto GetHeight() const {
+     return GetHeader().height;
+    }
+
+    uint64_t GetFOVAngle() const {
+     return GetHeader().fov_angle;
+    }
+  };
+
+} // namespace data
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/s11n/ImageSerializer.cpp

@@ -0,0 +1,21 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#include "carla/sensor/s11n/ImageSerializer.h"
+
+#include "carla/sensor/data/Image.h"
+
+namespace carla {
+namespace sensor {
+namespace s11n {
+
+  SharedPtr<SensorData> ImageSerializer::Deserialize(DataMessage message) {
+    return SharedPtr<data::Image>(new data::Image{std::move(message)});
+  }
+
+} // namespace s11n
+} // namespace sensor
+} // namespace carla

LibCarla/source/carla/sensor/s11n/ImageSerializer.h

@@ -6,44 +6,54 @@
 
 #pragma once
 
-#include "carla/Buffer.h"
+#include "carla/Memory.h"
+#include "carla/sensor/DataMessage.h"
 
 #include <cstdint>
 #include <cstring>
 
 namespace carla {
 namespace sensor {
+
+  class SensorData;
+
 namespace s11n {
 
   class ImageSerializer {
+  public:
 
 #pragma pack(push, 1)
     struct ImageHeader {
       uint32_t width;
       uint32_t height;
-      uint32_t type;
+      float fov_angle;
-      float fov;
     };
 #pragma pack(pop)
 
-  public:
+    constexpr static auto header_offset = sizeof(ImageHeader);
 
-    constexpr static auto offset_size = sizeof(ImageHeader);
+    static const ImageHeader &DeserializeHeader(const DataMessage &message) {
+      return *reinterpret_cast<const ImageHeader *>(message.begin());
+    }
 
     template <typename Sensor>
-    static Buffer Serialize(const Sensor &sensor, Buffer bitmap) {
+    static Buffer Serialize(const Sensor &sensor, Buffer bitmap);
-      DEBUG_ASSERT(bitmap.size() > sizeof(ImageHeader));
+
-      ImageHeader header = {
+    static SharedPtr<SensorData> Deserialize(DataMessage message);
-        sensor.GetImageWidth(),
-        sensor.GetImageHeight(),
-        0u, /// @todo
-        sensor.GetFOVAngle()
-      };
-      std::memcpy(bitmap.data(), reinterpret_cast<const void *>(&header), sizeof(header));
-      return bitmap;
-    }
   };
 
+  template <typename Sensor>
+  inline Buffer ImageSerializer::Serialize(const Sensor &sensor, Buffer bitmap) {
+    DEBUG_ASSERT(bitmap.size() > sizeof(ImageHeader));
+    ImageHeader header = {
+      sensor.GetImageWidth(),
+      sensor.GetImageHeight(),
+      sensor.GetFOVAngle()
+    };
+    std::memcpy(bitmap.data(), reinterpret_cast<const void *>(&header), sizeof(header));
+    return bitmap;
+  }
+
 } // namespace s11n
 } // namespace sensor
 } // namespace carla

LibCarla/source/carla/sensor/s11n/SensorHeaderSerializer.cpp

@@ -17,27 +17,14 @@ namespace s11n {
     return pool.Pop();
   }
 
-#pragma pack(push, 1)
-    struct Header {
-      uint64_t sensor_type;
-      uint64_t frame_counter;
-      float sensor_transform[6u];
-    };
-#pragma pack(pop)
-
   Buffer SensorHeaderSerializer::Serialize(
       const uint64_t index,
       const uint64_t frame,
       const rpc::Transform transform) {
     Header h;
     h.sensor_type = index;
-    h.frame_counter = frame;
+    h.frame_number = frame;
-    h.sensor_transform[0u] = transform.location.x;
+    h.sensor_transform = transform;
-    h.sensor_transform[1u] = transform.location.y;
-    h.sensor_transform[2u] = transform.location.z;
-    h.sensor_transform[3u] = transform.rotation.pitch;
-    h.sensor_transform[4u] = transform.rotation.yaw;
-    h.sensor_transform[5u] = transform.rotation.roll;
     auto buffer = PopBufferFromPool();
     buffer.copy_from(reinterpret_cast<const unsigned char *>(&h), sizeof(h));
     return buffer;

LibCarla/source/carla/sensor/s11n/SensorHeaderSerializer.h

@@ -8,7 +8,6 @@
 
 #include "carla/Buffer.h"
 #include "carla/rpc/Transform.h"
-#include "carla/sensor/SensorRegistry.h"
 
 namespace carla {
 namespace sensor {
@@ -17,17 +16,21 @@ namespace s11n {
   class SensorHeaderSerializer {
   public:
 
-    template <typename Sensor>
+#pragma pack(push, 1)
-    static Buffer Serialize(const Sensor &sensor, uint16_t frame_counter) {
+    struct Header {
-      return Serialize(
+      uint64_t sensor_type;
-          SensorRegistry::template get<Sensor*>::index,
+      uint64_t frame_number;
-          frame_counter,
+      rpc::Transform sensor_transform;
-          sensor.GetActorTransform());
+    };
-    }
+#pragma pack(pop)
 
-  private:
+    constexpr static auto header_offset = sizeof(Header);
 
     static Buffer Serialize(uint64_t index, uint64_t frame, rpc::Transform transform);
+
+    static const Header &Deserialize(const Buffer &message) {
+      return *reinterpret_cast<const Header *>(message.data());
+    }
   };
 
 } // namespace s11n

PythonAPI/source/libcarla/Actor.cpp

@@ -5,8 +5,6 @@
 // For a copy, see <https://opensource.org/licenses/MIT>.
 
 #include <carla/client/Actor.h>
-#include <carla/client/Image.h>
-#include <carla/client/Sensor.h>
 #include <carla/client/Vehicle.h>
 
 #include <boost/python.hpp>
@@ -14,25 +12,6 @@
 #include <ostream>
 #include <iostream>
 
-/// Aquires a lock on the Python's Global Interpreter Lock, necessary for
-/// calling Python code from a different thread.
-///
-/// https://wiki.python.org/moin/GlobalInterpreterLock
-class GILLockGuard {
-public:
-
-  GILLockGuard()
-    : _state(PyGILState_Ensure()) {}
-
-  ~GILLockGuard() {
-    PyGILState_Release(_state);
-  }
-
-private:
-
-  PyGILState_STATE _state;
-};
-
 namespace carla {
 namespace client {
 
@@ -46,39 +25,12 @@ namespace client {
     return out;
   }
 
-  std::ostream &operator<<(std::ostream &out, const Sensor &sensor) {
-    out << "Sensor(id=" << sensor.GetId() << ", type=" << sensor.GetTypeId() << ')';
-    return out;
-  }
-
 } // namespace client
 } // namespace carla
 
 void export_actor() {
   using namespace boost::python;
   namespace cc = carla::client;
-  namespace cr = carla::rpc;
-
-  class_<cc::Image, boost::noncopyable, boost::shared_ptr<cc::Image>>("Image", no_init)
-    .add_property("frame_number", &cc::Image::GetFrameNumber)
-    .add_property("width", &cc::Image::GetWidth)
-    .add_property("height", &cc::Image::GetHeight)
-    .add_property("type", +[](const cc::Image &self) -> std::string {
-      return self.GetType();
-    })
-    .add_property("fov", &cc::Image::GetFOV)
-    .add_property("raw_data", +[](cc::Image &self) {
-
-#if PYTHON3X
-      //NOTE(Andrei): python37
-      auto *ptr = PyMemoryView_FromMemory(reinterpret_cast<char *>(self.GetData()), self.GetSize(), PyBUF_READ);
-#else
-      //NOTE(Andrei): python2.7
-      auto *ptr = PyBuffer_FromMemory(self.GetData(), self.GetSize());
-#endif
-      return object(handle<>(ptr));
-    })
-  ;
 
   class_<cc::Actor, boost::noncopyable, boost::shared_ptr<cc::Actor>>("Actor", no_init)
     .add_property("id", &cc::Actor::GetId)
@@ -99,34 +51,4 @@ void export_actor() {
     .def("set_autopilot", &cc::Vehicle::SetAutopilot, (arg("enabled")=true))
     .def(self_ns::str(self_ns::self))
   ;
-
-  class_<cc::Sensor, bases<cc::Actor>, boost::noncopyable, boost::shared_ptr<cc::Sensor>>("Sensor", no_init)
-    .def("listen", +[](cc::Sensor &self, boost::python::object callback) {
-      // Make sure the callback is actually callable.
-      if (!PyCallable_Check(callback.ptr())) {
-        PyErr_SetString(PyExc_TypeError, "callback argument must be callable!");
-        throw_error_already_set();
-        return;
-      }
-
-      // Subscribe to the sensor.
-      self.Listen([callback](auto message) {
-        cc::SharedPtr<cc::Image> image;
-        try {
-          image = cc::Image::FromBuffer(std::move(message));
-        } catch (const std::exception &e) {
-          std::cerr << "exception while parsing the image: " << e.what() << std::endl;
-          return;
-        }
-
-        GILLockGuard gil_lock;
-        try {
-          boost::python::call<void>(callback.ptr(), boost::python::object(image));
-        } catch (const boost::python::error_already_set &e) {
-          PyErr_Print();
-        }
-      });
-    }, (arg("callback")))
-    .def(self_ns::str(self_ns::self))
-  ;
 }

PythonAPI/source/libcarla/Sensor.cpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma
+// de Barcelona (UAB).
+//
+// This work is licensed under the terms of the MIT license.
+// For a copy, see <https://opensource.org/licenses/MIT>.
+
+#include <carla/client/Sensor.h>
+#include <carla/sensor/SensorData.h>
+#include <carla/sensor/data/Image.h>
+
+#include <boost/python.hpp>
+
+#include <ostream>
+#include <iostream>
+
+/// Aquires a lock on the Python's Global Interpreter Lock, necessary for
+/// calling Python code from a different thread.
+///
+/// https://wiki.python.org/moin/GlobalInterpreterLock
+class GILLockGuard {
+public:
+
+  GILLockGuard()
+    : _state(PyGILState_Ensure()) {}
+
+  ~GILLockGuard() {
+    PyGILState_Release(_state);
+  }
+
+private:
+
+  PyGILState_STATE _state;
+};
+
+namespace carla {
+namespace client {
+
+  std::ostream &operator<<(std::ostream &out, const Sensor &sensor) {
+    out << "Sensor(id=" << sensor.GetId() << ", type=" << sensor.GetTypeId() << ')';
+    return out;
+  }
+
+} // namespace client
+
+namespace sensor {
+namespace data {
+
+  std::ostream &operator<<(std::ostream &out, const Image &image) {
+    out << "Image(" << image.GetWidth() << 'x' << image.GetHeight()
+        << ", frame=" << image.GetFrameNumber() << ')';
+    return out;
+  }
+
+} // namespace data
+} // namespace sensor
+} // namespace carla
+
+void export_sensor() {
+  using namespace boost::python;
+  namespace cc = carla::client;
+  namespace cs = carla::sensor;
+  namespace csd = carla::sensor::data;
+
+  class_<cs::SensorData, boost::noncopyable, boost::shared_ptr<cs::SensorData>>("SensorData", no_init)
+    .add_property("frame_number", &cs::SensorData::GetFrameNumber)
+    .add_property("transform", +[](const cs::SensorData &self) -> carla::rpc::Transform {
+      return self.GetSensorTransform();
+    })
+  ;
+
+  class_<csd::Image, bases<cs::SensorData>, boost::noncopyable, boost::shared_ptr<csd::Image>>("Image", no_init)
+    .add_property("width", &csd::Image::GetWidth)
+    .add_property("height", &csd::Image::GetHeight)
+    .add_property("fov", &csd::Image::GetFOVAngle)
+    .add_property("raw_data", +[](csd::Image &self) {
+      // An image is an Array<Color>, convert back to buffer of chars.
+      auto *data = reinterpret_cast<unsigned char *>(self.data());
+      auto size = sizeof(csd::Image::value_type) * self.size();
+#if PYTHON3X // NOTE(Andrei): python37
+      auto *ptr = PyMemoryView_FromMemory(reinterpret_cast<char *>(data), size, PyBUF_READ);
+#else        // NOTE(Andrei): python2.7
+      auto *ptr = PyBuffer_FromMemory(data, size);
+#endif
+      return object(handle<>(ptr));
+    })
+    .def(self_ns::str(self_ns::self))
+  ;
+
+  class_<cc::Sensor, bases<cc::Actor>, boost::noncopyable, boost::shared_ptr<cc::Sensor>>("Sensor", no_init)
+    .def("listen", +[](cc::Sensor &self, boost::python::object callback) {
+      // Make sure the callback is actually callable.
+      if (!PyCallable_Check(callback.ptr())) {
+        PyErr_SetString(PyExc_TypeError, "callback argument must be callable!");
+        throw_error_already_set();
+        return;
+      }
+
+      // Subscribe to the sensor.
+      self.Listen([callback](auto message) { // Here we retrieve already the deserialized object.
+        GILLockGuard gil_lock;
+        try {
+          boost::python::call<void>(callback.ptr(), boost::python::object(message));
+        } catch (const boost::python::error_already_set &e) {
+          PyErr_Print();
+        }
+      });
+    }, (arg("callback")))
+    .def(self_ns::str(self_ns::self))
+  ;
+}

PythonAPI/source/libcarla/World.cpp

@@ -17,8 +17,8 @@ void export_world() {
     .def("get_blueprint_library", &cc::World::GetBlueprintLibrary)
     .def("get_spectator", &cc::World::GetSpectator)
     .def("try_spawn_actor", &cc::World::TrySpawnActor,
-        (arg("blueprint"), arg("transform"), arg("attach_to")=cc::SharedPtr<cc::Actor>()))
+        (arg("blueprint"), arg("transform"), arg("attach_to")=carla::SharedPtr<cc::Actor>()))
     .def("spawn_actor", &cc::World::SpawnActor,
-        (arg("blueprint"), arg("transform"), arg("attach_to")=cc::SharedPtr<cc::Actor>()))
+        (arg("blueprint"), arg("transform"), arg("attach_to")=carla::SharedPtr<cc::Actor>()))
   ;
 }

PythonAPI/source/libcarla/libcarla.cpp

@@ -11,6 +11,7 @@
 #include "Client.cpp"
 #include "Control.cpp"
 #include "Exception.cpp"
+#include "Sensor.cpp"
 #include "Transform.cpp"
 #include "World.cpp"
 
@@ -22,6 +23,7 @@ BOOST_PYTHON_MODULE(libcarla) {
   export_control();
   export_blueprint();
   export_actor();
+  export_sensor();
   export_world();
   export_client();
   export_exception();

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DataStream.h

@@ -73,7 +73,10 @@ inline FSensorMessageHeader FDataStream::MakeHeader(const TSensor &Sensor)
 {
   check(IsInGameThread());
   using Serializer = carla::sensor::s11n::SensorHeaderSerializer;
-  return {Serializer::Serialize(Sensor, GFrameCounter)};
+  return {Serializer::Serialize(
+      carla::sensor::SensorRegistry::template get<TSensor*>::index,
+      GFrameCounter,
+      Sensor.GetActorTransform())};
 }
 
 inline carla::Buffer FDataStream::PopBufferFromPool()

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor.h

@@ -22,7 +22,7 @@ class UTextureRenderTarget2D;
 /// SetUpSceneCaptureComponent function.
 ///
 /// @warning All the setters should be called before BeginPlay.
-UCLASS(Abstract)
+UCLASS()
 class CARLA_API ASceneCaptureSensor : public ASensor
 {
   GENERATED_BODY()

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SceneCaptureSensor_Tick.cpp

@@ -10,7 +10,7 @@
 // Offset to keep clear in the buffer for later adding the image header by the
 // serializer.
 constexpr auto BUFFER_OFFSET =
-    carla::sensor::SensorRegistry::get<ASceneCaptureSensor *>::type::offset_size;
+    carla::sensor::SensorRegistry::get<ASceneCaptureSensor *>::type::header_offset;
 
 // =============================================================================
 // -- Local static methods -----------------------------------------------------

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SensorFactory.cpp

@@ -8,14 +8,14 @@
 #include "Carla/Sensor/SensorFactory.h"
 
 #include "Carla/Actor/ActorBlueprintFunctionLibrary.h"
-#include "Carla/Sensor/SceneCaptureCamera.h"
+#include "Carla/Sensor/SceneCaptureSensor.h"
 
 TArray<FActorDefinition> ADeprecatedSensorFactory::GetDefinitions()
 {
   FActorDefinition Cameras;
   bool Success = false;
   UActorBlueprintFunctionLibrary::MakeCameraDefinition(
-      {TEXT("camera"), ASceneCaptureCamera::StaticClass()},
+      {TEXT("camera"), ASceneCaptureSensor::StaticClass()},
       Success,
       Cameras);
   check(Success);
@@ -32,7 +32,7 @@ FActorSpawnResult ADeprecatedSensorFactory::SpawnActor(
   {
     return {};
   }
-  auto *Sensor = World->SpawnActorDeferred<ASceneCaptureCamera>(
+  auto *Sensor = World->SpawnActorDeferred<ASceneCaptureSensor>(
       Description.Class,
       Transform,
       this,
@@ -46,9 +46,10 @@ FActorSpawnResult ADeprecatedSensorFactory::SpawnActor(
         ABFL::RetrieveActorAttributeToInt("image_size_y", Description.Variations, 600));
     Sensor->SetFOVAngle(
         ABFL::RetrieveActorAttributeToFloat("fov", Description.Variations, 90.0f));
-    Sensor->SetPostProcessEffect(
+    /// @todo No post-process effects available.
-        PostProcessEffect::FromString(
+    // Sensor->SetPostProcessEffect(
-            ABFL::RetrieveActorAttributeToString("post_processing", Description.Variations, "SceneFinal")));
+    //     PostProcessEffect::FromString(
+    //         ABFL::RetrieveActorAttributeToString("post_processing", Description.Variations, "SceneFinal")));
   }
   UGameplayStatics::FinishSpawningActor(Sensor, Transform);
   return FActorSpawnResult{Sensor};

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/TheNewCarlaServer.cpp

@@ -7,7 +7,7 @@
 #include "Carla.h"
 #include "Carla/Server/TheNewCarlaServer.h"
 
-#include "Carla/Sensor/DeprecatedSensor.h"
+#include "Carla/Sensor/Sensor.h"
 
 #include "GameFramework/SpectatorPawn.h"
 
@@ -40,35 +40,6 @@ static void AttachActors(AActor *Child, AActor *Parent)
   Child->SetOwner(Parent);
 }
 
-// =============================================================================
-// -- FStreamingSensorDataSink -------------------------------------------------
-// =============================================================================
-
-class FStreamingSensorDataSink : public ISensorDataSink {
-public:
-
-  FStreamingSensorDataSink(carla::streaming::Stream InStream)
-    : TheStream(InStream) {}
-
-  ~FStreamingSensorDataSink()
-  {
-    UE_LOG(LogCarlaServer, Log, TEXT("Destroying sensor data sink"));
-  }
-
-  void Write(const FSensorDataView &SensorData) final {
-    auto MakeBuffer = [](FReadOnlyBufferView View) {
-      return carla::Buffer(boost::asio::buffer(View.GetData(), View.GetSize()));
-    };
-    TheStream.Write(
-        MakeBuffer(SensorData.GetHeader()),
-        MakeBuffer(SensorData.GetData()));
-  }
-
-private:
-
-  carla::streaming::Stream TheStream;
-};
-
 // =============================================================================
 // -- FTheNewCarlaServer::FPimpl -----------------------------------------------
 // =============================================================================
@@ -138,12 +109,12 @@ private:
   {
     if (ActorView.IsValid())
     {
-      auto *Sensor = Cast<ADeprecatedSensor>(ActorView.GetActor());
+      auto *Sensor = Cast<ASensor>(ActorView.GetActor());
       if (Sensor != nullptr)
       {
         UE_LOG(LogCarlaServer, Log, TEXT("Making a new sensor stream for actor '%s'"), *ActorView.GetActorDescription()->Id);
         auto Stream = StreamingServer.MakeStream();
-        Sensor->SetSensorDataSink(MakeShared<FStreamingSensorDataSink>(Stream));
+        Sensor->SetDataStream(Stream);
         return {ActorView, Stream.token()};
       }
     }