Merge pull request #348 from carla-simulator/issue#332
Add "frame number" to measurements and sensor data
This commit is contained in:
Néstor Subirón
GitHub
commit
c5ccf00134
|
|
@ -35,6 +35,11 @@ to a more human readable palette of colors. It can be found at
|
|||
["Util/ImageConverter"][imgconvlink]. Alternatively, they can also be converted
|
||||
using the functions at `carla.image_converter` Python module.
|
||||
|
||||
Note that all the sensor data comes with a _frame number_ stamp, this _frame
|
||||
number_ matches the one received in the measurements. This is especially useful
|
||||
for running the simulator in asynchronous mode and synchronize sensor data on
|
||||
the client side.
|
||||
|
||||
[clientexamplelink]: https://github.com/carla-simulator/carla/blob/master/PythonClient/client_example.py
|
||||
[settingslink]: https://github.com/carla-simulator/carla/blob/master/Docs/Example.CarlaSettings.ini
|
||||
[imgconvlink]: https://github.com/carla-simulator/carla/tree/master/Util/ImageConverter
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ to the client. This document describes the details of these measurements.
|
|||
Time-stamps
|
||||
-----------
|
||||
|
||||
Since CARLA can be run at fixed-frame rate, we keep track of two different
|
||||
time-stamps.
|
||||
Every frame is described by three different counters/time-stamps
|
||||
|
||||
Key | Type | Units | Description
|
||||
-------------------------- | --------- | ------------ | ------------
|
||||
frame_number | uint64 | | Frame counter (it is **not** restarted on each episode).
|
||||
platform_timestamp | uint32 | milliseconds | Time-stamp of the current frame, as given by the OS.
|
||||
game_timestamp | uint32 | milliseconds | In-game time-stamp, elapsed since the beginning of the current level.
|
||||
game_timestamp | uint32 | milliseconds | In-game time-stamp, elapsed since the beginning of the current episode.
|
||||
|
||||
In real-time mode, the elapsed time between two time steps should be similar
|
||||
both platform and game time-stamps. When run in fixed-time step, the game
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
|||
name='carla_server.proto',
|
||||
package='carla_server',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x12\x63\x61rla_server.proto\x12\x0c\x63\x61rla_server\"+\n\x08Vector3D\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"6\n\nRotation3D\x12\r\n\x05pitch\x18\x01 \x01(\x02\x12\x0b\n\x03yaw\x18\x02 \x01(\x02\x12\x0c\n\x04roll\x18\x03 \x01(\x02\"\x92\x01\n\tTransform\x12(\n\x08location\x18\x01 \x01(\x0b\x32\x16.carla_server.Vector3D\x12/\n\x0borientation\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3DB\x02\x18\x01\x12*\n\x08rotation\x18\x03 \x01(\x0b\x32\x18.carla_server.Rotation3D\"\x80\x01\n\x06Sensor\x12\n\n\x02id\x18\x01 \x01(\x07\x12\'\n\x04type\x18\x02 \x01(\x0e\x32\x19.carla_server.Sensor.Type\x12\x0c\n\x04name\x18\x03 \x01(\t\"3\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06\x43\x41MERA\x10\x01\x12\x12\n\x0eLIDAR_RAY_CAST\x10\x02\"x\n\x07Vehicle\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x03 \x01(\x02\"{\n\nPedestrian\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x03 \x01(\x02\"\x94\x01\n\x0cTrafficLight\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12/\n\x05state\x18\x02 \x01(\x0e\x32 .carla_server.TrafficLight.State\"\'\n\x05State\x12\t\n\x05GREEN\x10\x00\x12\n\n\x06YELLOW\x10\x01\x12\x07\n\x03RED\x10\x02\"Q\n\x0eSpeedLimitSign\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12\x13\n\x0bspeed_limit\x18\x02 \x01(\x02\"\xe5\x01\n\x05\x41gent\x12\n\n\x02id\x18\x01 \x01(\x07\x12(\n\x07vehicle\x18\x02 \x01(\x0b\x32\x15.carla_server.VehicleH\x00\x12.\n\npedestrian\x18\x03 \x01(\x0b\x32\x18.carla_server.PedestrianH\x00\x12\x33\n\rtraffic_light\x18\x04 \x01(\x0b\x32\x1a.carla_server.TrafficLightH\x00\x12\x38\n\x10speed_limit_sign\x18\x05 \x01(\x0b\x32\x1c.carla_server.SpeedLimitSignH\x00\x42\x07\n\x05\x61gent\"%\n\x11RequestNewEpisode\x12\x10\n\x08ini_file\x18\x01 \x01(\t\"n\n\x10SceneDescription\x12\x33\n\x12player_start_spots\x18\x01 \x03(\x0b\x32\x17.carla_server.Transform\x12%\n\x07sensors\x18\x02 \x03(\x0b\x32\x14.carla_server.Sensor\"/\n\x0c\x45pisodeStart\x12\x1f\n\x17player_start_spot_index\x18\x01 \x01(\r\"\x1d\n\x0c\x45pisodeReady\x12\r\n\x05ready\x18\x01 \x01(\x08\"^\n\x07\x43ontrol\x12\r\n\x05steer\x18\x01 \x01(\x02\x12\x10\n\x08throttle\x18\x02 \x01(\x02\x12\r\n\x05\x62rake\x18\x03 \x01(\x02\x12\x12\n\nhand_brake\x18\x04 \x01(\x08\x12\x0f\n\x07reverse\x18\x05 \x01(\x08\"\xb6\x04\n\x0cMeasurements\x12\x1a\n\x12platform_timestamp\x18\x01 \x01(\r\x12\x16\n\x0egame_timestamp\x18\x02 \x01(\r\x12J\n\x13player_measurements\x18\x03 \x01(\x0b\x32-.carla_server.Measurements.PlayerMeasurements\x12.\n\x11non_player_agents\x18\x04 \x03(\x0b\x32\x13.carla_server.Agent\x1a\xf5\x02\n\x12PlayerMeasurements\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x0b \x01(\x0b\x32\x16.carla_server.Vector3D\x12,\n\x0c\x61\x63\x63\x65leration\x18\x03 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x04 \x01(\x02\x12\x1a\n\x12\x63ollision_vehicles\x18\x05 \x01(\x02\x12\x1d\n\x15\x63ollision_pedestrians\x18\x06 \x01(\x02\x12\x17\n\x0f\x63ollision_other\x18\x07 \x01(\x02\x12\x1e\n\x16intersection_otherlane\x18\x08 \x01(\x02\x12\x1c\n\x14intersection_offroad\x18\t \x01(\x02\x12\x30\n\x11\x61utopilot_control\x18\n \x01(\x0b\x32\x15.carla_server.ControlB\x03\xf8\x01\x01\x62\x06proto3')
|
||||
serialized_pb=_b('\n\x12\x63\x61rla_server.proto\x12\x0c\x63\x61rla_server\"+\n\x08Vector3D\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\"6\n\nRotation3D\x12\r\n\x05pitch\x18\x01 \x01(\x02\x12\x0b\n\x03yaw\x18\x02 \x01(\x02\x12\x0c\n\x04roll\x18\x03 \x01(\x02\"\x92\x01\n\tTransform\x12(\n\x08location\x18\x01 \x01(\x0b\x32\x16.carla_server.Vector3D\x12/\n\x0borientation\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3DB\x02\x18\x01\x12*\n\x08rotation\x18\x03 \x01(\x0b\x32\x18.carla_server.Rotation3D\"\x80\x01\n\x06Sensor\x12\n\n\x02id\x18\x01 \x01(\x07\x12\'\n\x04type\x18\x02 \x01(\x0e\x32\x19.carla_server.Sensor.Type\x12\x0c\n\x04name\x18\x03 \x01(\t\"3\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06\x43\x41MERA\x10\x01\x12\x12\n\x0eLIDAR_RAY_CAST\x10\x02\"x\n\x07Vehicle\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x03 \x01(\x02\"{\n\nPedestrian\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x02 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x03 \x01(\x02\"\x94\x01\n\x0cTrafficLight\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12/\n\x05state\x18\x02 \x01(\x0e\x32 .carla_server.TrafficLight.State\"\'\n\x05State\x12\t\n\x05GREEN\x10\x00\x12\n\n\x06YELLOW\x10\x01\x12\x07\n\x03RED\x10\x02\"Q\n\x0eSpeedLimitSign\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12\x13\n\x0bspeed_limit\x18\x02 \x01(\x02\"\xe5\x01\n\x05\x41gent\x12\n\n\x02id\x18\x01 \x01(\x07\x12(\n\x07vehicle\x18\x02 \x01(\x0b\x32\x15.carla_server.VehicleH\x00\x12.\n\npedestrian\x18\x03 \x01(\x0b\x32\x18.carla_server.PedestrianH\x00\x12\x33\n\rtraffic_light\x18\x04 \x01(\x0b\x32\x1a.carla_server.TrafficLightH\x00\x12\x38\n\x10speed_limit_sign\x18\x05 \x01(\x0b\x32\x1c.carla_server.SpeedLimitSignH\x00\x42\x07\n\x05\x61gent\"%\n\x11RequestNewEpisode\x12\x10\n\x08ini_file\x18\x01 \x01(\t\"n\n\x10SceneDescription\x12\x33\n\x12player_start_spots\x18\x01 \x03(\x0b\x32\x17.carla_server.Transform\x12%\n\x07sensors\x18\x02 \x03(\x0b\x32\x14.carla_server.Sensor\"/\n\x0c\x45pisodeStart\x12\x1f\n\x17player_start_spot_index\x18\x01 \x01(\r\"\x1d\n\x0c\x45pisodeReady\x12\r\n\x05ready\x18\x01 \x01(\x08\"^\n\x07\x43ontrol\x12\r\n\x05steer\x18\x01 \x01(\x02\x12\x10\n\x08throttle\x18\x02 \x01(\x02\x12\r\n\x05\x62rake\x18\x03 \x01(\x02\x12\x12\n\nhand_brake\x18\x04 \x01(\x08\x12\x0f\n\x07reverse\x18\x05 \x01(\x08\"\xcc\x04\n\x0cMeasurements\x12\x14\n\x0c\x66rame_number\x18\x05 \x01(\x04\x12\x1a\n\x12platform_timestamp\x18\x01 \x01(\r\x12\x16\n\x0egame_timestamp\x18\x02 \x01(\r\x12J\n\x13player_measurements\x18\x03 \x01(\x0b\x32-.carla_server.Measurements.PlayerMeasurements\x12.\n\x11non_player_agents\x18\x04 \x03(\x0b\x32\x13.carla_server.Agent\x1a\xf5\x02\n\x12PlayerMeasurements\x12*\n\ttransform\x18\x01 \x01(\x0b\x32\x17.carla_server.Transform\x12*\n\nbox_extent\x18\x0b \x01(\x0b\x32\x16.carla_server.Vector3D\x12,\n\x0c\x61\x63\x63\x65leration\x18\x03 \x01(\x0b\x32\x16.carla_server.Vector3D\x12\x15\n\rforward_speed\x18\x04 \x01(\x02\x12\x1a\n\x12\x63ollision_vehicles\x18\x05 \x01(\x02\x12\x1d\n\x15\x63ollision_pedestrians\x18\x06 \x01(\x02\x12\x17\n\x0f\x63ollision_other\x18\x07 \x01(\x02\x12\x1e\n\x16intersection_otherlane\x18\x08 \x01(\x02\x12\x1c\n\x14intersection_offroad\x18\t \x01(\x02\x12\x30\n\x11\x61utopilot_control\x18\n \x01(\x0b\x32\x15.carla_server.ControlB\x03\xf8\x01\x01\x62\x06proto3')
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -766,8 +766,8 @@ _MEASUREMENTS_PLAYERMEASUREMENTS = _descriptor.Descriptor(
|
|||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=1651,
|
||||
serialized_end=2024,
|
||||
serialized_start=1673,
|
||||
serialized_end=2046,
|
||||
)
|
||||
|
||||
_MEASUREMENTS = _descriptor.Descriptor(
|
||||
|
|
@ -778,28 +778,35 @@ _MEASUREMENTS = _descriptor.Descriptor(
|
|||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='platform_timestamp', full_name='carla_server.Measurements.platform_timestamp', index=0,
|
||||
name='frame_number', full_name='carla_server.Measurements.frame_number', index=0,
|
||||
number=5, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='platform_timestamp', full_name='carla_server.Measurements.platform_timestamp', index=1,
|
||||
number=1, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='game_timestamp', full_name='carla_server.Measurements.game_timestamp', index=1,
|
||||
name='game_timestamp', full_name='carla_server.Measurements.game_timestamp', index=2,
|
||||
number=2, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='player_measurements', full_name='carla_server.Measurements.player_measurements', index=2,
|
||||
name='player_measurements', full_name='carla_server.Measurements.player_measurements', index=3,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='non_player_agents', full_name='carla_server.Measurements.non_player_agents', index=3,
|
||||
name='non_player_agents', full_name='carla_server.Measurements.non_player_agents', index=4,
|
||||
number=4, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
|
|
@ -818,7 +825,7 @@ _MEASUREMENTS = _descriptor.Descriptor(
|
|||
oneofs=[
|
||||
],
|
||||
serialized_start=1458,
|
||||
serialized_end=2024,
|
||||
serialized_end=2046,
|
||||
)
|
||||
|
||||
_TRANSFORM.fields_by_name['location'].message_type = _VECTOR3D
|
||||
|
|
|
|||
|
|
@ -183,31 +183,36 @@ class CarlaClient(object):
|
|||
def _make_sensor_parsers(sensors):
|
||||
image_types = ['None', 'SceneFinal', 'Depth', 'SemanticSegmentation']
|
||||
getimgtype = lambda id: image_types[id] if len(image_types) > id else 'Unknown'
|
||||
getint = lambda data, index: struct.unpack('<L', data[index*4:index*4+4])[0]
|
||||
getint32 = lambda data, index: struct.unpack('<L', data[index*4:index*4+4])[0]
|
||||
getint64 = lambda data, index: struct.unpack('<Q', data[index*4:index*4+8])[0]
|
||||
getfloat = lambda data, index: struct.unpack('<f', data[index*4:index*4+4])[0]
|
||||
|
||||
def parse_image(data):
|
||||
width = getint(data, 0)
|
||||
height = getint(data, 1)
|
||||
image_type = getimgtype(getint(data, 2))
|
||||
fov = getfloat(data, 3)
|
||||
return sensor.Image(width, height, image_type, fov, data[16:])
|
||||
frame_number = getint64(data, 0)
|
||||
width = getint32(data, 2)
|
||||
height = getint32(data, 3)
|
||||
image_type = getimgtype(getint32(data, 4))
|
||||
fov = getfloat(data, 5)
|
||||
return sensor.Image(frame_number, width, height, image_type, fov, data[24:])
|
||||
|
||||
def parse_lidar(data):
|
||||
horizontal_angle = getfloat(data, 0)
|
||||
channels = getint(data, 1)
|
||||
frame_number = getint64(data, 0)
|
||||
horizontal_angle = getfloat(data, 2)
|
||||
channels = getint32(data, 3)
|
||||
header_size = 16
|
||||
point_count_by_channel = numpy.frombuffer(
|
||||
data[8:8+channels*4],
|
||||
data[header_size:header_size+channels*4],
|
||||
dtype=numpy.dtype('uint32'))
|
||||
points = numpy.frombuffer(
|
||||
data[8+channels*4:],
|
||||
data[header_size+channels*4:],
|
||||
dtype=numpy.dtype('f4'))
|
||||
points = numpy.reshape(points, (int(points.shape[0]/3), 3))
|
||||
return sensor.LidarMeasurement(
|
||||
frame_number,
|
||||
horizontal_angle,
|
||||
channels,
|
||||
point_count_by_channel,
|
||||
sensor.PointCloud(points))
|
||||
sensor.PointCloud(frame_number, points))
|
||||
|
||||
class SensorDefinition(object):
|
||||
def __init__(self, s):
|
||||
|
|
|
|||
|
|
@ -154,6 +154,9 @@ def depth_to_local_point_cloud(image, color=None, max_depth=0.9):
|
|||
# [[X1,Y1,Z1,R1,G1,B1],[X2,Y2,Z2,R2,G2,B2], ... [Xn,Yn,Zn,Rn,Gn,Bn]]
|
||||
if color is not None:
|
||||
# numpy.concatenate((numpy.transpose(p3d), color), axis=1)
|
||||
return sensor.PointCloud(numpy.transpose(p3d), color_array=color)
|
||||
return sensor.PointCloud(
|
||||
image.frame_number,
|
||||
numpy.transpose(p3d),
|
||||
color_array=color)
|
||||
# [[X1,Y1,Z1],[X2,Y2,Z2], ... [Xn,Yn,Zn]]
|
||||
return sensor.PointCloud(numpy.transpose(p3d))
|
||||
return sensor.PointCloud(image.frame_number, numpy.transpose(p3d))
|
||||
|
|
|
|||
|
|
@ -134,15 +134,18 @@ class Lidar(Sensor):
|
|||
# -- SensorData ----------------------------------------------------------------
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
class SensorData(object):
|
||||
"""Base class for sensor data returned from the server."""
|
||||
pass
|
||||
def __init__(self, frame_number):
|
||||
self.frame_number = frame_number
|
||||
|
||||
|
||||
class Image(SensorData):
|
||||
"""Data generated by a Camera."""
|
||||
|
||||
def __init__(self, width, height, image_type, fov, raw_data):
|
||||
def __init__(self, frame_number, width, height, image_type, fov, raw_data):
|
||||
super(Image, self).__init__(frame_number=frame_number)
|
||||
assert len(raw_data) == 4 * width * height
|
||||
self.width = width
|
||||
self.height = height
|
||||
|
|
@ -195,7 +198,8 @@ class Image(SensorData):
|
|||
class PointCloud(SensorData):
|
||||
"""A list of points."""
|
||||
|
||||
def __init__(self, array, color_array=None):
|
||||
def __init__(self, frame_number, array, color_array=None):
|
||||
super(PointCloud, self).__init__(frame_number=frame_number)
|
||||
self._array = array
|
||||
self._color_array = color_array
|
||||
self._has_colors = color_array is not None
|
||||
|
|
@ -306,7 +310,8 @@ class PointCloud(SensorData):
|
|||
class LidarMeasurement(SensorData):
|
||||
"""Data generated by a Lidar."""
|
||||
|
||||
def __init__(self, horizontal_angle, channels, point_count_by_channel, point_cloud):
|
||||
def __init__(self, frame_number, horizontal_angle, channels, point_count_by_channel, point_cloud):
|
||||
super(LidarMeasurement, self).__init__(frame_number=frame_number)
|
||||
assert numpy.sum(point_count_by_channel) == len(point_cloud.array)
|
||||
self.horizontal_angle = horizontal_angle
|
||||
self.channels = channels
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
#include "Carla.h"
|
||||
#include "CarlaPlayerState.h"
|
||||
|
||||
#include "CoreGlobals.h"
|
||||
|
||||
void ACarlaPlayerState::Reset()
|
||||
{
|
||||
Super::Reset();
|
||||
|
|
@ -25,6 +27,7 @@ void ACarlaPlayerState::CopyProperties(APlayerState *PlayerState)
|
|||
ACarlaPlayerState *Other = Cast<ACarlaPlayerState>(PlayerState);
|
||||
if (Other != nullptr)
|
||||
{
|
||||
FrameNumber = Other->FrameNumber;
|
||||
FramesPerSecond = Other->FramesPerSecond;
|
||||
PlatformTimeStamp = Other->PlatformTimeStamp;
|
||||
GameTimeStamp = Other->GameTimeStamp;
|
||||
|
|
@ -74,6 +77,7 @@ static int32 RoundToMilliseconds(float Seconds)
|
|||
|
||||
void ACarlaPlayerState::UpdateTimeStamp(float DeltaSeconds)
|
||||
{
|
||||
FrameNumber = GFrameCounter;
|
||||
FramesPerSecond = 1.0f / DeltaSeconds;
|
||||
PlatformTimeStamp = RoundToMilliseconds(FPlatformTime::Seconds());
|
||||
GameTimeStamp += RoundToMilliseconds(DeltaSeconds);
|
||||
|
|
|
|||
|
|
@ -40,6 +40,11 @@ public:
|
|||
// ===========================================================================
|
||||
/// @{
|
||||
|
||||
uint64 GetFrameNumber() const
|
||||
{
|
||||
return FrameNumber;
|
||||
}
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
float GetFramesPerSecond() const
|
||||
{
|
||||
|
|
@ -214,6 +219,9 @@ private:
|
|||
// If you add another variable here, don't forget to copy it inside
|
||||
// CopyProperties if necessary.
|
||||
|
||||
UPROPERTY(VisibleAnywhere)
|
||||
uint64 FrameNumber;
|
||||
|
||||
UPROPERTY(VisibleAnywhere)
|
||||
float FramesPerSecond;
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ void ALidar::ReadPoints(const float DeltaTime)
|
|||
}
|
||||
|
||||
const float HorizontalAngle = std::fmod(CurrentHorizontalAngle + AngleDistanceOfTick, 360.0f);
|
||||
LidarMeasurement.SetFrameNumber(GFrameCounter);
|
||||
LidarMeasurement.SetHorizontalAngle(HorizontalAngle);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
/// The header consists of an array of uint32's in the following layout
|
||||
///
|
||||
/// {
|
||||
/// Frame number (uint64)
|
||||
/// Horizontal angle (float),
|
||||
/// Channel count,
|
||||
/// Point count of channel 0,
|
||||
|
|
@ -37,8 +38,8 @@ public:
|
|||
explicit FLidarMeasurement(uint32 SensorId = 0u, uint32 ChannelCount = 0u)
|
||||
: SensorId(SensorId)
|
||||
{
|
||||
Header.AddDefaulted(2u + ChannelCount);
|
||||
Header[1] = ChannelCount;
|
||||
Header.AddDefaulted(4u + ChannelCount);
|
||||
Header[3] = ChannelCount;
|
||||
}
|
||||
|
||||
FLidarMeasurement &operator=(FLidarMeasurement &&Other)
|
||||
|
|
@ -50,31 +51,36 @@ public:
|
|||
return *this;
|
||||
}
|
||||
|
||||
void SetFrameNumber(uint64 FrameNumber)
|
||||
{
|
||||
std::memcpy(Header.GetData(), reinterpret_cast<const void *>(&FrameNumber), 2u);
|
||||
}
|
||||
|
||||
float GetHorizontalAngle() const
|
||||
{
|
||||
return reinterpret_cast<const float &>(Header[0]);
|
||||
return reinterpret_cast<const float &>(Header[2]);
|
||||
}
|
||||
|
||||
void SetHorizontalAngle(float HorizontalAngle)
|
||||
{
|
||||
Header[0] = reinterpret_cast<const uint32 &>(HorizontalAngle);
|
||||
Header[2] = reinterpret_cast<const uint32 &>(HorizontalAngle);
|
||||
}
|
||||
|
||||
uint32 GetChannelCount() const
|
||||
{
|
||||
return Header[1];
|
||||
return Header[3];
|
||||
}
|
||||
|
||||
void Reset(uint32 TotalPointCount)
|
||||
{
|
||||
std::memset(Header.GetData() + 2u, 0, sizeof(uint32) * GetChannelCount());
|
||||
std::memset(Header.GetData() + 4u, 0, sizeof(uint32) * GetChannelCount());
|
||||
Points.Reset(3u * TotalPointCount);
|
||||
}
|
||||
|
||||
void WritePoint(uint32 Channel, const FVector &Point)
|
||||
{
|
||||
check(Header[1] > Channel);
|
||||
Header[2u + Channel] += 1u;
|
||||
check(Header[3] > Channel);
|
||||
Header[4u + Channel] += 1u;
|
||||
constexpr float TO_METERS = 1e-2f;
|
||||
Points.Emplace(TO_METERS * Point.X);
|
||||
Points.Emplace(TO_METERS * Point.Y);
|
||||
|
|
|
|||
|
|
@ -8,40 +8,63 @@
|
|||
#include "SceneCaptureCamera.h"
|
||||
|
||||
#include "Sensor/SensorDataView.h"
|
||||
#include "Game/CarlaGameInstance.h"
|
||||
#include "Settings/CarlaSettings.h"
|
||||
|
||||
#include "Components/DrawFrustumComponent.h"
|
||||
#include "Components/SceneCaptureComponent2D.h"
|
||||
#include "Components/StaticMeshComponent.h"
|
||||
#include "ConstructorHelpers.h"
|
||||
#include "CoreGlobals.h"
|
||||
#include "Engine/CollisionProfile.h"
|
||||
#include "Engine/TextureRenderTarget2D.h"
|
||||
#include "Materials/Material.h"
|
||||
#include "Game/CarlaGameInstance.h"
|
||||
#include "Kismet/KismetSystemLibrary.h"
|
||||
#include <memory>
|
||||
#include "ConstructorHelpers.h"
|
||||
#include "Materials/Material.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
// =============================================================================
|
||||
// -- Local static variables ---------------------------------------------------
|
||||
// =============================================================================
|
||||
|
||||
static constexpr auto DEPTH_MAT_PATH =
|
||||
#if PLATFORM_LINUX
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/DepthEffectMaterial_GLSL.DepthEffectMaterial_GLSL'");
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/DepthEffectMaterial_GLSL.DepthEffectMaterial_GLSL'");
|
||||
#elif PLATFORM_WINDOWS
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/DepthEffectMaterial.DepthEffectMaterial'");
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/DepthEffectMaterial.DepthEffectMaterial'");
|
||||
#else
|
||||
# error No depth material defined for this platform
|
||||
#endif
|
||||
|
||||
static constexpr auto SEMANTIC_SEGMENTATION_MAT_PATH =
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/GTMaterial.GTMaterial'");
|
||||
TEXT("Material'/Carla/PostProcessingMaterials/GTMaterial.GTMaterial'");
|
||||
|
||||
// =============================================================================
|
||||
// -- Local static methods and types -------------------------------------------
|
||||
// =============================================================================
|
||||
|
||||
struct FImageHeaderData
|
||||
{
|
||||
uint64 FrameNumber;
|
||||
uint32 Width;
|
||||
uint32 Height;
|
||||
uint32 Type;
|
||||
float FOV;
|
||||
};
|
||||
|
||||
static void RemoveShowFlags(FEngineShowFlags &ShowFlags);
|
||||
|
||||
// =============================================================================
|
||||
// -- ASceneCaptureCamera ------------------------------------------------------
|
||||
// =============================================================================
|
||||
|
||||
uint32 ASceneCaptureCamera::NumSceneCapture = 0;
|
||||
|
||||
ASceneCaptureCamera::ASceneCaptureCamera(const FObjectInitializer& ObjectInitializer) :
|
||||
Super(ObjectInitializer),
|
||||
SizeX(720u),
|
||||
SizeY(512u),
|
||||
PostProcessEffect(EPostProcessEffect::SceneFinal)
|
||||
ASceneCaptureCamera::ASceneCaptureCamera(const FObjectInitializer &ObjectInitializer)
|
||||
: Super(ObjectInitializer),
|
||||
SizeX(720u),
|
||||
SizeY(512u),
|
||||
PostProcessEffect(EPostProcessEffect::SceneFinal)
|
||||
{
|
||||
PrimaryActorTick.bCanEverTick = true;
|
||||
PrimaryActorTick.TickGroup = TG_PrePhysics;
|
||||
|
|
@ -59,24 +82,28 @@ ASceneCaptureCamera::ASceneCaptureCamera(const FObjectInitializer& ObjectInitial
|
|||
DrawFrustum->bIsEditorOnly = true;
|
||||
DrawFrustum->SetupAttachment(MeshComp);
|
||||
|
||||
CaptureRenderTarget = CreateDefaultSubobject<UTextureRenderTarget2D>(FName(*FString::Printf(TEXT("CaptureRenderTarget%d"),NumSceneCapture)));
|
||||
#if WITH_EDITORONLY_DATA
|
||||
CaptureRenderTarget->CompressionNoAlpha = true;
|
||||
CaptureRenderTarget->MipGenSettings = TextureMipGenSettings::TMGS_NoMipmaps;
|
||||
CaptureRenderTarget->bUseLegacyGamma = false;
|
||||
#endif
|
||||
CaptureRenderTarget = CreateDefaultSubobject<UTextureRenderTarget2D>(
|
||||
FName(*FString::Printf(TEXT("CaptureRenderTarget%d"), NumSceneCapture)));
|
||||
#if WITH_EDITORONLY_DATA
|
||||
CaptureRenderTarget->CompressionNoAlpha = true;
|
||||
CaptureRenderTarget->MipGenSettings = TextureMipGenSettings::TMGS_NoMipmaps;
|
||||
CaptureRenderTarget->bUseLegacyGamma = false;
|
||||
#endif
|
||||
CaptureRenderTarget->CompressionSettings = TextureCompressionSettings::TC_Default;
|
||||
CaptureRenderTarget->SRGB = false;
|
||||
CaptureRenderTarget->bAutoGenerateMips = false;
|
||||
CaptureRenderTarget->AddressX = TextureAddress::TA_Clamp;
|
||||
CaptureRenderTarget->AddressY = TextureAddress::TA_Clamp;
|
||||
CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(TEXT("SceneCaptureComponent2D"));
|
||||
CaptureComponent2D->SetupAttachment(MeshComp);
|
||||
|
||||
CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(
|
||||
TEXT("SceneCaptureComponent2D"));
|
||||
CaptureComponent2D->SetupAttachment(MeshComp);
|
||||
|
||||
// Load post-processing materials.
|
||||
static ConstructorHelpers::FObjectFinder<UMaterial> DEPTH(DEPTH_MAT_PATH);
|
||||
static ConstructorHelpers::FObjectFinder<UMaterial> DEPTH(
|
||||
DEPTH_MAT_PATH);
|
||||
PostProcessDepth = DEPTH.Object;
|
||||
static ConstructorHelpers::FObjectFinder<UMaterial> SEMANTIC_SEGMENTATION(SEMANTIC_SEGMENTATION_MAT_PATH);
|
||||
static ConstructorHelpers::FObjectFinder<UMaterial> SEMANTIC_SEGMENTATION(
|
||||
SEMANTIC_SEGMENTATION_MAT_PATH);
|
||||
PostProcessSemanticSegmentation = SEMANTIC_SEGMENTATION.Object;
|
||||
NumSceneCapture++;
|
||||
}
|
||||
|
|
@ -87,18 +114,23 @@ void ASceneCaptureCamera::PostActorCreated()
|
|||
|
||||
// no need load the editor mesh when there is no editor
|
||||
#if WITH_EDITOR
|
||||
if(MeshComp)
|
||||
if (MeshComp)
|
||||
{
|
||||
if (!IsRunningCommandlet())
|
||||
{
|
||||
if( !MeshComp->GetStaticMesh())
|
||||
if (!MeshComp->GetStaticMesh())
|
||||
{
|
||||
UStaticMesh* CamMesh = LoadObject<UStaticMesh>(NULL, TEXT("/Engine/EditorMeshes/MatineeCam_SM.MatineeCam_SM"), NULL, LOAD_None, NULL);
|
||||
UStaticMesh *CamMesh = LoadObject<UStaticMesh>(
|
||||
NULL,
|
||||
TEXT("/Engine/EditorMeshes/MatineeCam_SM.MatineeCam_SM"),
|
||||
NULL,
|
||||
LOAD_None,
|
||||
NULL);
|
||||
MeshComp->SetStaticMesh(CamMesh);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // WITH_EDITOR
|
||||
#endif // WITH_EDITOR
|
||||
|
||||
// Sync component with CameraActor frustum settings.
|
||||
UpdateDrawFrustum();
|
||||
|
|
@ -111,51 +143,53 @@ void ASceneCaptureCamera::BeginPlay()
|
|||
// Setup render target.
|
||||
const bool bInForceLinearGamma = bRemovePostProcessing;
|
||||
CaptureRenderTarget->InitCustomFormat(SizeX, SizeY, PF_B8G8R8A8, bInForceLinearGamma);
|
||||
if(!IsValid(CaptureComponent2D)||CaptureComponent2D->IsPendingKill())
|
||||
if (!IsValid(CaptureComponent2D) || CaptureComponent2D->IsPendingKill())
|
||||
{
|
||||
CaptureComponent2D = NewObject<USceneCaptureComponent2D>(this,TEXT("SceneCaptureComponent2D"));
|
||||
CaptureComponent2D = NewObject<USceneCaptureComponent2D>(this, TEXT("SceneCaptureComponent2D"));
|
||||
CaptureComponent2D->SetupAttachment(MeshComp);
|
||||
}
|
||||
CaptureComponent2D->Deactivate();
|
||||
CaptureComponent2D->TextureTarget = CaptureRenderTarget;
|
||||
|
||||
// Setup camera post-processing depending on the quality level:
|
||||
const UCarlaGameInstance* GameInstance = Cast<UCarlaGameInstance>(GetWorld()->GetGameInstance());
|
||||
check(GameInstance!=nullptr);
|
||||
const UCarlaSettings& CarlaSettings = GameInstance->GetCarlaSettings();
|
||||
switch(PostProcessEffect)
|
||||
// Setup camera post-processing depending on the quality level:
|
||||
const UCarlaGameInstance *GameInstance = Cast<UCarlaGameInstance>(GetWorld()->GetGameInstance());
|
||||
check(GameInstance != nullptr);
|
||||
const UCarlaSettings &CarlaSettings = GameInstance->GetCarlaSettings();
|
||||
switch (PostProcessEffect)
|
||||
{
|
||||
case EPostProcessEffect::None: break;
|
||||
case EPostProcessEffect::SceneFinal:
|
||||
{
|
||||
//we set LDR for high quality because it will include post-fx
|
||||
//and HDR for low quality to avoid high contrast
|
||||
switch(CarlaSettings.GetQualitySettingsLevel())
|
||||
{
|
||||
case EQualitySettingsLevel::Low:
|
||||
CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_SceneColorHDRNoAlpha;
|
||||
break;
|
||||
default:
|
||||
//LDR is faster than HDR (smaller bitmap array)
|
||||
CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR;
|
||||
break;
|
||||
}
|
||||
case EPostProcessEffect::None:
|
||||
break;
|
||||
case EPostProcessEffect::SceneFinal:
|
||||
{
|
||||
// We set LDR for high quality because it will include post-fx and HDR for
|
||||
// low quality to avoid high contrast.
|
||||
switch (CarlaSettings.GetQualitySettingsLevel())
|
||||
{
|
||||
case EQualitySettingsLevel::Low:
|
||||
CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_SceneColorHDRNoAlpha;
|
||||
break;
|
||||
default:
|
||||
// LDR is faster than HDR (smaller bitmap array).
|
||||
CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
CaptureComponent2D->CaptureSource = SCS_FinalColorLDR;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
CaptureComponent2D->CaptureSource = SCS_FinalColorLDR;
|
||||
break;
|
||||
}
|
||||
|
||||
if (bRemovePostProcessing)
|
||||
if (bRemovePostProcessing)
|
||||
{
|
||||
RemoveShowFlags(CaptureComponent2D->ShowFlags);
|
||||
}
|
||||
|
||||
if (PostProcessEffect == EPostProcessEffect::Depth)
|
||||
if (PostProcessEffect == EPostProcessEffect::Depth)
|
||||
{
|
||||
CaptureComponent2D->PostProcessSettings.AddBlendable(PostProcessDepth, 1.0f);
|
||||
} else if (PostProcessEffect == EPostProcessEffect::SemanticSegmentation)
|
||||
}
|
||||
else if (PostProcessEffect == EPostProcessEffect::SemanticSegmentation)
|
||||
{
|
||||
CaptureComponent2D->PostProcessSettings.AddBlendable(PostProcessSemanticSegmentation, 1.0f);
|
||||
}
|
||||
|
|
@ -163,37 +197,50 @@ void ASceneCaptureCamera::BeginPlay()
|
|||
CaptureComponent2D->UpdateContent();
|
||||
CaptureComponent2D->Activate();
|
||||
|
||||
//Make sure that there is enough time in the render queue
|
||||
UKismetSystemLibrary::ExecuteConsoleCommand(GetWorld(), FString("g.TimeoutForBlockOnRenderFence 300000"));
|
||||
// Make sure that there is enough time in the render queue.
|
||||
UKismetSystemLibrary::ExecuteConsoleCommand(
|
||||
GetWorld(),
|
||||
FString("g.TimeoutForBlockOnRenderFence 300000"));
|
||||
|
||||
Super::BeginPlay();
|
||||
}
|
||||
|
||||
void ASceneCaptureCamera::EndPlay(const EEndPlayReason::Type EndPlayReason)
|
||||
{
|
||||
if(NumSceneCapture!=0) NumSceneCapture = 0;
|
||||
if (NumSceneCapture != 0)
|
||||
{
|
||||
NumSceneCapture = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void ASceneCaptureCamera::Tick(const float DeltaSeconds)
|
||||
{
|
||||
Super::Tick(DeltaSeconds);
|
||||
if(IsVulkanPlatform(GMaxRHIShaderPlatform))
|
||||
|
||||
const auto FrameNumber = GFrameCounter;
|
||||
|
||||
if (IsVulkanPlatform(GMaxRHIShaderPlatform))
|
||||
{
|
||||
auto fn = [=](FRHICommandListImmediate& RHICmdList){WritePixelsNonBlocking(DeltaSeconds,RHICmdList);};
|
||||
auto fn = [=](FRHICommandListImmediate &RHICmdList) {
|
||||
WritePixelsNonBlocking(FrameNumber, RHICmdList);
|
||||
};
|
||||
ENQUEUE_UNIQUE_RENDER_COMMAND_ONEPARAMETER(
|
||||
FWritePixelsNonBlocking,
|
||||
decltype(fn),write_function_vulkan,fn,
|
||||
{
|
||||
write_function_vulkan(RHICmdList);
|
||||
});
|
||||
} else
|
||||
{
|
||||
auto fn = [=](){WritePixels(DeltaSeconds);};
|
||||
ENQUEUE_UNIQUE_RENDER_COMMAND_ONEPARAMETER(
|
||||
FWritePixels,
|
||||
decltype(fn),write_function,fn,
|
||||
FWritePixelsNonBlocking,
|
||||
decltype(fn), write_function_vulkan, fn,
|
||||
{
|
||||
write_function();
|
||||
write_function_vulkan(RHICmdList);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
auto fn = [=]() {
|
||||
WritePixels(FrameNumber);
|
||||
};
|
||||
ENQUEUE_UNIQUE_RENDER_COMMAND_ONEPARAMETER(
|
||||
FWritePixels,
|
||||
decltype(fn), write_function, fn,
|
||||
{
|
||||
write_function();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -214,7 +261,8 @@ void ASceneCaptureCamera::SetPostProcessEffect(EPostProcessEffect otherPostProce
|
|||
{
|
||||
PostProcessEffect = otherPostProcessEffect;
|
||||
auto &PostProcessSettings = CaptureComponent2D->PostProcessSettings;
|
||||
if (PostProcessEffect != EPostProcessEffect::SceneFinal) {
|
||||
if (PostProcessEffect != EPostProcessEffect::SceneFinal)
|
||||
{
|
||||
PostProcessSettings.bOverride_AutoExposureMethod = false;
|
||||
PostProcessSettings.bOverride_AutoExposureMinBrightness = false;
|
||||
PostProcessSettings.bOverride_AutoExposureMaxBrightness = false;
|
||||
|
|
@ -238,7 +286,8 @@ void ASceneCaptureCamera::Set(const UCameraDescription &CameraDescription)
|
|||
{
|
||||
Super::Set(CameraDescription);
|
||||
|
||||
if (CameraDescription.bOverrideCameraPostProcessParameters) {
|
||||
if (CameraDescription.bOverrideCameraPostProcessParameters)
|
||||
{
|
||||
auto &Override = CameraDescription.CameraPostProcessParameters;
|
||||
auto &PostProcessSettings = CaptureComponent2D->PostProcessSettings;
|
||||
PostProcessSettings.bOverride_AutoExposureMethod = true;
|
||||
|
|
@ -257,13 +306,15 @@ void ASceneCaptureCamera::Set(const UCameraDescription &CameraDescription)
|
|||
|
||||
bool ASceneCaptureCamera::ReadPixels(TArray<FColor> &BitMap) const
|
||||
{
|
||||
if(!CaptureRenderTarget)
|
||||
if (!CaptureRenderTarget)
|
||||
{
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target"));
|
||||
return false;
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target"));
|
||||
return false;
|
||||
}
|
||||
FTextureRenderTargetResource* RTResource = CaptureRenderTarget->GameThread_GetRenderTargetResource();
|
||||
if (RTResource == nullptr) {
|
||||
FTextureRenderTargetResource *RTResource =
|
||||
CaptureRenderTarget->GameThread_GetRenderTargetResource();
|
||||
if (RTResource == nullptr)
|
||||
{
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target"));
|
||||
return false;
|
||||
}
|
||||
|
|
@ -272,76 +323,62 @@ bool ASceneCaptureCamera::ReadPixels(TArray<FColor> &BitMap) const
|
|||
return RTResource->ReadPixels(BitMap, ReadPixelFlags);
|
||||
}
|
||||
|
||||
void ASceneCaptureCamera::WritePixelsNonBlocking(float DeltaTime, FRHICommandListImmediate& rhi_cmd_list) const
|
||||
{
|
||||
void ASceneCaptureCamera::WritePixelsNonBlocking(
|
||||
const uint64 FrameNumber,
|
||||
FRHICommandListImmediate &rhi_cmd_list) const
|
||||
{
|
||||
check(IsInRenderingThread());
|
||||
if(!CaptureRenderTarget)
|
||||
if (!CaptureRenderTarget)
|
||||
{
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target"));
|
||||
return ;
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target"));
|
||||
return;
|
||||
}
|
||||
FTextureRenderTarget2DResource* RenderResource = (FTextureRenderTarget2DResource*)CaptureRenderTarget->Resource;
|
||||
FTextureRenderTarget2DResource *RenderResource =
|
||||
(FTextureRenderTarget2DResource *) CaptureRenderTarget->Resource;
|
||||
FTextureRHIParamRef texture = RenderResource->GetRenderTargetTexture();
|
||||
if(!texture)
|
||||
if (!texture)
|
||||
{
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target texture"));
|
||||
return;
|
||||
}
|
||||
struct {
|
||||
uint32 Width;
|
||||
uint32 Height;
|
||||
uint32 Type;
|
||||
float FOV;
|
||||
} ImageHeader = {
|
||||
FImageHeaderData ImageHeader = {
|
||||
FrameNumber,
|
||||
SizeX,
|
||||
SizeY,
|
||||
PostProcessEffect::ToUInt(PostProcessEffect),
|
||||
CaptureComponent2D->FOVAngle
|
||||
};
|
||||
|
||||
struct FReadSurfaceContext
|
||||
{
|
||||
FRenderTarget* SrcRenderTarget;
|
||||
TArray<FColor>* OutData;
|
||||
FIntRect Rect;
|
||||
FReadSurfaceDataFlags Flags;
|
||||
};
|
||||
TArray<FColor> Pixels;
|
||||
|
||||
TArray<FColor> Pixels;
|
||||
rhi_cmd_list.ReadSurfaceData(
|
||||
texture,
|
||||
FIntRect(0, 0, RenderResource->GetSizeXY().X, RenderResource->GetSizeXY().Y),
|
||||
Pixels,
|
||||
FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX)
|
||||
);
|
||||
texture,
|
||||
FIntRect(0, 0, RenderResource->GetSizeXY().X, RenderResource->GetSizeXY().Y),
|
||||
Pixels,
|
||||
FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX));
|
||||
FSensorDataView DataView(
|
||||
GetId(),
|
||||
FReadOnlyBufferView{reinterpret_cast<const void *>(&ImageHeader), sizeof(ImageHeader)},
|
||||
FReadOnlyBufferView{Pixels}
|
||||
);
|
||||
WriteSensorData(DataView);
|
||||
|
||||
GetId(),
|
||||
FReadOnlyBufferView{reinterpret_cast<const void *>(&ImageHeader), sizeof(ImageHeader)},
|
||||
FReadOnlyBufferView{Pixels});
|
||||
WriteSensorData(DataView);
|
||||
}
|
||||
|
||||
void ASceneCaptureCamera::WritePixels(float DeltaTime) const
|
||||
void ASceneCaptureCamera::WritePixels(const uint64 FrameNumber) const
|
||||
{
|
||||
FRHITexture2D *texture = CaptureRenderTarget->GetRenderTargetResource()->GetRenderTargetTexture();
|
||||
if(!texture)
|
||||
if (!texture)
|
||||
{
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render texture"));
|
||||
return ;
|
||||
UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render texture"));
|
||||
return;
|
||||
}
|
||||
const uint32 num_bytes_per_pixel = 4; // PF_R8G8B8A8
|
||||
const uint32 width = texture->GetSizeX();
|
||||
const uint32 height = texture->GetSizeY();
|
||||
const uint32 dest_stride = width * height * num_bytes_per_pixel;
|
||||
uint32 src_stride;
|
||||
uint8 *src = reinterpret_cast<uint8*>(RHILockTexture2D(texture, 0, RLM_ReadOnly, src_stride, false));
|
||||
struct {
|
||||
uint32 Width;
|
||||
uint32 Height;
|
||||
uint32 Type;
|
||||
float FOV;
|
||||
} ImageHeader = {
|
||||
uint8 *src = reinterpret_cast<uint8 *>(
|
||||
RHILockTexture2D(texture, 0, RLM_ReadOnly, src_stride, false));
|
||||
FImageHeaderData ImageHeader = {
|
||||
FrameNumber,
|
||||
width,
|
||||
height,
|
||||
PostProcessEffect::ToUInt(PostProcessEffect),
|
||||
|
|
@ -349,49 +386,53 @@ void ASceneCaptureCamera::WritePixels(float DeltaTime) const
|
|||
};
|
||||
|
||||
std::unique_ptr<uint8[]> dest = nullptr;
|
||||
//Direct 3D uses additional rows in the buffer,so we need check the result stride from the lock:
|
||||
if(IsD3DPlatform(GMaxRHIShaderPlatform,false) && (dest_stride!=src_stride))
|
||||
// Direct 3D uses additional rows in the buffer,so we need check the result
|
||||
// stride from the lock:
|
||||
if (IsD3DPlatform(GMaxRHIShaderPlatform, false) && (dest_stride != src_stride))
|
||||
{
|
||||
const uint32 copy_row_stride = width * num_bytes_per_pixel;
|
||||
dest = std::make_unique<uint8[]>(dest_stride);
|
||||
// Copy per row
|
||||
uint8* dest_row = dest.get();
|
||||
uint8* src_row = src;
|
||||
uint8 *dest_row = dest.get();
|
||||
uint8 *src_row = src;
|
||||
for (uint32 Row = 0; Row < height; ++Row)
|
||||
{
|
||||
FMemory::Memcpy(dest_row, src_row, copy_row_stride);
|
||||
dest_row += copy_row_stride;
|
||||
src_row += src_stride;
|
||||
FMemory::Memcpy(dest_row, src_row, copy_row_stride);
|
||||
dest_row += copy_row_stride;
|
||||
src_row += src_stride;
|
||||
}
|
||||
src = dest.get();
|
||||
}
|
||||
}
|
||||
|
||||
const FSensorDataView DataView(
|
||||
GetId(),
|
||||
FReadOnlyBufferView{reinterpret_cast<const void *>(&ImageHeader), sizeof(ImageHeader)},
|
||||
FReadOnlyBufferView{src,dest_stride}
|
||||
);
|
||||
|
||||
GetId(),
|
||||
FReadOnlyBufferView{reinterpret_cast<const void *>(&ImageHeader), sizeof(ImageHeader)},
|
||||
FReadOnlyBufferView{src, dest_stride});
|
||||
|
||||
WriteSensorData(DataView);
|
||||
RHIUnlockTexture2D(texture, 0, false);
|
||||
}
|
||||
|
||||
|
||||
void ASceneCaptureCamera::UpdateDrawFrustum()
|
||||
{
|
||||
if(DrawFrustum && CaptureComponent2D)
|
||||
if (DrawFrustum && CaptureComponent2D)
|
||||
{
|
||||
DrawFrustum->FrustumStartDist = GNearClippingPlane;
|
||||
|
||||
// 1000 is the default frustum distance, ideally this would be infinite but that might cause rendering issues
|
||||
DrawFrustum->FrustumEndDist = (CaptureComponent2D->MaxViewDistanceOverride > DrawFrustum->FrustumStartDist)
|
||||
// 1000 is the default frustum distance, ideally this would be infinite but
|
||||
// that might cause rendering issues.
|
||||
DrawFrustum->FrustumEndDist =
|
||||
(CaptureComponent2D->MaxViewDistanceOverride > DrawFrustum->FrustumStartDist)
|
||||
? CaptureComponent2D->MaxViewDistanceOverride : 1000.0f;
|
||||
|
||||
DrawFrustum->FrustumAngle = CaptureComponent2D->FOVAngle;
|
||||
//DrawFrustum->FrustumAspectRatio = CaptureComponent2D->AspectRatio;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// -- Local static functions implementations -----------------------------------
|
||||
// =============================================================================
|
||||
|
||||
// Remove the show flags that might interfere with post-processing effects like
|
||||
// depth and semantic segmentation.
|
||||
static void RemoveShowFlags(FEngineShowFlags &ShowFlags)
|
||||
|
|
|
|||
|
|
@ -73,10 +73,16 @@ protected:
|
|||
static uint32 NumSceneCapture;
|
||||
|
||||
private:
|
||||
///Read the camera buffer and write it to the client with no lock of the resources (for Vulkan API)
|
||||
void WritePixelsNonBlocking(float DeltaTime,FRHICommandListImmediate& rhi_cmd_list) const;
|
||||
///Read the camera buffer and write it to the client with opengl or direct3d
|
||||
void WritePixels(float DeltaTime) const;
|
||||
|
||||
/// Read the camera buffer and write it to the client with no lock of the
|
||||
/// resources (for Vulkan API).
|
||||
void WritePixelsNonBlocking(
|
||||
uint64 FrameNumber,
|
||||
FRHICommandListImmediate& rhi_cmd_list) const;
|
||||
|
||||
/// Read the camera buffer and write it to the client with opengl or direct3d.
|
||||
void WritePixels(uint64 FrameNumber) const;
|
||||
|
||||
/// Used to synchronize the DrawFrustumComponent with the
|
||||
/// SceneCaptureComponent2D settings.
|
||||
void UpdateDrawFrustum();
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ void FCarlaEncoder::Encode(
|
|||
const ACarlaPlayerState &PlayerState,
|
||||
carla_measurements &Data)
|
||||
{
|
||||
Data.frame_number = PlayerState.GetFrameNumber();
|
||||
Data.platform_timestamp = PlayerState.GetPlatformTimeStamp();
|
||||
Data.game_timestamp = PlayerState.GetGameTimeStamp();
|
||||
auto &Player = Data.player_measurements;
|
||||
|
|
|
|||
|
|
@ -184,6 +184,8 @@ extern "C" {
|
|||
/* ======================================================================== */
|
||||
|
||||
struct carla_measurements {
|
||||
/** Frame counter. */
|
||||
uint32_t frame_number;
|
||||
/** Time-stamp of the current frame, in milliseconds as given by the OS. */
|
||||
uint32_t platform_timestamp;
|
||||
/** In-game time-stamp, milliseconds elapsed since the beginning of the current level. */
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ namespace server {
|
|||
std::string CarlaEncoder::Encode(const carla_measurements &values) {
|
||||
static thread_local auto *message = _protobuf.CreateMessage<cs::Measurements>();
|
||||
DEBUG_ASSERT(message != nullptr);
|
||||
message->set_frame_number(values.frame_number);
|
||||
message->set_platform_timestamp(values.platform_timestamp);
|
||||
message->set_game_timestamp(values.game_timestamp);
|
||||
// Player measurements.
|
||||
|
|
|
|||
|
|
@ -17,11 +17,12 @@ namespace test {
|
|||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
|
||||
const struct {
|
||||
uint64_t FrameNumber;
|
||||
uint32_t Width;
|
||||
uint32_t Height;
|
||||
uint32_t Type;
|
||||
float FOV;
|
||||
} ImageHeader = {300u, 200u, 1u, 90.0f};
|
||||
} ImageHeader = {++_frame_number, 300u, 200u, 1u, 90.0f};
|
||||
|
||||
_data.header_size = sizeof(ImageHeader);
|
||||
auto header = std::make_unique<unsigned char[]>(_data.header_size);
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ namespace test {
|
|||
|
||||
Sensor(uint32_t id);
|
||||
|
||||
uint64_t _frame_number = 0u;
|
||||
|
||||
mutable std::mutex _mutex;
|
||||
|
||||
const std::string _name;
|
||||
|
|
|
|||
|
|
@ -141,6 +141,8 @@ message Measurements {
|
|||
Control autopilot_control = 10;
|
||||
}
|
||||
|
||||
uint64 frame_number = 5;
|
||||
|
||||
uint32 platform_timestamp = 1;
|
||||
|
||||
uint32 game_timestamp = 2;
|
||||
|
|
|
|||
Loading…
Reference in New Issue