Improve benchmark script:

- Added option for sync mode (on by default)
 - Added a warm-up time after spawning actors to not contaminate the results
 - Added option for no rendering mode (on by default)
 - Added two new cases for Lidar (100k and 1M points)
 - Allow to cancel the execution like other scripts
 - Added the option to benchmark only a sublist of maps, sensors and
types of weather
 - Added a parameter to show all the maps, sensors and types of weather possible

PythonAPI/util/performance_benchmark.py

@@ -75,6 +75,13 @@ def define_weather():
     list_weather.append(weather01)
     list_weather.append(weather02)
 
+    if args.weather is not None:
+      try:
+        new_list = [list_weather[int(i)] for i in args.weather]
+        list_weather = new_list
+      except IndexError as error:
+        print("Warning!! The list of types of weather introduced is not valid. Using all available.")
+
   return list_weather
 
 
@@ -83,34 +90,48 @@ def define_sensors():
 
   if args.tm:
     sensors00 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                'width': 300, 'height': 200, 'fov': 100, 'label': '1. cam-300x200'}]
+                'width': 300, 'height': 200, 'fov': 100, 'label': '0. cam-300x200'}]
 
     list_sensor_specs.append(sensors00)
 
   else:
     sensors00 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                  'width': 300, 'height': 200, 'fov': 100, 'label': '1. cam-300x200'}]
+                  'width': 300, 'height': 200, 'fov': 100, 'label': '0. cam-300x200'}]
 
     sensors01 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                  'width': 800, 'height': 600, 'fov': 100, 'label': '2. cam-800x600'}]
+                  'width': 800, 'height': 600, 'fov': 100, 'label': '1. cam-800x600'}]
 
     sensors02 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                  'width': 1900, 'height': 1080, 'fov': 100, 'label': '3. cam-1900x1080'}]
+                  'width': 1900, 'height': 1080, 'fov': 100, 'label': '2. cam-1900x1080'}]
 
     sensors03 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                  'width': 300, 'height': 200, 'fov': 100, 'label': '4. cam-300x200'},
+                  'width': 300, 'height': 200, 'fov': 100, 'label': '3. cam-300x200'},
                  {'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.4, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0,
-                  'width': 300, 'height': 200, 'fov': 100, 'label': 'cam-300x200'},
+                  'width': 300, 'height': 200, 'fov': 100, 'label': 'cam-300x200'}]
-                 ]
 
     sensors04 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0,
-                  'label': '5. LIDAR'}]
+                  'pts_per_sec': '100000', 'label': '4. LIDAR: 100k'}]
+
+    sensors05 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0,
+                  'pts_per_sec': '500000', 'label': '5. LIDAR: 500k'}]
+
+    sensors06 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0,
+                  'pts_per_sec': '1000000', 'label': '6. LIDAR: 1M'}]
 
     list_sensor_specs.append(sensors00)
     list_sensor_specs.append(sensors01)
     list_sensor_specs.append(sensors02)
     list_sensor_specs.append(sensors03)
     list_sensor_specs.append(sensors04)
+    list_sensor_specs.append(sensors05)
+    list_sensor_specs.append(sensors06)
+
+    if args.sensors is not None:
+      try:
+        new_list = [list_sensor_specs[int(i)] for i in args.sensors]
+        list_sensor_specs = new_list
+      except IndexError as error:
+        print("Warning!! The list of sensors introduced is not valid. Using all available.")
 
   return list_sensor_specs
 
@@ -135,7 +156,18 @@ def define_environments():
 
   return list_env_specs
 
+def define_maps(client):
+  maps = [m.replace('/Game/Carla/Maps/', '') for m in client.get_available_maps()]
+  maps = sorted(maps)
 
+  if args.maps is not None:
+    all_good = all(elem in maps for elem in args.maps)
+    if all_good:
+      maps = sorted(args.maps)
+    else:
+      print("Warning!! The list of maps introduced is not valid. Using all available.")
+
+  return maps
 class CallBack(object):
     def __init__(self):
         self._lock = threading.Lock()
@@ -179,7 +211,7 @@ def create_environment(world, sensors, n_vehicles, n_walkers, spawn_points, clie
       bp.set_attribute('channels', '32')
       bp.set_attribute('upper_fov', '15')
       bp.set_attribute('lower_fov', '-30')
-      bp.set_attribute('points_per_second', '500000')
+      bp.set_attribute('points_per_second', str(sensor_spec['pts_per_sec']))
       sensor_location = carla.Location(
         x=sensor_spec['x'],
         y=sensor_spec['y'],
@@ -311,6 +343,21 @@ def create_environment(world, sensors, n_vehicles, n_walkers, spawn_points, clie
 # -- Benchmarking functions --------------------------------------------------------------------------------------------
 # ======================================================================================================================
 
+def set_world_settings(world, args = None):
+
+  if args == None:
+    settings = world.get_settings()
+    settings.synchronous_mode = False
+    settings.fixed_delta_seconds = 0.0
+    settings.no_rendering_mode = False
+    world.apply_settings(settings)
+  else:
+    settings = world.get_settings()
+    settings.synchronous_mode = args.sync
+    settings.fixed_delta_seconds = args.fixed_dt if args.sync else 0.0
+    settings.no_rendering_mode = args.no_render_mode
+    world.apply_settings(settings)
+
 def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False):
   global sensors_callback
 
@@ -319,14 +366,21 @@ def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False):
   list_fps = []
   sensor_list = None
 
+  tick = world.tick if args.sync else world.wait_for_tick
+  set_world_settings(world, args)
+
   vehicles_list, walkers_list, all_id, all_actors, sensors_ret = create_environment(world, sensors, n, n_walkers, spawn_points, client)
 
   if sensors_ret:
     sensor_list = sensors_ret
 
+  # Allow some time for the server to finish the initialization
+  for _i in range(0, 50):
+    tick()
+
   ticks = 0
   while ticks < int(args.ticks):
-    _ = world.wait_for_tick()
+    _ = tick()
     if debug:
       print("== Samples {} / {}".format(ticks + 1, args.ticks))
 
@@ -356,6 +410,8 @@ def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False):
   print('\ndestroying %d walkers' % len(walkers_list))
   client.apply_batch([carla.command.DestroyActor(x) for x in all_id])
 
+  set_world_settings(world)
+
   return list_fps
 
 
@@ -419,58 +475,86 @@ def get_system_specs():
   return str_system
 
 
+def show_benchmark_scenarios(maps):
+  print("Available maps")
+  for map in sorted(maps):
+    print("  - %s" % map)
+  print("Available sensors")
+  for i,sensors in enumerate(define_sensors()):
+    sensor_str = ""
+    for sensor in sensors:
+      sensor_str += (sensor['label'] + " ")
+    print('  - %s' % (sensor_str))
+  print("Available types of weather")
+  for i,weather in enumerate(define_weather()):
+    print('  - %i: %s' % (i, weather['name']))
+  print("Available Enviroments")
+  for i,env in enumerate(define_environments()):
+    print('  - %i: %s' % (i, str(env)))
+
+
 def main(args):
-  client = carla.Client(args.host, int(args.port))
-  client.set_timeout(60.0)
-  pygame.init()
 
-  records = {}
+  try:
-  maps = [m.replace('/Game/Carla/Maps/', '') for m in client.get_available_maps()]
+    client = carla.Client(args.host, int(args.port))
+    client.set_timeout(60.0)
+    pygame.init()
 
-  for town in sorted(maps):
+    records = {}
-    world = client.load_world(town)
+    maps = define_maps(client)
 
-    # set to async mode
+    if args.show_scenarios:
-    settings = world.get_settings()
+      show_benchmark_scenarios(maps)
-    settings.synchronous_mode = False
+      return
-    settings.fixed_delta_seconds = None
-    world.apply_settings(settings)
 
-    # spectator pointing to the sky to reduce rendering impact
+    #maps = ["Town04_Opt"]
-    spectator = world.get_spectator()
-    spectator.set_transform(carla.Transform(carla.Location(z=500), carla.Rotation(pitch=90)))
 
-    for weather in define_weather():
+    for town in maps:
-      world.set_weather(weather["parameter"])
+      world = client.load_world(town)
-      for env in define_environments():
-        for sensors in define_sensors():
-          list_fps = run_benchmark(world, sensors, env["vehicles"], env["walkers"], client)
-          mean, std = compute_mean_std(list_fps)
-          sensor_str = ""
-          for sensor in sensors:
-            sensor_str += (sensor['label'] + " ")
 
-          record = {
+      # set to async mode
-            'town': town,
+      set_world_settings(world)
-            'sensors': sensor_str,
-            'weather': weather["name"],
-            'n_vehicles': env["vehicles"],
-            'n_walkers': env["walkers"],
-            'samples': args.ticks,
-            'fps_mean': mean,
-            'fps_std': std
-          }
 
-          env_str = str(env["vehicles"]) + str(env["walkers"])
+      # spectator pointing to the sky to reduce rendering impact
+      spectator = world.get_spectator()
+      spectator.set_transform(carla.Transform(carla.Location(z=500), carla.Rotation(pitch=90)))
 
-          if env_str not in records:
+      for weather in define_weather():
-            records[env_str] = []
+        world.set_weather(weather["parameter"])
-          records[env_str].append(record)
+        for env in define_environments():
-          print(record)
+          for sensors in define_sensors():
+            list_fps = run_benchmark(world, sensors, env["vehicles"], env["walkers"], client)
+            mean, std = compute_mean_std(list_fps)
+            sensor_str = ""
+            for sensor in sensors:
+              sensor_str += (sensor['label'] + " ")
 
-  system_specs = get_system_specs()
+            record = {
-  serialize_records(records, system_specs, args.file)
+              'town': town,
-  pygame.quit()
+              'sensors': sensor_str,
+              'weather': weather["name"],
+              'n_vehicles': env["vehicles"],
+              'n_walkers': env["walkers"],
+              'samples': args.ticks,
+              'fps_mean': mean,
+              'fps_std': std
+            }
+
+            env_str = str(env["vehicles"]) + str(env["walkers"])
+
+            if env_str not in records:
+              records[env_str] = []
+            records[env_str].append(record)
+            print(record)
+
+    system_specs = get_system_specs()
+    serialize_records(records, system_specs, args.file)
+    pygame.quit()
+
+  except KeyboardInterrupt:
+      set_world_settings(world)
+      client.reload_world()
+      print('\nCancelled by user. Bye!')
 
 
 if __name__ == '__main__':
@@ -482,6 +566,15 @@ if __name__ == '__main__':
   parser.add_argument('--file', type=str, help='Write results into a txt file', default="benchmark.md")
   parser.add_argument('--tm', action='store_true', help='Switch to traffic manager benchmark')
   parser.add_argument('--ticks', default=100, help='Number of ticks for each scenario (default: 100)')
+  parser.add_argument('--sync', default=True, action='store_true', help='Synchronous mode execution (default)')
+  parser.add_argument('--async', dest='sync', action='store_false', help='Asynchronous mode execution')
+  parser.add_argument('--fixed_dt', type=float, default=0.05, help='Time interval for the simulator in synchronous mode (default: 0.05)')
+  parser.add_argument('--render_mode', dest='no_render_mode', action='store_false', help='Execute with spectator')
+  parser.add_argument('--no_render_mode', default=True, action='store_true', help='Execute in no rendering mode (default)')
+  parser.add_argument('--show_scenarios', default=False, action='store_true', help='Show the scenarios to benchmark and return (default=False)')
+  parser.add_argument('--sensors', nargs="+", default=None, help='List of sensors to benchmark, by default all defined ones')
+  parser.add_argument('--maps', nargs="+", default=None, help='List of maps to benchmark, by default all defined ones')
+  parser.add_argument('--weather', nargs="+", default=None, help='List of weather types to benchmark, by default all defined ones')
 
   args = parser.parse_args()