diff --git a/PythonAPI/util/performance_benchmark.py b/PythonAPI/util/performance_benchmark.py index c407d3c34..b9a0a2fae 100755 --- a/PythonAPI/util/performance_benchmark.py +++ b/PythonAPI/util/performance_benchmark.py @@ -75,6 +75,13 @@ def define_weather(): list_weather.append(weather01) list_weather.append(weather02) + if args.weather is not None: + try: + new_list = [list_weather[int(i)] for i in args.weather] + list_weather = new_list + except IndexError as error: + print("Warning!! The list of types of weather introduced is not valid. Using all available.") + return list_weather @@ -83,34 +90,48 @@ def define_sensors(): if args.tm: sensors00 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 300, 'height': 200, 'fov': 100, 'label': '1. cam-300x200'}] + 'width': 300, 'height': 200, 'fov': 100, 'label': '0. cam-300x200'}] list_sensor_specs.append(sensors00) else: sensors00 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 300, 'height': 200, 'fov': 100, 'label': '1. cam-300x200'}] + 'width': 300, 'height': 200, 'fov': 100, 'label': '0. cam-300x200'}] sensors01 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 800, 'height': 600, 'fov': 100, 'label': '2. cam-800x600'}] + 'width': 800, 'height': 600, 'fov': 100, 'label': '1. cam-800x600'}] sensors02 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 1900, 'height': 1080, 'fov': 100, 'label': '3. cam-1900x1080'}] + 'width': 1900, 'height': 1080, 'fov': 100, 'label': '2. cam-1900x1080'}] sensors03 = [{'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 300, 'height': 200, 'fov': 100, 'label': '4. cam-300x200'}, + 'width': 300, 'height': 200, 'fov': 100, 'label': '3. cam-300x200'}, {'type': 'sensor.camera.rgb', 'x': 0.7, 'y': 0.4, 'z': 1.60, 'roll': 0.0, 'pitch': 0.0, 'yaw': 0.0, - 'width': 300, 'height': 200, 'fov': 100, 'label': 'cam-300x200'}, - ] + 'width': 300, 'height': 200, 'fov': 100, 'label': 'cam-300x200'}] sensors04 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0, - 'label': '5. LIDAR'}] + 'pts_per_sec': '100000', 'label': '4. LIDAR: 100k'}] + + sensors05 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0, + 'pts_per_sec': '500000', 'label': '5. LIDAR: 500k'}] + + sensors06 = [{'type': 'sensor.lidar.ray_cast', 'x': 0.7, 'y': 0.0, 'z': 1.60, 'yaw': 0.0, 'pitch': 0.0, 'roll': 0.0, + 'pts_per_sec': '1000000', 'label': '6. LIDAR: 1M'}] list_sensor_specs.append(sensors00) list_sensor_specs.append(sensors01) list_sensor_specs.append(sensors02) list_sensor_specs.append(sensors03) list_sensor_specs.append(sensors04) + list_sensor_specs.append(sensors05) + list_sensor_specs.append(sensors06) + + if args.sensors is not None: + try: + new_list = [list_sensor_specs[int(i)] for i in args.sensors] + list_sensor_specs = new_list + except IndexError as error: + print("Warning!! The list of sensors introduced is not valid. Using all available.") return list_sensor_specs @@ -135,7 +156,18 @@ def define_environments(): return list_env_specs +def define_maps(client): + maps = [m.replace('/Game/Carla/Maps/', '') for m in client.get_available_maps()] + maps = sorted(maps) + if args.maps is not None: + all_good = all(elem in maps for elem in args.maps) + if all_good: + maps = sorted(args.maps) + else: + print("Warning!! The list of maps introduced is not valid. Using all available.") + + return maps class CallBack(object): def __init__(self): self._lock = threading.Lock() @@ -179,7 +211,7 @@ def create_environment(world, sensors, n_vehicles, n_walkers, spawn_points, clie bp.set_attribute('channels', '32') bp.set_attribute('upper_fov', '15') bp.set_attribute('lower_fov', '-30') - bp.set_attribute('points_per_second', '500000') + bp.set_attribute('points_per_second', str(sensor_spec['pts_per_sec'])) sensor_location = carla.Location( x=sensor_spec['x'], y=sensor_spec['y'], @@ -311,6 +343,21 @@ def create_environment(world, sensors, n_vehicles, n_walkers, spawn_points, clie # -- Benchmarking functions -------------------------------------------------------------------------------------------- # ====================================================================================================================== +def set_world_settings(world, args = None): + + if args == None: + settings = world.get_settings() + settings.synchronous_mode = False + settings.fixed_delta_seconds = 0.0 + settings.no_rendering_mode = False + world.apply_settings(settings) + else: + settings = world.get_settings() + settings.synchronous_mode = args.sync + settings.fixed_delta_seconds = args.fixed_dt if args.sync else 0.0 + settings.no_rendering_mode = args.no_render_mode + world.apply_settings(settings) + def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False): global sensors_callback @@ -319,14 +366,21 @@ def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False): list_fps = [] sensor_list = None + tick = world.tick if args.sync else world.wait_for_tick + set_world_settings(world, args) + vehicles_list, walkers_list, all_id, all_actors, sensors_ret = create_environment(world, sensors, n, n_walkers, spawn_points, client) if sensors_ret: sensor_list = sensors_ret + # Allow some time for the server to finish the initialization + for _i in range(0, 50): + tick() + ticks = 0 while ticks < int(args.ticks): - _ = world.wait_for_tick() + _ = tick() if debug: print("== Samples {} / {}".format(ticks + 1, args.ticks)) @@ -356,6 +410,8 @@ def run_benchmark(world, sensors, n_vehicles, n_walkers, client, debug=False): print('\ndestroying %d walkers' % len(walkers_list)) client.apply_batch([carla.command.DestroyActor(x) for x in all_id]) + set_world_settings(world) + return list_fps @@ -419,58 +475,86 @@ def get_system_specs(): return str_system +def show_benchmark_scenarios(maps): + print("Available maps") + for map in sorted(maps): + print(" - %s" % map) + print("Available sensors") + for i,sensors in enumerate(define_sensors()): + sensor_str = "" + for sensor in sensors: + sensor_str += (sensor['label'] + " ") + print(' - %s' % (sensor_str)) + print("Available types of weather") + for i,weather in enumerate(define_weather()): + print(' - %i: %s' % (i, weather['name'])) + print("Available Enviroments") + for i,env in enumerate(define_environments()): + print(' - %i: %s' % (i, str(env))) + + def main(args): - client = carla.Client(args.host, int(args.port)) - client.set_timeout(60.0) - pygame.init() - records = {} - maps = [m.replace('/Game/Carla/Maps/', '') for m in client.get_available_maps()] + try: + client = carla.Client(args.host, int(args.port)) + client.set_timeout(60.0) + pygame.init() - for town in sorted(maps): - world = client.load_world(town) + records = {} + maps = define_maps(client) - # set to async mode - settings = world.get_settings() - settings.synchronous_mode = False - settings.fixed_delta_seconds = None - world.apply_settings(settings) + if args.show_scenarios: + show_benchmark_scenarios(maps) + return - # spectator pointing to the sky to reduce rendering impact - spectator = world.get_spectator() - spectator.set_transform(carla.Transform(carla.Location(z=500), carla.Rotation(pitch=90))) + #maps = ["Town04_Opt"] - for weather in define_weather(): - world.set_weather(weather["parameter"]) - for env in define_environments(): - for sensors in define_sensors(): - list_fps = run_benchmark(world, sensors, env["vehicles"], env["walkers"], client) - mean, std = compute_mean_std(list_fps) - sensor_str = "" - for sensor in sensors: - sensor_str += (sensor['label'] + " ") + for town in maps: + world = client.load_world(town) - record = { - 'town': town, - 'sensors': sensor_str, - 'weather': weather["name"], - 'n_vehicles': env["vehicles"], - 'n_walkers': env["walkers"], - 'samples': args.ticks, - 'fps_mean': mean, - 'fps_std': std - } + # set to async mode + set_world_settings(world) - env_str = str(env["vehicles"]) + str(env["walkers"]) + # spectator pointing to the sky to reduce rendering impact + spectator = world.get_spectator() + spectator.set_transform(carla.Transform(carla.Location(z=500), carla.Rotation(pitch=90))) - if env_str not in records: - records[env_str] = [] - records[env_str].append(record) - print(record) + for weather in define_weather(): + world.set_weather(weather["parameter"]) + for env in define_environments(): + for sensors in define_sensors(): + list_fps = run_benchmark(world, sensors, env["vehicles"], env["walkers"], client) + mean, std = compute_mean_std(list_fps) + sensor_str = "" + for sensor in sensors: + sensor_str += (sensor['label'] + " ") - system_specs = get_system_specs() - serialize_records(records, system_specs, args.file) - pygame.quit() + record = { + 'town': town, + 'sensors': sensor_str, + 'weather': weather["name"], + 'n_vehicles': env["vehicles"], + 'n_walkers': env["walkers"], + 'samples': args.ticks, + 'fps_mean': mean, + 'fps_std': std + } + + env_str = str(env["vehicles"]) + str(env["walkers"]) + + if env_str not in records: + records[env_str] = [] + records[env_str].append(record) + print(record) + + system_specs = get_system_specs() + serialize_records(records, system_specs, args.file) + pygame.quit() + + except KeyboardInterrupt: + set_world_settings(world) + client.reload_world() + print('\nCancelled by user. Bye!') if __name__ == '__main__': @@ -482,6 +566,15 @@ if __name__ == '__main__': parser.add_argument('--file', type=str, help='Write results into a txt file', default="benchmark.md") parser.add_argument('--tm', action='store_true', help='Switch to traffic manager benchmark') parser.add_argument('--ticks', default=100, help='Number of ticks for each scenario (default: 100)') + parser.add_argument('--sync', default=True, action='store_true', help='Synchronous mode execution (default)') + parser.add_argument('--async', dest='sync', action='store_false', help='Asynchronous mode execution') + parser.add_argument('--fixed_dt', type=float, default=0.05, help='Time interval for the simulator in synchronous mode (default: 0.05)') + parser.add_argument('--render_mode', dest='no_render_mode', action='store_false', help='Execute with spectator') + parser.add_argument('--no_render_mode', default=True, action='store_true', help='Execute in no rendering mode (default)') + parser.add_argument('--show_scenarios', default=False, action='store_true', help='Show the scenarios to benchmark and return (default=False)') + parser.add_argument('--sensors', nargs="+", default=None, help='List of sensors to benchmark, by default all defined ones') + parser.add_argument('--maps', nargs="+", default=None, help='List of maps to benchmark, by default all defined ones') + parser.add_argument('--weather', nargs="+", default=None, help='List of weather types to benchmark, by default all defined ones') args = parser.parse_args()