From 9b7cdbc15e91798bf4e50ad6bdb0e7e7a0048403 Mon Sep 17 00:00:00 2001 From: felipecode Date: Thu, 19 Apr 2018 15:11:29 +0200 Subject: [PATCH] benchmark --- Docs/benchmark_basic_results_town01.md | 192 ++++++++++++------------ Docs/benchmark_basic_results_town02.md | 193 ++++++++++++------------- Docs/benchmark_creating.md | 4 + PythonClient/client_example.py | 5 +- 4 files changed, 198 insertions(+), 196 deletions(-) diff --git a/Docs/benchmark_basic_results_town01.md b/Docs/benchmark_basic_results_town01.md index 2675a503f..adf013c97 100644 --- a/Docs/benchmark_basic_results_town01.md +++ b/Docs/benchmark_basic_results_town01.md @@ -22,75 +22,75 @@ example. Average Percentage of Distance to Goal Travelled Weather: Clear Noon - Task: 0 -> 0.9642745353103807 - Task: 1 -> 0.6793899335407552 - Task: 2 -> 0.659387057876586 - Task: 3 -> 0.6997037642505417 + Task: 0 -> 0.9643630125892909 + Task: 1 -> 0.6794216252808839 + Task: 2 -> 0.6593855166486696 + Task: 3 -> 0.6646695325122313 Average Between Weathers - Task 0 -> 0.9642745353103807 - Task 1 -> 0.6793899335407552 - Task 2 -> 0.659387057876586 - Task 3 -> 0.6997037642505417 + Task 0 -> 0.9643630125892909 + Task 1 -> 0.6794216252808839 + Task 2 -> 0.6593855166486696 + Task 3 -> 0.6646695325122313 Avg. Kilometers driven before a collision to a PEDESTRIAN Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Avg. Kilometers driven before a collision to a VEHICLE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.11491704214531683 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.11491704214531683 Avg. Kilometers driven before a collision to a STATIC OBSTACLE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.07269845062683851 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.07269845062683851 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.22983408429063365 Avg. Kilometers driven before going OUTSIDE OF THE ROAD Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> 0.12350085985904342 + Task 2 -> 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> 0.12350085985904342 + Task 2 -> 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Avg. Kilometers driven before invading the OPPOSITE LANE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> 0.1235917529074286 - Task 2 -> 0.010437001976169077 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> 0.1235917529074286 - Task 2 -> 0.010437001976169077 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 @@ -115,75 +115,75 @@ example. Average Percentage of Distance to Goal Travelled Weather: Clear Noon - Task: 0 -> 0.9642745353103807 - Task: 1 -> 0.6793899335407552 - Task: 2 -> 0.659387057876586 - Task: 3 -> 0.6997037642505417 + Task: 0 -> 0.9643630125892909 + Task: 1 -> 0.6794216252808839 + Task: 2 -> 0.6593855166486696 + Task: 3 -> 0.6646695325122313 Average Between Weathers - Task 0 -> 0.9642745353103807 - Task 1 -> 0.6793899335407552 - Task 2 -> 0.659387057876586 - Task 3 -> 0.6997037642505417 + Task 0 -> 0.9643630125892909 + Task 1 -> 0.6794216252808839 + Task 2 -> 0.6593855166486696 + Task 3 -> 0.6646695325122313 Avg. Kilometers driven before a collision to a PEDESTRIAN Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Avg. Kilometers driven before a collision to a VEHICLE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.11491704214531683 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.11491704214531683 Avg. Kilometers driven before a collision to a STATIC OBSTACLE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.07269845062683851 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.07269845062683851 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> 0.22983408429063365 Avg. Kilometers driven before going OUTSIDE OF THE ROAD Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> 0.12350085985904342 + Task 2 -> 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> more than 0.1235917529074286 - Task 2 -> more than 0.2400510454518888 - Task 3 -> 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> 0.12350085985904342 + Task 2 -> 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Avg. Kilometers driven before invading the OPPOSITE LANE Weather: Clear Noon - Task 0 -> more than 0.043162963265066225 - Task 1 -> 0.1235917529074286 - Task 2 -> 0.010437001976169077 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 Average Between Weathers - Task 0 -> more than 0.043162963265066225 - Task 1 -> 0.1235917529074286 - Task 2 -> 0.010437001976169077 - Task 3 -> more than 0.21809535188051554 + Task 0 -> more than 0.04316352371637994 + Task 1 -> more than 0.12350085985904342 + Task 2 -> more than 0.2400373917146113 + Task 3 -> more than 0.22983408429063365 diff --git a/Docs/benchmark_basic_results_town02.md b/Docs/benchmark_basic_results_town02.md index e05f2efdc..8bba9b4da 100644 --- a/Docs/benchmark_basic_results_town02.md +++ b/Docs/benchmark_basic_results_town02.md @@ -23,75 +23,75 @@ example. Average Percentage of Distance to Goal Travelled Weather: Clear Noon - Task: 0 -> 0.8118712628185436 - Task: 1 -> 0.10663609976621982 - Task: 2 -> -0.20449201041073187 - Task: 3 -> -0.20447646890991833 + Task: 0 -> 0.8127653637426329 + Task: 1 -> 0.10658303206448155 + Task: 2 -> -0.20448736444348714 + Task: 3 -> -0.20446966646041384 Average Between Weathers - Task 0 -> 0.8118712628185436 - Task 1 -> 0.10663609976621982 - Task 2 -> -0.20449201041073187 - Task 3 -> -0.20447646890991833 + Task 0 -> 0.8127653637426329 + Task 1 -> 0.10658303206448155 + Task 2 -> -0.20448736444348714 + Task 3 -> -0.20446966646041384 Avg. Kilometers driven before a collision to a PEDESTRIAN Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> more than 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - More than 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Avg. Kilometers driven before a collision to a VEHICLE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> more than 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - More than 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Avg. Kilometers driven before a collision to a STATIC OBSTACLE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> 0.019708680616537626 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> 0.019641485501456352 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - Task 3 -> 0.019708680616537626 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> 0.019641485501456352 Avg. Kilometers driven before going OUTSIDE OF THE ROAD Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> 0.03856641710143665 + Task 2 -> 0.03928511962584409 + Task 3 -> 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> 0.03856641710143665 + Task 2 -> 0.03928511962584409 + Task 3 -> 0.039282971002912705 Avg. Kilometers driven before invading the OPPOSITE LANE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 @@ -116,73 +116,72 @@ example. Average Percentage of Distance to Goal Travelled Weather: Clear Noon - Task: 0 -> 0.8118712628185436 - Task: 1 -> 0.10663609976621982 - Task: 2 -> -0.20449201041073187 - Task: 3 -> -0.20447646890991833 + Task: 0 -> 0.8127653637426329 + Task: 1 -> 0.10658303206448155 + Task: 2 -> -0.20448736444348714 + Task: 3 -> -0.20446966646041384 Average Between Weathers - Task 0 -> 0.8118712628185436 - Task 1 -> 0.10663609976621982 - Task 2 -> -0.20449201041073187 - Task 3 -> -0.20447646890991833 + Task 0 -> 0.8127653637426329 + Task 1 -> 0.10658303206448155 + Task 2 -> -0.20448736444348714 + Task 3 -> -0.20446966646041384 Avg. Kilometers driven before a collision to a PEDESTRIAN Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> more than 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - More than 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Avg. Kilometers driven before a collision to a VEHICLE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> more than 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - More than 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Avg. Kilometers driven before a collision to a STATIC OBSTACLE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> more than 0.038524663402911156 - Task 2 -> more than 0.039421759075344356 - Task 3 -> 0.019708680616537626 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> 0.019641485501456352 Average Between Weathers - More than 0 -> 0.007091656691537298 - More than 1 -> 0.038524663402911156 - More than 2 -> 0.039421759075344356 - Task 3 -> 0.019708680616537626 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> 0.019641485501456352 Avg. Kilometers driven before going OUTSIDE OF THE ROAD Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> 0.03856641710143665 + Task 2 -> 0.03928511962584409 + Task 3 -> 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> 0.03856641710143665 + Task 2 -> 0.03928511962584409 + Task 3 -> 0.039282971002912705 Avg. Kilometers driven before invading the OPPOSITE LANE Weather: Clear Noon - Task 0 -> more than 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 Average Between Weathers - More than 0 -> 0.007091656691537298 - Task 1 -> 0.038524663402911156 - Task 2 -> 0.039421759075344356 - Task 3 -> 0.03941736123307525 - \ No newline at end of file + Task 0 -> more than 0.0071004936693366055 + Task 1 -> more than 0.03856641710143665 + Task 2 -> more than 0.03928511962584409 + Task 3 -> more than 0.039282971002912705 diff --git a/Docs/benchmark_creating.md b/Docs/benchmark_creating.md index 0bc5fe626..6121a2c24 100644 --- a/Docs/benchmark_creating.md +++ b/Docs/benchmark_creating.md @@ -219,6 +219,10 @@ The full code could be found at [basic_experiment_suite.py](https://github.com/c #### Expected Results +To run the examples, to get similar you need a CARLA Simulator server running with fixed simulation step + +sh CarlaUE4.sh /Game/Maps/Town01 -windowed -world-port=2000 -benchmark -fps=10 + The presented example can be execute for Town01 as: ./driving_benchmark_example.py -c Town01 diff --git a/PythonClient/client_example.py b/PythonClient/client_example.py index 1ecf7fd05..402fa0bcc 100755 --- a/PythonClient/client_example.py +++ b/PythonClient/client_example.py @@ -24,8 +24,8 @@ from carla.util import print_over_same_line def run_carla_client(args): # Here we will run 3 episodes with 300 frames each. - number_of_episodes = 10 - frames_per_episode = 30 + number_of_episodes = 3 + frames_per_episode = 300 # We assume the CARLA server is already waiting for a client to connect at # host:port. To create a connection we can use the `make_carla_client` @@ -155,7 +155,6 @@ def run_carla_client(args): control.steer += random.uniform(-0.1, 0.1) client.send_control(control) - time.sleep(0.05)