diff --git a/PythonClient/benchmarks/benchmark.py b/PythonClient/benchmarks/benchmark.py index 26d4b840c..82f4554a5 100644 --- a/PythonClient/benchmarks/benchmark.py +++ b/PythonClient/benchmarks/benchmark.py @@ -14,7 +14,7 @@ sldist = lambda c1, c2: math.sqrt((c2[0] - c1[0])**2 + (c2[1] - c1[1])**2) class Benchmark(object,): # Param @name to be used for saving purposes - def __init__(self,city_name,name): + def __init__(self,city_name,name,starting_position=0): self._city_name = city_name # The name of the city that is going to be used. self._base_name = name # Sends a base name, the rest will be saved with respect to what the episode was about self._dict_stats = {'exp_id':-1, @@ -45,6 +45,11 @@ class Benchmark(object,): 'pos_y':-1 } + # Returns a experiment class that is build from a benchmark inherited class + self._experiments = self._build_experiments() + + self._suffix_name = self._get_experiments_names(self._experiments) + def run_navigation_episode(self, agent,carla, time_out, target): @@ -104,12 +109,10 @@ class Benchmark(object,): return (0, measurement_vec, time_out,distance) - def benchmark_agent(self,agent,carla,starting_position=0): + def benchmark_agent(self,agent,carla): - + # The fixed name considering all the experiments being run - experiments = self._build_experiments() # Returns a experiment class that is build from a benchmark inherited class - self._suffix_name = self._get_experiments_names(experiments[starting_position:]) # The fixed name considering all the experiments being run with open(self._base_name + self._suffix_name , 'wb') as ofd: w = csv.DictWriter(ofd, self._dict_stats.keys()) @@ -126,7 +129,7 @@ class Benchmark(object,): self.write_experiment() # write the experiment being run - for experiment in experiments[starting_position:]: + for experiment in self._experiments[starting_position:]: @@ -208,7 +211,7 @@ class Benchmark(object,): with open('rewards_' + self._base_name + self._suffix_name , 'a+') as rfd: rw = csv.DictWriter(rfd, self._dict_rewards.keys()) - rw.writeheader() + for i in range(len(reward_vec)): @@ -258,7 +261,7 @@ class Benchmark(object,): def _get_experiments_names(self,experiments): name_cat ='_t' - for experiment in experiments: + for experiment in experiments: name_cat += str(experiment.id) + '.' diff --git a/PythonClient/benchmarks/corl.py b/PythonClient/benchmarks/corl.py index 17fe61d21..ecd522e1b 100644 --- a/PythonClient/benchmarks/corl.py +++ b/PythonClient/benchmarks/corl.py @@ -1,3 +1,11 @@ +# Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma de +# Barcelona (UAB), and the INTEL Visual Computing Lab. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see . + +# CORL experiment set. + from .benchmark import Benchmark from .experiment import Experiment @@ -11,130 +19,136 @@ from carla.settings import CarlaSettings class CoRL(Benchmark): - def _calculate_time_out(self,path_distance): - return ((path_distance/100000.0)/10.0)*3600.0 + 10.0 + def _calculate_time_out(self,path_distance): + return ((path_distance/100000.0)/10.0)*3600.0 + 10.0 - def _poses_town01(self): + def _poses_town01(self): - return [ [[36,40],[39,35],[110,114],[7,3],[0,4],\ - [68,50],[61,59],[47,64],[147,90],[33,87],\ - [26,19],[80,76],[45,49],[55,44],[29,107],\ - [95,104],[34,84],[51,67],[22,17],[91,148],\ - [20,107],[78,70],[95,102],[68,44],[45,69]], + return [ [[36,40],[39,35],[110,114],[7,3],[0,4],\ + [68,50],[61,59],[47,64],[147,90],[33,87],\ + [26,19],[80,76],[45,49],[55,44],[29,107],\ + [95,104],[34,84],[51,67],[22,17],[91,148],\ + [20,107],[78,70],[95,102],[68,44],[45,69]], - [[138,17],[46,16],[26,9],[42,49],[140,26],\ - [85,97],[65,133],[137,51],[76,66],[46,39],\ - [40,60],[1,28],[4,129],[121,107],[2,129],\ - [78,44],[68,85],[41,102],[95,70],[68,129],\ - [84,69],[47,79],[110,15],[130,17],[0,17]], + [[138,17],[46,16],[26,9],[42,49],[140,26],\ + [85,97],[65,133],[137,51],[76,66],[46,39],\ + [40,60],[1,28],[4,129],[121,107],[2,129],\ + [78,44],[68,85],[41,102],[95,70],[68,129],\ + [84,69],[47,79],[110,15],[130,17],[0,17]], - [[105,29],[27,130],[102,87],[132,27],[24,44],\ - [96,26],[34,67],[28,1],[140,134],[105,9],\ - [148,129],[65,18],[21,16],[147,97],[42,51],\ - [30,41],[18,107],[69,45],[102,95],[18,145],\ - [111,64],[79,45],[84,69],[73,31],[37,81]], + [[105,29],[27,130],[102,87],[132,27],[24,44],\ + [96,26],[34,67],[28,1],[140,134],[105,9],\ + [148,129],[65,18],[21,16],[147,97],[42,51],\ + [30,41],[18,107],[69,45],[102,95],[18,145],\ + [111,64],[79,45],[84,69],[73,31],[37,81]], - [[105,29],[27,130],[102,87],[132,27],[24,44],\ - [96,26],[34,67],[28,1],[140,134],[105,9],\ - [148,129],[65,18],[21,16],[147,97],[42,51],\ - [30,41],[18,107],[69,45],[102,95],[18,145],\ - [111,64],[79,45],[84,69],[73,31],[37,81]] + [[105,29],[27,130],[102,87],[132,27],[24,44],\ + [96,26],[34,67],[28,1],[140,134],[105,9],\ + [148,129],[65,18],[21,16],[147,97],[42,51],\ + [30,41],[18,107],[69,45],[102,95],[18,145],\ + [111,64],[79,45],[84,69],[73,31],[37,81]] - ] + ] - def _poses_town02(self): + def _poses_town02(self): - return [ [[38, 34], [4, 2], [12, 10], [62, 55], [43, 47],\ - [64, 66], [78, 76],[59,57],[61,18],[35,39],\ - [12,8],[0,18],[75,68],[54,60],[45,49],\ - [46,42],[53,46],[80,29],[65,63],[0,81],\ - [54,63],[51,42],[16,19],[17,26],[77,68]], + return [ [[38, 34], [4, 2], [12, 10], [62, 55], [43, 47],\ + [64, 66], [78, 76],[59,57],[61,18],[35,39],\ + [12,8],[0,18],[75,68],[54,60],[45,49],\ + [46,42],[53,46],[80,29],[65,63],[0,81],\ + [54,63],[51,42],[16,19],[17,26],[77,68]], - [[37, 76], [8, 24], [60, 69], [38, 10], [21, 1],\ - [58,71],[74,32],[44,0],[71,16],[14,24],\ - [34,11],[43,14],[75,16],[80,21],[3,23],\ - [75,59],[50,47],[11,19],[77,34],[79,25] ,\ - [40,63],[58,76],[79,55],[16,61],[27,11]], + [[37, 76], [8, 24], [60, 69], [38, 10], [21, 1],\ + [58,71],[74,32],[44,0],[71,16],[14,24],\ + [34,11],[43,14],[75,16],[80,21],[3,23],\ + [75,59],[50,47],[11,19],[77,34],[79,25] ,\ + [40,63],[58,76],[79,55],[16,61],[27,11]], - [[19,66],[79,14],[19,57],[23,1],\ - [53,76],[42,13],[31,71],[33,5],\ - [54,30],[10,61],[66,3],[27,12],\ - [79,19],[2,29],[16,14],[5,57],\ - [70,73],[46,67],[57,50],[61,49],[21,12],\ - [51,81],[77,68],[56,65],[43,54]], + [[19,66],[79,14],[19,57],[23,1],\ + [53,76],[42,13],[31,71],[33,5],\ + [54,30],[10,61],[66,3],[27,12],\ + [79,19],[2,29],[16,14],[5,57],\ + [70,73],[46,67],[57,50],[61,49],[21,12],\ + [51,81],[77,68],[56,65],[43,54]], - [[19,66],[79,14],[19,57],[23,1],\ - [53,76],[42,13],[31,71],[33,5],\ - [54,30],[10,61],[66,3],[27,12],\ - [79,19],[2,29],[16,14],[5,57],\ - [70,73],[46,67],[57,50],[61,49],[21,12],\ - [51,81],[77,68],[56,65],[43,54]] + [[19,66],[79,14],[19,57],[23,1],\ + [53,76],[42,13],[31,71],[33,5],\ + [54,30],[10,61],[66,3],[27,12],\ + [79,19],[2,29],[16,14],[5,57],\ + [70,73],[46,67],[57,50],[61,49],[21,12],\ + [51,81],[77,68],[56,65],[43,54]] - ] + ] - def _build_experiments(self): + def _build_experiments(self): + """ Creates the whole set of experiment objects, + The experiments created depend on the selected Town. + """ + + # We set the camera + # This single RGB camera is used on every experiment + + camera = Camera('CameraRGB') + camera.set(CameraFOV=100) - # We set the camera that is going to be used for all experiments - camera = Camera('CameraRGB') - camera.set(CameraFOV=100) + camera.set_image_size(800, 600) + camera.set_position(200, 0, 140) + camera.set_rotation(-15.0,0,0) - camera.set_image_size(800, 600) - - camera.set_position(200, 0, 140) - camera.set_rotation(-15.0,0,0) - - weathers = [1,3,6,8,4,14] - if self._city_name == 'Town01': - poses_tasks = self._poses_town01() - vehicles_tasks =[0,0,0,20] - pedestrians_tasks = [0,0,0,50] - else: - poses_tasks = self._poses_town02() - vehicles_tasks =[0,0,0,15] - pedestrians_tasks = [0,0,0,50] + weathers = [1,3,6,8,4,14] + if self._city_name == 'Town01': + poses_tasks = self._poses_town01() + vehicles_tasks =[0,0,0,20] + pedestrians_tasks = [0,0,0,50] + else: + poses_tasks = self._poses_town02() + vehicles_tasks =[0,0,0,15] + pedestrians_tasks = [0,0,0,50] - experiments_vector = [] + experiments_vector = [] - for weather in weathers: + for weather in weathers: - for iteration in range(len(poses_tasks)): + for iteration in range(len(poses_tasks)): - poses = poses_tasks[iteration] - vehicles = vehicles_tasks[iteration] - pedestrians = pedestrians_tasks[iteration] + poses = poses_tasks[iteration] + vehicles = vehicles_tasks[iteration] + pedestrians = pedestrians_tasks[iteration] - conditions = CarlaSettings() - conditions.set( - SynchronousMode=True, - SendNonPlayerAgentsInfo=True, - NumberOfVehicles=vehicles, - NumberOfPedestrians=pedestrians, - WeatherId=weather, - SeedVehicles=123456789, - SeedPedestrians=123456789 - ) - # Add all the cameras that were set for this experiments + conditions = CarlaSettings() + conditions.set( + SynchronousMode=True, + SendNonPlayerAgentsInfo=True, + NumberOfVehicles=vehicles, + NumberOfPedestrians=pedestrians, + WeatherId=weather, + SeedVehicles=123456789, + SeedPedestrians=123456789 + ) + # Add all the cameras that were set for this experiments - conditions.add_sensor(camera) + conditions.add_sensor(camera) - experiment = Experiment() - experiment.set( - Conditions=conditions, - Poses= poses, - Id = iteration, - Repetitions = 1 - ) - experiments_vector.append(experiment) + experiment = Experiment() + experiment.set( + Conditions=conditions, + Poses= poses, + Id = iteration, + Repetitions = 1 + ) + experiments_vector.append(experiment) - return experiments_vector - def _get_details(self): # Function to get automatic information from the experiment for writing purposes - return 'corl_' + self._city_name \ No newline at end of file + return experiments_vector + + def _get_details(self): + # Function to get automatic information from the experiment for writing purposes + return 'corl_' + self._city_name \ No newline at end of file diff --git a/PythonClient/benchmarks/metrics.py b/PythonClient/benchmarks/metrics.py index 3e8bf2859..bd3856ad5 100644 --- a/PythonClient/benchmarks/metrics.py +++ b/PythonClient/benchmarks/metrics.py @@ -278,12 +278,7 @@ def plot_summary(file,summary_weathers): print 'Avg completion', summaries[sw]['completion']/summaries[sw]['num_weathers'] print 'Avg infractions' print_infractions(summaries[sw]['infractions']/summaries[sw]['num_weathers']) - # - # - #infractions_vec = [sum(x) for x in zip(infractions, infractions_vec)] - #print 'Non_Colisions/Km', (infractions[1]+ infractions[0])/2.0 ,'Lane Cross/Km ',infractions[0],'Side Cross/Km ',infractions[1],'Col Gen /Km ',infractions[2]\ - #,'Col Ped /Km ',infractions[3],'Col Ped /Km ',infractions[4], 'Acidents/Km ', (infractions[4] +infractions[2] + infractions[3])/3,\ - #'total', 1/((infractions[4] +infractions[2] + infractions[3] + infractions[1] + infractions[0])/5.0) + return summaries