Renaming, extra final refactoring

Docs/benchmark.md

@@ -1,4 +1,4 @@
-Benchmark
+Agent Benchmark
 ===============
 
 Short introduction ( Similar to the blog post but shorter)
@@ -29,7 +29,7 @@ Run the help command to see options available.
 
 Now we will show 
 
-Benchmark Structure
+Agent Benchmark Structure
 -------------------
 
 
@@ -39,27 +39,34 @@ Benchmark Structure
 
 
 
-The *benchmark*  is composed by set of *Experiments*.
-Each *experiment* is related to *Task* that contains a set of poses.
-These *poses* are  are tuples containing a start and an end point.
+The *benchmark* is the module responsible for evaluating a certain
+*agent* in a *experiment suite*. 
+  
+The *experiment suite* is composed by set of *experiments*.
+Each *experiment* is related to *task* that contains a set of poses.
+These *poses* are  are tuples containing a start and end point of a navigation
+task.
 The *experiments* also are associated with a *condition* which is
 a [carla settings](carla_settings.md) object. The conditions specify
 simulation parameters such as: weather, sensor suite, number of
 vehicles and pedestrians, etc.
 
 
-
-The base benchmark class contains the general execution process.
-The actual set of experiments must be defined in the derived 
-class. 
-
-The user also should define the agent class. The agent is the active
-part which will be benchmarked.
+The user must define its own experiment suite to be tested
+on an agent. The user also should define the agent class. The agent is the active
+part which will be evaluated by the agent benchmark module.
 
 
+To know how see: creating your set of experiments tutorial.
+
+
+CORL 2017 
+----------------------
+
+We already provide the CORL 2017 experiment suite used to benchmark the
+agents for the CORL 2017 paper.
 
 
-Creating a benchmark tutorial.
 
 
 

Docs/creating_benchmark.md

@@ -0,0 +1,2 @@
+
+

PythonClient/benchmark_example.py

@@ -10,8 +10,7 @@ import argparse
 import logging
 import time
 
-from carla.benchmarks.agent import Agent
-from carla.benchmarks.corl_2017 import CoRL2017
+from carla.agent_benchmark.agent import Agent
 
 from carla.client import make_carla_client, VehicleControl
 from carla.tcp import TCPConnectionError
@@ -29,6 +28,35 @@ class Manual(Agent):
         return control
 
 
+
+# Here I will more or less build a new class.
+def run_benchmark():
+
+
+    while True:
+        try:
+
+            with make_carla_client(args.host, args.port) as client:
+
+                #corl = CoRL2017(city_name=args.city_name, name_to_save=args.log_name)
+
+                benchmark = Benchmark_AI(experi)
+
+                agent = Manual(args.city_name)
+                results = corl.benchmark_agent(agent, client)
+                corl.plot_summary_test()
+                corl.plot_summary_train()
+
+                break
+
+        except TCPConnectionError as error:
+            logging.error(error)
+            time.sleep(1)
+
+
+
+
+
 if __name__ == '__main__':
 
     argparser = argparse.ArgumentParser(description=__doc__)
@@ -65,6 +93,13 @@ if __name__ == '__main__':
         default='test',
         help='The name of the log file to be created by the benchmark'
         )
+    argparser.add_argument(
+        '-n', '--log_name',
+        metavar='T',
+        default='test',
+        help='The name of the log file to be created by the benchmark'
+        )
+
 
     args = argparser.parse_args()
     if args.debug:
@@ -77,20 +112,3 @@ if __name__ == '__main__':
     logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level)
     logging.info('listening to server %s:%s', args.host, args.port)
 
-
-    while True:
-        try:
-
-            with make_carla_client(args.host, args.port) as client:
-
-                corl = CoRL2017(city_name=args.city_name, name_to_save=args.log_name)
-                agent = Manual(args.city_name)
-                results = corl.benchmark_agent(agent, client)
-                corl.plot_summary_test()
-                corl.plot_summary_train()
-
-                break
-
-        except TCPConnectionError as error:
-            logging.error(error)
-            time.sleep(1)

PythonClient/carla/agent_benchmark/benchmark.py

@@ -25,8 +25,10 @@ def sldist(c1, c2):
 
 class Benchmark(object):
     """
-    The Benchmark class, controls the execution of the benchmark by an
-    Agent class.
+    The Benchmark class, controls the execution of the benchmark interfacing
+    an Agent class with a set Suite.
+
+
     The benchmark class must be inherited with a class that defines the
     all the experiments to be run by the agent
     """

PythonClient/carla/agent_benchmark/experiment_suite/corl_2017.py

@@ -11,12 +11,12 @@ from __future__ import print_function
 import os
 import numpy as np
 
-from .benchmark import Benchmark
-from .experiment import Experiment
+from carla.agent_benchmark.benchmark import Benchmark
+from carla.agent_benchmark.experiment import Experiment
 from carla.sensor import Camera
 from carla.settings import CarlaSettings
 
-from .metrics import Metrics
+from carla.agent_benchmark.metrics import Metrics
 
 
 class CoRL2017(Benchmark):

PythonClient/carla/agent_benchmark/experiment_suite/sample.py

@@ -11,14 +11,14 @@ from __future__ import print_function
 import os
 import numpy as np
 
-from .benchmark import Benchmark
-from .experiment import Experiment
+from carla.agent_benchmark.benchmark import Benchmark
+from carla.agent_benchmark.experiment import Experiment
 from carla.sensor import Camera
 from carla.settings import CarlaSettings
 
 from carla.planner.planner import Planner
 
-from .metrics import Metrics
+from carla.agent_benchmark.metrics import Metrics
 
 
 class SampleBenchmark(Benchmark):

PythonClient/setup.py

@@ -5,7 +5,7 @@ from setuptools import setup
 setup(
     name='carla_client',
     version='0.7.1',
-    packages=['carla', 'carla.benchmarks', 'carla.planner'],
+    packages=['carla', 'carla.agent_benchmark', 'carla.planner'],
     license='MIT License',
     description='Python API for communicating with the CARLA server.',
     url='https://github.com/carla-simulator/carla',

PythonClient/test/acceptance_tests/test_benchmark_module.py

@@ -8,11 +8,6 @@
 
 from __future__ import print_function
 
-import os
-import numpy as np
-
-
-import argparse
 import logging
 import time
 
@@ -21,10 +16,9 @@ import time
 
 import unittest
 
-from carla.benchmarks.agent import Agent
-from carla.benchmarks.benchmark import Benchmark
+from carla.agent_benchmark.agent import Agent
 
-from carla.benchmarks.sample_benchmark import SampleBenchmark
+from carla.agent_benchmark.experiment_suite.sample import SampleBenchmark
 
 from carla.client import make_carla_client, VehicleControl
 from carla.tcp import TCPConnectionError

PythonClient/test/unit_tests/test_metrics.py

@@ -1,8 +1,8 @@
 import os
 import numpy as np
 import unittest
-from carla.benchmarks.metrics import Metrics
-from carla.benchmarks.recording import Recording
+from carla.agent_benchmark.metrics import Metrics
+from carla.agent_benchmark.recording import Recording
 
 
 
@@ -54,7 +54,7 @@ class testMetrics(unittest.TestCase):
 
 
 
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
         from carla.carla_server_pb2 import Measurements
         from carla.carla_server_pb2 import Control
 

PythonClient/test/unit_tests/test_recording.py

@@ -1,6 +1,6 @@
 
 import unittest
-from carla.benchmarks.recording import Recording
+from carla.agent_benchmark.recording import Recording
 
 class testRecording(unittest.TestCase):
 
@@ -29,7 +29,7 @@ class testRecording(unittest.TestCase):
     def test_write_summary_results(self):
 
         import os
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
 
         recording = Recording(name_to_save='Test1'
                               , continue_experiment=False, save_images=True
@@ -58,7 +58,7 @@ class testRecording(unittest.TestCase):
     def teste_write_measurements_results(self):
 
         import os
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
         from carla.carla_server_pb2 import Measurements
         from carla.carla_server_pb2 import Control
 
@@ -101,7 +101,7 @@ class testRecording(unittest.TestCase):
         # If you don't want to continue, should return also one
         self.assertEqual(recording._continue_experiment(False)[1], 1)
 
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
 
         recording.write_summary_results(experiment=Experiment(), pose=[24, 32], rep=1,
                                          path_distance=200, remaining_distance=0,
@@ -126,7 +126,7 @@ class testRecording(unittest.TestCase):
 
 
 
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
 
 
         pose, experiment = recording.get_pose_and_experiment(25)
@@ -169,7 +169,7 @@ class testRecording(unittest.TestCase):
     def test_get_pose_and_experiment_corner(self):
         print 'Pose '
 
-        from carla.benchmarks.experiment import Experiment
+        from carla.agent_benchmark.experiment import Experiment
 
         recording = Recording( name_to_save='Test1'
                               , continue_experiment=False, save_images=True