Merge pull request #1367 from carla-simulator/jenkins/smoke-tests

Adding smoke tests to Jenkins

Jenkinsfile

@@ -61,6 +61,18 @@ pipeline {
             }
         }
 
+        stage('Smoke Tests') {
+            steps {
+                sh 'DISPLAY= ./Dist/*/LinuxNoEditor/CarlaUE4.sh --carla-rpc-port=3654 --carla-streaming-port=0 > CarlaUE4.log &'
+                sh 'make smoke_tests ARGS="--xml"'
+            }
+            post {
+                always {
+                    archiveArtifacts 'CarlaUE4.log'
+                    junit 'Build/test-results/smoke-tests-*.xml'
+                }
+            }
+        }
     }
 
     post {

PythonAPI/test/smoke/__init__.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+import glob
+import os
+import sys
+import unittest
+
+try:
+    sys.path.append(glob.glob('../../dist/carla-*%d.%d-%s.egg' % (
+        sys.version_info.major,
+        sys.version_info.minor,
+        'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
+except IndexError:
+    pass
+
+import carla
+
+
+TESTING_ADDRESS=('localhost', 3654)
+
+
+class SmokeTest(unittest.TestCase):
+    def setUp(self):
+        self.client = carla.Client(*TESTING_ADDRESS)
+        self.client.set_timeout(2.0)
+
+    def tearDown(self):
+        self.client = None

PythonAPI/test/smoke/test_blueprint.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+import re
+
+from . import SmokeTest
+
+
+class TestBlueprintLibrary(SmokeTest):
+    def test_blueprint_ids(self):
+        library = self.client.get_world().get_blueprint_library()
+        self.assertTrue([x for x in library])
+        self.assertTrue([x for x in library.filter('sensor.*')])
+        self.assertTrue([x for x in library.filter('static.*')])
+        self.assertTrue([x for x in library.filter('vehicle.*')])
+        self.assertTrue([x for x in library.filter('walker.*')])
+        rgx = re.compile(r'\S+\.\S+\.\S+')
+        for bp in library:
+            self.assertTrue(rgx.match(bp.id))
+        rgx = re.compile(r'(vehicle)\.\S+\.\S+')
+        for bp in library.filter('vehicle.*'):
+            self.assertTrue(rgx.match(bp.id))

PythonAPI/test/smoke/test_client.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+import random
+
+from . import SmokeTest
+
+
+class TestClient(SmokeTest):
+    def test_version(self):
+        self.assertEqual(self.client.get_client_version(), self.client.get_server_version())
+
+    def test_reload_world(self):
+        map_name = self.client.get_world().get_map().name
+        world = self.client.reload_world()
+        self.assertEqual(map_name, world.get_map().name)
+
+    def test_load_all_maps(self):
+        map_names = list(self.client.get_available_maps())
+        random.shuffle(map_names)
+        for map_name in map_names:
+            world = self.client.load_world(map_name)
+            self.assertEqual(map_name.split('/')[-1], world.get_map().name)

PythonAPI/test/smoke/test_sync.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+from . import SmokeTest
+
+import carla
+
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+
+
+class TestSynchronousMode(SmokeTest):
+    def setUp(self):
+        super(TestSynchronousMode, self).setUp()
+        self.world = self.client.get_world()
+        self.settings = self.world.get_settings()
+        settings = carla.WorldSettings(
+            no_rendering_mode=False,
+            synchronous_mode=True)
+        self.world.apply_settings(settings)
+
+    def tearDown(self):
+        self.world.apply_settings(self.settings)
+        self.settings = None
+        self.world = None
+        super(TestSynchronousMode, self).tearDown()
+
+
+    def test_camera_on_synchronous_mode(self):
+        cam_bp = self.world.get_blueprint_library().find('sensor.camera.rgb')
+        t = carla.Transform(carla.Location(z=10))
+        camera = self.world.spawn_actor(cam_bp, t)
+        try:
+
+            image_queue = queue.Queue()
+            camera.listen(image_queue.put)
+
+            frame = None
+
+            for _ in range(0, 100):
+                self.world.tick()
+                ts = self.world.wait_for_tick()
+
+                if frame is not None:
+                    self.assertEqual(ts.frame_count, frame + 1)
+
+                frame = ts.frame_count
+
+                image = image_queue.get()
+                self.assertEqual(image.frame_number, ts.frame_count)
+
+        finally:
+            camera.destroy()

PythonAPI/test/test_connection.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Barcelona (UAB).
+#
+# This work is licensed under the terms of the MIT license.
+# For a copy, see <https://opensource.org/licenses/MIT>.
+
+"""Blocks until the simulator is ready or the time-out is met."""
+
+import glob
+import os
+import sys
+
+try:
+    sys.path.append(glob.glob('../dist/carla-*%d.%d-%s.egg' % (
+        sys.version_info.major,
+        sys.version_info.minor,
+        'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
+except IndexError:
+    pass
+
+
+import carla
+
+import argparse
+import time
+
+
+def main():
+    argparser = argparse.ArgumentParser(
+        description=__doc__)
+    argparser.add_argument(
+        '--host',
+        metavar='H',
+        default='127.0.0.1',
+        help='IP of the host server (default: 127.0.0.1)')
+    argparser.add_argument(
+        '-p', '--port',
+        metavar='P',
+        default=2000,
+        type=int,
+        help='TCP port to listen to (default: 2000)')
+    argparser.add_argument(
+        '--timeout',
+        metavar='T',
+        default=10.0,
+        type=float,
+        help='time-out in seconds (default: 10)')
+    args = argparser.parse_args()
+
+    t0 = time.time()
+
+    while args.timeout > (time.time() - t0):
+        try:
+            client = carla.Client(args.host, args.port)
+            client.set_timeout(0.1)
+            print('CARLA %s connected at %s:%d.' % (client.get_server_version(), args.host, args.port))
+            return 0
+        except RuntimeError:
+            pass
+
+    print('Failed to connect to %s:%d.' % (args.host, args.port))
+    return 1
+
+
+if __name__ == '__main__':
+
+    sys.exit(main())

PythonAPI/test/unit/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
 # Barcelona (UAB).
 #
 # This work is licensed under the terms of the MIT license.
@@ -9,7 +9,7 @@ import os
 import sys
 
 try:
-    sys.path.append(glob.glob('../dist/carla-*%d.%d-%s.egg' % (
+    sys.path.append(glob.glob('../../dist/carla-*%d.%d-%s.egg' % (
         sys.version_info.major,
         sys.version_info.minor,
         'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])

PythonAPI/test/unit/test_client.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
 # Barcelona (UAB).
 #
 # This work is licensed under the terms of the MIT license.
@@ -12,7 +12,7 @@ import unittest
 from subprocess import check_output
 
 
-class testClient(unittest.TestCase):
+class TestClient(unittest.TestCase):
     def test_client_version(self):
         c = carla.Client('localhost', 8080)
         v = c.get_client_version()

PythonAPI/test/unit/test_transform.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
 # Barcelona (UAB).
 #
 # This work is licensed under the terms of the MIT license.
@@ -9,7 +9,7 @@ import carla
 import unittest
 
 
-class testLocation(unittest.TestCase):
+class TestLocation(unittest.TestCase):
     def test_default_values(self):
         location = carla.Location()
         self.assertEqual(location.x, 0.0)
@@ -47,7 +47,7 @@ class testLocation(unittest.TestCase):
         self.assertEqual(location.z, 3.0)
 
 
-class testRotation(unittest.TestCase):
+class TestRotation(unittest.TestCase):
     def test_default_values(self):
         rotation = carla.Rotation()
         self.assertEqual(rotation.pitch, 0.0)
@@ -85,7 +85,7 @@ class testRotation(unittest.TestCase):
         self.assertEqual(rotation.roll, 3.0)
 
 
-class testTransform(unittest.TestCase):
+class TestTransform(unittest.TestCase):
     def test_values(self):
         t = carla.Transform()
         self.assertEqual(t.location.x, 0.0)

PythonAPI/test/unit/test_vehicle.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma de
+# Copyright (c) 2019 Computer Vision Center (CVC) at the Universitat Autonoma de
 # Barcelona (UAB).
 #
 # This work is licensed under the terms of the MIT license.
@@ -9,7 +9,7 @@ import carla
 import unittest
 
 
-class testVehicleControl(unittest.TestCase):
+class TestVehicleControl(unittest.TestCase):
     def test_default_values(self):
         c = carla.VehicleControl()
         self.assertEqual(c.throttle, 0.0)
@@ -38,7 +38,7 @@ class testVehicleControl(unittest.TestCase):
         self.assertEqual(c.reverse, True)
 
 
-class testVehiclePhysicsControl(unittest.TestCase):
+class TestVehiclePhysicsControl(unittest.TestCase):
     def test_named_args(self):
 
         torque_curve = [[0, 400],

PythonAPI/test/unit/unittest.cfg

@@ -0,0 +1,4 @@
+[unittest]
+plugins = nose2.plugins.junitxml
+[junit-xml]
+path = test-results.xml

Util/BuildTools/Check.sh

@@ -37,9 +37,11 @@ LIBCARLA_RELEASE=false
 LIBCARLA_DEBUG=false
 PYTHON_API_2=false
 PYTHON_API_3=false
+SMOKE_TESTS_2=false
+SMOKE_TESTS_3=false
 RUN_BENCHMARK=false
 
-OPTS=`getopt -o h --long help,gdb,xml,gtest_args:,all,libcarla-release,libcarla-debug,python-api-2,python-api-3,benchmark -n 'parse-options' -- "$@"`
+OPTS=`getopt -o h --long help,gdb,xml,gtest_args:,all,libcarla-release,libcarla-debug,python-api-2,python-api-3,smoke-2,smoke-3,benchmark -n 'parse-options' -- "$@"`
 
 if [ $? != 0 ] ; then echo "$USAGE_STRING" ; exit 2 ; fi
 
@@ -55,7 +57,7 @@ while true; do
       shift ;;
     --gtest_args )
       GTEST_ARGS="$2";
-      shift ;;
+      shift 2 ;;
     --all )
       LIBCARLA_RELEASE=true;
       LIBCARLA_DEBUG=true;
@@ -74,6 +76,12 @@ while true; do
     --python-api-3 )
       PYTHON_API_3=true;
       shift ;;
+    --smoke-2 )
+      SMOKE_TESTS_2=true;
+      shift ;;
+    --smoke-3 )
+      SMOKE_TESTS_3=true;
+      shift ;;
     --benchmark )
       LIBCARLA_RELEASE=true;
       RUN_BENCHMARK=true;
@@ -89,7 +97,7 @@ while true; do
   esac
 done
 
-if ! { ${LIBCARLA_RELEASE} || ${LIBCARLA_DEBUG} || ${PYTHON_API_2} || ${PYTHON_API_3}; }; then
+if ! { ${LIBCARLA_RELEASE} || ${LIBCARLA_DEBUG} || ${PYTHON_API_2} || ${PYTHON_API_3} || ${SMOKE_TESTS_2} || ${SMOKE_TESTS_3}; }; then
   fatal_error "Nothing selected to be done."
 fi
 
@@ -138,10 +146,10 @@ if ${LIBCARLA_RELEASE} ; then
 fi
 
 # ==============================================================================
-# -- Run Python API tests ------------------------------------------------------
+# -- Run Python API unit tests -------------------------------------------------
 # ==============================================================================
 
-pushd "${CARLA_PYTHONAPI_ROOT_FOLDER}/test" >/dev/null
+pushd "${CARLA_PYTHONAPI_ROOT_FOLDER}/test/unit" >/dev/null
 
 if ${XML_OUTPUT} ; then
   EXTRA_ARGS="-X"
@@ -175,6 +183,51 @@ fi
 
 popd >/dev/null
 
+# ==============================================================================
+# -- Run smoke tests -----------------------------------------------------------
+# ==============================================================================
+
+if ${SMOKE_TESTS_2} || ${SMOKE_TESTS_3} ; then
+  pushd "${CARLA_PYTHONAPI_ROOT_FOLDER}/test" >/dev/null
+    log "Checking connection with the simulator."
+    ./test_connection.py -p 3654 --timeout=60.0
+  popd >/dev/null
+fi
+
+pushd "${CARLA_PYTHONAPI_ROOT_FOLDER}/test/smoke" >/dev/null
+
+if ${XML_OUTPUT} ; then
+  EXTRA_ARGS="-X"
+else
+  EXTRA_ARGS=
+fi
+
+if ${SMOKE_TESTS_2} ; then
+
+  log "Running smoke tests for Python 2."
+
+  /usr/bin/env python2 -m nose2 ${EXTRA_ARGS}
+
+  if ${XML_OUTPUT} ; then
+    mv test-results.xml ${CARLA_TEST_RESULTS_FOLDER}/smoke-tests-2.xml
+  fi
+
+fi
+
+if ${SMOKE_TESTS_3} ; then
+
+  log "Running smoke tests for Python 3."
+
+  /usr/bin/env python3 -m nose2 ${EXTRA_ARGS}
+
+  if ${XML_OUTPUT} ; then
+    mv test-results.xml ${CARLA_TEST_RESULTS_FOLDER}/smoke-tests-3.xml
+  fi
+
+fi
+
+popd >/dev/null
+
 # ==============================================================================
 # -- ...and we are done --------------------------------------------------------
 # ==============================================================================

Util/BuildTools/Linux.mk

@@ -60,6 +60,9 @@ benchmark: LibCarla.release
 	@${CARLA_BUILD_TOOLS_FOLDER}/Check.sh --benchmark $(ARGS)
 	@cat profiler.csv
 
+smoke_tests:
+	@${CARLA_BUILD_TOOLS_FOLDER}/Check.sh --smoke-2 --smoke-3 $(ARGS)
+
 CarlaUE4Editor: LibCarla.server.release
 	@${CARLA_BUILD_TOOLS_FOLDER}/BuildCarlaUE4.sh --build