Use a GABBI_TEMPEST_PATH to locate gabbits

In this way the plugin becomes generic: point it at some directories
that contain gabbi YAML files that use the provided environment
variables (see the README.rst), what them run.

One concern that will need to be address is dealing with parallelism:
tempest doesn't make it easy to manipulate the group_regex.

README.rst

@@ -4,12 +4,25 @@ Gabbi + Tempest
 
 This is an exploration of running gabbi_ as a tempest plugin. This
 code is based entirely on the work of Mehdi Abaakouk who made a
-tempest plugin for gnocchi_. This code models that but will try to
-be more generic, eventually.
+tempest plugin for gnocchi_. This code models that but tries to
+be more generic: it allows you to set a ``GABBI_TEMPEST_PATH``
+environment variable pointing to multiple directories containing
+gabbi YAML files.
 
-For the time being it works with Nova.
+The test harness sets a series of enviornment variables that can
+be used in the YAML to reach the available services. These will
+eventually need to be extended (or can be extended by subclasses).
 
-To experiment with it you need a working tempest installation and
+For each service in the service catalog there are
+``<SERVICE_TYPE>_SERVICE`` and ``<SERVICE_TYPE>_BASE`` variables
+(e.g., ``PLACEMENT_SERVICE`` and ``PLACEMENT_BASE``). A useful
+``SERVICE_TOKEN``, ``IMAGE_REF``, ``FLAVOR_REF`` and ``FLAVOR_REF_ALT``
+are also available.
+
+Trying It
+---------
+
+To experiment with this you need a working tempest installation and
 configuration. I used devstack with::
 
     enable_service tempest
@@ -26,17 +39,18 @@ If you are using virtualenvs or need sudo, your form will be
 different.
 
 Go to the tempest directory (often ``/opt/stack/tempest``) and run
-tempest limiting the test run to gabbi related tests::
+tempest limiting the test run to gabbi related tests and setting
+the PATH variable::
 
-    tempest run --regex gabbi
-
-You can be more specific if you like::
-
-    tempest run --regex PlacementNovaGabbi
+    GABBI_TEMPEST_PATH=/path/one:/path/two tempest run --regex gabbi
 
 This will run the tests described by the YAML files in
-``gabbi_tempest/tests/scenario/gabbits/``. Edit those files and run
-the tempest command again for fun and adventure.
+``/path/one`` and ``/path/two``.
+
+There is a sample files in ``samples`` in the repo which you can try
+with::
+
+    GABBI_TEMPEST_PATH=/path/to/samples tempest run --regex gabbi
 
 .. _gnocchi: https://review.openstack.org/#/c/301585/
 .. _gabbi: https://gabbi.readthedocs.org/

gabbi_tempest/tests/scenario/__init__.py

@@ -22,27 +22,36 @@ import tempest.test
 CONF = config.CONF
 
 class GenericGabbiTest(tempest.test.BaseTestCase):
-    credentials = []
-    service_name = None
-    service_type = None
-
-    @classmethod
-    def skip_checks(cls):
-        service = cls.service_name
-        super(GenericGabbiTest, cls).skip_checks()
-        if not service:
-            # FIXME(cdent): Hack to work around discoverability
-            # weirdness
-            raise cls.skipException('skipping the base class fake test')
-        if not CONF.service_available.get(service):
-            raise cls.skipException('%s support is required' % service)
+    credentials = ['admin']
 
     @classmethod
     def resource_setup(cls):
         super(GenericGabbiTest, cls).resource_setup()
-        endpoints, token = cls._get_service_auth()
 
+        cls._set_environ()
+
+        # We support all enabled services, so use base hosts.
+        host = 'stub'
+        url = None
+
+        fallback_dir = os.path.join(os.getcwd(), 'gabbits')
+        gabbi_paths = os.environ.get('GABBI_TEMPEST_PATH', fallback_dir)
+
+        top_suite = unittest.TestSuite()
+
+        for test_dir in gabbi_paths.split(':'):
+            dotted_path = test_dir.replace(os.path.sep, '.')
+            inner_suite = driver.build_tests(
+                    test_dir, unittest.TestLoader(), host=host, url=url,
+                    test_loader_name='tempest.scenario%s' % dotted_path)
+            top_suite.addTest(inner_suite)
+
+        cls.tests = top_suite
+
+    @classmethod
+    def _set_environ(cls):
         # Set test ENVIRON substitutions.
+        endpoints, token = cls._get_service_auth()
         for service_type, url in endpoints.items():
             name = '%s_SERVICE' % service_type.upper()
             os.environ[name] = url
@@ -53,20 +62,6 @@ class GenericGabbiTest(tempest.test.BaseTestCase):
         os.environ['FLAVOR_REF'] = CONF.compute.flavor_ref
         os.environ['FLAVOR_REF_ALT'] = CONF.compute.flavor_ref_alt
 
-        if cls.service_type in endpoints:
-            host = None
-            url = endpoints[cls.service_type]
-        else:
-            host = 'stub'
-            url = None
-
-        test_dir = os.path.join(os.path.dirname(__file__), 'gabbits',
-                                cls.service_type)
-        cls.tests = driver.build_tests(
-            test_dir, unittest.TestLoader(), host=host, url=url,
-            test_loader_name='tempest.scenario.%s.%s' % (
-                cls.__name__, cls.service_type))
-
     @classmethod
     def clear_credentials(cls):
         # FIXME(sileht): We don't want the token to be invalided, but
@@ -104,21 +99,3 @@ class GenericGabbiTest(tempest.test.BaseTestCase):
         # NOTE(sileht): A fake test is needed to have the class loaded
         # by the test runner
         pass
-
-
-class NovaGabbiTest(GenericGabbiTest):
-    credentials = ['admin']
-    service_name = 'nova'
-    service_type = 'compute'
-
-
-class GlanceGabbiTest(GenericGabbiTest):
-    credentials = ['admin']
-    service_name = 'glance'
-    service_type = 'image'
-
-
-class PlacementNovaGabbiTest(GenericGabbiTest):
-    credentials = ['admin']
-    service_name = 'nova'
-    service_type = 'multi'

gabbi_tempest/tests/scenario/gabbits/compute/base.yaml

@@ -1,76 +0,0 @@
-
-defaults:
-    request_headers:
-        x-auth-token: $ENVIRON['SERVICE_TOKEN']
-
-# NOTE(cdent): The use of complex JSONPath to determine URLs is bad
-# for readability. 
-
-tests:
-
-- name: retrieve root
-  GET: /
-  response_json_paths:
-      # $NETLOC contains the /v2.1 prefix
-      $.version.links[?rel = "self"].href: /$ENVIRON['COMPUTE_SERVICE']/
-      $.version.status: CURRENT
-
-- name: retrieve empty servers
-  GET: /servers
-  response_json_paths:
-      $.servers: []
-
-- name: try bad accept
-  desc: https://bugs.launchpad.net/nova/+bug/1567966
-  xfail: True
-  GET: /servers
-  request_headers:
-      accept: text/plain
-  status: 406
-
-- name: try bad method
-  desc: https://bugs.launchpad.net/nova/+bug/1567970
-  xfail: True
-  DELETE: /servers
-  status: 405
-
-- name: post bad content-type
-  desc: https://bugs.launchpad.net/nova/+bug/1567977
-  xfail: True
-  POST: /servers
-  request_headers:
-      content-type: text/plain
-  data: I want a server so badly
-  status: 415
-
-- name: create server
-  POST: /servers
-  request_headers:
-      content-type: application/json
-  data:
-      server:
-          name: new-server-one
-          imageRef: $ENVIRON['IMAGE_REF']
-          flavorRef: $ENVIRON['FLAVOR_REF']
-  status: 202
-  response_headers:
-      location: //servers/[a-f0-9-]+/
-
-- name: wait for the server to be done
-  GET: $LOCATION
-  request_headers:
-      content-type: application/json
-  poll:
-      count: 10
-      delay: .5
-  response_json_paths:
-      $.server.status: ACTIVE
-
-# bookmark link, whatever it is, is busted. Goes to bad version
-# - name: get bookmark
-#   GET: $RESPONSE['$.server.links[?rel = "bookmark"].href']
-
-- name: get server
-  GET: $RESPONSE['$.server.links[?rel = "self"].href']
-  response_json_paths:
-      $.server.name: new-server-one

gabbi_tempest/tests/scenario/gabbits/image/base.yaml

@@ -1,34 +0,0 @@
-
-defaults:
-    request_headers:
-        x-auth-token: $ENVIRON['SERVICE_TOKEN']
-
-# NOTE(cdent): The use of complex JSONPath to determine URLs is bad
-# for readability. 
-
-tests:
-
-- name: retrieve root
-  GET: /
-  status: 300
-
-- name: choose current api
-  desc: apparently the /v2/ url is a 404, despite being right there in discovery
-  xfail: true
-  GET: $RESPONSE['$.versions[?status = "CURRENT"].links[?rel = "self"].href']
-
-- name: get images
-  GET: /v2/images
-  
-- name: get one image
-  GET: $RESPONSE['$.images[0].self']
-  response_json_paths:
-      $.status: active
-      $.schema: /v2/schemas/image
-
-- name: get the schema
-  GET: $RESPONSE['$.schema']
-  response_json_paths:
-      $.name: image
-
-# TODO(cdent): add some images