tempest: rework gabbi setup

The current approach is a bit hacky and create one
tempest scenario per gabbi HTTP call.

The side effect is that tests continue to run even the previous
one have fail.

This changes the approach by running gabbi to have one scenario
per yaml file from tempest point of view.

This will make easier to debug the scenario in case of failure.

Change-Id: Ice4be203798a222b4b521540e637fb6ec0c9c01e

gnocchi/tempest/scenario/__init__.py

@@ -15,31 +15,35 @@ from __future__ import absolute_import
 import os
 import unittest
 
-from gabbi import driver
+from gabbi import runner
+from gabbi import suitemaker
+from gabbi import utils
 import six.moves.urllib.parse as urlparse
 from tempest import config
 import tempest.test
 
 CONF = config.CONF
 
+TEST_DIR = os.path.join(os.path.dirname(__file__), '..', '..',
+                        'tests', 'functional_live', 'gabbits')
+
 
 class GnocchiGabbiTest(tempest.test.BaseTestCase):
     credentials = ['admin']
 
+    TIMEOUT_SCALING_FACTOR = 5
+
     @classmethod
     def skip_checks(cls):
         super(GnocchiGabbiTest, cls).skip_checks()
         if not CONF.service_available.gnocchi:
             raise cls.skipException("Gnocchi support is required")
 
-    @classmethod
-    def resource_setup(cls):
-        super(GnocchiGabbiTest, cls).resource_setup()
-
-        url = cls.os_admin.auth_provider.base_url(
+    def _do_test(self, filename):
+        token = self.os_admin.auth_provider.get_token()
+        url = self.os_admin.auth_provider.base_url(
             {'service':  CONF.metric.catalog_type,
              'endpoint_type': CONF.metric.endpoint_type})
-        token = cls.os_admin.auth_provider.get_token()
 
         parsed_url = urlparse.urlsplit(url)
         prefix = parsed_url.path.rstrip('/')  # turn it into a prefix
@@ -53,34 +57,54 @@ class GnocchiGabbiTest(tempest.test.BaseTestCase):
         if parsed_url.port:
             port = parsed_url.port
 
-        test_dir = os.path.join(os.path.dirname(__file__), '..', '..',
-                                'tests', 'functional_live', 'gabbits')
-        cls.tests = driver.build_tests(
-            test_dir, unittest.TestLoader(),
-            host=host, port=port, prefix=prefix,
-            test_loader_name='tempest.scenario.gnocchi.test',
-            require_ssl=require_ssl)
-
         os.environ["GNOCCHI_SERVICE_TOKEN"] = token
         os.environ["GNOCCHI_AUTHORIZATION"] = "not used"
 
-    @classmethod
-    def clear_credentials(cls):
-        # FIXME(sileht): We don't want the token to be invalided, but
-        # for some obcurs reason, clear_credentials is called before/during run
-        # So, make the one used by tearDropClass a dump, and call it manually
-        # in run()
-        pass
+        with file(os.path.join(TEST_DIR, filename)) as f:
+            suite_dict = utils.load_yaml(f)
+            suite_dict.setdefault('defaults', {})['ssl'] = require_ssl
+            test_suite = suitemaker.test_suite_from_dict(
+                loader=unittest.defaultTestLoader,
+                test_base_name="gabbi",
+                suite_dict=suite_dict,
+                test_directory=TEST_DIR,
+                host=host, port=port,
+                fixture_module=None,
+                intercept=None,
+                prefix=prefix,
+                handlers=runner.initialize_handlers([]),
+                test_loader_name="tempest")
 
-    def run(self, result=None):
-        self.setUp()
-        try:
-            self.tests.run(result)
-        finally:
-            super(GnocchiGabbiTest, self).clear_credentials()
-            self.tearDown()
+            # NOTE(sileht): We hide stdout/stderr and reraise the failure
+            # manually, tempest will print it itself.
+            with open(os.devnull, 'w') as stream:
+                result = unittest.TextTestRunner(
+                    stream=stream, verbosity=0, failfast=True,
+                ).run(test_suite)
 
-    def test_fake(self):
-        # NOTE(sileht): A fake test is needed to have the class loaded
-        # by the test runner
-        pass
+            if not result.wasSuccessful():
+                failures = (result.errors + result.failures +
+                            result.unexpectedSuccesses)
+                if failures:
+                    test, bt = failures[0]
+                    name = test.test_data.get('name', test.id())
+                    msg = 'From test "%s" :\n%s' % (name, bt)
+                    self.fail(msg)
+
+            self.assertTrue(result.wasSuccessful())
+
+
+def test_maker(name, filename):
+    def test(self):
+        self._do_test(filename)
+        test.__name__ = name
+    return test
+
+
+# Create one scenario per yaml file
+for filename in os.listdir(TEST_DIR):
+    if not filename.endswith('.yaml'):
+        continue
+    name = "test_%s" % filename[:-5].lower().replace("-", "_")
+    setattr(GnocchiGabbiTest, name,
+            test_maker(name, filename))