Initial stab and a generic gabbi plugin for tempest

This is entirely the result of @sileht, who got things working for
gnocchi.

The end game here is a generic way to have a suite of YAML files
sitting around somewhere and run them, via tempest, and for it
to just work. This is currently some distance away from that
(because I got distracted finding nova bugs).

.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+.coverage
+.testrepository/
+.tox
+doc/build/*
+*.egg*

README.rst

@@ -0,0 +1,37 @@
+===============
+Gabbi + Tempest
+===============
+
+This is an exploration of running gabbi_ as a tempest plugin. This
+code is based entirely on the work of Mehdi Abaakouk who made a
+tempest plugin for gnocchi_. This code models that but will try to
+be more generic, eventually.
+
+For the time being it works with Nova.
+
+To experiment with it you need a working tempest installation and
+configuration. I used devstack with::
+
+    enable_service tempest
+
+in local.conf.
+
+Once tempest is confirmed to be working, make a clone of this repo,
+cd into it and do the equivalent of::
+
+    pip install -e .
+
+If you are using virtualenvs or need sudo, your form will be
+different.
+
+Go to the tempest directory and run testr limit the test run to
+gabbi related tests::
+
+    testr run gabbi --subunit |subunit-trace
+
+This will run the tests described by the YAML files in
+``gabbi_tempest/tests/scenario/gabbits/``. Edit those files and run
+the testr command again for fun and adventure.
+
+.. _gnocchi: https://review.openstack.org/#/c/301585/
+.. _gabbi: https://gabbi.readthedocs.org/

gabbi_tempest/config.py

@@ -0,0 +1,11 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

gabbi_tempest/plugin.py

@@ -0,0 +1,34 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os
+
+from tempest import config
+from tempest.test_discover import plugins
+
+from gabbi_tempest import config as project_config
+
+
+class GabbiTempestPlugin(plugins.TempestPlugin):
+    def load_tests(self):
+        base_path = os.path.split(os.path.dirname(
+            os.path.abspath(__file__)))[0]
+        test_dir = "gabbi_tempest/tests"
+        full_test_dir = os.path.join(base_path, test_dir)
+        return full_test_dir, base_path
+
+    def register_opts(self, conf):
+        pass
+
+    def get_opt_lists(self):
+        pass

gabbi_tempest/tests/scenario/__init__.py

@@ -0,0 +1,90 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import unittest
+
+from gabbi import driver
+import six.moves.urllib.parse as urlparse
+from tempest import config
+import tempest.test
+
+CONF = config.CONF
+
+
+class GenericGabbiTest(tempest.test.BaseTestCase):
+    credentials = ['admin']
+    # NOTE(cdent): WTF? 'nova' being the thing in service_available?
+    # Boo!
+    service_name = 'nova'
+    service_type = 'compute'
+
+    @classmethod
+    def skip_checks(cls):
+        service = cls.service_name
+        super(GenericGabbiTest, cls).skip_checks()
+        if not CONF.service_available.get(service):
+            raise cls.skipException('%s support is required' % service)
+
+    @classmethod
+    def resource_setup(cls):
+        super(GenericGabbiTest, cls).resource_setup()
+        url, token = cls._get_service_auth()
+        parsed_url = urlparse.urlsplit(url)
+        prefix = parsed_url.path.rstrip('/')  # turn it into a prefix
+        port = 443 if parsed_url.scheme == 'https' else 80
+        host = parsed_url.hostname
+        if parsed_url.port:
+            port = parsed_url.port
+
+        test_dir = os.path.join(os.path.dirname(__file__), 'gabbits')
+        cls.tests = driver.build_tests(
+            test_dir, unittest.TestLoader(),
+            host=host, port=port, prefix=prefix,
+            test_loader_name='tempest.scenario.gabbi')
+
+        os.environ["SERVICE_TOKEN"] = token
+        # TODO(cdent): not very generic!
+        os.environ['IMAGE_REF'] = CONF.compute.image_ref
+        os.environ['FLAVOR_REF'] = CONF.compute.flavor_ref
+
+    @classmethod
+    def clear_credentials(cls):
+        # FIXME(sileht): We don't want the token to be invalided, but
+        # for some obscure reason, clear_credentials is called before/during
+        # run. So, make the one used by tearDropClass a dump, and call it
+        # manually in run()
+        pass
+
+    def run(self, result=None):
+        self.setUp()
+        try:
+            self.tests.run(result)
+        finally:
+            super(GenericGabbiTest, self).clear_credentials()
+            self.tearDown()
+
+    @classmethod
+    def _get_service_auth(cls):
+        endpoint_type = 'publicURL'
+
+        auth = cls.os_admin.auth_provider.get_auth()
+        endpoints = [e for e in auth[1]['serviceCatalog']
+                     if e['type'] == cls.service_type]
+        if not endpoints:
+            raise Exception("%s endpoint not found" % cls.service_type)
+        return endpoints[0]['endpoints'][0][endpoint_type], auth[0]
+
+    def test_fake(self):
+        # NOTE(sileht): A fake test is needed to have the class loaded
+        # by the test runner
+        pass

gabbi_tempest/tests/scenario/gabbits/base.yaml

@@ -0,0 +1,73 @@
+
+defaults:
+    request_headers:
+        x-auth-token: $ENVIRON['SERVICE_TOKEN']
+
+tests:
+
+- name: retrieve root
+  GET: /
+  response_json_paths:
+      # $NETLOC contains the /v2.1 prefix
+      $.version.links[?rel = "self"].href: $SCHEME://$NETLOC/
+      $.version.status: CURRENT
+
+- name: retrieve empty servers
+  GET: /servers
+  response_json_paths:
+      $.servers: []
+
+- name: try bad accept
+  desc: https://bugs.launchpad.net/nova/+bug/1567966
+  xfail: True
+  GET: /servers
+  request_headers:
+      accept: text/plain
+  status: 406
+
+- name: try bad method
+  desc: https://bugs.launchpad.net/nova/+bug/1567970
+  xfail: True
+  DELETE: /servers
+  status: 405
+
+- name: post bad content-type
+  desc: https://bugs.launchpad.net/nova/+bug/1567977
+  xfail: True
+  POST: /servers
+  request_headers:
+      content-type: text/plain
+  data: I want a server so badly
+  status: 415
+
+- name: create server
+  POST: /servers
+  request_headers:
+      content-type: application/json
+  data:
+      server:
+          name: new-server-one
+          imageRef: $ENVIRON['IMAGE_REF']
+          flavorRef: $ENVIRON['FLAVOR_REF']
+  status: 202
+  response_headers:
+      location: //servers/[a-f0-9-]+/
+
+- name: wait for the server to be done
+  GET: $LOCATION
+  request_headers:
+      content-type: application/json
+  poll:
+      count: 10
+      delay: .5
+  response_json_paths:
+      $.server.status: ACTIVE
+
+- name: get bookmark
+  GET: $RESPONSE['$.server.links[?rel = "bookmark"].href']
+  status: 300
+
+- name: get 2.1 server via bookmark
+  GET: $RESPONSE['$.choices[?status = "CURRENT"].links[?rel = "self"].href']
+  response_json_paths:
+      $.server.name: new-server-one

requirements.txt

@@ -0,0 +1 @@
+gabbi

setup.cfg

@@ -0,0 +1,33 @@
+[metadata]
+name = gabbi-tempest
+author = Chris Dent
+author-email = cdent@anticdent.org
+summary = Run Gabbi Tests as Tempest Plugin
+description-file = README.rst
+license = Apache-2
+home-page = https://github.com/cdent/gabbi-tempest
+classifier =
+    Intended Audience :: Developers
+    Intended Audience :: Information Technology
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
+    Topic :: Software Development :: Testing
+
+[files]
+packages =
+    gabbi_tempest
+
+[build_sphinx]
+all_files = 1
+build-dir = docs/build
+source-dir = docs/source
+
+[entry_points]
+tempest.test_plugins =
+    plugin_name = gabbi_tempest.plugin:GabbiTempestPlugin

setup.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import setuptools
+
+
+setuptools.setup(
+    setup_requires=['pbr'],
+    pbr=True)