From d570f5da52ba80b6d4b75b18e10897c49e9b6aed Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Wed, 3 Jan 2018 17:43:40 +0000 Subject: [PATCH] Use a GABBI_TEMPEST_PATH to locate gabbits In this way the plugin becomes generic: point it at some directories that contain gabbi YAML files that use the provided environment variables (see the README.rst), what them run. One concern that will need to be address is dealing with parallelism: tempest doesn't make it easy to manipulate the group_regex. --- README.rst | 38 +++++++--- gabbi_tempest/tests/scenario/__init__.py | 71 ++++++----------- .../tests/scenario/gabbits/compute/base.yaml | 76 ------------------- .../tests/scenario/gabbits/image/base.yaml | 34 --------- .../multi/base.yaml => samples/multi.yaml | 0 5 files changed, 50 insertions(+), 169 deletions(-) delete mode 100644 gabbi_tempest/tests/scenario/gabbits/compute/base.yaml delete mode 100644 gabbi_tempest/tests/scenario/gabbits/image/base.yaml rename gabbi_tempest/tests/scenario/gabbits/multi/base.yaml => samples/multi.yaml (100%) diff --git a/README.rst b/README.rst index e4d3cd3..775fa79 100644 --- a/README.rst +++ b/README.rst @@ -4,12 +4,25 @@ Gabbi + Tempest This is an exploration of running gabbi_ as a tempest plugin. This code is based entirely on the work of Mehdi Abaakouk who made a -tempest plugin for gnocchi_. This code models that but will try to -be more generic, eventually. +tempest plugin for gnocchi_. This code models that but tries to +be more generic: it allows you to set a ``GABBI_TEMPEST_PATH`` +environment variable pointing to multiple directories containing +gabbi YAML files. -For the time being it works with Nova. +The test harness sets a series of enviornment variables that can +be used in the YAML to reach the available services. These will +eventually need to be extended (or can be extended by subclasses). -To experiment with it you need a working tempest installation and +For each service in the service catalog there are +``_SERVICE`` and ``_BASE`` variables +(e.g., ``PLACEMENT_SERVICE`` and ``PLACEMENT_BASE``). A useful +``SERVICE_TOKEN``, ``IMAGE_REF``, ``FLAVOR_REF`` and ``FLAVOR_REF_ALT`` +are also available. + +Trying It +--------- + +To experiment with this you need a working tempest installation and configuration. I used devstack with:: enable_service tempest @@ -26,17 +39,18 @@ If you are using virtualenvs or need sudo, your form will be different. Go to the tempest directory (often ``/opt/stack/tempest``) and run -tempest limiting the test run to gabbi related tests:: +tempest limiting the test run to gabbi related tests and setting +the PATH variable:: - tempest run --regex gabbi - -You can be more specific if you like:: - - tempest run --regex PlacementNovaGabbi + GABBI_TEMPEST_PATH=/path/one:/path/two tempest run --regex gabbi This will run the tests described by the YAML files in -``gabbi_tempest/tests/scenario/gabbits/``. Edit those files and run -the tempest command again for fun and adventure. +``/path/one`` and ``/path/two``. + +There is a sample files in ``samples`` in the repo which you can try +with:: + + GABBI_TEMPEST_PATH=/path/to/samples tempest run --regex gabbi .. _gnocchi: https://review.openstack.org/#/c/301585/ .. _gabbi: https://gabbi.readthedocs.org/ diff --git a/gabbi_tempest/tests/scenario/__init__.py b/gabbi_tempest/tests/scenario/__init__.py index 0272a33..af1e26d 100755 --- a/gabbi_tempest/tests/scenario/__init__.py +++ b/gabbi_tempest/tests/scenario/__init__.py @@ -22,27 +22,36 @@ import tempest.test CONF = config.CONF class GenericGabbiTest(tempest.test.BaseTestCase): - credentials = [] - service_name = None - service_type = None - - @classmethod - def skip_checks(cls): - service = cls.service_name - super(GenericGabbiTest, cls).skip_checks() - if not service: - # FIXME(cdent): Hack to work around discoverability - # weirdness - raise cls.skipException('skipping the base class fake test') - if not CONF.service_available.get(service): - raise cls.skipException('%s support is required' % service) + credentials = ['admin'] @classmethod def resource_setup(cls): super(GenericGabbiTest, cls).resource_setup() - endpoints, token = cls._get_service_auth() + cls._set_environ() + + # We support all enabled services, so use base hosts. + host = 'stub' + url = None + + fallback_dir = os.path.join(os.getcwd(), 'gabbits') + gabbi_paths = os.environ.get('GABBI_TEMPEST_PATH', fallback_dir) + + top_suite = unittest.TestSuite() + + for test_dir in gabbi_paths.split(':'): + dotted_path = test_dir.replace(os.path.sep, '.') + inner_suite = driver.build_tests( + test_dir, unittest.TestLoader(), host=host, url=url, + test_loader_name='tempest.scenario%s' % dotted_path) + top_suite.addTest(inner_suite) + + cls.tests = top_suite + + @classmethod + def _set_environ(cls): # Set test ENVIRON substitutions. + endpoints, token = cls._get_service_auth() for service_type, url in endpoints.items(): name = '%s_SERVICE' % service_type.upper() os.environ[name] = url @@ -53,20 +62,6 @@ class GenericGabbiTest(tempest.test.BaseTestCase): os.environ['FLAVOR_REF'] = CONF.compute.flavor_ref os.environ['FLAVOR_REF_ALT'] = CONF.compute.flavor_ref_alt - if cls.service_type in endpoints: - host = None - url = endpoints[cls.service_type] - else: - host = 'stub' - url = None - - test_dir = os.path.join(os.path.dirname(__file__), 'gabbits', - cls.service_type) - cls.tests = driver.build_tests( - test_dir, unittest.TestLoader(), host=host, url=url, - test_loader_name='tempest.scenario.%s.%s' % ( - cls.__name__, cls.service_type)) - @classmethod def clear_credentials(cls): # FIXME(sileht): We don't want the token to be invalided, but @@ -104,21 +99,3 @@ class GenericGabbiTest(tempest.test.BaseTestCase): # NOTE(sileht): A fake test is needed to have the class loaded # by the test runner pass - - -class NovaGabbiTest(GenericGabbiTest): - credentials = ['admin'] - service_name = 'nova' - service_type = 'compute' - - -class GlanceGabbiTest(GenericGabbiTest): - credentials = ['admin'] - service_name = 'glance' - service_type = 'image' - - -class PlacementNovaGabbiTest(GenericGabbiTest): - credentials = ['admin'] - service_name = 'nova' - service_type = 'multi' diff --git a/gabbi_tempest/tests/scenario/gabbits/compute/base.yaml b/gabbi_tempest/tests/scenario/gabbits/compute/base.yaml deleted file mode 100644 index 075b4ab..0000000 --- a/gabbi_tempest/tests/scenario/gabbits/compute/base.yaml +++ /dev/null @@ -1,76 +0,0 @@ - -defaults: - request_headers: - x-auth-token: $ENVIRON['SERVICE_TOKEN'] - -# NOTE(cdent): The use of complex JSONPath to determine URLs is bad -# for readability. - -tests: - -- name: retrieve root - GET: / - response_json_paths: - # $NETLOC contains the /v2.1 prefix - $.version.links[?rel = "self"].href: /$ENVIRON['COMPUTE_SERVICE']/ - $.version.status: CURRENT - -- name: retrieve empty servers - GET: /servers - response_json_paths: - $.servers: [] - -- name: try bad accept - desc: https://bugs.launchpad.net/nova/+bug/1567966 - xfail: True - GET: /servers - request_headers: - accept: text/plain - status: 406 - -- name: try bad method - desc: https://bugs.launchpad.net/nova/+bug/1567970 - xfail: True - DELETE: /servers - status: 405 - -- name: post bad content-type - desc: https://bugs.launchpad.net/nova/+bug/1567977 - xfail: True - POST: /servers - request_headers: - content-type: text/plain - data: I want a server so badly - status: 415 - -- name: create server - POST: /servers - request_headers: - content-type: application/json - data: - server: - name: new-server-one - imageRef: $ENVIRON['IMAGE_REF'] - flavorRef: $ENVIRON['FLAVOR_REF'] - status: 202 - response_headers: - location: //servers/[a-f0-9-]+/ - -- name: wait for the server to be done - GET: $LOCATION - request_headers: - content-type: application/json - poll: - count: 10 - delay: .5 - response_json_paths: - $.server.status: ACTIVE - -# bookmark link, whatever it is, is busted. Goes to bad version -# - name: get bookmark -# GET: $RESPONSE['$.server.links[?rel = "bookmark"].href'] - -- name: get server - GET: $RESPONSE['$.server.links[?rel = "self"].href'] - response_json_paths: - $.server.name: new-server-one diff --git a/gabbi_tempest/tests/scenario/gabbits/image/base.yaml b/gabbi_tempest/tests/scenario/gabbits/image/base.yaml deleted file mode 100644 index acd89ab..0000000 --- a/gabbi_tempest/tests/scenario/gabbits/image/base.yaml +++ /dev/null @@ -1,34 +0,0 @@ - -defaults: - request_headers: - x-auth-token: $ENVIRON['SERVICE_TOKEN'] - -# NOTE(cdent): The use of complex JSONPath to determine URLs is bad -# for readability. - -tests: - -- name: retrieve root - GET: / - status: 300 - -- name: choose current api - desc: apparently the /v2/ url is a 404, despite being right there in discovery - xfail: true - GET: $RESPONSE['$.versions[?status = "CURRENT"].links[?rel = "self"].href'] - -- name: get images - GET: /v2/images - -- name: get one image - GET: $RESPONSE['$.images[0].self'] - response_json_paths: - $.status: active - $.schema: /v2/schemas/image - -- name: get the schema - GET: $RESPONSE['$.schema'] - response_json_paths: - $.name: image - -# TODO(cdent): add some images diff --git a/gabbi_tempest/tests/scenario/gabbits/multi/base.yaml b/samples/multi.yaml similarity index 100% rename from gabbi_tempest/tests/scenario/gabbits/multi/base.yaml rename to samples/multi.yaml