From 552c63849ed961035473bc8f72ed4bed1f5b2446 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 9 Jan 2015 19:32:55 +0000 Subject: [PATCH] Declarative HTTP testing for the Ceilometer API A 'gabbi' tox target is added which runs a declarative HTTP tests described in YAML files in 'ceilometer/tests/gabbi/gabbits' and loaded by 'ceilometer/tests/gabbi/test_gabbi.py'. These are driven by the 'gabbi' python package (available from PyPI). tox and testr are configured to start and run the tests efficiently: * a mongodb server, using multiple databases, is made available * the API wsgi application is used directly via 'wsgi-intercept', no web server required * each YAML file is run as a sequence and where number of processors allows, in a different test process * individual tests can be requested in the usual way: tox -egabbi -- If this is done, all the tests prior to the one requested, from its YAML file, will be run as ordered prerequisites. * tox targets that already run the tests in ceilometer/tests will also discover gabbi tests. If there is no mongodb, they will be skipped. A ConfigFixture does the necessary work of adjusting the configuration and pipeline to use the mongodb database and nothing else. An internal InterceptFixture uses wsgi-intercept to access the ceilometer API. Each yaml file has its own intercepted host. Fixtures are implemented as nested context managers that are declared per YAML file, see ceilometer/gabbi/fixtures.py and fixtures-samples.yaml for an example of how they can be used. Every yaml file uses at least ConfigFixture. YAML files can use a variety of strategies for formatting requests and evaluating the correctness of response. See: http://gabbi.readthedocs.org/en/latest/format.html The YAML files included here test simple API features for creating and retrieving samples. Subsequent patches can (and should) create additional YAML files to describe more complex scenarios that cover the entire API (for example alarms are not touched at all by this patch). Change-Id: I52551f88bc3beac4bf8a92afa45ac70cd97ffcec Implements: blueprint declarative-http-tests --- .testr.conf | 4 +- ceilometer/tests/gabbi/__init__.py | 0 ceilometer/tests/gabbi/fixtures.py | 116 ++++++++++++++++++ ceilometer/tests/gabbi/gabbits/basic.yaml | 24 ++++ .../tests/gabbi/gabbits/capabilities.yaml | 16 +++ .../tests/gabbi/gabbits/clean-samples.yaml | 75 +++++++++++ .../tests/gabbi/gabbits/fixture-samples.yaml | 18 +++ ceilometer/tests/gabbi/test_gabbi.py | 37 ++++++ doc/source/contributing/source.rst | 13 ++ etc/ceilometer/gabbi_pipeline.yaml | 19 +++ test-requirements.txt | 1 + tox.ini | 11 ++ 12 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 ceilometer/tests/gabbi/__init__.py create mode 100644 ceilometer/tests/gabbi/fixtures.py create mode 100644 ceilometer/tests/gabbi/gabbits/basic.yaml create mode 100644 ceilometer/tests/gabbi/gabbits/capabilities.yaml create mode 100644 ceilometer/tests/gabbi/gabbits/clean-samples.yaml create mode 100644 ceilometer/tests/gabbi/gabbits/fixture-samples.yaml create mode 100644 ceilometer/tests/gabbi/test_gabbi.py create mode 100644 etc/ceilometer/gabbi_pipeline.yaml diff --git a/.testr.conf b/.testr.conf index dbdbc68173..4b4a9d6551 100644 --- a/.testr.conf +++ b/.testr.conf @@ -2,6 +2,8 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-600} \ - ${PYTHON:-python} -m subunit.run discover ceilometer/tests -t . $LISTOPT $IDOPTION + ${PYTHON:-python} -m subunit.run discover ${OS_TEST_PATH:-./ceilometer/tests} -t . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list +# NOTE(chdent): Only used/matches on gabbi-related tests. +group_regex=(gabbi\.driver\.test_gabbi_[^_]+)_ diff --git a/ceilometer/tests/gabbi/__init__.py b/ceilometer/tests/gabbi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ceilometer/tests/gabbi/fixtures.py b/ceilometer/tests/gabbi/fixtures.py new file mode 100644 index 0000000000..a9c9d38a6b --- /dev/null +++ b/ceilometer/tests/gabbi/fixtures.py @@ -0,0 +1,116 @@ +# +# Copyright 2015 Red Hat. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Fixtures used during Gabbi-based test runs.""" + +from __future__ import print_function + +import datetime +import os +import random +from unittest import case +import uuid + +from gabbi import fixture +from oslo.config import fixture as fixture_config + +from ceilometer.publisher import utils +from ceilometer import sample +from ceilometer import service +from ceilometer import storage + + +# TODO(chdent): For now only MongoDB is supported, because of easy +# database name handling and intentional focus on the API, not the +# data store. +ENGINES = ['MONGODB'] + + +class ConfigFixture(fixture.GabbiFixture): + """Establish the relevant configuration for a test run.""" + + def start_fixture(self): + """Set up config.""" + + service.prepare_service([]) + conf = fixture_config.Config().conf + self.conf = conf + conf.import_opt('policy_file', 'ceilometer.openstack.common.policy') + conf.set_override('policy_file', + os.path.abspath('etc/ceilometer/policy.json')) + + # A special pipeline is required to use the direct publisher. + conf.set_override('pipeline_cfg_file', + 'etc/ceilometer/gabbi_pipeline.yaml') + + # Determine the database connection. + db_url = None + for engine in ENGINES: + try: + db_url = os.environ['CEILOMETER_TEST_%s_URL' % engine] + except KeyError: + pass + if db_url is None: + raise case.SkipTest('No database connection configured') + + database_name = '%s-%s' % (db_url, str(uuid.uuid4())) + conf.set_override('connection', database_name, group='database') + conf.set_override('metering_connection', '', group='database') + conf.set_override('event_connection', '', group='database') + conf.set_override('alarm_connection', '', group='database') + + conf.set_override('pecan_debug', True, group='api') + + def stop_fixture(self): + """Clean up the config.""" + self.conf.reset() + + +class SampleDataFixture(fixture.GabbiFixture): + """Instantiate some sample data for use in testing.""" + + def start_fixture(self): + """Create some samples.""" + conf = fixture_config.Config().conf + self.conn = storage.get_connection_from_config(conf) + timestamp = datetime.datetime.utcnow() + project_id = str(uuid.uuid4()) + self.source = str(uuid.uuid4()) + resource_metadata = {'farmed_by': 'nancy'} + + for name in ['cow', 'pig', 'sheep']: + resource_metadata.update({'breed': name}), + c = sample.Sample(name='livestock', + type='gauge', + unit='head', + volume=int(10 * random.random()), + user_id='farmerjon', + project_id=project_id, + resource_id=project_id, + timestamp=timestamp, + resource_metadata=resource_metadata, + source=self.source, + ) + data = utils.meter_message_from_counter( + c, conf.publisher.telemetry_secret) + self.conn.record_metering_data(data) + + def stop_fixture(self): + """Destroy the samples.""" + # NOTE(chdent): print here for sake of info during testing. + # This will go away eventually. + print('resource', + self.conn.db.resource.remove({'source': self.source})) + print('meter', self.conn.db.meter.remove({'source': self.source})) diff --git a/ceilometer/tests/gabbi/gabbits/basic.yaml b/ceilometer/tests/gabbi/gabbits/basic.yaml new file mode 100644 index 0000000000..2a67e51014 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/basic.yaml @@ -0,0 +1,24 @@ +# +# Some simple tests just to confirm that the system works. +# +fixtures: + - ConfigFixture + +tests: + +# Root gives us some information on where to go from here. +- name: quick root check + url: / + response_headers: + content-type: application/json; charset=UTF-8 + response_strings: + - '"base": "application/json"' + response_json_paths: + versions.values.[0].status: stable + versions.values.[0].media-types.[0].base: application/json + +# NOTE(chdent): Ideally since / has a links ref to /v2, /v2 ought not 404! +- name: v2 visit + desc: this demonstrates a bug in the info in / + url: $RESPONSE['versions.values.[0].links.[0].href'] + status: 404 diff --git a/ceilometer/tests/gabbi/gabbits/capabilities.yaml b/ceilometer/tests/gabbi/gabbits/capabilities.yaml new file mode 100644 index 0000000000..320fde47a3 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/capabilities.yaml @@ -0,0 +1,16 @@ +# +# Explore the capabilities API +# +fixtures: + - ConfigFixture + +tests: + +- name: get capabilities + desc: retrieve capabilities for the mongo store + url: /v2/capabilities + response_json_paths: + $.alarm_storage.['storage:production_ready']: true + $.event_storage.['storage:production_ready']: true + $.storage.['storage:production_ready']: true + $.api.['meters:pagination']: false diff --git a/ceilometer/tests/gabbi/gabbits/clean-samples.yaml b/ceilometer/tests/gabbi/gabbits/clean-samples.yaml new file mode 100644 index 0000000000..eff56efb05 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/clean-samples.yaml @@ -0,0 +1,75 @@ +# Post a simple sample, sir, and the retrieve it in various ways. +fixtures: + - ConfigFixture + +tests: +- name: post sample for meter + desc: post a single sample + url: /v2/meters/apples + method: POST + request_headers: + content-type: application/json + data: | + [ + { + "counter_name": "apples", + "project_id": "35b17138-b364-4e6a-a131-8f3099c5be68", + "user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff", + "counter_unit": "instance", + "counter_volume": 1, + "resource_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36", + "resource_metadata": { + "name2": "value2", + "name1": "value1" + }, + "counter_type": "gauge" + } + ] + + response_json_paths: + $.[0].counter_name: apples + status: 200 + response_headers: + content-type: application/json; charset=UTF-8 + +# TODO(chdent): No location header in the response!!!! +# - name: get sample +# desc: get the sample we just posted +# url: $LOCATION + +- name: get samples for meter + desc: get all the samples at that meter + url: /v2/meters/apples + response_json_paths: + $.[0].counter_name: apples + $.[0].counter_volume: 1 + $.[0].resource_metadata.name2: value2 + +- name: get resources + desc: get the resources that exist because of the sample + url: /v2/resources + response_json_paths: + $.[0].metadata.name2: value2 + +# NOTE(chdent): We assume that the first item in links is self. +# Need to determine how to express the more correct JSONPath here +# (if possible). +- name: get resource + desc: get just one of those resources via self + url: $RESPONSE['$[0].links[0].href'] + response_json_paths: + $.metadata.name2: value2 + +- name: get samples + desc: get all the created samples + url: /v2/samples + response_json_paths: + $.[0].metadata.name2: value2 + $.[0].meter: apples + +- name: get one sample + desc: get the one sample that exists + url: /v2/samples/$RESPONSE['$[0].id'] + response_json_paths: + $.metadata.name2: value2 + $.meter: apples diff --git a/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml b/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml new file mode 100644 index 0000000000..94369703b6 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml @@ -0,0 +1,18 @@ +# +# Demonstrate a simple sample fixture. +# +fixtures: + - ConfigFixture + - SampleDataFixture + +tests: +- name: get fixture samples + desc: get all the samples at livestock + url: /v2/meters/livestock + response_json_paths: + $.[0].counter_name: livestock + $.[1].counter_name: livestock + $.[2].counter_name: livestock + $.[2].user_id: farmerjon + $.[0].resource_metadata.breed: cow + $.[1].resource_metadata.farmed_by: nancy diff --git a/ceilometer/tests/gabbi/test_gabbi.py b/ceilometer/tests/gabbi/test_gabbi.py new file mode 100644 index 0000000000..e4c256a8fe --- /dev/null +++ b/ceilometer/tests/gabbi/test_gabbi.py @@ -0,0 +1,37 @@ +# +# Copyright 2015 Red Hat. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A test module to exercise the Ceilometer API with gabbi + +For the sake of exploratory development. +""" + +import os + +from gabbi import driver + +from ceilometer.api import app +from ceilometer.tests.gabbi import fixtures as fixture_module + + +TESTS_DIR = 'gabbits' + + +def load_tests(loader, tests, pattern): + """Provide a TestSuite to the discovery process.""" + test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR) + return driver.build_tests(test_dir, loader, host=None, + intercept=app.VersionSelectorApplication, + fixture_module=fixture_module) diff --git a/doc/source/contributing/source.rst b/doc/source/contributing/source.rst index 823650c274..35e0650d9b 100644 --- a/doc/source/contributing/source.rst +++ b/doc/source/contributing/source.rst @@ -88,6 +88,19 @@ run through tox_. For reference, the ``debug`` tox environment implements the instructions here: https://wiki.openstack.org/wiki/Testr#Debugging_.28pdb.29_Tests +5. There is a growing suite of tests which use a tool called `gabbi`_ to + test and validate the behavior of the Ceilometer API. These tests are run + when using the usual ``py27`` tox target but if desired they can be run by + themselves:: + + $ tox -e gabbi + + The YAML files used to drive the gabbi tests can be found in + ``ceilometer/tests/gabbi/gabbits``. If you are adding to or adjusting the + API you should consider adding tests here. + +.. _gabbi: https://gabbi.readthedocs.org/ + .. seealso:: * tox_ diff --git a/etc/ceilometer/gabbi_pipeline.yaml b/etc/ceilometer/gabbi_pipeline.yaml new file mode 100644 index 0000000000..e90516f0be --- /dev/null +++ b/etc/ceilometer/gabbi_pipeline.yaml @@ -0,0 +1,19 @@ +# A limited pipeline for use with the Gabbi spike. +# direct writes to the the metering database without using an +# intermediary dispatcher. +# +# This is one of several things that will need some extensive +# tidying to be more right. +--- +sources: + - name: meter_source + interval: 1 + meters: + - "*" + sinks: + - meter_sink +sinks: + - name: meter_sink + transformers: + publishers: + - direct:// diff --git a/test-requirements.txt b/test-requirements.txt index c5b6f3fa6c..d3d35303e5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,3 +25,4 @@ sphinxcontrib-pecanwsme>=0.8 testrepository>=0.0.18 testscenarios>=0.4 testtools>=0.9.36,!=1.2.0 +gabbi>=0.5.0 diff --git a/tox.ini b/tox.ini index d1958d73f1..cf21cea81d 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,17 @@ deps = -r{toxinidir}/requirements-py3.txt commands = python -m testtools.run \ ceilometer.tests.test_utils +# NOTE(chdent): The gabbi tests are also run under the primary tox +# targets. This target simply provides a target to directly run just +# gabbi tests without needing to discovery across the entire body of +# tests. +[testenv:gabbi] +setenv = OS_TEST_PATH=ceilometer/tests/gabbi +commands = + bash -x {toxinidir}/setup-test-env-mongodb.sh \ + python setup.py testr --testr-args="{posargs}" + + [testenv:cover] commands = bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --coverage --testr-args="{posargs}"