From 3dd88b6eb343298b8cd6d0a9db3dbc35472725d7 Mon Sep 17 00:00:00 2001 From: gord chung Date: Wed, 1 Feb 2017 09:26:58 -0500 Subject: [PATCH] backport lib caps and test patches to fix stable gate accommodate new oslo.config Ib8fe3f0e6f37f2ca5c0785e73b2c770b71fac9e6 commit 4637aeb2d3aae4dd4b980612f9728b002a9a6de5 fix oslo.db 4.15.0 breakage I69f8ff9f702064e8fc5bf4018ebc6f3b2a8ea1a8 commit f5794af695b9a105f500ef56751a13c8a1dbe6cf also commit b2d64d26acd0f525ef45721f3a07b269eec48e3d commit c062916a81af2961a1fbebda82ac8bef88ba62cc Fix expected content-type and move CORS tests to gabbi I0f23481d5f75694da23d05c9ef88005a0f2c27d7 note: don't include CORS test commit 5a6d3935bc935eb11f2c1dee9ba5ed9c8fe0e86e Put the regex first Ibaffa3a8568ea058d964463df4b76196c4d2bc7a commit e326a0612f21f353091d1d6aacc45167f8cee3a8 Fix some gabbi tests I9a60db296467b08ef32f956471ef5e4f81f462af note: does not include change to live.yaml commit 35c4a08dd4163e42d4af82695fb8642952b63b91 Rename gabbits with _ to have - instead Ida8ce49582d5b8d927858982fae9fdfc267e4be4 commit 6980d7acf44420f5a802d7cd8bb71b7eabc46da4 ceph: fix setup extra I21bfb7177c16b200dcbf1849176c50e1371128cc commit f1021318e783c8492c9a7c594ce48b1a68b78f98 test: move root tests to their own class I68aca7822c781d915d167b152aab4d7deec07200 10975c55ff3cb3dedc28ec1b3aadfb2686706194 sql: default to pymysql I00f050423fd86ffd51d7ef2bb7f5e5928620f412 commit 99d91a0b0b12df51af4327d03092213f2c929d1c cap gnocchiclient Change-Id: Ib2960423cc3b72e1b21a9d9da9b87cdede1ab44c commit 7bf0e3540a889a4943ce3fc2425557981927cb7c --- devstack/gate/post_test_hook.sh | 2 +- devstack/plugin.sh | 2 +- gnocchi/indexer/sqlalchemy.py | 19 ++++++++++++++++--- gnocchi/tests/gabbi/fixtures.py | 5 ++++- .../{archive_rule.yaml => archive-rule.yaml} | 0 gnocchi/tests/gabbi/gabbits/archive.yaml | 1 + gnocchi/tests/gabbi/gabbits/base.yaml | 6 +++--- ...atch_measures.yaml => batch-measures.yaml} | 0 gnocchi/tests/gabbi/gabbits/history.yaml | 2 +- ...anularity.yaml => metric-granularity.yaml} | 0 gnocchi/tests/gabbi/gabbits/metric.yaml | 3 +-- ...egation.yaml => resource-aggregation.yaml} | 0 ...{resource_type.yaml => resource-type.yaml} | 0 gnocchi/tests/gabbi/gabbits/resource.yaml | 9 +++------ ...{search_metric.yaml => search-metric.yaml} | 0 .../indexer/sqlalchemy/test_migrations.py | 13 +++++++++---- gnocchi/tests/test_rest.py | 10 ++++++---- gnocchi/tests/test_statsd.py | 2 +- run-tests.sh | 5 ++--- setup-test-env.sh | 13 ------------- setup.cfg | 11 ++++++----- tox.ini | 10 +++++----- 22 files changed, 60 insertions(+), 53 deletions(-) rename gnocchi/tests/gabbi/gabbits/{archive_rule.yaml => archive-rule.yaml} (100%) rename gnocchi/tests/gabbi/gabbits/{batch_measures.yaml => batch-measures.yaml} (100%) rename gnocchi/tests/gabbi/gabbits/{metric_granularity.yaml => metric-granularity.yaml} (100%) rename gnocchi/tests/gabbi/gabbits/{resource_aggregation.yaml => resource-aggregation.yaml} (100%) rename gnocchi/tests/gabbi/gabbits/{resource_type.yaml => resource-type.yaml} (100%) rename gnocchi/tests/gabbi/gabbits/{search_metric.yaml => search-metric.yaml} (100%) delete mode 100755 setup-test-env.sh diff --git a/devstack/gate/post_test_hook.sh b/devstack/gate/post_test_hook.sh index 6fe8e139..9c3ff39a 100755 --- a/devstack/gate/post_test_hook.sh +++ b/devstack/gate/post_test_hook.sh @@ -57,7 +57,7 @@ sudo chown -R tempest:stack $BASE/data/tempest # Run tests with tempst cd $BASE/new/tempest set +e -sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- --concurrency=$TEMPEST_CONCURRENCY gnocchi +sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- gnocchi --concurrency=$TEMPEST_CONCURRENCY TEMPEST_EXIT_CODE=$? set -e if [[ $TEMPEST_EXIT_CODE != 0 ]]; then diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 8fb0dc77..ebfb1b45 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -127,7 +127,7 @@ function install_gnocchiclient { git_clone_by_name python-gnocchiclient setup_dev_lib python-gnocchiclient else - pip_install gnocchiclient + pip_install 'gnocchiclient<3.0' fi } diff --git a/gnocchi/indexer/sqlalchemy.py b/gnocchi/indexer/sqlalchemy.py index 2414d532..02add6c5 100644 --- a/gnocchi/indexer/sqlalchemy.py +++ b/gnocchi/indexer/sqlalchemy.py @@ -36,7 +36,7 @@ except ImportError: pymysql = None import six import sqlalchemy -import sqlalchemy.engine.url as sqlalchemy_url +from sqlalchemy.engine import url as sqlalchemy_url import sqlalchemy.exc from sqlalchemy import types import sqlalchemy_utils @@ -221,14 +221,27 @@ class SQLAlchemyIndexer(indexer.IndexerDriver): @classmethod def _create_new_database(cls, url): """Used by testing to create a new database.""" - purl = sqlalchemy_url.make_url(url) + purl = sqlalchemy_url.make_url( + cls.dress_url( + url)) purl.database = purl.database + str(uuid.uuid4()).replace('-', '') new_url = str(purl) sqlalchemy_utils.create_database(new_url) return new_url + @staticmethod + def dress_url(url): + # If no explicit driver has been set, we default to pymysql + if url.startswith("mysql://"): + url = sqlalchemy_url.make_url(url) + url.drivername = "mysql+pymysql" + return str(url) + return url + def __init__(self, conf): - conf.set_override("connection", conf.indexer.url, "database") + conf.set_override("connection", + self.dress_url(conf.indexer.url), + "database") self.conf = conf self.facade = PerInstanceFacade(conf) diff --git a/gnocchi/tests/gabbi/fixtures.py b/gnocchi/tests/gabbi/fixtures.py index b6ee59c5..bbf460f2 100644 --- a/gnocchi/tests/gabbi/fixtures.py +++ b/gnocchi/tests/gabbi/fixtures.py @@ -28,6 +28,7 @@ import sqlalchemy.engine.url as sqlalchemy_url import sqlalchemy_utils from gnocchi import indexer +from gnocchi.indexer import sqlalchemy from gnocchi.rest import app from gnocchi import service from gnocchi import storage @@ -102,7 +103,9 @@ class ConfigFixture(fixture.GabbiFixture): # NOTE(jd) All of that is still very SQL centric but we only support # SQL for now so let's say it's good enough. - url = sqlalchemy_url.make_url(conf.indexer.url) + url = sqlalchemy_url.make_url( + sqlalchemy.SQLAlchemyIndexer.dress_url( + conf.indexer.url)) url.database = url.database + str(uuid.uuid4()).replace('-', '') db_url = str(url) diff --git a/gnocchi/tests/gabbi/gabbits/archive_rule.yaml b/gnocchi/tests/gabbi/gabbits/archive-rule.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/archive_rule.yaml rename to gnocchi/tests/gabbi/gabbits/archive-rule.yaml diff --git a/gnocchi/tests/gabbi/gabbits/archive.yaml b/gnocchi/tests/gabbi/gabbits/archive.yaml index e25bf865..0633ae9b 100644 --- a/gnocchi/tests/gabbi/gabbits/archive.yaml +++ b/gnocchi/tests/gabbi/gabbits/archive.yaml @@ -441,6 +441,7 @@ tests: url: /v1/archive_policy method: POST request_headers: + content-type: application/json x-user-id: b45187c5-150b-4730-bcb2-b5e04e234220 x-project-id: 16764ee0-bffe-4843-aa36-04b002cdbc7c data: diff --git a/gnocchi/tests/gabbi/gabbits/base.yaml b/gnocchi/tests/gabbi/gabbits/base.yaml index 31971bbd..c4b92b0d 100644 --- a/gnocchi/tests/gabbi/gabbits/base.yaml +++ b/gnocchi/tests/gabbi/gabbits/base.yaml @@ -7,7 +7,7 @@ tests: desc: Root URL must return information about API versions url: / response_headers: - content-type: application/json; charset=UTF-8 + content-type: /^application\/json/ response_json_paths: $.versions.[0].id: "v1.0" $.versions.[0].status: "CURRENT" @@ -25,7 +25,7 @@ tests: points: 20 status: 201 response_headers: - content-type: /application\/json/ + content-type: /^application\/json/ location: $SCHEME://$NETLOC/v1/archive_policy/test1 response_json_paths: $.name: test1 @@ -95,7 +95,7 @@ tests: desc: Resources index page should return list of type associated with a URL url: /v1/resource/ response_headers: - content-type: application/json; charset=UTF-8 + content-type: /^application\/json/ status: 200 response_json_paths: $.generic: $SCHEME://$NETLOC/v1/resource/generic diff --git a/gnocchi/tests/gabbi/gabbits/batch_measures.yaml b/gnocchi/tests/gabbi/gabbits/batch-measures.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/batch_measures.yaml rename to gnocchi/tests/gabbi/gabbits/batch-measures.yaml diff --git a/gnocchi/tests/gabbi/gabbits/history.yaml b/gnocchi/tests/gabbi/gabbits/history.yaml index b7c28bc4..1721be37 100644 --- a/gnocchi/tests/gabbi/gabbits/history.yaml +++ b/gnocchi/tests/gabbi/gabbits/history.yaml @@ -37,7 +37,7 @@ tests: status: 201 response_headers: location: $SCHEME://$NETLOC/v1/resource/generic/f93450f2-d8a5-4d67-9985-02511241e7d1 - content-type: application/json; charset=UTF-8 + content-type: /^application\/json/ response_json_paths: $.created_by_project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea $.created_by_user_id: 0fbb231484614b1a80131fc22f6afc9c diff --git a/gnocchi/tests/gabbi/gabbits/metric_granularity.yaml b/gnocchi/tests/gabbi/gabbits/metric-granularity.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/metric_granularity.yaml rename to gnocchi/tests/gabbi/gabbits/metric-granularity.yaml diff --git a/gnocchi/tests/gabbi/gabbits/metric.yaml b/gnocchi/tests/gabbi/gabbits/metric.yaml index 8ee0b490..23f2ee33 100644 --- a/gnocchi/tests/gabbi/gabbits/metric.yaml +++ b/gnocchi/tests/gabbi/gabbits/metric.yaml @@ -102,8 +102,7 @@ tests: method: POST request_headers: content-type: plain/text - data: - archive_policy_name: cookies + data: '{"archive_policy_name": "cookies"}' status: 415 - name: create valid metric diff --git a/gnocchi/tests/gabbi/gabbits/resource_aggregation.yaml b/gnocchi/tests/gabbi/gabbits/resource-aggregation.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/resource_aggregation.yaml rename to gnocchi/tests/gabbi/gabbits/resource-aggregation.yaml diff --git a/gnocchi/tests/gabbi/gabbits/resource_type.yaml b/gnocchi/tests/gabbi/gabbits/resource-type.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/resource_type.yaml rename to gnocchi/tests/gabbi/gabbits/resource-type.yaml diff --git a/gnocchi/tests/gabbi/gabbits/resource.yaml b/gnocchi/tests/gabbi/gabbits/resource.yaml index c7882494..e848dba3 100644 --- a/gnocchi/tests/gabbi/gabbits/resource.yaml +++ b/gnocchi/tests/gabbi/gabbits/resource.yaml @@ -135,7 +135,7 @@ tests: status: 201 response_headers: location: $SCHEME://$NETLOC/v1/resource/generic/f93450f2-d8a5-4d67-9985-02511241e7d1 - content-type: application/json; charset=UTF-8 + content-type: /^application\/json/ response_json_paths: $.created_by_project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea $.created_by_user_id: 0fbb231484614b1a80131fc22f6afc9c @@ -163,11 +163,7 @@ tests: x-user-id: 0fbb231484614b1a80131fc22f6afc9c x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea content-type: text/plain - data: - id: f93450f2-d8a5-4d67-9985-02511241e7d1 - started_at: "2014-01-03T02:02:02.000000" - user_id: 0fbb231484614b1a80131fc22f6afc9c - project_id: f3d41b770cc14f0bb94a1d5be9c0e3ea + data: '{"id": "f93450f2-d8a5-4d67-9985-02511241e7d1", "started_at": "2014-01-03T02:02:02.000000", "user_id": "0fbb231484614b1a80131fc22f6afc9c", "project_id": "f3d41b770cc14f0bb94a1d5be9c0e3ea"}' status: 415 # Create a new instance resource, demonstrate that including no data @@ -356,6 +352,7 @@ tests: request_headers: x-user-id: 0fbb231484614b1a80131fc22f6afc9c x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea + content-type: application/json data: host: compute2 diff --git a/gnocchi/tests/gabbi/gabbits/search_metric.yaml b/gnocchi/tests/gabbi/gabbits/search-metric.yaml similarity index 100% rename from gnocchi/tests/gabbi/gabbits/search_metric.yaml rename to gnocchi/tests/gabbi/gabbits/search-metric.yaml diff --git a/gnocchi/tests/indexer/sqlalchemy/test_migrations.py b/gnocchi/tests/indexer/sqlalchemy/test_migrations.py index df6adfcb..920aca77 100644 --- a/gnocchi/tests/indexer/sqlalchemy/test_migrations.py +++ b/gnocchi/tests/indexer/sqlalchemy/test_migrations.py @@ -15,6 +15,7 @@ import abc import mock +import oslo_db.exception from oslo_db.sqlalchemy import test_migrations import six import sqlalchemy_utils @@ -45,6 +46,14 @@ class ModelsMigrationsSync( self.index = indexer.get_driver(self.conf) self.index.connect() self.index.upgrade(nocreate=True, create_legacy_resource_types=True) + self.addCleanup(self._drop_database) + + def _drop_database(self): + try: + sqlalchemy_utils.drop_database(self.conf.indexer.url) + except oslo_db.exception.DBNonExistentDatabase: + # NOTE(sileht): oslo db >= 4.15.0 cleanup this for us + pass @staticmethod def get_metadata(): @@ -56,7 +65,3 @@ class ModelsMigrationsSync( @staticmethod def db_sync(engine): pass - - def tearDown(self): - sqlalchemy_utils.drop_database(self.conf.indexer.url) - super(ModelsMigrationsSync, self).tearDown() diff --git a/gnocchi/tests/test_rest.py b/gnocchi/tests/test_rest.py index d1466cad..d8511a94 100644 --- a/gnocchi/tests/test_rest.py +++ b/gnocchi/tests/test_rest.py @@ -138,6 +138,12 @@ class RestTest(tests_base.TestCase, testscenarios.TestWithScenarios): indexer=self.index, auth=self.auth) + @staticmethod + def runTest(): + pass + + +class RootTest(RestTest): def test_deserialize_force_json(self): with self.app.use_admin_user(): self.app.post( @@ -170,10 +176,6 @@ class RestTest(tests_base.TestCase, testscenarios.TestWithScenarios): self.assertIs(type(status['storage']['summary']['metrics']), int) self.assertIs(type(status['storage']['summary']['measures']), int) - @staticmethod - def runTest(): - pass - class ArchivePolicyTest(RestTest): """Test the ArchivePolicies REST API. diff --git a/gnocchi/tests/test_statsd.py b/gnocchi/tests/test_statsd.py index ec8b2e23..dfb4ed41 100644 --- a/gnocchi/tests/test_statsd.py +++ b/gnocchi/tests/test_statsd.py @@ -35,7 +35,7 @@ class TestStatsd(tests_base.TestCase): super(TestStatsd, self).setUp() self.conf.set_override("resource_id", - uuid.uuid4(), "statsd") + str(uuid.uuid4()), "statsd") self.conf.set_override("user_id", self.STATSD_USER_ID, "statsd") self.conf.set_override("project_id", diff --git a/run-tests.sh b/run-tests.sh index 492725da..06c35801 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -4,10 +4,9 @@ GNOCCHI_TEST_STORAGE_DRIVERS=${GNOCCHI_TEST_STORAGE_DRIVERS:-file} GNOCCHI_TEST_INDEXER_DRIVERS=${GNOCCHI_TEST_INDEXER_DRIVERS:-postgresql} for storage in ${GNOCCHI_TEST_STORAGE_DRIVERS} do + export GNOCCHI_TEST_STORAGE_DRIVER=$storage for indexer in ${GNOCCHI_TEST_INDEXER_DRIVERS} do - export GNOCCHI_TEST_INDEXER_DRIVER=$indexer - export GNOCCHI_TEST_STORAGE_DRIVER=$storage - ./setup-test-env.sh ./tools/pretty_tox.sh $* + pifpaf -g GNOCCHI_INDEXER_URL run $indexer -- ./tools/pretty_tox.sh $* done done diff --git a/setup-test-env.sh b/setup-test-env.sh deleted file mode 100755 index 7d7789fa..00000000 --- a/setup-test-env.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e -set -x -# Activate pifpaf for indexer -GNOCCHI_TEST_INDEXER_DRIVER=${GNOCCHI_TEST_INDEXER_DRIVER:-postgresql} -eval `pifpaf run $GNOCCHI_TEST_INDEXER_DRIVER` -kill_pifpaf () -{ - test -n "$PIFPAF_PID" && kill "$PIFPAF_PID" -} -trap kill_pifpaf EXIT -export GNOCCHI_INDEXER_URL=${PIFPAF_URL/#mysql:/mysql+pymysql:} -$* diff --git a/setup.cfg b/setup.cfg index 0ecc34b8..2603f764 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,13 +23,13 @@ keystone = keystonemiddleware>=4.0.0 mysql = pymysql - oslo.db>=1.8.0 + oslo.db>=1.8.0,!=4.13.1,!=4.13.2,!=4.15.0 sqlalchemy sqlalchemy-utils alembic>=0.7.6,!=0.8.1 postgresql = psycopg2 - oslo.db>=1.8.0 + oslo.db>=1.8.0,!=4.13.1,!=4.13.2,!=4.15.0 sqlalchemy sqlalchemy-utils alembic>=0.7.6,!=0.8.1 @@ -42,9 +42,9 @@ ceph = msgpack-python lz4 tooz>=1.30 -ceph-pre-jewel: +ceph_pre_jewel: cradox>=1.0.9 -ceph-jewel-and-later: +ceph_jewel_and_later: python-rados>=10.1.0 # not available on pypi file = msgpack-python @@ -57,7 +57,7 @@ doc = PyYAML Jinja2 test = - pifpaf + pifpaf>=0.1.0 gabbi>=0.101.2 coverage>=3.6 fixtures @@ -67,6 +67,7 @@ test = os-testr testrepository testscenarios + testresources>=0.2.4 # Apache-2.0/BSD testtools>=0.9.38 WebTest>=2.0.16 doc8 diff --git a/tox.ini b/tox.ini index a5153c7f..c7c1d493 100644 --- a/tox.ini +++ b/tox.ini @@ -47,15 +47,15 @@ commands = {toxinidir}/tools/pretty_tox.sh '{posargs}' deps = .[test,postgresql,file] setenv = OS_TEST_PATH=gnocchi/tests/gabbi basepython = python2.7 -commands = {toxinidir}/setup-test-env.sh {toxinidir}/tools/pretty_tox.sh '{posargs}' +commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- {toxinidir}/tools/pretty_tox.sh '{posargs}' [testenv:py27-cover] -commands = {toxinidir}/setup-test-env.sh python setup.py testr --coverage --testr-args="{posargs}" +commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py testr --coverage --testr-args="{posargs}" [testenv:venv] # This is used by the doc job on the gate deps = {[testenv:docs]deps} -commands = {toxinidir}/setup-test-env.sh {posargs} +commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- {posargs} [flake8] exclude = .tox,.eggs,doc @@ -73,9 +73,9 @@ deps = .[test,postgresql,file,doc] setenv = GNOCCHI_TEST_STORAGE_DRIVER=file GNOCCHI_TEST_INDEXER_DRIVER=postgresql commands = doc8 --ignore-path doc/source/rest.rst doc/source - {toxinidir}/setup-test-env.sh python setup.py build_sphinx + pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- python setup.py build_sphinx [testenv:docs-gnocchi.xyz] deps = .[file,postgresql,test,doc] sphinx_rtd_theme -commands = {toxinidir}/setup-test-env.sh sphinx-build -D html_theme=sphinx_rtd_theme doc/source doc/build/html +commands = pifpaf -g GNOCCHI_INDEXER_URL run postgresql -- sphinx-build -D html_theme=sphinx_rtd_theme doc/source doc/build/html