diff --git a/Makefile b/Makefile index c05df85..3138238 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ test: functional_test: @echo Starting amulet deployment tests... - @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700 + @tox -e func27 bin/charm_helpers_sync.py: @mkdir -p bin diff --git a/test-requirements.txt b/test-requirements.txt index 4faf254..74baa12 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,3 +7,19 @@ flake8>=2.2.4,<=2.4.1 os-testr>=0.4.1 charm-tools>=2.0.0 requests==2.6.0 +# BEGIN: Amulet OpenStack Charm Helper Requirements +# Liberty client lower constraints +amulet>=1.14.3,<2.0 +bundletester>=0.6.1,<1.0 +python-ceilometerclient>=1.5.0,<2.0 +python-cinderclient>=1.4.0,<2.0 +python-glanceclient>=1.1.0,<2.0 +python-heatclient>=0.8.0,<1.0 +python-keystoneclient>=1.7.1,<2.0 +python-neutronclient>=3.1.0,<4.0 +python-novaclient>=2.30.1,<3.0 +python-openstackclient>=1.7.0,<2.0 +python-swiftclient>=2.6.0,<3.0 +pika>=0.10.0,<1.0 +distro-info +# END: Amulet OpenStack Charm Helper Requirements diff --git a/tests/00-setup b/tests/00-setup deleted file mode 100755 index 14711e9..0000000 --- a/tests/00-setup +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -sudo add-apt-repository --yes ppa:juju/stable -sudo apt-get update --yes -sudo apt-get install --yes amulet \ - distro-info-data \ - python-cinderclient \ - python-distro-info \ - python-glanceclient \ - python-heatclient \ - python-keystoneclient \ - python-neutronclient \ - python-novaclient \ - python-pika \ - python-swiftclient diff --git a/tests/016-basic-trusty-juno b/tests/016-basic-trusty-juno deleted file mode 100755 index ae39b77..0000000 --- a/tests/016-basic-trusty-juno +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic cinder-backup deployment on trusty-juno.""" - -from basic_deployment import CinderBackupBasicDeployment - -if __name__ == '__main__': - deployment = CinderBackupBasicDeployment(series='trusty', - openstack='cloud:trusty-juno', - source='cloud:trusty-updates/juno') - deployment.run_tests() diff --git a/tests/README b/tests/README deleted file mode 100644 index 2e81818..0000000 --- a/tests/README +++ /dev/null @@ -1,68 +0,0 @@ -This directory provides Amulet tests that focus on verification of -cinder-backup deployments. - -test_* methods are called in lexical sort order. - -Test name convention to ensure desired test order: - 1xx service and endpoint checks - 2xx relation checks - 3xx config checks - 4xx functional checks - 9xx restarts and other final checks - -Common uses of backend relations in deployments: - - [ cinder, cinder-ceph ] - - [ cinder, cinder-backup ] - - [ cinder-ceph, ceph ] - - [ cinder-backup, ceph ] - -In order to run tests, you'll need charm-tools installed (in addition to -juju, of course): - sudo add-apt-repository ppa:juju/stable - sudo apt-get update - sudo apt-get install charm-tools - -If you use a web proxy server to access the web, you'll need to set the -AMULET_HTTP_PROXY environment variable to the http URL of the proxy server. - -The following examples demonstrate different ways that tests can be executed. -All examples are run from the charm's root directory. - - * To run all tests (starting with 00-setup): - - make test - - * To run a specific test module (or modules): - - juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse - - * To run a specific test module (or modules), and keep the environment - deployed after a failure: - - juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse - - * To re-run a test module against an already deployed environment (one - that was deployed by a previous call to 'juju test --set-e'): - - ./tests/15-basic-trusty-icehouse - -For debugging and test development purposes, all code should be idempotent. -In other words, the code should have the ability to be re-run without changing -the results beyond the initial run. This enables editing and re-running of a -test module against an already deployed environment, as described above. - -Manual debugging tips: - - * Set the following env vars before using the OpenStack CLI as admin: - export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 - export OS_TENANT_NAME=admin - export OS_USERNAME=admin - export OS_PASSWORD=openstack - export OS_REGION_NAME=RegionOne - - * Set the following env vars before using the OpenStack CLI as demoUser: - export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 - export OS_TENANT_NAME=demoTenant - export OS_USERNAME=demoUser - export OS_PASSWORD=password - export OS_REGION_NAME=RegionOne diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..046be7f --- /dev/null +++ b/tests/README.md @@ -0,0 +1,9 @@ +# Overview + +This directory provides Amulet tests to verify basic deployment functionality +from the perspective of this charm, its requirements and its features, as +exercised in a subset of the full OpenStack deployment test bundle topology. + +For full details on functional testing of OpenStack charms please refer to +the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing) +section of the OpenStack Charm Guide. diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 7b66066..43d63f9 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # @@ -45,6 +45,14 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): self._add_relations() self._configure_services() self._deploy() + + u.log.info('Waiting on extended status checks...') + + # XXX: cinder-backup workload status ignored until it grows support + # https://bugs.launchpad.net/bugs/1604580 + exclude_services = ['mysql', 'cinder-backup'] + self._auto_wait_for_status(exclude_services=exclude_services) + self._initialize_tests() def _add_services(self): @@ -118,25 +126,19 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): def _initialize_tests(self): """Perform final initialization before tests get run.""" # Access the sentries for inspecting service units - u.log.debug('!!!!!') - u.log.debug(dir(self.d.sentry)) - - self.mysql_sentry = self.d.sentry.unit['mysql/0'] - self.keystone_sentry = self.d.sentry.unit['keystone/0'] - self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] - self.cinder_sentry = self.d.sentry.unit['cinder/0'] - self.ceph0_sentry = self.d.sentry.unit['ceph/0'] - self.ceph1_sentry = self.d.sentry.unit['ceph/1'] - self.ceph2_sentry = self.d.sentry.unit['ceph/2'] - self.cinder_backup_sentry = self.d.sentry.unit['cinder-backup/0'] + self.mysql_sentry = self.d.sentry['mysql'][0] + self.keystone_sentry = self.d.sentry['keystone'][0] + self.rabbitmq_sentry = self.d.sentry['rabbitmq-server'][0] + self.cinder_sentry = self.d.sentry['cinder'][0] + self.ceph0_sentry = self.d.sentry['ceph'][0] + self.ceph1_sentry = self.d.sentry['ceph'][1] + self.ceph2_sentry = self.d.sentry['ceph'][2] + self.cinder_backup_sentry = self.d.sentry['cinder-backup'][0] u.log.debug('openstack release val: {}'.format( self._get_openstack_release())) u.log.debug('openstack release str: {}'.format( self._get_openstack_release_string())) - # Let things settle a bit original moving forward - time.sleep(30) - # Authenticate admin with keystone self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, user='admin', @@ -322,7 +324,14 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): def get_broker_response(self): broker_request = self.get_broker_request() - response_key = "broker-rsp-cinder-backup-0" + u.log.debug('Broker request: {}'.format(broker_request)) + + response_key = "broker-rsp-{}-{}".format( + self.cinder_backup_sentry.info['service'], + self.cinder_backup_sentry.info['unit'] + ) + u.log.debug('Checking response_key: {}'.format(response_key)) + ceph_sentrys = [self.ceph0_sentry, self.ceph1_sentry, self.ceph2_sentry] @@ -332,6 +341,7 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): broker_response = json.loads(relation_data[response_key]) if (broker_request['request-id'] == broker_response['request-id']): + u.log.debug('broker_response: {}'.format(broker_response)) return broker_response def test_200_cinderbackup_ceph_ceph_relation(self): @@ -374,11 +384,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): if ret: msg = u.relation_error('cinder cinder-backup backup-backend', ret) amulet.raise_status(amulet.FAIL, msg=msg) - broker_response = self.get_broker_response() - if not broker_response or broker_response['exit-code'] != 0: - msg = ('Broker request invalid' - ' or failed: {}'.format(broker_response)) - amulet.raise_status(amulet.FAIL, msg=msg) def test_202_cinderbackup_cinder_backend_relation(self): u.log.debug('Checking cinder-backup:backup-backend to ' @@ -525,15 +530,7 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): unit = self.cinder_sentry conf = '/etc/cinder/cinder.conf' unit_mq = self.rabbitmq_sentry - unit_ks = self.keystone_sentry rel_mq_ci = unit_mq.relation('amqp', 'cinder:amqp') - rel_ks_ci = unit_ks.relation('identity-service', - 'cinder:identity-service') - - auth_uri = 'http://' + rel_ks_ci['auth_host'] + \ - ':' + rel_ks_ci['service_port'] + '/' - auth_url = ('http://%s:%s/' % - (rel_ks_ci['auth_host'], rel_ks_ci['auth_port'])) expected = { 'DEFAULT': { @@ -549,12 +546,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): 'backup_ceph_pool': 'cinder-backup', 'backup_ceph_user': 'cinder-backup' }, - 'keystone_authtoken': { - 'admin_user': rel_ks_ci['service_username'], - 'admin_password': rel_ks_ci['service_password'], - 'admin_tenant_name': rel_ks_ci['service_tenant'], - 'auth_uri': auth_uri - }, 'cinder-ceph': { 'volume_backend_name': 'cinder-ceph', 'volume_driver': 'cinder.volume.drivers.rbd.RBDDriver', @@ -569,18 +560,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): 'rabbit_password': rel_mq_ci['password'], 'rabbit_host': rel_mq_ci['hostname'], } - if self._get_openstack_release() >= self.trusty_liberty: - expected['keystone_authtoken'] = { - 'auth_uri': auth_uri.rstrip('/'), - 'auth_url': auth_url.rstrip('/'), - 'auth_plugin': 'password', - 'project_domain_id': 'default', - 'user_domain_id': 'default', - 'project_name': 'services', - 'username': rel_ks_ci['service_username'], - 'password': rel_ks_ci['service_password'], - 'signing_dir': '/var/cache/cinder' - } if self._get_openstack_release() >= self.trusty_kilo: # Kilo or later @@ -588,8 +567,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): else: # Juno or earlier expected['DEFAULT'].update(expected_rmq) - expected['keystone_authtoken']['auth_host'] = \ - rel_ks_ci['auth_host'] for section, pairs in expected.iteritems(): ret = u.validate_config_data(unit, conf, section, pairs) @@ -631,7 +608,15 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment): u.log.debug('Cinder api check (volumes.list): {}'.format(check)) assert(check == []) - def test_401_create_delete_volume(self): + def test_401_check_broker_reponse(self): + u.log.debug('Checking broker response') + broker_response = self.get_broker_response() + if not broker_response or broker_response['exit-code'] != 0: + msg = ('Broker request invalid' + ' or failed: {}'.format(broker_response)) + amulet.raise_status(amulet.FAIL, msg=msg) + + def test_402_create_delete_volume(self): """Create a cinder volume and delete it.""" u.log.debug('Creating, checking and deleting cinder volume...') vol_new = u.create_cinder_volume(self.cinder) diff --git a/tests/018-basic-vivid-kilo b/tests/dev-basic-xenial-newton similarity index 68% rename from tests/018-basic-vivid-kilo rename to tests/dev-basic-xenial-newton index 9efad00..93fcc99 100755 --- a/tests/018-basic-vivid-kilo +++ b/tests/dev-basic-xenial-newton @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # @@ -14,10 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Amulet tests on a basic cinder-backup deployment on vivid-kilo.""" +"""Amulet tests on a basic cinder backup deployment on xenial-newton.""" from basic_deployment import CinderBackupBasicDeployment if __name__ == '__main__': - deployment = CinderBackupBasicDeployment(series='vivid') + deployment = CinderBackupBasicDeployment(series='xenial', + openstack='cloud:xenial-newton', + source='cloud:xenial-updates/newton') deployment.run_tests() diff --git a/tests/019-basic-wily-liberty b/tests/dev-basic-yakkety-newton similarity index 81% rename from tests/019-basic-wily-liberty rename to tests/dev-basic-yakkety-newton index 966396e..c79d7bd 100755 --- a/tests/019-basic-wily-liberty +++ b/tests/dev-basic-yakkety-newton @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # @@ -14,10 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Amulet tests on a basic cinder-backup deployment on wily-liberty.""" +"""Amulet tests on a basic Cinder backup deployment on yakkety-newton.""" from basic_deployment import CinderBackupBasicDeployment if __name__ == '__main__': - deployment = CinderBackupBasicDeployment(series='wily') + deployment = CinderBackupBasicDeployment(series='yakkety') deployment.run_tests() diff --git a/tests/014-basic-precise-icehouse b/tests/gate-basic-precise-icehouse similarity index 97% rename from tests/014-basic-precise-icehouse rename to tests/gate-basic-precise-icehouse index 7e624ae..06843b7 100755 --- a/tests/014-basic-precise-icehouse +++ b/tests/gate-basic-precise-icehouse @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # diff --git a/tests/015-basic-trusty-icehouse b/tests/gate-basic-trusty-icehouse similarity index 97% rename from tests/015-basic-trusty-icehouse rename to tests/gate-basic-trusty-icehouse index 5009514..0e9863a 100755 --- a/tests/015-basic-trusty-icehouse +++ b/tests/gate-basic-trusty-icehouse @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # diff --git a/tests/017-basic-trusty-kilo b/tests/gate-basic-trusty-kilo similarity index 97% rename from tests/017-basic-trusty-kilo rename to tests/gate-basic-trusty-kilo index 3752f3c..ac759ca 100755 --- a/tests/017-basic-trusty-kilo +++ b/tests/gate-basic-trusty-kilo @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # diff --git a/tests/021-basic-xenial-mitaka b/tests/gate-basic-xenial-mitaka similarity index 97% rename from tests/021-basic-xenial-mitaka rename to tests/gate-basic-xenial-mitaka index ccd6fce..b61481a 100755 --- a/tests/021-basic-xenial-mitaka +++ b/tests/gate-basic-xenial-mitaka @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright 2016 Canonical Ltd # diff --git a/tests/tests.yaml b/tests/tests.yaml index 64e3e2d..e3185c6 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -1,20 +1,17 @@ -bootstrap: true -reset: true -virtualenv: true -makefile: - - lint - - test -sources: - - ppa:juju/stable -packages: - - amulet - - distro-info-data - - python-cinderclient - - python-distro-info - - python-glanceclient - - python-heatclient - - python-keystoneclient - - python-neutronclient - - python-novaclient - - python-pika - - python-swiftclient +# Bootstrap the model if necessary. +bootstrap: True +# Re-use bootstrap node instead of destroying/re-bootstrapping. +reset: True +# Use tox/requirements to drive the venv instead of bundletester's venv feature. +virtualenv: False +# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet. +makefile: [] +# Do not specify juju PPA sources. Juju is presumed to be pre-installed +# and configured in all test runner environments. +#sources: +# Do not specify or rely on system packages. +#packages: +# Do not specify python packages here. Use test-requirements.txt +# and tox instead. ie. The venv is constructed before bundletester +# is invoked. +#python-packages: diff --git a/tox.ini b/tox.ini index a4d9c0d..39b3aad 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ skipsdist = True [testenv] setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 + AMULET_SETUP_TIMEOUT=2700 +passenv = HOME TERM AMULET_HTTP_PROXY AMULET_OS_VIP install_command = pip install --allow-unverified python-apt {opts} {packages} commands = ostestr {posargs} @@ -18,12 +20,56 @@ deps = -r{toxinidir}/requirements.txt basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = flake8 {posargs} hooks unit_tests tests +commands = flake8 {posargs} --exclude */charmhelpers hooks unit_tests tests charm-proof [testenv:venv] commands = {posargs} +[testenv:func27-noop] +# DRY RUN - For Debug +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy + +[testenv:func27] +# Charm Functional Test +# Run all gate tests which are +x (expected to always pass) +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy + +[testenv:func27-smoke] +# Charm Functional Test +# Run a specific test as an Amulet smoke test (expected to always pass) +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy + +[testenv:func27-dfs] +# Charm Functional Test +# Run all deploy-from-source tests which are +x (may not always pass!) +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy + +[testenv:func27-dev] +# Charm Functional Test +# Run all development test targets which are +x (may not always pass!) +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy + [flake8] ignore = E402,E226 exclude = hooks/charmhelpers