From 490cddccaac907faed5e809485769c41467daac6 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 15 Nov 2016 19:57:51 +0000 Subject: [PATCH] First commit of manila charm This patchset contains the code/charm for a working manila fileshare service. On its own, the fileshare the charm installs is not functional as a backend is needed. The charm-manila-generic plugin charm is provided separately to provide the generic NFS file share service. This patchset also includes the amulet/bundle tests to test that the charm installs the manila software and can get it running. However, no functional tests of the actual manila software are done. This patchset is dependent on the interface-manila-plugin interface and an updated version of charms.openstack that provides the 'options' member. It also depends on a slight change to the interface-neutron-plugin which adds a requires.py to allow it to be used to plugin to principal charms: these are declared below. Change-Id: Ie9bb7af1baab8b3bc20d26d907d9b51957eb326e Depends-On: Ied0ad014ab7b1d4778113b0d3f2bbae08075372e Depends-On: If6d103b4f62c95b0fa76562a18e418e0d319e987 Depends-On: I8760f2f9bec85ccc8b149b9560a6eed3e9ab418b --- .gitignore | 7 + .testr.conf | 8 + HACKING.md | 10 + LICENSE | 202 ++++++ Makefile | 11 + README.md | 26 + TODO.md | 50 ++ requirements.txt | 2 + src/README.md | 173 ++++++ src/config.yaml | 63 ++ src/layer.yaml | 8 + src/lib/__init__.py | 13 + src/lib/charm/__init__.py | 13 + src/lib/charm/openstack/__init__.py | 13 + src/lib/charm/openstack/manila.py | 334 ++++++++++ src/metadata.yaml | 29 + src/reactive/__init__.py | 13 + src/reactive/manila_handlers.py | 115 ++++ src/templates/mitaka/api-paste.ini | 59 ++ src/templates/mitaka/logging.conf | 77 +++ src/templates/mitaka/manila.conf | 81 +++ src/test-requirements.txt | 22 + src/tests/README.md | 9 + src/tests/basic_deployment.py | 583 ++++++++++++++++++ src/tests/gate-basic-xenial-mitaka | 10 + src/tests/tests.yaml | 17 + src/tox.ini | 53 ++ test-requirements.txt | 7 + tox.ini | 53 ++ unit_tests/__init__.py | 45 ++ unit_tests/test_lib_charm_openstack_manila.py | 283 +++++++++ unit_tests/test_manila_handlers.py | 90 +++ 32 files changed, 2479 insertions(+) create mode 100644 .gitignore create mode 100644 .testr.conf create mode 100644 HACKING.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 TODO.md create mode 100644 requirements.txt create mode 100644 src/README.md create mode 100644 src/config.yaml create mode 100644 src/layer.yaml create mode 100644 src/lib/__init__.py create mode 100644 src/lib/charm/__init__.py create mode 100644 src/lib/charm/openstack/__init__.py create mode 100644 src/lib/charm/openstack/manila.py create mode 100644 src/metadata.yaml create mode 100644 src/reactive/__init__.py create mode 100644 src/reactive/manila_handlers.py create mode 100644 src/templates/mitaka/api-paste.ini create mode 100644 src/templates/mitaka/logging.conf create mode 100644 src/templates/mitaka/manila.conf create mode 100644 src/test-requirements.txt create mode 100644 src/tests/README.md create mode 100644 src/tests/basic_deployment.py create mode 100755 src/tests/gate-basic-xenial-mitaka create mode 100644 src/tests/tests.yaml create mode 100644 src/tox.ini create mode 100644 test-requirements.txt create mode 100644 tox.ini create mode 100644 unit_tests/__init__.py create mode 100644 unit_tests/test_lib_charm_openstack_manila.py create mode 100644 unit_tests/test_manila_handlers.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a68c28d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +build +.tox +layers +interfaces +trusty +.testrepository +__pycache__ diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..801646b --- /dev/null +++ b/.testr.conf @@ -0,0 +1,8 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..8249e9d --- /dev/null +++ b/HACKING.md @@ -0,0 +1,10 @@ +# Overview + +This charm is developed as part of the OpenStack Charms project, and as such you +should refer to the [OpenStack Charm Development Guide](https://github.com/openstack/charm-guide) for details on how +to contribute to this charm. + +You can find its source code here: . + + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fd13171 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +#!/usr/bin/make + +clean: + find . -iname '*.pyc' -delete + find . -iname '__pycache__' -delete + +default: + echo "Doing nothing -- run 'make clean'" + +all: default + diff --git a/README.md b/README.md new file mode 100644 index 0000000..6764edc --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# Manila Source Charm + +THIS CHARM IS FOR EXPERIMENTAL USE AT PRESENT. This is a pre-release charm for +the Manila service to enable testing and to inform further development. It +shouldn't be used in production environments yet. Note that the OpenStack +manila service *is* production ready (according to their website). + +This repository is for the reactive, layered, +[Manila](https://wiki.openstack.org/wiki/Manila) _source_ charm. + +Please see the src/README.md for details on the built Manila charm and how to +use it. + +## Building the charm + +To build the charm run the following command in the root of the repository: + +```bash +$ tox -e build +``` + +The resultant built charm will be in the builds directory. + +## Development/Hacking of the charm + +Please see HACKING.md in this directory. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..78e7735 --- /dev/null +++ b/TODO.md @@ -0,0 +1,50 @@ +TODO +==== + + * Add roles to the manila charm: api, scheduler, data, process, (all) + * Ensure that HA is supported properly on the charm (and tested). + * Pause/Resume - implement the pause/resume actions that other charms support. + +## Add roles: + +It's necessary for the manila charm to be able to install itself as one of a +number of roles: + + 1. The manila-api: this provides the API to the rest of OpenStack. Until this + is HA aware, only ONE manila-api can be provisioned because it registers + itself with the keystone identity-service. + 2. The manila-scheduler: Responsible for scheduling/routing requests to the + appropriate manila-share service. It does that by picking one back-end + while filtering all except one back-end. + 3. The manila-share process: Responsible for managing Shared File Service + devices, specifically the back-end devices. + 4. The manila-data process: This is responsible, in the manila system, for + data operations such as copying, migration, backups, etc. It's not clear + how far progressed that this service is. + +Currently, the manila charm installs exactly one unit with all of the shared +services on the same unit. This is fine for testing, but won't be particularly +suited for a production environment. + +So, it is proposed to enable configuration of the charm to enable it to install +any/all of the roles. This will then allow two manila (juju) applications to +be installed, such that (say) manila-api and manila-scheduler roles can be +configured as one (juju) application, and the manila-share as a seperate +application. Manila allows for serveral, different, manila-share instances to +be deployed, which would mean a single manila-api/scheduler (juju) application +and several (juju) applications, one each for each different manila-share +instance in the OpenStack cloud. + +## Support HA Mode + +The charm has been implemented using the `HAOpenStackCharm` class, which means +that the plumbing is available to support multiple juju units, each with a +manila instance, with the API endpoints provided via an vip. However, this has +not been tested, and before it can be declared 'production ready', this HA +modes need to be tested alongside the 'roles' discussed above. + +## Pause/Resume + +The charm does not implement the Pause/Resume actions that the other OpenStack +charms support. This needs to be implemented if the charm will be a +well-behaved citizen like the other charms. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..96d5c76 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +charm-tools +simplejson diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..74070af --- /dev/null +++ b/src/README.md @@ -0,0 +1,173 @@ +# Overview + +Pre-release charm for testing: + +This charm provides the Manila shared file service for an OpenStack Cloud. It +installs a single instance that, on its own, can't be used. + +In order to use the manila charm, a suitable backend charm is needed to +configure a share backend. At the time of writing (Dec 2016) the only backend +charm available for testing is the 'generic backend' charm called +'manila-generic'. This is used to configure a generic fileshare backend that +can implement an NFS server that then uses a cinder backend block storage +service to provide the share instances. + +Without a backend subordinate charm related to the manila-charm there will be +no manila backends configured; the manila charm will be stuck in the blocked +state. + + +## Manila share backends are configured using subordinate charms + +It's necessary to have the ability to configure a share backend independently +of the main charm. This means that plugin charms will be used to configure +each backend. Multiple backend charms can be related to the manila charm to +allow a manaila (juju) application to support multiple share backends. + +Essentially, a plugin needs to be able to configure: + + - it's section in the manila.conf along with any network plugin's that it + needs (assuming that it's a share that manages it's own share-instance). + - ensure that the relevant services are restarted. + +This pre-release of manila provides (in the charm store): + + - charm-manila: the main charm, + - interface-manila-plugin : the interface for plugging in the generic + backend (and other interfaces), + - charm-manila-generic: the plugin for configuring the generic backend. + +The backend provides a piece of the manila.conf configuration file with +the sections necessary to configure the backend. This is mostly for the share, +rather than the api level. + +# Usage + +Manila (plus manila-generic) relies on services from the mysql/percona, +rabbitmq-server, keystone charms, and a storage backend charm. The following +yaml file will create a small, unconfigured, OpenStack system with the +necessary components to start testing with Manila. Note that these target the +'next' OpenStack charms which are essentially 'edge' charms. + +```yaml + +# vim: set ts=2 et: +# Juju 2.0 deploy bundle for development ('next') charms +# UOSCI relies on this for OS-on-OS deployment testing +series: xenial +automatically-retry-hooks: False +services: + mysql: + charm: cs:~openstack-charmers/xenial/percona-cluster + num_units: 1 + constraints: mem=1G + options: + dataset-size: 50% + root-password: mysql + rabbitmq-server: + charm: cs:~openstack-charmers/xenial/rabbitmq-server + num_units: 1 + constraints: mem=1G + keystone: + charm: cs:~openstack-charmers/xenial/keystone + num_units: 1 + constraints: mem=1G + options: + admin-password: openstack + admin-token: ubuntutesting + preferred-api-version: "2" + glance: + charm: cs:~openstack-charmers/xenial/glance + num_units: 1 + constraints: mem=1G + nova-cloud-controller: + charm: cs:~openstack-charmers/xenial/nova-cloud-controller + num_units: 1 + constraints: mem=1G + options: + network-manager: Neutron + nova-compute: + charm: cs:~openstack-charmers/xenial/nova-compute + num_units: 1 + constraints: mem=4G + neutron-gateway: + charm: cs:~openstack-charmers/xenial/neutron-gateway + num_units: 1 + constraints: mem=1G + options: + bridge-mappings: physnet1:br-ex + instance-mtu: 1300 + neutron-api: + charm: cs:~openstack-charmers/xenial/neutron-api + num_units: 1 + constraints: mem=1G + options: + neutron-security-groups: True + flat-network-providers: physnet1 + neutron-openvswitch: + charm: cs:~openstack-charmers/xenial/neutron-openvswitch + cinder: + charm: cs:~openstack-charmers/xenial/cinder + num_units: 1 + constraints: mem=1G + options: + block-device: vdb + glance-api-version: 2 + overwrite: 'true' + ephemeral-unmount: /mnt + manila: + charm: cs:~openstack-charmers/xenial/manila + num_units: 1 + options: + debug: True + manila-generic: + charm: cs:~openstack-charmers/xenial/manila-generic + options: + debug: True +relations: + - [ keystone, mysql ] + - [ manila, mysql ] + - [ manila, rabbitmq-server ] + - [ manila, keystone ] + - [ manila, manila-generic ] + - [ glance, keystone] + - [ glance, mysql ] + - [ glance, "cinder:image-service" ] + - [ nova-compute, "rabbitmq-server:amqp" ] + - [ nova-compute, glance ] + - [ nova-cloud-controller, rabbitmq-server ] + - [ nova-cloud-controller, mysql ] + - [ nova-cloud-controller, keystone ] + - [ nova-cloud-controller, glance ] + - [ nova-cloud-controller, nova-compute ] + - [ cinder, keystone ] + - [ cinder, mysql ] + - [ cinder, rabbitmq-server ] + - [ cinder, nova-cloud-controller ] + - [ "neutron-gateway:amqp", "rabbitmq-server:amqp" ] + - [ neutron-gateway, nova-cloud-controller ] + - [ neutron-api, mysql ] + - [ neutron-api, rabbitmq-server ] + - [ neutron-api, nova-cloud-controller ] + - [ neutron-api, neutron-openvswitch ] + - [ neutron-api, keystone ] + - [ neutron-api, neutron-gateway ] + - [ neutron-openvswitch, nova-compute ] + - [ neutron-openvswitch, rabbitmq-server ] + - [ neutron-openvswitch, manila ] +``` + +and then (with juju 2.x): + +```bash + juju deploy manila.yaml +``` + +Note that this OpenStack system will need to be configured (in terms of +networking, images, etc.) before testing can commence. + +# Bugs + +Please report bugs on [Launchpad](https://bugs.launchpad.net/charm-manila/+filebug). + +For general questions please refer to the OpenStack [Charm Guide](https://github.com/openstack/charm-guide). diff --git a/src/config.yaml b/src/config.yaml new file mode 100644 index 0000000..01ade67 --- /dev/null +++ b/src/config.yaml @@ -0,0 +1,63 @@ +options: + openstack-origin: + default: distro + type: string + description: | + Repository from which to install. May be one of the following: + distro (default), ppa:somecustom/ppa, a deb url sources entry, + or a supported Cloud Archive release pocket. + + Supported Cloud Archive sources include: cloud:precise-folsom, + cloud:precise-folsom/updates, cloud:precise-folsom/staging, + cloud:precise-folsom/proposed. + + Note that updating this setting to a source that is known to + provide a later version of OpenStack will trigger a software + upgrade. + rabbit-user: + default: manila + type: string + description: Username used to access rabbitmq queue + rabbit-vhost: + default: openstack + type: string + description: Rabbitmq vhost + database-user: + default: manila + type: string + description: Username for Manila database access + database: + default: manila + type: string + description: Database name for Manila + debug: + default: False + type: boolean + description: Enable debug logging + verbose: + default: False + type: boolean + description: Enable verbose logging + region: + default: RegionOne + type: string + description: OpenStack Region + share-protocols: + type: string + default: NFS CIFS + description: | + The share protocols that the backends will be able to provide. The + default is good for the generic backends. Other backends may not support + both NFS and CIFS. This is a space delimited list of protocols. + default-share-backend: + type: string + default: "" + description: | + The default backend for this manila set. Must be one of the + 'share-backends' or the charm will block. + default-share-type: + type: string + default: default_share_type + description: | + The 'default_share_type' must match the the configured default_share_type + set up in manila using 'manila create-type'. diff --git a/src/layer.yaml b/src/layer.yaml new file mode 100644 index 0000000..236f474 --- /dev/null +++ b/src/layer.yaml @@ -0,0 +1,8 @@ +includes: + - layer:openstack-api + - interface:mysql-shared + - interface:rabbitmq + - interface:keystone + - interface:neutron-plugin + - interface:manila-plugin +repo: https://github.com/openstack/charm-manila diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/src/lib/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/__init__.py b/src/lib/charm/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/src/lib/charm/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/openstack/__init__.py b/src/lib/charm/openstack/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/src/lib/charm/openstack/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/lib/charm/openstack/manila.py b/src/lib/charm/openstack/manila.py new file mode 100644 index 0000000..348ed1f --- /dev/null +++ b/src/lib/charm/openstack/manila.py @@ -0,0 +1,334 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# The manila handlers class + +# bare functions are provided to the reactive handlers to perform the functions +# needed on the class. +from __future__ import absolute_import + +import re +import subprocess + +import charmhelpers.core.hookenv as hookenv + +import charms_openstack.charm +import charms_openstack.adapters +import charms_openstack.ip as os_ip + +# note that manila-common is pulled in via the other packages. +PACKAGES = ['manila-api', + 'manila-data', + 'manila-scheduler', + 'manila-share', + 'python-pymysql', + 'python-apt', # for subordinate neutron-openvswitch if needed. + ] + +MANILA_DIR = '/etc/manila/' +MANILA_CONF = MANILA_DIR + "manila.conf" +MANILA_LOGGING_CONF = MANILA_DIR + "logging.conf" +MANILA_API_PASTE_CONF = MANILA_DIR + "api-paste.ini" + +# select the default release function and ssl feature +charms_openstack.charm.use_defaults('charm.default-select-release') + + +def strip_join(s, divider=" "): + """Cleanup the string passed, split on whitespace and then rejoin it + cleanly + + :param s: A sting to cleanup, remove non alpha chars and then represent the + string. + :param divider: The joining string to put the bits back together again. + :returns: string + """ + return divider.join( + re.split(r'\s+', re.sub(r'([^\s\w-])+', '', (s or "")))) + + +### +# Compute some options to help with template rendering +@charms_openstack.adapters.config_property +def computed_share_backends(config): + """Determine the backend protocols that are provided as a string. + + This asks the charm class what the backend protocols are, and then provides + it as a space separated list of backends. + + :param config: the config option on which to look up config options + :returns: string + """ + return ' '.join(config.charm_instance.configured_backends) + + +@charms_openstack.adapters.config_property +def computed_share_protocols(config): + """Return a list of protocols as a comma (no space) separated list. + The default protocols are CIFS,NFS. + + :param config: the config option on which to look up config options + :returns: string + """ + return strip_join(config.share_protocols, ',').upper() + + +@charms_openstack.adapters.config_property +def computed_backend_lines_manila_conf(config): + """Return the list of lines from the backends that need to go into the + various configuration files. + + This one is for manila.conf + :returns list of lines: the config for the manila.conf file + """ + return config.charm_instance.config_lines_for(MANILA_CONF) + + +@charms_openstack.adapters.config_property +def computed_debug_level(config): + """Return NONE, INFO, WARNING, DEBUG depending on the settings of + options.debug and options.level + :returns: string, NONE, WARNING, DEBUG + """ + if not config.debug: + return "NONE" + if config.verbose: + return "DEBUG" + return "WARNING" + + +### +# Implementation of the Manila Charm classes + +class ManilaCharm(charms_openstack.charm.HAOpenStackCharm): + """ManilaCharm provides the specialisation of the OpenStackCharm + functionality to manage a manila unit. + """ + + release = 'mitaka' + name = 'manila' + packages = PACKAGES + api_ports = { + 'manila-api': { + os_ip.PUBLIC: 8786, + os_ip.ADMIN: 8786, + os_ip.INTERNAL: 8786, + }, + } + service_type = 'manila' + # manila needs a second service type as well - there is a custom connect + # function to set both service types. + service_type_v2 = 'manilav2' + + default_service = 'manila-api' + services = ['manila-api', + 'manila-scheduler', + 'manila-share', + 'manila-data'] + + # Note that the hsm interface is optional - defined in config.yaml + required_relations = ['shared-db', 'amqp', 'identity-service'] + + restart_map = { + MANILA_CONF: services, + MANILA_API_PASTE_CONF: services, + MANILA_LOGGING_CONF: services, + } + + # This is the command to sync the database + sync_cmd = ['sudo', 'manila-manage', 'db', 'sync'] + + # ha_resources = ['vips', 'haproxy'] + + # Custom charm configuration + + def install(self): + """Called when the charm is being installed or upgraded. + + The available configuration options need to be check AFTER the charm is + installed to check to see whether it is blocked or can go into service. + """ + super().install() + # this creates the /etc/nova directory for the + # neutron-openvswitch plugin if needed. + subprocess.check_call(["mkdir", "-p", "/etc/nova"]) + self.assess_status() + + def custom_assess_status_check(self): + """Verify that the configuration provided is valid and thus the service + is ready to go. This will return blocked if the configuraiton is not + valid for the service. + + :returns (status: string, message: string): the status, and message if + there is a problem. Or (None, None) if there are no issues. + """ + options = self.options # tiny optimisation for less typing. + backends = options.computed_share_backends + if not backends: + return 'blocked', 'No share backends configured' + default_share_backend = options.default_share_backend + if not default_share_backend: + return 'blocked', "'default-share-backend' is not set" + if default_share_backend not in backends: + return ('blocked', + "'default-share-backend:{}' is not a configured backend" + .format(default_share_backend)) + return None, None + + def get_amqp_credentials(self): + """Provide the default amqp username and vhost as a tuple. + + :returns (username, host): two strings to send to the amqp provider. + """ + return (self.options.rabbit_user, self.options.rabbit_vhost) + + def get_database_setup(self): + """Provide the default database credentials as a list of 3-tuples + + returns a structure of: + [ + {'database': , + 'username': , + 'hostname': + 'prefix': , }, + ] + + :returns [{'database': ...}, ...]: credentials for multiple databases + """ + return [ + dict( + database=self.options.database, + username=self.options.database_user, + hostname=hookenv.unit_private_ip(), ) + ] + + def register_endpoints(self, keystone): + """Custom function to register the TWO keystone endpoints that this + charm requires. 'charm' and 'charmv2'. + + :param keystone: the keystone relation on which to setup the endpoints + """ + # regsiter the first endpoint + self._custom_register_endpoints(keystone, 'v1', + self.service_type, + self.region, + self.public_url, + self.internal_url, + self.admin_url) + # regsiter the second endpoint + self._custom_register_endpoints(keystone, 'v2', + self.service_type_v2, + self.region, + self.public_url_v2, + self.internal_url_v2, + self.admin_url_v2) + + @staticmethod + def _custom_register_endpoints(keystone, prefix, service, region, + public_url, internal_url, admin_url): + """Custom function to enable registering of multiple endpoints. + + Keystone charm understands multiple endpoints if they are prefixed with + a string_ as in 'v1_service' and 'v2_service', etc. However, the + keystone interface doesn't know how to do this. Therefore, this + function duplicates part of that functionality but enables the + 'multiple' endpoints to be set + + :param keystone: the relation that is keystone. + :param prefix: the prefix to prepend to '_' + :param service: the service to set + :param region: the OS region + :param public_url: the public_url + :param internal_url: the internal_url + :prarm admin_url: the admin url. + """ + relation_info = { + '{}_service'.format(prefix): service, + '{}_public_url'.format(prefix): public_url, + '{}_internal_url'.format(prefix): internal_url, + '{}_admin_url'.format(prefix): admin_url, + '{}_region'.format(prefix): region, + } + keystone.set_local(**relation_info) + keystone.set_remote(**relation_info) + + @property + def public_url(self): + return super().public_url + "/v1/%(tenant_id)s" + + @property + def admin_url(self): + return super().admin_url + "/v1/%(tenant_id)s" + + @property + def internal_url(self): + return super().internal_url + "/v1/%(tenant_id)s" + + @property + def public_url_v2(self): + return super().public_url + "/v2/%(tenant_id)s" + + @property + def admin_url_v2(self): + return super().admin_url + "/v2/%(tenant_id)s" + + @property + def internal_url_v2(self): + return super().internal_url + "/v2/%(tenant_id)s" + + @property + def configured_backends(self): + """Return a list of configured backends that come from the associated + 'manila-share.available' state.. + + TODO: Note that the first backend that becomes 'available' will set + this state. It's not clear how multiple backends will interact yet! + + :returns: list of strings: backend sections that are configured. + """ + adapter = self.get_adapter('manila-plugin.available') + if adapter is None: + return [] + # adapter.names is a property that provides a list of backend manila + # plugin names for the sections + return adapter.relation.names + + def config_lines_for(self, config_file): + """Return the list of configuration lines for `config_file` as returned + by manila-plugin backend charms. + + TODO: Note that it is not clear how we get this from multiple plugin + charms -- still to be worked out + + :param config_file: string, filename for configuration lines + :returns: list of strings: config lines for `config_file` + """ + adapter = self.get_adapter('manila-plugin.available') + if adapter is not None: + # get the configuration data for all plugins + config_data = adapter.relation.get_configuration_data() + if config_file not in config_data: + return [] + config_lines = [] + for section, lines in config_data[config_file].items(): + if section == 'complete': + # if the 'lines' is not truthy, then this conf isn't + # complete, so just break out. + if not lines: + break + continue + config_lines.append(section) + config_lines.extend(lines) + config_lines.append('') + return config_lines + return [] diff --git a/src/metadata.yaml b/src/metadata.yaml new file mode 100644 index 0000000..4a05cc2 --- /dev/null +++ b/src/metadata.yaml @@ -0,0 +1,29 @@ +name: manila +summary: A REST API for folder shares +maintainer: OpenStack Charmers +description: | + Shared File Systems service provides a set of services for management of + shared file systems in a multi-tenant cloud environment. The service resembles + OpenStack block-based storage management from the OpenStack Block Storage + service project. With the Shared File Systems service, you can create a remote + file system, mount the file system on your instances, and then read and write + data from your instances to and from your file system. +tags: + - openstack +series: + - xenial + - yakkety +subordinate: false +requires: + shared-db: + interface: mysql-shared + amqp: + interface: rabbitmq + identity-service: + interface: keystone + neutron-plugin: + interface: neutron-plugin + scope: container + manila-plugin: + interface: manila-plugin + scope: container diff --git a/src/reactive/__init__.py b/src/reactive/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/src/reactive/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/reactive/manila_handlers.py b/src/reactive/manila_handlers.py new file mode 100644 index 0000000..ddba19e --- /dev/null +++ b/src/reactive/manila_handlers.py @@ -0,0 +1,115 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this is just for the reactive handlers and calls into the charm. +from __future__ import absolute_import + +import charms.reactive +import charms_openstack.charm + +# This charm's library contains all of the handler code associated with +# manila -- we need to import it to get the definitions for the charm. +import charm.openstack.manila # noqa + + +# Use the charms.openstack defaults for common states and hooks +charms_openstack.charm.use_defaults( + 'charm.installed', + 'amqp.connected', + 'shared-db.connected', + # 'identity-service.connected', + 'identity-service.available', # enables SSL support + # 'config.changed', + # 'update-status' +) + + +@charms.reactive.when('identity-service.connected') +def register_endpoints(keystone): + """Register the endpoints when the identity-service connects. + Note that this charm doesn't use the default endpoint registration function + as it needs to register multiple endpoints, and thus needs a custom + function in the charm. + """ + with charms_openstack.charm.provide_charm_instance() as manila_charm: + manila_charm.register_endpoints(keystone) + manila_charm.assess_status() + + +@charms.reactive.when('identity-service.connected', + 'manila-plugin.connected') +def share_to_manila_plugins_auth(keystone, manila_plugin): + """When we have the identity-service and (a) backend plugin, share the auth + plugin with the back end. + + TODO: if we have multiple manila-plugin's does this get called for each + relation that gets connected? + """ + data = { + 'username': keystone.service_username(), + 'password': keystone.service_password(), + 'project_domain_id': 'default', + 'project_name': 'services', + 'user_domain_id': 'default', + 'auth_uri': ("{protocol}://{host}:{port}" + .format(protocol=keystone.service_protocol(), + host=keystone.service_host(), + port=keystone.service_port())), + 'auth_url': ("{protocol}://{host}:{port}" + .format(protocol=keystone.auth_protocol(), + host=keystone.auth_host(), + port=keystone.auth_port())), + 'auth_type': 'password', + } + # Set the auth data to be the same for all plugins + manila_plugin.set_authentication_data(data) + + +@charms.reactive.when('shared-db.available', + 'manila.config.rendered') +def maybe_do_syncdb(shared_db): + """Sync the database when the shared-db becomes available. Note that the + charms.openstack.OpenStackCharm.db_sync() default method checks that only + the leader does the sync. As manila uses alembic to do the database + migration, it doesn't matter if it's done more than once, so we don't have + to gate it in the charm. + """ + with charms_openstack.charm.provide_charm_instance() as manila_charm: + manila_charm.db_sync() + + +@charms.reactive.when('shared-db.available', + 'identity-service.available', + 'amqp.available') +def render_stuff(*args): + """Render the configuration for Manila when all the interfaces are + available. + """ + with charms_openstack.charm.provide_charm_instance() as manila_charm: + manila_charm.render_with_interfaces(args) + manila_charm.assess_status() + charms.reactive.set_state('manila.config.rendered') + + +@charms.reactive.when('config.changed', + 'shared-db.available', + 'identity-service.available', + 'amqp.available') +def config_changed(*args): + """When the configuration is changed, check that we have all the interfaces + and then re-render all the configuration files. Note that this means that + the configuration files won't be written until all the interfaces are + available and STAY available. + """ + render_stuff(*args) diff --git a/src/templates/mitaka/api-paste.ini b/src/templates/mitaka/api-paste.ini new file mode 100644 index 0000000..4e0f32e --- /dev/null +++ b/src/templates/mitaka/api-paste.ini @@ -0,0 +1,59 @@ +############# +# OpenStack # +############# + +[composite:osapi_share] +use = call:manila.api:root_app_factory +/: apiversions +/v1: openstack_share_api +/v2: openstack_share_api_v2 + +[composite:openstack_share_api] +use = call:manila.api.middleware.auth:pipeline_factory +noauth = cors faultwrap ssl sizelimit noauth api +keystone = cors faultwrap ssl sizelimit authtoken keystonecontext api +keystone_nolimit = cors faultwrap ssl sizelimit authtoken keystonecontext api + +[composite:openstack_share_api_v2] +use = call:manila.api.middleware.auth:pipeline_factory +noauth = cors faultwrap ssl sizelimit noauth apiv2 +keystone = cors faultwrap ssl sizelimit authtoken keystonecontext apiv2 +keystone_nolimit = cors faultwrap ssl sizelimit authtoken keystonecontext apiv2 + +[filter:faultwrap] +paste.filter_factory = manila.api.middleware.fault:FaultWrapper.factory + +[filter:noauth] +paste.filter_factory = manila.api.middleware.auth:NoAuthMiddleware.factory + +[filter:sizelimit] +paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory + +[filter:ssl] +paste.filter_factory = oslo_middleware.ssl:SSLMiddleware.factory + +[app:api] +paste.app_factory = manila.api.v1.router:APIRouter.factory + +[app:apiv2] +paste.app_factory = manila.api.v2.router:APIRouter.factory + +[pipeline:apiversions] +pipeline = cors faultwrap osshareversionapp + +[app:osshareversionapp] +paste.app_factory = manila.api.versions:VersionsRouter.factory + +########## +# Shared # +########## + +[filter:keystonecontext] +paste.filter_factory = manila.api.middleware.auth:ManilaKeystoneContext.factory + +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:cors] +paste.filter_factory = oslo_middleware.cors:filter_factory +oslo_config_project = manila diff --git a/src/templates/mitaka/logging.conf b/src/templates/mitaka/logging.conf new file mode 100644 index 0000000..8de2024 --- /dev/null +++ b/src/templates/mitaka/logging.conf @@ -0,0 +1,77 @@ +[loggers] +keys = root, manila + +[handlers] +keys = stderr, stdout, watchedfile, syslog, null + +[formatters] +keys = legacymanila, default + +[logger_root] +level = {{ options.computed_debug_level }} +handlers = null + +[logger_manila] +# level = INFO +level = {{ options.computed_debug_level }} +handlers = stderr +qualname = manila + +[logger_amqplib] +level = WARNING +handlers = stderr +qualname = amqplib + +[logger_sqlalchemy] +level = WARNING +handlers = stderr +qualname = sqlalchemy +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARNING" logs neither. (Recommended for production systems.) + +[logger_boto] +level = WARNING +handlers = stderr +qualname = boto + +[logger_suds] +level = INFO +handlers = stderr +qualname = suds + +[logger_eventletwsgi] +level = WARNING +handlers = stderr +qualname = eventlet.wsgi.server + +[handler_stderr] +class = StreamHandler +args = (sys.stderr,) +formatter = legacymanila + +[handler_stdout] +class = StreamHandler +args = (sys.stdout,) +formatter = legacymanila + +[handler_watchedfile] +class = handlers.WatchedFileHandler +args = ('manila.log',) +formatter = legacymanila + +[handler_syslog] +class = handlers.SysLogHandler +args = ('/dev/log', handlers.SysLogHandler.LOG_USER) +formatter = legacymanila + +[handler_null] +class = manila.common.openstack.NullHandler +formatter = default +args = () + +[formatter_legacymanila] +class = manila.openstack.common.log.LegacyFormatter + +[formatter_default] +format = %(message)s diff --git a/src/templates/mitaka/manila.conf b/src/templates/mitaka/manila.conf new file mode 100644 index 0000000..0dc4acd --- /dev/null +++ b/src/templates/mitaka/manila.conf @@ -0,0 +1,81 @@ +# Note that the original manila.conf file is extensive and has many options +# that the charm does not set. Please refer to that file if there are options +# that you think the charm should set, but doesn't, or provide options for. +# Please file a bug at: https://bugs.launchpad.net/charm-barbican/+filebug for +# any changes you need made or intend to modify in the charm. + +[DEFAULT] + +# This all needs to be configurable +enabled_share_backends = {{ options.computed_share_backends }} + +# enabled_share_protocols = NFS,CIFS +enabled_share_protocols = {{ options.computed_share_protocols }} + +#default_share_type = default_share_type +default_share_type = {{ options.default_share_type }} + +state_path = /var/local/manila +osapi_share_extension = manila.api.contrib.standard_extenstions +rootwrap_config = /etc/manila/rootwrap.conf +api_paste_config = /etc/manila/api-paste.ini +share_name_template = share-%s + +scheduler_driver = manila.scheduler.drivers.filter.FilterScheduler + +debug = {{ options.debug }} + + +[cors] + +# +# From oslo.middleware.cors +# + +[cors.subdomain] + +# +# From oslo.middleware.cors +# + +# parts/section-database includes the [database] section identifier +{% include "parts/section-database" %} + + +# parts/section-keystone-authtoken includes the [keystone_authtoken] section +# identifier +{% include "parts/section-keystone-authtoken" %} + + + +[matchmaker_redis] + +# +# From oslo.messaging +# + +[oslo_messaging_amqp] + +# +# From oslo.messaging +# + +[oslo_messaging_notifications] + +# +# From oslo.messaging +# + + +# parts/section-rabbitmq-olso include the [oslo_messaging_rabbit] section +# identifier +{% include "parts/section-rabbitmq-oslo" %} + + +# +# Now configuration from the backend manila-plugin charms +# + +{% for line in options.computed_backend_lines_manila_conf %} +{{ line }} +{%- endfor %} diff --git a/src/test-requirements.txt b/src/test-requirements.txt new file mode 100644 index 0000000..ec50ece --- /dev/null +++ b/src/test-requirements.txt @@ -0,0 +1,22 @@ +# charm-proof +charm-tools>=2.0.0 +# amulet deployment helpers +bzr+lp:charm-helpers#egg=charmhelpers +# BEGIN: Amulet OpenStack Charm Helper Requirements +# Liberty client lower constraints +amulet>=1.14.3,<2.0 +bundletester>=0.6.1,<1.0 +python-keystoneclient>=1.7.1,<2.0 +python-barbicanclient>=4.0.1,<5.0 +python-designateclient>=1.5,<2.0 +python-cinderclient>=1.4.0,<2.0 +python-glanceclient>=1.1.0,<2.0 +python-heatclient>=0.8.0,<1.0 +python-neutronclient>=3.1.0,<4.0 +python-novaclient>=2.30.1,<3.0 +python-openstackclient>=1.7.0,<2.0 +python-swiftclient>=2.6.0,<3.0 +python-manilaclient>=1.8.1,<2.0 +pika>=0.10.0,<1.0 +distro-info +# END: Amulet OpenStack Charm Helper Requirements diff --git a/src/tests/README.md b/src/tests/README.md new file mode 100644 index 0000000..046be7f --- /dev/null +++ b/src/tests/README.md @@ -0,0 +1,9 @@ +# Overview + +This directory provides Amulet tests to verify basic deployment functionality +from the perspective of this charm, its requirements and its features, as +exercised in a subset of the full OpenStack deployment test bundle topology. + +For full details on functional testing of OpenStack charms please refer to +the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing) +section of the OpenStack Charm Guide. diff --git a/src/tests/basic_deployment.py b/src/tests/basic_deployment.py new file mode 100644 index 0000000..e9b2687 --- /dev/null +++ b/src/tests/basic_deployment.py @@ -0,0 +1,583 @@ +import amulet +import json +import subprocess +import time + +from keystoneclient import session as keystone_session +from keystoneclient.auth import identity as keystone_identity +import keystoneclient.exceptions +from keystoneclient.v2_0 import client as keystone_v2_0_client +from keystoneclient.v3 import client as keystone_v3_client +from manilaclient.v1 import client as manila_client + +from charmhelpers.contrib.openstack.amulet.deployment import ( + OpenStackAmuletDeployment +) + +from charmhelpers.contrib.openstack.amulet.utils import ( + OpenStackAmuletUtils, + DEBUG, +) + +# Use DEBUG to turn on debug logging +u = OpenStackAmuletUtils(DEBUG) + + +class ManilaBasicDeployment(OpenStackAmuletDeployment): + """Amulet tests on a basic Manila deployment. + + Note that these tests don't attempt to do a functional test on Manila, + merely to demonstrate that the relations work and that they transfer the + correct information across them. + + A functional test will be performed by a mojo or tempest test. + """ + + def __init__(self, series, openstack=None, source=None, stable=False): + """Deploy the entire test environment. + """ + super(ManilaBasicDeployment, self).__init__( + series, openstack, source, stable) + self._keystone_version = '2' + self._add_services() + self._add_relations() + self._configure_services() + self._deploy() + + u.log.info('Waiting on extended status checks...') + exclude_services = ['mysql', ] + self._auto_wait_for_status(exclude_services=exclude_services) + + self._initialize_tests() + + def _add_services(self): + """Add services + + Add the services that we're testing, where manila is local, + and the rest of the service are from lp branches that are + compatible with the local charm (e.g. stable or next). + """ + this_service = {'name': 'manila'} + other_services = [ + {'name': 'mysql', + 'location': 'cs:percona-cluster', + 'constraints': {'mem': '3072M'}}, + {'name': 'rabbitmq-server'}, + {'name': 'keystone'}, + {'name': 'manila-generic', + 'location': 'cs:~ajkavanagh/xenial/manila-generic-1'} + ] + super(ManilaBasicDeployment, self)._add_services( + this_service, other_services) + + def _add_relations(self): + """Add all of the relations for the services.""" + relations = { + 'manila:shared-db': 'mysql:shared-db', + 'manila:amqp': 'rabbitmq-server:amqp', + 'manila:identity-service': 'keystone:identity-service', + 'manila:manila-plugin': 'manila-generic:manila-plugin', + 'keystone:shared-db': 'mysql:shared-db', + } + super(ManilaBasicDeployment, self)._add_relations(relations) + + def _configure_services(self): + """Configure all of the services.""" + keystone_config = { + 'admin-password': 'openstack', + 'admin-token': 'ubuntutesting', + } + manila_config = { + 'default-share-backend': 'generic', + } + manila_generic_config = { + 'driver-handles-share-servers': False, + } + configs = { + 'keystone': keystone_config, + 'manila': manila_config, + 'manila-generic': manila_generic_config, + } + super(ManilaBasicDeployment, self)._configure_services(configs) + + def _initialize_tests(self): + """Perform final initialization before tests get run.""" + # Access the sentries for inspecting service units + self.manila_sentry = self.d.sentry['manila'][0] + self.mysql_sentry = self.d.sentry['mysql'][0] + self.keystone_sentry = self.d.sentry['keystone'][0] + self.rabbitmq_sentry = self.d.sentry['rabbitmq-server'][0] + u.log.debug('openstack release val: {}'.format( + self._get_openstack_release())) + u.log.debug('openstack release str: {}'.format( + self._get_openstack_release_string())) + + keystone_ip = self.keystone_sentry.relation( + 'shared-db', 'mysql:shared-db')['private-address'] + + # We need to auth either to v2.0 or v3 keystone + if self._keystone_version == '2': + ep = ("http://{}:35357/v2.0" + .format(keystone_ip.strip().decode('utf-8'))) + auth = keystone_identity.v2.Password( + username='admin', + password='openstack', + tenant_name='admin', + auth_url=ep) + keystone_client_lib = keystone_v2_0_client + elif self._keystone_version == '3': + ep = ("http://{}:35357/v3" + .format(keystone_ip.strip().decode('utf-8'))) + auth = keystone_identity.v3.Password( + user_domain_name='admin_domain', + username='admin', + password='openstack', + domain_name='admin_domain', + auth_url=ep) + keystone_client_lib = keystone_v3_client + else: + raise RuntimeError("keystone version must be '2' or '3'") + + sess = keystone_session.Session(auth=auth) + self.keystone = keystone_client_lib.Client(session=sess) + # The service_catalog is missing from V3 keystone client when auth is + # done with session (via authenticate_keystone_admin() + # See https://bugs.launchpad.net/python-keystoneclient/+bug/1508374 + # using session construct client will miss service_catalog property + # workaround bug # 1508374 by forcing a pre-auth and therefore, getting + # the service-catalog -- + # see https://bugs.launchpad.net/python-keystoneclient/+bug/1547331 + self.keystone.auth_ref = auth.get_access(sess) + + def _run_action(self, unit_id, action, *args): + command = ["juju", "action", "do", "--format=json", unit_id, action] + command.extend(args) + print("Running command: %s\n" % " ".join(command)) + output = subprocess.check_output(command) + output_json = output.decode(encoding="UTF-8") + data = json.loads(output_json) + action_id = data[u'Action queued with id'] + return action_id + + def _wait_on_action(self, action_id): + command = ["juju", "action", "fetch", "--format=json", action_id] + while True: + try: + output = subprocess.check_output(command) + except Exception as e: + print(e) + return False + output_json = output.decode(encoding="UTF-8") + data = json.loads(output_json) + if data[u"status"] == "completed": + return True + elif data[u"status"] == "failed": + return False + time.sleep(2) + + def test_100_services(self): + """Verify the expected services are running on the corresponding + service units.""" + u.log.debug('Checking system services on units...') + + manila_svcs = [ + 'manila-api', + 'manila-scheduler', + 'manila-share', + 'manila-data', + ] + + service_names = { + self.manila_sentry: manila_svcs, + } + + ret = u.validate_services_by_name(service_names) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + u.log.debug('OK') + + def test_110_service_catalog(self): + """Verify that the service catalog endpoint data is valid.""" + u.log.debug('Checking keystone service catalog data...') + + actual = self.keystone.service_catalog.get_endpoints() + + if self._keystone_version == '2': + endpoint_check = [{ + 'adminURL': u.valid_url, + 'id': u.not_null, + 'region': 'RegionOne', + 'publicURL': u.valid_url, + 'internalURL': u.valid_url, + }] + validate_catalog = u.validate_svc_catalog_endpoint_data + else: + # v3 endpoint check + endpoint_check = [ + { + 'id': u.not_null, + 'interface': interface, + 'region': 'RegionOne', + 'region_id': 'RegionOne', + 'url': u.valid_url, + } + for interface in ('admin', 'public', 'internal')] + validate_catalog = u.validate_v3_svc_catalog_endpoint_data + + expected = { + 'sharev2': endpoint_check, + } + + ret = validate_catalog(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + u.log.debug('OK') + + def test_114_manila_api_endpoint(self): + """Verify the manila api endpoint data.""" + u.log.debug('Checking manila api endpoint data...') + endpoints = self.keystone.endpoints.list() + u.log.debug(endpoints) + admin_port = '8786' + internal_port = public_port = admin_port + if self._keystone_version == '2': + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'adminurl': u.valid_url, + 'internalurl': u.valid_url, + 'publicurl': u.valid_url, + 'service_id': u.not_null} + + ret = u.validate_endpoint_data( + endpoints, admin_port, internal_port, public_port, expected) + elif self._keystone_version == '3': + # For keystone v3 it's slightly different. + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'region_id': 'RegionOne', + 'url': u.valid_url, + 'interface': u.not_null, # we match this in the test + 'service_id': u.not_null} + + ret = u.validate_v3_endpoint_data( + endpoints, admin_port, internal_port, public_port, expected) + else: + raise RuntimeError("Unexpected self._keystone_version: {}" + .format(self._keystone_version)) + + if ret: + message = 'manila endpoint: {}'.format(ret) + amulet.raise_status(amulet.FAIL, msg=message) + + u.log.debug('OK') + + def test_200_manila_identity_relation(self): + """Verify the manila to keystone identity-service relation data""" + u.log.debug('Checking manila to keystone identity-service ' + 'relation data...') + unit = self.manila_sentry + relation = ['identity-service', 'keystone:identity-service'] + manila_ip = unit.relation(*relation)['private-address'] + manila_v1_endpoint = ("http://{}:8786/v1/%(tenant_id)s" + .format(manila_ip)) + manila_v2_endpoint = ("http://{}:8786/v2/%(tenant_id)s" + .format(manila_ip)) + + expected = { + 'private-address': manila_ip, + 'v1_region': 'RegionOne', + 'v1_admin_url': manila_v1_endpoint, + 'v1_internal_url': manila_v1_endpoint, + 'v1_public_url': manila_v1_endpoint, + 'v1_service': 'manila', + 'v2_region': 'RegionOne', + 'v2_admin_url': manila_v2_endpoint, + 'v2_internal_url': manila_v2_endpoint, + 'v2_public_url': manila_v2_endpoint, + 'v2_service': 'manilav2', + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('manila identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + u.log.debug('OK') + + def test_201_keystone_manila_identity_relation(self): + """Verify the keystone to manila identity-service relation data""" + u.log.debug('Checking keystone:manila identity relation data...') + unit = self.keystone_sentry + relation = ['identity-service', 'manila:identity-service'] + id_relation = unit.relation(*relation) + id_ip = id_relation['private-address'] + expected = { + 'admin_token': 'ubuntutesting', + 'auth_host': id_ip, + 'auth_port': "35357", + 'auth_protocol': 'http', + 'private-address': id_ip, + 'service_host': id_ip, + 'service_password': u.not_null, + 'service_port': "5000", + 'service_protocol': 'http', + 'service_tenant': 'services', + 'service_tenant_id': u.not_null, + 'service_username': 'manila_manilav2', # oddness, but registers 2 + } + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('keystone identity-service', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + u.log.debug('OK') + + def test_203_manila_amqp_relation(self): + """Verify the manila to rabbitmq-server amqp relation data""" + u.log.debug('Checking manila:rabbitmq amqp relation data...') + unit = self.manila_sentry + relation = ['amqp', 'rabbitmq-server:amqp'] + expected = { + 'username': 'manila', + 'private-address': u.valid_ip, + 'vhost': 'openstack' + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('manila amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + u.log.debug('OK') + + def test_204_manila_amqp_relation(self): + """Verify the rabbitmq-server to manila amqp relation data""" + u.log.debug('Checking rabbitmq:manila manila relation data...') + unit = self.rabbitmq_sentry + relation = ['amqp', 'manila:amqp'] + expected = { + 'hostname': u.valid_ip, + 'private-address': u.valid_ip, + 'password': u.not_null, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('rabbitmq manila', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + u.log.debug('OK') + + @staticmethod + def _find_or_create(items, key, create): + """Find or create the thing in the items + + :param items: the items to search using the key + :param key: a function that key(item) -> boolean if found. + :param create: a function to call if the key() never was true. + :returns: the item that was either found or created. + """ + for i in items: + if key(i): + return i + return create() + + def test_400_api_connection(self): + """Simple api calls to check service is up and responding""" + u.log.debug('Checking api functionality...') + + # This handles both keystone v2 and v3. + # For keystone v2 we need a user: + # - 'demo' user + # - has a project 'demo' + # - in the 'demo' project + # - with an 'admin' role + # For keystone v3 we need a user: + # - 'default' domain + # - 'demo' user + # - 'demo' project + # - 'admin' role -- to be able to delete. + + # manila requires a user with creator or admin role on the project + # when creating a secret (which this test does). Therefore, we create + # a demo user, demo project, and then get a demo manila client and do + # the secret. ensure that the default domain is created. + + if self._keystone_version == '2': + # find or create the 'demo' tenant (project) + tenant = self._find_or_create( + items=self.keystone.tenants.list(), + key=lambda t: t.name == 'demo', + create=lambda: self.keystone.tenants.create( + tenant_name="demo", + description="Demo for testing manila", + enabled=True)) + # find or create the demo user + demo_user = self._find_or_create( + items=self.keystone.users.list(), + key=lambda u: u.name == 'demo', + create=lambda: self.keystone.users.create( + name='demo', + password='pass', + tenant_id=tenant.id)) + # find the admin role + # already be created - if not, then this will fail later. + admin_role = self._find_or_create( + items=self.keystone.roles.list(), + key=lambda r: r.name.lower() == 'admin', + create=lambda: None) + # grant the role if it isn't already created. + # now grant the creator role to the demo user. + self._find_or_create( + items=self.keystone.roles.roles_for_user( + demo_user, tenant=tenant), + key=lambda r: r.name.lower() == admin_role.name.lower(), + create=lambda: self.keystone.roles.add_user_role( + demo_user, admin_role, tenant=tenant)) + # now we can finally get the manila client and create the secret + keystone_ep = self.keystone.service_catalog.url_for( + service_type='identity', endpoint_type='publicURL') + auth = keystone_identity.v2.Password( + username=demo_user.name, + password='pass', + tenant_name=tenant.name, + auth_url=keystone_ep) + + else: + # find or create the 'default' domain + domain = self._find_or_create( + items=self.keystone.domains.list(), + key=lambda u: u.name == 'default', + create=lambda: self.keystone.domains.create( + "default", + description="domain for manila testing", + enabled=True)) + # find or create the 'demo' user + demo_user = self._find_or_create( + items=self.keystone.users.list(domain=domain.id), + key=lambda u: u.name == 'demo', + create=lambda: self.keystone.users.create( + 'demo', + domain=domain.id, + description="Demo user for manila tests", + enabled=True, + email="demo@example.com", + password="pass")) + # find or create the 'demo' project + demo_project = self._find_or_create( + items=self.keystone.projects.list(domain=domain.id), + key=lambda x: x.name == 'demo', + create=lambda: self.keystone.projects.create( + 'demo', + domain=domain.id, + description='manila testing project', + enabled=True)) + # create the role for the user - needs to be admin so that the + # secret can be deleted - note there is only one admin role, and it + # should already be created - if not, then this will fail later. + admin_role = self._find_or_create( + items=self.keystone.roles.list(), + key=lambda r: r.name.lower() == 'admin', + create=lambda: None) + # now grant the creator role to the demo user. + try: + self.keystone.roles.check( + role=admin_role, + user=demo_user, + project=demo_project) + except keystoneclient.exceptions.NotFound: + # create it if it isn't found + self.keystone.roles.grant( + role=admin_role, + user=demo_user, + project=demo_project) + # now we can finally get the manila client and create the secret + keystone_ep = self.keystone.service_catalog.url_for( + service_type='identity', endpoint_type='publicURL') + auth = keystone_identity.v3.Password( + user_domain_name=domain.name, + username=demo_user.name, + password='pass', + project_domain_name=domain.name, + project_name=demo_project.name, + auth_url=keystone_ep) + + # Now we carry on with common v2 and v3 code + sess = keystone_session.Session(auth=auth) + # Authenticate admin with manila endpoint + manila_ep = self.keystone.service_catalog.url_for( + service_type='share', endpoint_type='publicURL') + manila = manila_client.Client(session=sess, + endpoint=manila_ep) + # now just try a list the shares + manila.shares.list() + u.log.debug('OK') + + def test_900_restart_on_config_change(self): + """Verify that the specified services are restarted when the config + is changed. + """ + sentry = self.manila_sentry + juju_service = 'manila' + + # Expected default and alternate values + set_default = {'debug': 'False'} + set_alternate = {'debug': 'True'} + + # Services which are expected to restart upon config change, + # and corresponding config files affected by the change + conf_file = '/etc/manila/manila.conf' + services = { + 'manila-api': conf_file, + } + + # Make config change, check for service restarts + u.log.debug('Making config change on {}...'.format(juju_service)) + mtime = u.get_sentry_time(sentry) + self.d.configure(juju_service, set_alternate) + + sleep_time = 40 + for s, conf_file in services.iteritems(): + u.log.debug("Checking that service restarted: {}".format(s)) + if not u.validate_service_config_changed(sentry, mtime, s, + conf_file, + retry_count=4, + retry_sleep_time=20, + sleep_time=sleep_time): + self.d.configure(juju_service, set_default) + msg = "service {} didn't restart after config change".format(s) + amulet.raise_status(amulet.FAIL, msg=msg) + sleep_time = 0 + + self.d.configure(juju_service, set_default) + u.log.debug('OK') + + def _test_910_pause_and_resume(self): + """The services can be paused and resumed. """ + # test disabled as feature is not implemented yet - kept for future + # usage. + return + u.log.debug('Checking pause and resume actions...') + unit_name = "manila/0" + juju_service = 'manila' + unit = self.d.sentry[juju_service][0] + + assert u.status_get(unit)[0] == "active" + + action_id = self._run_action(unit_name, "pause") + assert self._wait_on_action(action_id), "Pause action failed." + assert u.status_get(unit)[0] == "maintenance" + + # trigger config-changed to ensure that services are still stopped + u.log.debug("Making config change on manila ...") + self.d.configure(juju_service, {'debug': 'True'}) + assert u.status_get(unit)[0] == "maintenance" + self.d.configure(juju_service, {'debug': 'False'}) + assert u.status_get(unit)[0] == "maintenance" + + action_id = self._run_action(unit_name, "resume") + assert self._wait_on_action(action_id), "Resume action failed." + assert u.status_get(unit)[0] == "active" + u.log.debug('OK') diff --git a/src/tests/gate-basic-xenial-mitaka b/src/tests/gate-basic-xenial-mitaka new file mode 100755 index 0000000..c022641 --- /dev/null +++ b/src/tests/gate-basic-xenial-mitaka @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +"""Amulet tests on a basic manila deployment on xenial-mitaka. +""" + +from basic_deployment import ManilaBasicDeployment + +if __name__ == '__main__': + deployment = ManilaBasicDeployment(series='xenial') + deployment.run_tests() diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml new file mode 100644 index 0000000..e3185c6 --- /dev/null +++ b/src/tests/tests.yaml @@ -0,0 +1,17 @@ +# Bootstrap the model if necessary. +bootstrap: True +# Re-use bootstrap node instead of destroying/re-bootstrapping. +reset: True +# Use tox/requirements to drive the venv instead of bundletester's venv feature. +virtualenv: False +# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet. +makefile: [] +# Do not specify juju PPA sources. Juju is presumed to be pre-installed +# and configured in all test runner environments. +#sources: +# Do not specify or rely on system packages. +#packages: +# Do not specify python packages here. Use test-requirements.txt +# and tox instead. ie. The venv is constructed before bundletester +# is invoked. +#python-packages: diff --git a/src/tox.ini b/src/tox.ini new file mode 100644 index 0000000..479d7bb --- /dev/null +++ b/src/tox.ini @@ -0,0 +1,53 @@ +# Source charm: ./src/tox.ini +# This file is managed centrally by release-tools and should not be modified +# within individual charm repos. +[tox] +envlist = pep8 +skipsdist = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 + AMULET_SETUP_TIMEOUT=2700 +whitelist_externals = juju +passenv = HOME TERM AMULET_* +deps = -r{toxinidir}/test-requirements.txt +install_command = + pip install --allow-unverified python-apt {opts} {packages} + +[testenv:pep8] +basepython = python2.7 +commands = charm-proof + +[testenv:func27-noop] +# DRY RUN - For Debug +basepython = python2.7 +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy + +[testenv:func27] +# Run all gate tests which are +x (expected to always pass) +basepython = python2.7 +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy + +[testenv:func27-smoke] +# Run a specific test as an Amulet smoke test (expected to always pass) +basepython = python2.7 +commands = + bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy + +[testenv:func27-dfs] +# Run all deploy-from-source tests which are +x (may not always pass!) +basepython = python2.7 +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy + +[testenv:func27-dev] +# Run all development test targets which are +x (may not always pass!) +basepython = python2.7 +commands = + bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy + +[testenv:venv] +commands = {posargs} diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..368dbf2 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +# Lint and unit test requirements +flake8 +os-testr>=0.4.1 +charms.reactive +mock>=1.2 +coverage>=3.6 +git+https://github.com/openstack/charms.openstack.git#egg=charms-openstack diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0f8d053 --- /dev/null +++ b/tox.ini @@ -0,0 +1,53 @@ +[tox] +skipsdist = True +envlist = pep8,py34,py35 +skip_missing_interpreters = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 + TERM=linux + INTERFACE_PATH={toxinidir}/interfaces + LAYER_PATH={toxinidir}/layers + INTERFACE_PATH={toxinidir}/interfaces + JUJU_REPOSITORY={toxinidir}/build +passenv = http_proxy https_proxy +install_command = + pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt + +[testenv:build] +basepython = python2.7 +commands = + charm-build --log-level DEBUG -o {toxinidir}/build src {posargs} + +[testenv:py27] +basepython = python2.7 +# Reactive source charms are Python3-only, but a py27 unit test target +# is required by OpenStack Governance. Remove this shim as soon as +# permitted. http://governance.openstack.org/reference/cti/python_cti.html +whitelist_externals = true +commands = true + +[testenv:py34] +basepython = python3.4 +deps = -r{toxinidir}/test-requirements.txt +commands = ostestr {posargs} + +[testenv:py35] +basepython = python3.5 +deps = -r{toxinidir}/test-requirements.txt +commands = ostestr {posargs} + +[testenv:pep8] +basepython = python2.7 +deps = -r{toxinidir}/test-requirements.txt +commands = flake8 {posargs} src unit_tests + +[testenv:venv] +commands = {posargs} + +[flake8] +# E402 ignore necessary for path append before sys module import in actions +ignore = E402 diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py new file mode 100644 index 0000000..ad8caed --- /dev/null +++ b/unit_tests/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import mock + +sys.path.append('src') +sys.path.append('src/lib') + +# Mock out charmhelpers so that we can test without it. +# also stops sideeffects from occuring. +charmhelpers = mock.MagicMock() +apt_pkg = mock.MagicMock() +sys.modules['apt_pkg'] = apt_pkg +sys.modules['charmhelpers'] = charmhelpers +sys.modules['charmhelpers.core'] = charmhelpers.core +sys.modules['charmhelpers.core.hookenv'] = charmhelpers.core.hookenv +sys.modules['charmhelpers.core.host'] = charmhelpers.core.host +sys.modules['charmhelpers.core.unitdata'] = charmhelpers.core.unitdata +sys.modules['charmhelpers.core.templating'] = charmhelpers.core.templating +sys.modules['charmhelpers.contrib'] = charmhelpers.contrib +sys.modules['charmhelpers.contrib.openstack'] = charmhelpers.contrib.openstack +sys.modules['charmhelpers.contrib.openstack.utils'] = ( + charmhelpers.contrib.openstack.utils) +sys.modules['charmhelpers.contrib.openstack.templating'] = ( + charmhelpers.contrib.openstack.templating) +sys.modules['charmhelpers.contrib.network'] = charmhelpers.contrib.network +sys.modules['charmhelpers.contrib.network.ip'] = ( + charmhelpers.contrib.network.ip) +sys.modules['charmhelpers.fetch'] = charmhelpers.fetch +sys.modules['charmhelpers.cli'] = charmhelpers.cli +sys.modules['charmhelpers.contrib.hahelpers'] = charmhelpers.contrib.hahelpers +sys.modules['charmhelpers.contrib.hahelpers.cluster'] = ( + charmhelpers.contrib.hahelpers.cluster) diff --git a/unit_tests/test_lib_charm_openstack_manila.py b/unit_tests/test_lib_charm_openstack_manila.py new file mode 100644 index 0000000..6a4868c --- /dev/null +++ b/unit_tests/test_lib_charm_openstack_manila.py @@ -0,0 +1,283 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import print_function + +import mock + +import charm.openstack.manila as manila + +import charms_openstack.test_utils as test_utils + + +class Helper(test_utils.PatchHelper): + + def setUp(self): + super().setUp() + self.patch_release(manila.ManilaCharm.release) + + +class TestManilaCharmUtilities(Helper): + + def test_strip_join(self): + tests1 = ( + ("this is the one", "this is the one"), + ("this, is, the one", "this is the one"), + ("this is, the one", "this is the one")) + for (t, r) in tests1: + self.assertEqual(r, manila.strip_join(t)) + tests2 = ( + ("this is the one", "this, is, the, one"), + ("this, is, the one", "this, is, the, one"), + ("this is, the one", "this, is, the, one")) + for (t, r) in tests2: + self.assertEqual(r, manila.strip_join(t, divider=", ")) + + +class TestManilaCharmConfigProperties(Helper): + + def test_computed_share_backends(self): + config = mock.MagicMock() + config.charm_instance.configured_backends = ["a", "c", "b"] + self.assertEqual(manila.computed_share_backends(config), "a c b") + + def test_computed_share_protocols(self): + config = mock.MagicMock() + config.share_protocols = "a c b" + self.assertEqual(manila.computed_share_protocols(config), "A,C,B") + + def test_computed_backend_lines_manila_conf(self): + config = mock.MagicMock() + config.share_protocols = "a c b" + config.charm_instance.config_lines_for.return_value = "Test Value" + self.assertEqual(manila.computed_backend_lines_manila_conf(config), + "Test Value") + config.charm_instance.config_lines_for.assert_called_once_with( + manila.MANILA_CONF) + + def test_computed_debug_level(self): + config = mock.MagicMock() + config.debug = False + config.verbose = False + self.assertEqual(manila.computed_debug_level(config), "NONE") + config.verbose = True + self.assertEqual(manila.computed_debug_level(config), "NONE") + config.debug = True + config.verbose = False + self.assertEqual(manila.computed_debug_level(config), "WARNING") + config.verbose = True + self.assertEqual(manila.computed_debug_level(config), "DEBUG") + + +class TestManilaCharm(Helper): + + def _patch_config_and_charm(self, config): + self.patch_object(manila.hookenv, 'config') + + def cf(key=None): + if key is not None: + return config[key] + return config + + self.config.side_effect = cf + c = manila.ManilaCharm() + return c + + def test_install(self): + self.patch("charms_openstack.charm.OpenStackCharm.install", + name="install") + self.patch("subprocess.check_call", name="check_call") + self.patch("charms_openstack.charm.OpenStackCharm.assess_status", + name="assess_status") + c = manila.ManilaCharm() + c.install() + self.install.assert_called_once_with() + self.check_call.assert_called_once_with(["mkdir", "-p", "/etc/nova"]) + self.assess_status.assert_called_once_with() + + def _patch_get_adapter(self, c): + self.patch_object(c, 'get_adapter') + + def _helper(x): + self.var = x + return self.out + + self.get_adapter.side_effect = _helper + + def test_custom_assess_status_check1(self): + config = { + 'default-share-backend': '', + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + self.out = None + + self.assertEqual(c.configured_backends, []) + self.assertEqual(c.custom_assess_status_check(), + ('blocked', 'No share backends configured')) + self.out = mock.Mock() + self.out.relation.names = ['name1'] + self.assertEqual(c.custom_assess_status_check(), + ('blocked', "'default-share-backend' is not set")) + self.assertEqual(self.var, 'manila-plugin.available') + + def test_custom_assess_status_check2(self): + config = { + 'default-share-backend': 'name2', + } + c = self._patch_config_and_charm(config) + self._patch_get_adapter(c) + self.out = mock.Mock() + self.out.relation.names = ['name1'] + self.assertEqual( + c.custom_assess_status_check(), + ('blocked', + "'default-share-backend:name2' is not a configured backend")) + self.out.relation.names = ['name1', 'name2'] + self.assertEqual(c.custom_assess_status_check(), (None, None)) + + def test_get_amqp_credentials(self): + config = { + 'rabbit-user': 'rabbit1', + 'rabbit-vhost': 'password' + } + c = self._patch_config_and_charm(config) + self.assertEqual(c.get_amqp_credentials(), ('rabbit1', 'password')) + + def test_get_database_setup(self): + config = { + 'database': 'db1', + 'database-user': 'user1' + } + c = self._patch_config_and_charm(config) + self.patch_object(manila.hookenv, 'unit_private_ip') + self.unit_private_ip.return_value = 'ip1' + self.assertEqual( + c.get_database_setup(), + [dict(database='db1', username='user1', hostname='ip1')]) + + def test_register_endpoints(self): + # note that this also tests _custom_register_endpoints() indirectly, + # which means it doesn't require a separate test. + keystone = mock.MagicMock() + config = { + 'region': 'the_region', + } + c = self._patch_config_and_charm(config) + self.patch_object(manila.ManilaCharm, + 'public_url', new_callable=mock.PropertyMock) + self.patch_object(manila.ManilaCharm, + 'internal_url', new_callable=mock.PropertyMock) + self.patch_object(manila.ManilaCharm, + 'admin_url', new_callable=mock.PropertyMock) + self.patch_object(manila.ManilaCharm, + 'public_url_v2', new_callable=mock.PropertyMock) + self.patch_object(manila.ManilaCharm, + 'internal_url_v2', new_callable=mock.PropertyMock) + self.patch_object(manila.ManilaCharm, + 'admin_url_v2', new_callable=mock.PropertyMock) + self.public_url.return_value = 'p1' + self.internal_url.return_value = 'i1' + self.admin_url.return_value = 'a1' + self.public_url_v2.return_value = 'p2' + self.internal_url_v2.return_value = 'i2' + self.admin_url_v2.return_value = 'a2' + c.register_endpoints(keystone) + v1 = mock.call(v1_admin_url='a1', + v1_internal_url='i1', + v1_public_url='p1', + v1_region='the_region', + v1_service='manila') + v2 = mock.call(v2_admin_url='a2', + v2_internal_url='i2', + v2_public_url='p2', + v2_region='the_region', + v2_service='manilav2') + calls = [v1, v2] + keystone.set_local.assert_has_calls(calls) + keystone.set_remote.assert_has_calls(calls) + + def test_url_endpoints_creation(self): + # Tests that the endpoint functions call through to the baseclass + self.patch_object(manila.charms_openstack.charm.OpenStackCharm, + 'public_url', new_callable=mock.PropertyMock) + self.patch_object(manila.charms_openstack.charm.OpenStackCharm, + 'internal_url', new_callable=mock.PropertyMock) + self.patch_object(manila.charms_openstack.charm.OpenStackCharm, + 'admin_url', new_callable=mock.PropertyMock) + self.public_url.return_value = 'p1' + self.internal_url.return_value = 'i1' + self.admin_url.return_value = 'a1' + c = self._patch_config_and_charm({}) + self.assertEqual(c.public_url, 'p1/v1/%(tenant_id)s') + self.assertEqual(c.internal_url, 'i1/v1/%(tenant_id)s') + self.assertEqual(c.admin_url, 'a1/v1/%(tenant_id)s') + self.assertEqual(c.public_url_v2, 'p1/v2/%(tenant_id)s') + self.assertEqual(c.internal_url_v2, 'i1/v2/%(tenant_id)s') + self.assertEqual(c.admin_url_v2, 'a1/v2/%(tenant_id)s') + + def test_configured_backends(self): + c = self._patch_config_and_charm({}) + self._patch_get_adapter(c) + self.out = None + self.assertEqual(c.configured_backends, []) + self.assertEqual(self.var, 'manila-plugin.available') + self.out = mock.Mock() + self.out.relation.names = ['a', 'b'] + self.assertEqual(c.configured_backends, ['a', 'b']) + + def test_config_lines_for(self): + c = self._patch_config_and_charm({}) + self._patch_get_adapter(c) + self.out = None + self.assertEqual(c.config_lines_for('conf'), []) + self.assertEqual(self.var, 'manila-plugin.available') + self.out = mock.Mock() + self.out.relation.get_configuration_data.return_value = {} + self.assertEqual(c.config_lines_for('conf'), []) + config = { + 'conf': { + 'complete': True, + '[section1]': ( + 'line1', 'line2'), + '[section2]': ( + 'line3', ), + }, + 'conf2': { + 'complete': True, + '[section3]': ( + 'line4', 'line5'), + }, + 'conf3': { + 'complete': False, + '[section4]': ( + 'line6', 'line7'), + } + } + self.out.relation.get_configuration_data.return_value = config + self.assertEqual(c.config_lines_for('conf'), [ + '[section1]', + 'line1', + 'line2', + '', + '[section2]', + 'line3', + '']) + self.assertEqual(c.config_lines_for('conf2'), [ + '[section3]', + 'line4', + 'line5', + '']) + self.assertEqual(c.config_lines_for('conf3'), []) diff --git a/unit_tests/test_manila_handlers.py b/unit_tests/test_manila_handlers.py new file mode 100644 index 0000000..b3431ee --- /dev/null +++ b/unit_tests/test_manila_handlers.py @@ -0,0 +1,90 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import print_function + +import mock + +import reactive.manila_handlers as handlers + +import charms_openstack.test_utils as test_utils + + +class TestRegisteredHooks(test_utils.TestRegisteredHooks): + + def test_hooks(self): + defaults = [ + 'charm.installed', + 'amqp.connected', + 'shared-db.connected', + 'identity-service.available', # enables SSL support + ] + hook_set = { + 'when': { + 'render_stuff': ('shared-db.available', + 'identity-service.available', + 'amqp.available', ), + 'register_endpoints': ('identity-service.connected', ), + 'share_to_manila_plugins_auth': ('identity-service.connected', + 'manila-plugin.connected', ), + 'maybe_do_syncdb': ('shared-db.available', + 'manila.config.rendered', ), + 'config_changed': ('config.changed', + 'shared-db.available', + 'identity-service.available', + 'amqp.available', ) + } + } + # test that the hooks were registered via the + # reactive.barbican_handlers + self.registered_hooks_test_helper(handlers, hook_set, defaults) + + +class TestRenderStuff(test_utils.PatchHelper): + + def _patch_provide_charm_instance(self): + manila_charm = mock.MagicMock() + self.patch('charms_openstack.charm.provide_charm_instance', + name='provide_charm_instance', + new=mock.MagicMock()) + self.provide_charm_instance().__enter__.return_value = manila_charm + self.provide_charm_instance().__exit__.return_value = None + return manila_charm + + def test_register_endpoints(self): + manila_charm = self._patch_provide_charm_instance() + handlers.register_endpoints('keystone') + manila_charm.register_endpoints.assert_called_once_with('keystone') + manila_charm.assess_status.assert_called_once_with() + + def test_maybe_do_syncdb(self): + manila_charm = self._patch_provide_charm_instance() + handlers.maybe_do_syncdb('shared_db') + manila_charm.db_sync.assert_called_once_with() + + def test_render_stuff(self): + manila_charm = self._patch_provide_charm_instance() + self.patch('charms.reactive.set_state', name='set_state') + + handlers.render_stuff('arg1', 'arg2') + manila_charm.render_with_interfaces.assert_called_once_with( + ('arg1', 'arg2', )) + manila_charm.assess_status.assert_called_once_with() + self.set_state.assert_called_once_with('manila.config.rendered') + + def test_config_changed(self): + self.patch_object(handlers, 'render_stuff') + handlers.config_changed('hello', 'there') + self.render_stuff.assert_called_once_with('hello', 'there')