Add .gitreview and clean up repo

Several updates to make this consistent with other OpenStack
charms repos, and to enable Gerrit + OSCI test gates.

- Update .gitignore.

- Add .gitreview for gerrit.

- Add copyright file, and copyright headers.

- Add tox.ini and src/tox.ini

- Add tests README

- Fix layer.yaml

- Add unit tests.

- Add amulet tests.

- fixed lint.

See running bug/TODO list:

- https://bugs.launchpad.net/charm-ceph-fs

Change-Id: Iaec5f25706387f9b59b36e307e954318b23c9417
Depends-On: I3f78a5a3231377f5542b81cec27dd4531144cfd1
Closes-Bug: 1640895
Closes-Bug: 1640896
Closes-Bug: 1640898
Closes-Bug: 1640900
Closes-Bug: 1640902
Closes-Bug: 1653767

.gitignore

@@ -1 +1,8 @@
+build
+.tox
+layers
+interfaces
+.testrepository
+__pycache__
+*.pyc
 .idea

.gitreview

@@ -0,0 +1,4 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=openstack/charm-ceph-fs.git

.testr.conf

@@ -0,0 +1,8 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION
+
+test_id_option=--load-list $IDFILE
+test_list_option=--list

requirements.txt

@@ -0,0 +1,4 @@
+# Requirements to build the charm
+charm-tools
+simplejson
+flake8

src/actions.yaml

@@ -2,7 +2,7 @@ get-quota:
   description: View quota settings on a directory
   params:
     max-files:
-      type: boolean
+      type: integer
       description: |
         The limit of how many files can be written. Use either this or
         max-bytes but not both. The action tries max-files first and then
@@ -23,7 +23,7 @@ remove-quota:
   description: Remove a quota on a directory
   params:
     max-files:
-      type: boolean
+      type: integer
       description: |
         The limit of how many files can be written. Use either this or
         max-bytes but not both. The action tries max-files first and then

src/actions/__init__.py

@@ -0,0 +1 @@
+__author__ = 'Chris Holcombe <chris.holcombe@canonical.com>'

src/actions/get-quota

@@ -1 +1 @@
-get-quota.py
+get_quota.py

src/actions/get_quota.py

@@ -29,9 +29,9 @@ def get_quota():
         action_fail("Directory must exist before setting quota")
     attr = "ceph.quota.{}"
     if max_files:
-        attr.format("max_files")
+        attr = attr.format("max_files")
     elif max_bytes:
-        attr.format("max_bytes")
+        attr = attr.format("max_bytes")
 
     try:
         quota_value = xattr.getxattr(directory, attr)

src/actions/remove-quota

@@ -1 +1 @@
-remove-quota.py
+remove_quota.py

src/actions/remove_quota.py

@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from charmhelpers.core.hookenv import action_get, action_fail, action_set
+
+from charmhelpers.core.hookenv import action_get, action_fail
 import xattr
 
 __author__ = 'Chris Holcombe <chris.holcombe@canonical.com>'
@@ -29,9 +30,9 @@ def remove_quota():
         action_fail("Directory must exist before setting quota")
     attr = "ceph.quota.{}"
     if max_files:
-        attr.format("max_files")
+        attr = attr.format("max_files")
     elif max_bytes:
-        attr.format("max_bytes")
+        attr = attr.format("max_bytes")
 
     try:
         xattr.setxattr(directory, attr, str(0))

src/actions/set-quota

@@ -1 +1 @@
-set-quota.py
+set_quota.py

src/actions/set_quota.py

@@ -30,10 +30,10 @@ def set_quota():
     attr = "ceph.quota.{}"
     value = None
     if max_files:
-        attr.format("max_files")
+        attr = attr.format("max_files")
         value = str(max_files)
     elif max_bytes:
-        attr.format("max_bytes")
+        attr = attr.format("max_bytes")
         value = str(max_bytes)
 
     try:

src/config.yaml

@@ -1,7 +1,4 @@
 options:
-  apt:
-    packages:
-      - python3-pyxattr
   ceph-public-network:
     type: string
     default:

src/copyright

@@ -0,0 +1,6 @@
+Format: http://dep.debian.net/deps/dep5/
+
+Files: *
+Copyright: Copyright 2016, Canonical Ltd
+License: Apache-2.0
+

src/layer.yaml

@@ -1,2 +1,6 @@
-includes: ['layer:apt', 'layer:ceph-base', 'interface:/home/chris/repos/juju-interface-ceph-mds']  # if you use any interfaces, add them here
-repo: git@github.com:cholcombe973/charm-ceph-fs.git
+includes: ['layer:apt', 'layer:ceph-base', 'interface:ceph-mds']  # if you use any interfaces, add them here
+options:
+  apt:
+    packages:
+      - python3-pyxattr
+repo: https://git.openstack.org/openstack/charm-ceph-fs

src/metadata.yaml

@@ -11,6 +11,7 @@ tags:
   - misc
 series:
   - xenial
+  - yakkety
 subordinate: false
 requires:
   ceph-mds:

src/reactive/ceph_fs.py

@@ -1,19 +1,38 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import socket
 import subprocess
 
 from charms.reactive import when, when_not, set_state
 from charmhelpers.core.hookenv import (
-    config, log, ERROR, service_name)
+    application_version_set, config, log, ERROR)
 from charmhelpers.core.host import service_restart
 from charmhelpers.contrib.network.ip import (
     get_address_in_network
 )
+from charmhelpers.fetch import (
+    get_upstream_version,
+)
+
 import jinja2
 
 from charms.apt import queue_install
 
 TEMPLATES_DIR = 'templates'
+VERSION_PACKAGE = 'ceph-common'
 
 
 def render_template(template_name, context, template_dir=TEMPLATES_DIR):
@@ -35,6 +54,7 @@ def setup_mds(relation):
     try:
         service_restart('ceph-mds')
         set_state('cephfs.started')
+        application_version_set(get_upstream_version(VERSION_PACKAGE))
     except subprocess.CalledProcessError as err:
         log(message='Error: {}'.format(err), level=ERROR)
 
@@ -77,10 +97,8 @@ def config_changed(ceph_client):
             key_file.write("[mds.{}]\n\tkey = {}\n".format(
                 socket.gethostname(),
                 ceph_client.mds_key()
-                # ceph_client.mds_bootstrap_key()
             ))
     except IOError as err:
-
         log("IOError writing mds-a.keyring: {}".format(err))
     set_state('cephfs.configured')
 

src/test-requirements.txt

@@ -0,0 +1,23 @@
+# charm-proof
+charm-tools>=2.0.0
+# amulet deployment helpers
+bzr+lp:charm-helpers#egg=charmhelpers
+# BEGIN: Amulet OpenStack Charm Helper Requirements
+# Liberty client lower constraints
+amulet>=1.14.3,<2.0
+bundletester>=0.6.1,<1.0
+aodhclient>=0.1.0
+python-ceilometerclient>=1.5.0,<2.0
+python-cinderclient>=1.4.0,<2.0
+python-glanceclient>=1.1.0,<2.0
+python-heatclient>=0.8.0,<1.0
+python-keystoneclient>=1.7.1,<2.0
+python-neutronclient>=3.1.0,<4.0
+python-novaclient>=2.30.1,<3.0
+python-openstackclient>=1.7.0,<2.0
+python-swiftclient>=2.6.0,<3.0
+pika>=0.10.0,<1.0
+distro-info
+# END: Amulet OpenStack Charm Helper Requirements
+# NOTE: workaround for 14.04 pip/tox
+pytz

src/tests/00-setup

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-sudo add-apt-repository ppa:juju/stable -y
-sudo apt-get update
-sudo apt-get install amulet python-requests -y

src/tests/10-deploy

@@ -1,31 +0,0 @@
-#!/usr/bin/python3
-
-import amulet
-import requests
-import unittest
-
-
-class TestCharm(unittest.TestCase):
-    def setUp(self):
-        self.d = amulet.Deployment()
-
-        self.d.add('charm-ceph-fs')
-        self.d.expose('charm-ceph-fs')
-
-        self.d.setup(timeout=900)
-        self.d.sentry.wait()
-
-        self.unit = self.d.sentry['charm-ceph-fs'][0]
-
-    def test_service(self):
-        # test we can access over http
-        page = requests.get('http://{}'.format(self.unit.info['public-address']))
-        self.assertEqual(page.status_code, 200)
-        # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform
-        # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods:
-        # - .info - An array of the information of that unit from Juju
-        # - .file(PATH) - Get the details of a file on that unit
-        # - .file_contents(PATH) - Get plain text output of PATH file from that unit
-        # - .directory(PATH) - Get details of directory
-        # - .directory_contents(PATH) - List files and folders in PATH on that unit
-        # - .relation(relation, service:rel) - Get relation data from return service

src/tests/README.md

@@ -0,0 +1,9 @@
+# Overview
+
+This directory provides Amulet tests to verify basic deployment functionality
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
+
+For full details on functional testing of OpenStack charms please refer to
+the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing)
+section of the OpenStack Charm Guide.

src/tests/basic_deployment.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import amulet
+
+from charmhelpers.contrib.openstack.amulet.deployment import (
+    OpenStackAmuletDeployment
+)
+from charmhelpers.contrib.openstack.amulet.utils import (  # noqa
+    OpenStackAmuletUtils,
+    DEBUG,
+    )
+
+# Use DEBUG to turn on debug logging
+u = OpenStackAmuletUtils(DEBUG)
+
+
+class CephFsBasicDeployment(OpenStackAmuletDeployment):
+    """Amulet tests on a basic ceph deployment."""
+
+    def __init__(self, series=None, openstack=None, source=None, stable=False):
+        """Deploy the entire test environment."""
+        super(CephFsBasicDeployment, self).__init__(series,
+                                                    openstack,
+                                                    source,
+                                                    stable)
+        self._add_services()
+        self._add_relations()
+        self._configure_services()
+        self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+        exclude_services = []
+
+        # Wait for deployment ready msgs, except exclusions
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
+        self.d.sentry.wait()
+        self._initialize_tests()
+
+    def _add_services(self, **kwargs):
+        """Add services
+
+           Add the services that we're testing, where cephfs is local,
+           and the rest of the service are from lp branches that are
+           compatible with the local charm (e.g. stable or next).
+        :param **kwargs:
+           """
+        this_service = {'name': 'ceph-fs', 'units': 1}
+        other_services = [
+            {'name': 'ceph-mon', 'units': 3},
+            {'name': 'ceph-osd', 'units': 3},
+        ]
+        super(CephFsBasicDeployment, self)._add_services(this_service,
+                                                         other_services)
+
+    def _add_relations(self, **kwargs):
+        """Add all of the relations for the services.
+        :param **kwargs:
+        """
+        relations = {
+            'ceph-osd:mon': 'ceph-mon:osd',
+            'ceph-fs:ceph-mds': 'ceph-mon:mds',
+        }
+        super(CephFsBasicDeployment, self)._add_relations(relations)
+
+    def _configure_services(self, **kwargs):
+        """Configure all of the services.
+        :param **kwargs:
+        """
+        # Include a non-existent device as osd-devices is a whitelist,
+        # and this will catch cases where proposals attempt to change that.
+        ceph_mon_config = {
+            'monitor-count': '3',
+            'auth-supported': 'none',
+            'fsid': '6547bd3e-1397-11e2-82e5-53567c8d32dc',
+        }
+
+        # Include a non-existent device as osd-devices is a whitelist,
+        # and this will catch cases where proposals attempt to change that.
+        ceph_osd_config = {
+            'osd-reformat': 'yes',
+            'ephemeral-unmount': '/mnt',
+            'osd-devices': '/dev/vdb /srv/ceph /dev/test-non-existent'
+        }
+
+        configs = {
+            'ceph-mon': ceph_mon_config,
+            'ceph-osd': ceph_osd_config}
+        super(CephFsBasicDeployment, self)._configure_services(configs)
+
+    def _initialize_tests(self):
+        """Perform final initialization before tests get run."""
+        # Access the sentries for inspecting service units
+        self.ceph_osd0_sentry = self.d.sentry['ceph-osd'][0]
+        self.ceph_osd1_sentry = self.d.sentry['ceph-osd'][1]
+        self.ceph_osd2_sentry = self.d.sentry['ceph-osd'][2]
+
+        self.ceph_mon0_sentry = self.d.sentry['ceph-mon'][0]
+        self.ceph_mon1_sentry = self.d.sentry['ceph-mon'][1]
+        self.ceph_mon2_sentry = self.d.sentry['ceph-mon'][2]
+
+        self.ceph_mds_sentry = self.d.sentry['ceph-fs'][0]
+
+    def test_100_ceph_processes(self):
+        """Verify that the expected service processes are running
+        on each ceph unit."""
+
+        # Process name and quantity of processes to expect on each unit
+        ceph_mon_processes = {
+            'ceph-mon': 1
+        }
+        ceph_osd_processes = {
+            'ceph-osd': 2
+        }
+        ceph_mds_processes = {
+            'ceph-mds': 1
+        }
+
+        # Units with process names and PID quantities expected
+        expected_processes = {
+            self.ceph_mon0_sentry: ceph_mon_processes,
+            self.ceph_mon1_sentry: ceph_mon_processes,
+            self.ceph_mon2_sentry: ceph_mon_processes,
+            self.ceph_osd0_sentry: ceph_osd_processes,
+            self.ceph_osd1_sentry: ceph_osd_processes,
+            self.ceph_osd2_sentry: ceph_osd_processes,
+            self.ceph_mds_sentry: ceph_mds_processes
+        }
+
+        actual_pids = u.get_unit_process_ids(expected_processes)
+        ret = u.validate_unit_process_ids(expected_processes, actual_pids)
+        if ret:
+            amulet.raise_status(amulet.FAIL, msg=ret)
+
+    def test_102_services(self):
+        """Verify the expected services are running on the service units."""
+        services = {}
+
+        if self._get_openstack_release() < self.xenial_mitaka:
+            # For upstart systems only.  Ceph services under systemd
+            # are checked by process name instead.
+            ceph_services = [
+                'ceph-mon-all',
+                'ceph-mon id=`hostname`'
+            ]
+            services[self.ceph_mon0_sentry] = ceph_services
+            services[self.ceph_mon1_sentry] = ceph_services
+            services[self.ceph_mon2_sentry] = ceph_services
+
+            ceph_osd_services = [
+                'ceph-osd id={}'.format(u.get_ceph_osd_id_cmd(0)),
+            ]
+
+            services[self.ceph_osd0_sentry] = ceph_osd_services
+
+        ret = u.validate_services_by_name(services)
+        if ret:
+            amulet.raise_status(amulet.FAIL, msg=ret)
+
+    def test_300_ceph_config(self):
+        """Verify the data in the ceph config file."""
+        u.log.debug('Checking ceph config file data...')
+        unit = self.ceph_mon0_sentry
+        conf = '/etc/ceph/ceph.conf'
+        expected = {
+            'global': {
+                'keyring': '/etc/ceph/$cluster.$name.keyring',
+                'fsid': '6547bd3e-1397-11e2-82e5-53567c8d32dc',
+                'log to syslog': 'false',
+                'err to syslog': 'false',
+                'clog to syslog': 'false',
+                'mon cluster log to syslog': 'false',
+                'auth cluster required': 'none',
+                'auth service required': 'none',
+                'auth client required': 'none'
+            },
+            'mon': {
+                'keyring': '/var/lib/ceph/mon/$cluster-$id/keyring'
+            },
+            'mds': {
+                'keyring': '/var/lib/ceph/mds/$cluster-$id/keyring'
+            },
+        }
+
+        for section, pairs in expected.iteritems():
+            ret = u.validate_config_data(unit, conf, section, pairs)
+            if ret:
+                message = "ceph config error: {}".format(ret)
+                amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_400_ceph_check_osd_pools(self):
+        """Check osd pools on all ceph units, expect them to be
+        identical, and expect specific pools to be present."""
+        u.log.debug('Checking pools on ceph units...')
+        expected_pools = ['rbd', 'ceph-fs_data', 'ceph-fs_metadata']
+        results = []
+        sentries = [
+            self.ceph_mon0_sentry,
+            self.ceph_mon1_sentry,
+            self.ceph_mon2_sentry
+        ]
+
+        # Check for presence of expected pools on each unit
+        u.log.debug('Expected pools: {}'.format(expected_pools))
+        for sentry_unit in sentries:
+            pools = u.get_ceph_pools(sentry_unit)
+            results.append(pools)
+
+            for expected_pool in expected_pools:
+                if expected_pool not in pools:
+                    msg = ('{} does not have pool: '
+                           '{}'.format(sentry_unit.info['unit_name'],
+                                       expected_pool))
+                    amulet.raise_status(amulet.FAIL, msg=msg)
+            u.log.debug('{} has (at least) the expected '
+                        'pools.'.format(sentry_unit.info['unit_name']))
+
+        # Check that all units returned the same pool name:id data
+        ret = u.validate_list_of_identical_dicts(results)
+        if ret:
+            u.log.debug('Pool list results: {}'.format(results))
+            msg = ('{}; Pool list results are not identical on all '
+                   'ceph units.'.format(ret))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        else:
+            u.log.debug('Pool list on all ceph units produced the '
+                        'same results (OK).')
+
+    def test_499_ceph_cmds_exit_zero(self):
+        """Check basic functionality of ceph cli commands against
+        all ceph units."""
+        sentry_units = [
+            self.ceph_mon0_sentry,
+            self.ceph_mon1_sentry,
+            self.ceph_mon2_sentry
+        ]
+        commands = [
+            'sudo ceph health',
+            'sudo ceph mds stat',
+            'sudo ceph pg stat',
+            'sudo ceph osd stat',
+            'sudo ceph mon stat',
+            'sudo ceph fs ls',
+        ]
+        ret = u.check_commands_on_units(commands, sentry_units)
+        if ret:
+            amulet.raise_status(amulet.FAIL, msg=ret)
+
+            # FYI: No restart check as ceph services do not restart
+            # when charm config changes, unless monitor count increases.

src/tests/charmhelpers/__init__.py

@@ -0,0 +1,36 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bootstrap charm-helpers, installing its dependencies if necessary using
+# only standard libraries.
+import subprocess
+import sys
+
+try:
+    import six  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
+    import six  # flake8: noqa
+
+try:
+    import yaml  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+    import yaml  # flake8: noqa

src/tests/charmhelpers/contrib/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/tests/charmhelpers/contrib/amulet/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/tests/charmhelpers/contrib/amulet/deployment.py

@@ -0,0 +1,97 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import amulet
+import os
+import six
+
+
+class AmuletDeployment(object):
+    """Amulet deployment.
+
+       This class provides generic Amulet deployment and test runner
+       methods.
+       """
+
+    def __init__(self, series=None):
+        """Initialize the deployment environment."""
+        self.series = None
+
+        if series:
+            self.series = series
+            self.d = amulet.Deployment(series=self.series)
+        else:
+            self.d = amulet.Deployment()
+
+    def _add_services(self, this_service, other_services):
+        """Add services.
+
+           Add services to the deployment where this_service is the local charm
+           that we're testing and other_services are the other services that
+           are being used in the local amulet tests.
+           """
+        if this_service['name'] != os.path.basename(os.getcwd()):
+            s = this_service['name']
+            msg = "The charm's root directory name needs to be {}".format(s)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        if 'units' not in this_service:
+            this_service['units'] = 1
+
+        self.d.add(this_service['name'], units=this_service['units'],
+                   constraints=this_service.get('constraints'))
+
+        for svc in other_services:
+            if 'location' in svc:
+                branch_location = svc['location']
+            elif self.series:
+                branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
+            else:
+                branch_location = None
+
+            if 'units' not in svc:
+                svc['units'] = 1
+
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+                       constraints=svc.get('constraints'))
+
+    def _add_relations(self, relations):
+        """Add all of the relations for the services."""
+        for k, v in six.iteritems(relations):
+            self.d.relate(k, v)
+
+    def _configure_services(self, configs):
+        """Configure all of the services."""
+        for service, config in six.iteritems(configs):
+            self.d.configure(service, config)
+
+    def _deploy(self):
+        """Deploy environment and wait for all hooks to finish executing."""
+        timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 900))
+        try:
+            self.d.setup(timeout=timeout)
+            self.d.sentry.wait(timeout=timeout)
+        except amulet.helpers.TimeoutError:
+            amulet.raise_status(
+                amulet.FAIL,
+                msg="Deployment timed out ({}s)".format(timeout)
+            )
+        except Exception:
+            raise
+
+    def run_tests(self):
+        """Run all of the methods that are prefixed with 'test_'."""
+        for test in dir(self):
+            if test.startswith('test_'):
+                getattr(self, test)()

src/tests/charmhelpers/contrib/amulet/utils.py

@@ -0,0 +1,827 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import json
+import logging
+import os
+import re
+import socket
+import subprocess
+import sys
+import time
+import uuid
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
+
+
+class AmuletUtils(object):
+    """Amulet utilities.
+
+       This class provides common utility functions that are used by Amulet
+       tests.
+       """
+
+    def __init__(self, log_level=logging.ERROR):
+        self.log = self.get_logger(level=log_level)
+        self.ubuntu_releases = self.get_ubuntu_releases()
+
+    def get_logger(self, name="amulet-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
+    def valid_ip(self, ip):
+        if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
+            return True
+        else:
+            return False
+
+    def valid_url(self, url):
+        p = re.compile(
+            r'^(?:http|ftp)s?://'
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # noqa
+            r'localhost|'
+            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
+            r'(?::\d+)?'
+            r'(?:/?|[/?]\S+)$',
+            re.IGNORECASE)
+        if p.match(url):
+            return True
+        else:
+            return False
+
+    def get_ubuntu_release_from_sentry(self, sentry_unit):
+        """Get Ubuntu release codename from sentry unit.
+
+        :param sentry_unit: amulet sentry/service unit pointer
+        :returns: list of strings - release codename, failure message
+        """
+        msg = None
+        cmd = 'lsb_release -cs'
+        release, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} lsb_release: {}'.format(
+                sentry_unit.info['unit_name'], release))
+        else:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, release, code))
+        if release not in self.ubuntu_releases:
+            msg = ("Release ({}) not found in Ubuntu releases "
+                   "({})".format(release, self.ubuntu_releases))
+        return release, msg
+
+    def validate_services(self, commands):
+        """Validate that lists of commands succeed on service units.  Can be
+           used to verify system services are running on the corresponding
+           service units.
+
+        :param commands: dict with sentry keys and arbitrary command list vals
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # /!\ DEPRECATION WARNING (beisner):
+        # New and existing tests should be rewritten to use
+        # validate_services_by_name() as it is aware of init systems.
+        self.log.warn('DEPRECATION WARNING:  use '
+                      'validate_services_by_name instead of validate_services '
+                      'due to init system differences.')
+
+        for k, v in six.iteritems(commands):
+            for cmd in v:
+                output, code = k.run(cmd)
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(k.info['unit_name'],
+                                           cmd, code))
+                if code != 0:
+                    return "command `{}` returned {}".format(cmd, str(code))
+        return None
+
+    def validate_services_by_name(self, sentry_services):
+        """Validate system service status by service name, automatically
+           detecting init system based on Ubuntu release codename.
+
+        :param sentry_services: dict with sentry keys and svc list values
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # Point at which systemd became a thing
+        systemd_switch = self.ubuntu_releases.index('vivid')
+
+        for sentry_unit, services_list in six.iteritems(sentry_services):
+            # Get lsb_release codename from unit
+            release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+            if ret:
+                return ret
+
+            for service_name in services_list:
+                if (self.ubuntu_releases.index(release) >= systemd_switch or
+                        service_name in ['rabbitmq-server', 'apache2']):
+                    # init is systemd (or regular sysv)
+                    cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
+                elif self.ubuntu_releases.index(release) < systemd_switch:
+                    # init is upstart
+                    cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
+
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(sentry_unit.info['unit_name'],
+                                           cmd, code))
+                if not service_running:
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
+        return None
+
+    def _get_config(self, unit, filename):
+        """Get a ConfigParser object for parsing a unit's config file."""
+        file_contents = unit.file_contents(filename)
+
+        # NOTE(beisner):  by default, ConfigParser does not handle options
+        # with no value, such as the flags used in the mysql my.cnf file.
+        # https://bugs.python.org/issue7005
+        config = configparser.ConfigParser(allow_no_value=True)
+        config.readfp(io.StringIO(file_contents))
+        return config
+
+    def validate_config_data(self, sentry_unit, config_file, section,
+                             expected):
+        """Validate config file data.
+
+           Verify that the specified section of the config file contains
+           the expected option key:value pairs.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluates a variable and returns a
+           bool.
+           """
+        self.log.debug('Validating config file data ({} in {} on {})'
+                       '...'.format(section, config_file,
+                                    sentry_unit.info['unit_name']))
+        config = self._get_config(sentry_unit, config_file)
+
+        if section != 'DEFAULT' and not config.has_section(section):
+            return "section [{}] does not exist".format(section)
+
+        for k in expected.keys():
+            if not config.has_option(section, k):
+                return "section [{}] is missing option {}".format(section, k)
+
+            actual = config.get(section, k)
+            v = expected[k]
+            if (isinstance(v, six.string_types) or
+                    isinstance(v, bool) or
+                    isinstance(v, six.integer_types)):
+                # handle explicit values
+                if actual != v:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
+            # handle function pointers, such as not_null or valid_ip
+            elif not v(actual):
+                return "section [{}] {}:{} != expected {}:{}".format(
+                       section, k, actual, k, expected[k])
+        return None
+
+    def _validate_dict_data(self, expected, actual):
+        """Validate dictionary data.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluates a variable and returns a
+           bool.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        self.log.debug('expected: {}'.format(repr(expected)))
+
+        for k, v in six.iteritems(expected):
+            if k in actual:
+                if (isinstance(v, six.string_types) or
+                        isinstance(v, bool) or
+                        isinstance(v, six.integer_types)):
+                    # handle explicit values
+                    if v != actual[k]:
+                        return "{}:{}".format(k, actual[k])
+                # handle function pointers, such as not_null or valid_ip
+                elif not v(actual[k]):
+                    return "{}:{}".format(k, actual[k])
+            else:
+                return "key '{}' does not exist".format(k)
+        return None
+
+    def validate_relation_data(self, sentry_unit, relation, expected):
+        """Validate actual relation data based on expected relation data."""
+        actual = sentry_unit.relation(relation[0], relation[1])
+        return self._validate_dict_data(expected, actual)
+
+    def _validate_list_data(self, expected, actual):
+        """Compare expected list vs actual list data."""
+        for e in expected:
+            if e not in actual:
+                return "expected item {} not found in actual list".format(e)
+        return None
+
+    def not_null(self, string):
+        if string is not None:
+            return True
+        else:
+            return False
+
+    def _get_file_mtime(self, sentry_unit, filename):
+        """Get last modification time of file."""
+        return sentry_unit.file_stat(filename)['mtime']
+
+    def _get_dir_mtime(self, sentry_unit, directory):
+        """Get last modification time of directory."""
+        return sentry_unit.directory_stat(directory)['mtime']
+
+    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
+        """Get start time of a process based on the last modification time
+           of the /proc/pid directory.
+
+        :sentry_unit:  The sentry unit to check for the service on
+        :service:  service name to look for in process table
+        :pgrep_full:  [Deprecated] Use full command line search mode with pgrep
+        :returns:  epoch time of service process start
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns:  None if successful; Failure message otherwise
+        """
+        if pgrep_full is not None:
+            # /!\ DEPRECATION WARNING (beisner):
+            # No longer implemented, as pidof is now used instead of pgrep.
+            # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+            self.log.warn('DEPRECATION WARNING:  pgrep_full bool is no '
+                          'longer implemented re: lp 1474030.')
+
+        pid_list = self.get_process_id_list(sentry_unit, service)
+        pid = pid_list[0]
+        proc_dir = '/proc/{}'.format(pid)
+        self.log.debug('Pid for {} on {}: {}'.format(
+            service, sentry_unit.info['unit_name'], pid))
+
+        return self._get_dir_mtime(sentry_unit, proc_dir)
+
+    def service_restarted(self, sentry_unit, service, filename,
+                          pgrep_full=None, sleep_time=20):
+        """Check if service was restarted.
+
+           Compare a service's start time vs a file's last modification time
+           (such as a config file for that service) to determine if the service
+           has been restarted.
+           """
+        # /!\ DEPRECATION WARNING (beisner):
+        # This method is prone to races in that no before-time is known.
+        # Use validate_service_config_changed instead.
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+        self.log.warn('DEPRECATION WARNING:  use '
+                      'validate_service_config_changed instead of '
+                      'service_restarted due to known races.')
+
+        time.sleep(sleep_time)
+        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+                self._get_file_mtime(sentry_unit, filename)):
+            return True
+        else:
+            return False
+
+    def service_restarted_since(self, sentry_unit, mtime, service,
+                                pgrep_full=None, sleep_time=20,
+                                retry_count=30, retry_sleep_time=10):
+        """Check if service was been started after a given time.
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check for the service on
+          mtime (float): The epoch time to check against
+          service (string): service name to look for in process table
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
+
+        Returns:
+          bool: True if service found and its start time it newer than mtime,
+                False if service is older than mtime or if service was
+                not found.
+        """
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s service restarted since %s on '
+                       '%s' % (service, mtime, unit_name))
+        time.sleep(sleep_time)
+        proc_start_time = None
+        tries = 0
+        while tries <= retry_count and not proc_start_time:
+            try:
+                proc_start_time = self._get_proc_start_time(sentry_unit,
+                                                            service,
+                                                            pgrep_full)
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'OK'.format(tries, service, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, proc may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'failed\n{}'.format(tries, service,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
+
+        if not proc_start_time:
+            self.log.warn('No proc start time found, assuming service did '
+                          'not start')
+            return False
+        if proc_start_time >= mtime:
+            self.log.debug('Proc start time is newer than provided mtime'
+                           '(%s >= %s) on %s (OK)' % (proc_start_time,
+                                                      mtime, unit_name))
+            return True
+        else:
+            self.log.warn('Proc start time (%s) is older than provided mtime '
+                          '(%s) on %s, service did not '
+                          'restart' % (proc_start_time, mtime, unit_name))
+            return False
+
+    def config_updated_since(self, sentry_unit, filename, mtime,
+                             sleep_time=20, retry_count=30,
+                             retry_sleep_time=10):
+        """Check if file was modified after a given time.
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check the file mtime on
+          filename (string): The file to check mtime of
+          mtime (float): The epoch time to check against
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
+
+        Returns:
+          bool: True if file was modified more recently than mtime, False if
+                file was modified before mtime, or if file not found.
+        """
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s updated since %s on '
+                       '%s' % (filename, mtime, unit_name))
+        time.sleep(sleep_time)
+        file_mtime = None
+        tries = 0
+        while tries <= retry_count and not file_mtime:
+            try:
+                file_mtime = self._get_file_mtime(sentry_unit, filename)
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'OK'.format(tries, filename, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, file may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'failed\n{}'.format(tries, filename,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
+
+        if not file_mtime:
+            self.log.warn('Could not determine file mtime, assuming '
+                          'file does not exist')
+            return False
+
+        if file_mtime >= mtime:
+            self.log.debug('File mtime is newer than provided mtime '
+                           '(%s >= %s) on %s (OK)' % (file_mtime,
+                                                      mtime, unit_name))
+            return True
+        else:
+            self.log.warn('File mtime is older than provided mtime'
+                          '(%s < on %s) on %s' % (file_mtime,
+                                                  mtime, unit_name))
+            return False
+
+    def validate_service_config_changed(self, sentry_unit, mtime, service,
+                                        filename, pgrep_full=None,
+                                        sleep_time=20, retry_count=30,
+                                        retry_sleep_time=10):
+        """Check service and file were updated after mtime
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check for the service on
+          mtime (float): The epoch time to check against
+          service (string): service name to look for in process table
+          filename (string): The file to check mtime of
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep in seconds to pass to test helpers
+          retry_count (int): If service is not found, how many times to retry
+          retry_sleep_time (int): Time in seconds to wait between retries
+
+        Typical Usage:
+            u = OpenStackAmuletUtils(ERROR)
+            ...
+            mtime = u.get_sentry_time(self.cinder_sentry)
+            self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
+            if not u.validate_service_config_changed(self.cinder_sentry,
+                                                     mtime,
+                                                     'cinder-api',
+                                                     '/etc/cinder/cinder.conf')
+                amulet.raise_status(amulet.FAIL, msg='update failed')
+        Returns:
+          bool: True if both service and file where updated/restarted after
+                mtime, False if service is older than mtime or if service was
+                not found or if filename was modified before mtime.
+        """
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        service_restart = self.service_restarted_since(
+            sentry_unit, mtime,
+            service,
+            pgrep_full=pgrep_full,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
+        config_update = self.config_updated_since(
+            sentry_unit,
+            filename,
+            mtime,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
+        return service_restart and config_update
+
+    def get_sentry_time(self, sentry_unit):
+        """Return current epoch time on a sentry"""
+        cmd = "date +'%s'"
+        return float(sentry_unit.run(cmd)[0])
+
+    def relation_error(self, name, data):
+        return 'unexpected relation data in {} - {}'.format(name, data)
+
+    def endpoint_error(self, name, data):
+        return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+    def get_ubuntu_releases(self):
+        """Return a list of all Ubuntu releases in order of release."""
+        _d = distro_info.UbuntuDistroInfo()
+        _release_list = _d.all
+        return _release_list
+
+    def file_to_url(self, file_rel_path):
+        """Convert a relative file path to a file URL."""
+        _abs_path = os.path.abspath(file_rel_path)
+        return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+    def check_commands_on_units(self, commands, sentry_units):
+        """Check that all commands in a list exit zero on all
+        sentry units in a list.
+
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns: None if successful; Failure message otherwise
+        """
+        self.log.debug('Checking exit codes for {} commands on {} '
+                       'sentry units...'.format(len(commands),
+                                                len(sentry_units)))
+        for sentry_unit in sentry_units:
+            for cmd in commands:
+                output, code = sentry_unit.run(cmd)
+                if code == 0:
+                    self.log.debug('{} `{}` returned {} '
+                                   '(OK)'.format(sentry_unit.info['unit_name'],
+                                                 cmd, code))
+                else:
+                    return ('{} `{}` returned {} '
+                            '{}'.format(sentry_unit.info['unit_name'],
+                                        cmd, code, output))
+        return None
+
+    def get_process_id_list(self, sentry_unit, process_name,
+                            expect_success=True):
+        """Get a list of process ID(s) from a single sentry juju unit
+        for a single process name.
+
+        :param sentry_unit: Amulet sentry instance (juju unit)
+        :param process_name: Process name
+        :param expect_success: If False, expect the PID to be missing,
+            raise if it is present.
+        :returns: List of process IDs
+        """
+        cmd = 'pidof -x {}'.format(process_name)
+        if not expect_success:
+            cmd += " || exit 0 && exit 1"
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output).split()
+
+    def get_unit_process_ids(self, unit_processes, expect_success=True):
+        """Construct a dict containing unit sentries, process names, and
+        process IDs.
+
+        :param unit_processes: A dictionary of Amulet sentry instance
+            to list of process names.
+        :param expect_success: if False expect the processes to not be
+            running, raise if they are.
+        :returns: Dictionary of Amulet sentry instance to dictionary
+            of process names to PIDs.
+        """
+        pid_dict = {}
+        for sentry_unit, process_list in six.iteritems(unit_processes):
+            pid_dict[sentry_unit] = {}
+            for process in process_list:
+                pids = self.get_process_id_list(
+                    sentry_unit, process, expect_success=expect_success)
+                pid_dict[sentry_unit].update({process: pids})
+        return pid_dict
+
+    def validate_unit_process_ids(self, expected, actual):
+        """Validate process id quantities for services on units."""
+        self.log.debug('Checking units for running processes...')
+        self.log.debug('Expected PIDs: {}'.format(expected))
+        self.log.debug('Actual PIDs: {}'.format(actual))
+
+        if len(actual) != len(expected):
+            return ('Unit count mismatch.  expected, actual: {}, '
+                    '{} '.format(len(expected), len(actual)))
+
+        for (e_sentry, e_proc_names) in six.iteritems(expected):
+            e_sentry_name = e_sentry.info['unit_name']
+            if e_sentry in actual.keys():
+                a_proc_names = actual[e_sentry]
+            else:
+                return ('Expected sentry ({}) not found in actual dict data.'
+                        '{}'.format(e_sentry_name, e_sentry))
+
+            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+                return ('Process name count mismatch.  expected, actual: {}, '
+                        '{}'.format(len(expected), len(actual)))
+
+            for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
+                    zip(e_proc_names.items(), a_proc_names.items()):
+                if e_proc_name != a_proc_name:
+                    return ('Process name mismatch.  expected, actual: {}, '
+                            '{}'.format(e_proc_name, a_proc_name))
+
+                a_pids_length = len(a_pids)
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+                            '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+                                                 e_pids, a_pids_length,
+                                                 a_pids))
+
+                # If expected is a list, ensure at least one PID quantity match
+                if isinstance(e_pids, list) and \
+                        a_pids_length not in e_pids:
+                    return fail_msg
+                # If expected is not bool and not list,
+                # ensure PID quantities match
+                elif not isinstance(e_pids, bool) and \
+                        not isinstance(e_pids, list) and \
+                        a_pids_length != e_pids:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids, bool) and \
+                        e_pids is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids, bool) and \
+                        e_pids is False and a_pids_length != 0:
+                    return fail_msg
+                else:
+                    self.log.debug('PID check OK: {} {} {}: '
+                                   '{}'.format(e_sentry_name, e_proc_name,
+                                               e_pids, a_pids))
+        return None
+
+    def validate_list_of_identical_dicts(self, list_of_dicts):
+        """Check that all dicts within a list are identical."""
+        hashes = []
+        for _dict in list_of_dicts:
+            hashes.append(hash(frozenset(_dict.items())))
+
+        self.log.debug('Hashes: {}'.format(hashes))
+        if len(set(hashes)) == 1:
+            self.log.debug('Dicts within list are identical')
+        else:
+            return 'Dicts within list are not identical'
+
+        return None
+
+    def validate_sectionless_conf(self, file_contents, expected):
+        """A crude conf parser.  Useful to inspect configuration files which
+        do not have section headers (as would be necessary in order to use
+        the configparser).  Such as openstack-dashboard or rabbitmq confs."""
+        for line in file_contents.split('\n'):
+            if '=' in line:
+                args = line.split('=')
+                if len(args) <= 1:
+                    continue
+                key = args[0].strip()
+                value = args[1].strip()
+                if key in expected.keys():
+                    if expected[key] != value:
+                        msg = ('Config mismatch.  Expected, actual:  {}, '
+                               '{}'.format(expected[key], value))
+                        amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def get_unit_hostnames(self, units):
+        """Return a dict of juju unit names to hostnames."""
+        host_names = {}
+        for unit in units:
+            host_names[unit.info['unit_name']] = \
+                str(unit.file_contents('/etc/hostname').strip())
+        self.log.debug('Unit host names: {}'.format(host_names))
+        return host_names
+
+    def run_cmd_unit(self, sentry_unit, cmd):
+        """Run a command on a unit, return the output and exit code."""
+        output, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} `{}` command returned {} '
+                           '(OK)'.format(sentry_unit.info['unit_name'],
+                                         cmd, code))
+        else:
+            msg = ('{} `{}` command returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output), code
+
+    def file_exists_on_unit(self, sentry_unit, file_name):
+        """Check if a file exists on a unit."""
+        try:
+            sentry_unit.file_stat(file_name)
+            return True
+        except IOError:
+            return False
+        except Exception as e:
+            msg = 'Error checking file {}: {}'.format(file_name, e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def file_contents_safe(self, sentry_unit, file_name,
+                           max_wait=60, fatal=False):
+        """Get file contents from a sentry unit.  Wrap amulet file_contents
+        with retry logic to address races where a file checks as existing,
+        but no longer exists by the time file_contents is called.
+        Return None if file not found. Optionally raise if fatal is True."""
+        unit_name = sentry_unit.info['unit_name']
+        file_contents = False
+        tries = 0
+        while not file_contents and tries < (max_wait / 4):
+            try:
+                file_contents = sentry_unit.file_contents(file_name)
+            except IOError:
+                self.log.debug('Attempt {} to open file {} from {} '
+                               'failed'.format(tries, file_name,
+                                               unit_name))
+                time.sleep(4)
+                tries += 1
+
+        if file_contents:
+            return file_contents
+        elif not fatal:
+            return None
+        elif fatal:
+            msg = 'Failed to get file contents from unit.'
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def port_knock_tcp(self, host="localhost", port=22, timeout=15):
+        """Open a TCP socket to check for a listening sevice on a host.
+
+        :param host: host name or IP address, default to localhost
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :returns: True if successful, False if connect failed
+        """
+
+        # Resolve host name if possible
+        try:
+            connect_host = socket.gethostbyname(host)
+            host_human = "{} ({})".format(connect_host, host)
+        except socket.error as e:
+            self.log.warn('Unable to resolve address: '
+                          '{} ({}) Trying anyway!'.format(host, e))
+            connect_host = host
+            host_human = connect_host
+
+        # Attempt socket connection
+        try:
+            knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            knock.settimeout(timeout)
+            knock.connect((connect_host, port))
+            knock.close()
+            self.log.debug('Socket connect OK for host '
+                           '{} on port {}.'.format(host_human, port))
+            return True
+        except socket.error as e:
+            self.log.debug('Socket connect FAIL for'
+                           ' {} port {} ({})'.format(host_human, port, e))
+            return False
+
+    def port_knock_units(self, sentry_units, port=22,
+                         timeout=15, expect_success=True):
+        """Open a TCP socket to check for a listening sevice on each
+        listed juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :expect_success: True by default, set False to invert logic
+        :returns: None if successful, Failure message otherwise
+        """
+        for unit in sentry_units:
+            host = unit.info['public-address']
+            connected = self.port_knock_tcp(host, port, timeout)
+            if not connected and expect_success:
+                return 'Socket connect failed.'
+            elif connected and not expect_success:
+                return 'Socket connected unexpectedly.'
+
+    def get_uuid_epoch_stamp(self):
+        """Returns a stamp string based on uuid4 and epoch time.  Useful in
+        generating test messages which need to be unique-ish."""
+        return '[{}-{}]'.format(uuid.uuid4(), time.time())
+
+# amulet juju action helpers:
+    def run_action(self, unit_sentry, action,
+                   _check_output=subprocess.check_output,
+                   params=None):
+        """Run the named action on a given unit sentry.
+
+        params a dict of parameters to use
+        _check_output parameter is used for dependency injection.
+
+        @return action_id.
+        """
+        unit_id = unit_sentry.info["unit_name"]
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        if params is not None:
+            for key, value in params.iteritems():
+                command.append("{}={}".format(key, value))
+        self.log.info("Running command: %s\n" % " ".join(command))
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+        """Wait for a given action, returning if it completed or not.
+
+        _check_output parameter is used for dependency injection.
+        """
+        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                   action_id]
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        return data.get(u"status") == "completed"
+
+    def status_get(self, unit):
+        """Return the current service status of this unit."""
+        raw_status, return_code = unit.run(
+            "status-get --format=json --include-data")
+        if return_code != 0:
+            return ("unknown", "")
+        status = json.loads(raw_status)
+        return (status["status"], status["message"])

src/tests/charmhelpers/contrib/openstack/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/tests/charmhelpers/contrib/openstack/amulet/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -0,0 +1,345 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+import sys
+import six
+from collections import OrderedDict
+from charmhelpers.contrib.amulet.deployment import (
+    AmuletDeployment
+)
+
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
+
+class OpenStackAmuletDeployment(AmuletDeployment):
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
+
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
+        """Initialize the deployment environment."""
+        super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
+        self.openstack = openstack
+        self.source = source
+        self.stable = stable
+
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
+    def _determine_branch_locations(self, other_services):
+        """Determine the branch locations for the other services.
+
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
+        # Charms outside the ~openstack-charmers
+        base_charms = {
+            'mysql': ['precise', 'trusty'],
+            'mongodb': ['precise', 'trusty'],
+            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
+        }
+
+        for svc in other_services:
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if svc['name'] in base_charms:
+                # NOTE: not all charms have support for all series we
+                #       want/need to test against, so fix to most recent
+                #       that each base charm supports
+                target_series = self.series
+                if self.series not in base_charms[svc['name']]:
+                    target_series = base_charms[svc['name']][-1]
+                svc['location'] = 'cs:{}/{}'.format(target_series,
+                                                    svc['name'])
+            elif self.stable:
+                svc['location'] = 'cs:{}/{}'.format(self.series,
+                                                    svc['name'])
+            else:
+                svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
+                    self.series,
+                    svc['name']
+                )
+
+        return other_services
+
+    def _add_services(self, this_service, other_services, use_source=None,
+                      no_origin=None):
+        """Add services to the deployment and optionally set
+        openstack-origin/source.
+
+        :param this_service dict: Service dictionary describing the service
+                                  whose amulet tests are being run
+        :param other_services dict: List of service dictionaries describing
+                                    the services needed to support the target
+                                    service
+        :param use_source list: List of services which use the 'source' config
+                                option rather than 'openstack-origin'
+        :param no_origin list: List of services which do not support setting
+                               the Cloud Archive.
+        Service Dict:
+            {
+                'name': str charm-name,
+                'units': int number of units,
+                'constraints': dict of juju constraints,
+                'location': str location of charm,
+            }
+        eg
+        this_service = {
+            'name': 'openvswitch-odl',
+            'constraints': {'mem': '8G'},
+        }
+        other_services = [
+            {
+                'name': 'nova-compute',
+                'units': 2,
+                'constraints': {'mem': '4G'},
+                'location': cs:~bob/xenial/nova-compute
+            },
+            {
+                'name': 'mysql',
+                'constraints': {'mem': '2G'},
+            },
+            {'neutron-api-odl'}]
+        use_source = ['mysql']
+        no_origin = ['neutron-api-odl']
+        """
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
+        other_services = self._determine_branch_locations(other_services)
+
+        super(OpenStackAmuletDeployment, self)._add_services(this_service,
+                                                             other_services)
+
+        services = other_services
+        services.append(this_service)
+
+        use_source = use_source or []
+        no_origin = no_origin or []
+
+        # Charms which should use the source config option
+        use_source = list(set(
+            use_source + ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                          'ceph-osd', 'ceph-radosgw', 'ceph-mon',
+                          'ceph-proxy', 'percona-cluster', 'lxd']))
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = list(set(
+            no_origin + ['cinder-ceph', 'hacluster', 'neutron-openvswitch',
+                         'nrpe', 'openvswitch-odl', 'neutron-api-odl',
+                         'odl-controller', 'cinder-backup', 'nexentaedge-data',
+                         'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
+                         'cinder-nexentaedge', 'nexentaedge-mgmt']))
+
+        if self.openstack:
+            for svc in services:
+                if svc['name'] not in use_source + no_origin:
+                    config = {'openstack-origin': self.openstack}
+                    self.d.configure(svc['name'], config)
+
+        if self.source:
+            for svc in services:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
+                    config = {'source': self.source}
+                    self.d.configure(svc['name'], config)
+
+    def _configure_services(self, configs):
+        """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
+        for service, config in six.iteritems(configs):
+            self.d.configure(service, config)
+
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
+    def _get_openstack_release(self):
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        # Must be ordered by OpenStack release (not by Ubuntu release):
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty, self.trusty_mitaka,
+         self.xenial_mitaka, self.xenial_newton,
+         self.yakkety_newton) = range(16)
+
+        releases = {
+            ('precise', None): self.precise_essex,
+            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
+            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
+            ('precise', 'cloud:precise-havana'): self.precise_havana,
+            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+            ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty,
+            ('xenial', None): self.xenial_mitaka,
+            ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+            ('yakkety', None): self.yakkety_newton,
+        }
+        return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+            ('wily', 'liberty'),
+            ('xenial', 'mitaka'),
+            ('yakkety', 'newton'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools

src/tests/gate-basic-xenial-mitaka

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic ceph deployment on xenial-mitaka."""
+
+from basic_deployment import CephFsBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephFsBasicDeployment(series='xenial')
+deployment.run_tests()

src/tests/tests.yaml

@@ -0,0 +1,17 @@
+# Bootstrap the model if necessary.
+bootstrap: True
+# Re-use bootstrap node instead of destroying/re-bootstrapping.
+reset: True
+# Use tox/requirements to drive the venv instead of bundletester's venv feature.
+virtualenv: False
+# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet.
+makefile: []
+# Do not specify juju PPA sources.  Juju is presumed to be pre-installed
+# and configured in all test runner environments.
+#sources:
+# Do not specify or rely on system packages.
+#packages:
+# Do not specify python packages here.  Use test-requirements.txt
+# and tox instead.  ie. The venv is constructed before bundletester
+# is invoked.
+#python-packages:

src/tox.ini

@@ -0,0 +1,53 @@
+# Source charm: ./src/tox.ini
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.
+[tox]
+envlist = pep8
+skipsdist = True
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONHASHSEED=0
+         AMULET_SETUP_TIMEOUT=2700
+whitelist_externals = juju
+passenv = HOME TERM AMULET_*
+deps = -r{toxinidir}/test-requirements.txt
+install_command =
+  pip install --allow-unverified python-apt {opts} {packages}
+
+[testenv:pep8]
+basepython = python2.7
+commands = charm-proof
+
+[testenv:func27-noop]
+# DRY RUN - For Debug
+basepython = python2.7
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy
+
+[testenv:func27]
+# Run all gate tests which are +x (expected to always pass)
+basepython = python2.7
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy
+
+[testenv:func27-smoke]
+# Run a specific test as an Amulet smoke test (expected to always pass)
+basepython = python2.7
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+
+[testenv:func27-dfs]
+# Run all deploy-from-source tests which are +x (may not always pass!)
+basepython = python2.7
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy
+
+[testenv:func27-dev]
+# Run all development test targets which are +x (may not always pass!)
+basepython = python2.7
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy
+
+[testenv:venv]
+commands = {posargs}

test-requirements.txt

@@ -0,0 +1,7 @@
+# Unit test requirements
+flake8>=2.2.4,<=2.4.1
+os-testr>=0.4.1
+charms.reactive
+mock>=1.2
+coverage>=3.6
+git+https://github.com/openstack/charms.openstack#egg=charms.openstack

tox.ini

@@ -1,41 +1,55 @@
+# Source charm: ./tox.ini
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.
 [tox]
-envlist = pep8,py34
 skipsdist = True
+envlist = pep8,py34,py35
 skip_missing_interpreters = True
 
 [testenv]
-basepython = python2.7
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
          TERM=linux
-         INTERFACE_PATH={toxinidir}/interfaces
          LAYER_PATH={toxinidir}/layers
+         INTERFACE_PATH={toxinidir}/interfaces
          JUJU_REPOSITORY={toxinidir}/build
 passenv = http_proxy https_proxy
 install_command =
   pip install {opts} {packages}
 deps =
-  -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements.txt
 
 [testenv:build]
+basepython = python2.7
 commands =
-  charm-build --log-level DEBUG -o {toxinidir}/build --name ceph-mon src
+    charm-build --log-level DEBUG -o {toxinidir}/build src {posargs}
 
-[testenv:venv]
-commands = {posargs}
-
-[testenv:pep8]
-commands = flake8 {posargs} reactive lib unit_tests
+[testenv:py27]
+basepython = python2.7
+# Reactive source charms are Python3-only, but a py27 unit test target
+# is required by OpenStack Governance.  Remove this shim as soon as
+# permitted.  http://governance.openstack.org/reference/cti/python_cti.html
+whitelist_externals = true
+commands = true
 
 [testenv:py34]
 basepython = python3.4
 deps = -r{toxinidir}/test-requirements.txt
-commands = ostestr -p {toxinidir}/build/trusty/ceph-mon {posargs}
+commands = ostestr {posargs}
 
 [testenv:py35]
 basepython = python3.5
 deps = -r{toxinidir}/test-requirements.txt
-commands = ostestr -p {toxinidir}/build/trusty/ceph-mon {posargs}
+commands = ostestr {posargs}
+
+[testenv:pep8]
+basepython = python3.5
+deps = -r{toxinidir}/test-requirements.txt
+commands = flake8 {posargs} src unit_tests
+
+[testenv:venv]
+commands = {posargs}
 
 [flake8]
-ignore = E402,E226
+# E402 ignore necessary for path append before sys module import in actions
+ignore = E402

unit_tests/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import mock
+
+sys.path.append('src')
+
+apt_pkg = mock.MagicMock()
+charmhelpers = mock.MagicMock()
+sys.modules['apt_pkg'] = apt_pkg
+sys.modules['charmhelpers'] = charmhelpers
+sys.modules['charmhelpers.core'] = charmhelpers.core
+sys.modules['charmhelpers.core.hookenv'] = charmhelpers.core.hookenv
+sys.modules['charmhelpers.core.host'] = charmhelpers.core.host
+sys.modules['charmhelpers.core.unitdata'] = charmhelpers.core.unitdata
+sys.modules['charmhelpers.core.templating'] = charmhelpers.core.templating
+sys.modules['charmhelpers.contrib'] = charmhelpers.contrib
+sys.modules['charmhelpers.contrib.openstack'] = charmhelpers.contrib.openstack
+sys.modules['charmhelpers.contrib.openstack.utils'] = (
+    charmhelpers.contrib.openstack.utils)
+sys.modules['charmhelpers.contrib.openstack.templating'] = (
+    charmhelpers.contrib.openstack.templating)
+sys.modules['charmhelpers.contrib.network'] = charmhelpers.contrib.network
+sys.modules['charmhelpers.contrib.network.ip'] = (
+    charmhelpers.contrib.network.ip)
+sys.modules['charmhelpers.fetch'] = charmhelpers.fetch
+sys.modules['charmhelpers.cli'] = charmhelpers.cli
+sys.modules['charmhelpers.contrib.hahelpers'] = charmhelpers.contrib.hahelpers
+sys.modules['charmhelpers.contrib.hahelpers.cluster'] = (
+    charmhelpers.contrib.hahelpers.cluster)

unit_tests/test_actions.py

@@ -0,0 +1,79 @@
+import sys
+
+sys.path.append('src/actions')
+import unittest
+from mock import patch, call, Mock
+
+__author__ = 'Chris Holcombe <chris.holcombe@canonical.com>'
+
+sys.modules['action_set'] = Mock()
+sys.modules['action_get'] = Mock()
+sys.modules['action_fail'] = Mock()
+sys.modules['xattr'] = Mock()
+from get_quota import get_quota
+from remove_quota import remove_quota
+from set_quota import set_quota
+
+
+def action_get_side_effect(*args):
+    if args[0] == 'max-files':
+        return 1024
+    elif args[0] == 'max-bytes':
+        return 1024
+    elif args[0] == 'directory':
+        return 'foo'
+
+
+class CephActionsTestCase(unittest.TestCase):
+    @patch('get_quota.action_fail')
+    @patch('get_quota.action_set')
+    @patch('get_quota.action_get')
+    @patch('get_quota.os')
+    @patch('get_quota.xattr')
+    def test_get_quota(self, xattr, os, action_get, action_set, action_fail):
+        action_get.side_effect = action_get_side_effect
+        os.path.exists.return_value = True
+        xattr.getxattr.return_value = "1024"
+        get_quota()
+        action_get.assert_has_calls(
+            [call('max-files'),
+             call('max-bytes'),
+             call('directory')])
+        action_fail.assert_not_called()
+        xattr.getxattr.assert_called_with('foo',
+                                          'ceph.quota.max_files')
+        action_set.assert_called_with({'foo quota': "1024"})
+
+    @patch('set_quota.action_fail')
+    @patch('set_quota.action_get')
+    @patch('set_quota.os')
+    @patch('set_quota.xattr')
+    def test_set_quota(self, xattr, os, action_get, action_fail):
+        action_get.side_effect = action_get_side_effect
+        os.path.exists.return_value = True
+        set_quota()
+        xattr.setxattr.assert_called_with('foo',
+                                          'ceph.quota.max_files',
+                                          '1024')
+        action_get.assert_has_calls(
+            [call('max-files'),
+             call('max-bytes'),
+             call('directory')])
+        action_fail.assert_not_called()
+
+    @patch('remove_quota.action_fail')
+    @patch('remove_quota.action_get')
+    @patch('remove_quota.os')
+    @patch('remove_quota.xattr')
+    def test_remove_quota(self, xattr, os, action_get, action_fail):
+        action_get.side_effect = action_get_side_effect
+        os.path.exists.return_value = True
+        remove_quota()
+        xattr.setxattr.assert_called_with('foo',
+                                          'ceph.quota.max_files',
+                                          '0')
+        action_get.assert_has_calls(
+            [call('max-files'),
+             call('max-bytes'),
+             call('directory')])
+        action_fail.assert_not_called()

unit_tests/test_utils.py

@@ -0,0 +1,116 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+import os
+import yaml
+
+from mock import patch
+
+
+def load_config():
+    '''
+    Walk backwords from __file__ looking for config.yaml, load and return the
+    'options' section'
+    '''
+    config = None
+    f = __file__
+    while config is None:
+        d = os.path.dirname(f)
+        if os.path.isfile(os.path.join(d, 'config.yaml')):
+            config = os.path.join(d, 'config.yaml')
+            break
+        f = d
+
+    if not config:
+        logging.error('Could not find config.yaml in any parent directory '
+                      'of %s. ' % f)
+        raise Exception
+
+    return yaml.safe_load(open(config).read())['options']
+
+
+def get_default_config():
+    '''
+    Load default charm config from config.yaml return as a dict.
+    If no default is set in config.yaml, its value is None.
+    '''
+    default_config = {}
+    config = load_config()
+    for k, v in config.iteritems():
+        if 'default' in v:
+            default_config[k] = v['default']
+        else:
+            default_config[k] = None
+    return default_config
+
+
+class CharmTestCase(unittest.TestCase):
+
+    def setUp(self, obj, patches):
+        super(CharmTestCase, self).setUp()
+        self.patches = patches
+        self.obj = obj
+        self.test_config = TestConfig()
+        self.test_relation = TestRelation()
+        self.patch_all()
+
+    def patch(self, method):
+        _m = patch.object(self.obj, method)
+        mock = _m.start()
+        self.addCleanup(_m.stop)
+        return mock
+
+    def patch_all(self):
+        for method in self.patches:
+            setattr(self, method, self.patch(method))
+
+
+class TestConfig(object):
+
+    def __init__(self):
+        self.config = get_default_config()
+
+    def get(self, attr=None):
+        if not attr:
+            return self.get_all()
+        try:
+            return self.config[attr]
+        except KeyError:
+            return None
+
+    def get_all(self):
+        return self.config
+
+    def set(self, attr, value):
+        if attr not in self.config:
+            raise KeyError
+        self.config[attr] = value
+
+
+class TestRelation(object):
+
+    def __init__(self, relation_data={}):
+        self.relation_data = relation_data
+
+    def set(self, relation_data):
+        self.relation_data = relation_data
+
+    def get(self, attr=None, unit=None, rid=None):
+        if attr is None:
+            return self.relation_data
+        elif attr in self.relation_data:
+            return self.relation_data[attr]
+        return None