Update amulet test definitions

This change requires the following charm-helpers change
to land first:

- https://github.com/juju/charm-helpers/pull/32

Change-Id: I7c1c2399cb38fd48139f3d42854442796e0d3a3b

hooks/charmhelpers/contrib/openstack/utils.py

@@ -392,6 +392,8 @@ def get_swift_codename(version):
             releases = UBUNTU_OPENSTACK_RELEASE
             release = [k for k, v in six.iteritems(releases) if codename in v]
             ret = subprocess.check_output(['apt-cache', 'policy', 'swift'])
+            if six.PY3:
+                ret = ret.decode('UTF-8')
             if codename in ret or release[0] in ret:
                 return codename
     elif len(codenames) == 1:

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -377,12 +377,12 @@ def get_mon_map(service):
         try:
             return json.loads(mon_status)
         except ValueError as v:
-            log("Unable to parse mon_status json: {}. Error: {}".format(
-                mon_status, v.message))
+            log("Unable to parse mon_status json: {}. Error: {}"
+                .format(mon_status, str(v)))
             raise
     except CalledProcessError as e:
-        log("mon_status command failed with message: {}".format(
-            e.message))
+        log("mon_status command failed with message: {}"
+            .format(str(e)))
         raise
 
 

hooks/charmhelpers/contrib/storage/linux/lvm.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 from subprocess import (
     CalledProcessError,
     check_call,
@@ -101,3 +102,52 @@ def create_lvm_volume_group(volume_group, block_device):
     :block_device: str: Full path of PV-initialized block device.
     '''
     check_call(['vgcreate', volume_group, block_device])
+
+
+def list_logical_volumes(select_criteria=None, path_mode=False):
+    '''
+    List logical volumes
+
+    :param select_criteria: str: Limit list to those volumes matching this
+                                 criteria (see 'lvs -S help' for more details)
+    :param path_mode: bool: return logical volume name in 'vg/lv' format, this
+                            format is required for some commands like lvextend
+    :returns: [str]: List of logical volumes
+    '''
+    lv_diplay_attr = 'lv_name'
+    if path_mode:
+        # Parsing output logic relies on the column order
+        lv_diplay_attr = 'vg_name,' + lv_diplay_attr
+    cmd = ['lvs', '--options', lv_diplay_attr, '--noheadings']
+    if select_criteria:
+        cmd.extend(['--select', select_criteria])
+    lvs = []
+    for lv in check_output(cmd).decode('UTF-8').splitlines():
+        if not lv:
+            continue
+        if path_mode:
+            lvs.append('/'.join(lv.strip().split()))
+        else:
+            lvs.append(lv.strip())
+    return lvs
+
+
+list_thin_logical_volume_pools = functools.partial(
+    list_logical_volumes,
+    select_criteria='lv_attr =~ ^t')
+
+list_thin_logical_volumes = functools.partial(
+    list_logical_volumes,
+    select_criteria='lv_attr =~ ^V')
+
+
+def extend_logical_volume_by_device(lv_name, block_device):
+    '''
+    Extends the size of logical volume lv_name by the amount of free space on
+    physical volume block_device.
+
+    :param lv_name: str: name of logical volume to be extended (vg/lv format)
+    :param block_device: str: name of block_device to be allocated to lv_name
+    '''
+    cmd = ['lvextend', lv_name, block_device]
+    check_call(cmd)

hooks/charmhelpers/core/host.py

@@ -549,6 +549,8 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
         with open(path, 'wb') as target:
             os.fchown(target.fileno(), uid, gid)
             os.fchmod(target.fileno(), perms)
+            if six.PY3 and isinstance(content, six.string_types):
+                content = content.encode('UTF-8')
             target.write(content)
         return
     # the contents were the same, but we might still need to change the

hooks/charmhelpers/core/host_factory/ubuntu.py

@@ -20,6 +20,7 @@ UBUNTU_RELEASES = (
     'yakkety',
     'zesty',
     'artful',
+    'bionic',
 )
 
 

tests/basic_deployment.py

@@ -60,7 +60,7 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
            """
         this_service = {'name': 'ceph', 'units': 3}
         other_services = [
-            {'name': 'percona-cluster', 'constraints': {'mem': '3072M'}},
+            {'name': 'percona-cluster'},
             {'name': 'keystone'},
             {'name': 'rabbitmq-server'},
             {'name': 'nova-compute'},
@@ -99,7 +99,7 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
         pxc_config = {
-            'dataset-size': '25%',
+            'innodb-buffer-pool-size': '256M',
             'max-connections': 1000,
             'root-password': 'ChangeMe123',
             'sst-password': 'ChangeMe123',
@@ -555,11 +555,14 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-        # Validate ceph cinder pool disk space usage samples over time
-        ret = u.validate_ceph_pool_samples(pool_size_samples,
-                                           "cinder pool disk usage")
-        if ret:
-            amulet.raise_status(amulet.FAIL, msg=ret)
+        # Luminous (pike) ceph seems more efficient at disk usage so we cannot
+        # grantee the ordering of kb_used
+        if self._get_openstack_release() < self.xenial_pike:
+            # Validate ceph cinder pool disk space usage samples over time
+            ret = u.validate_ceph_pool_samples(pool_size_samples,
+                                               "cinder pool disk usage")
+            if ret:
+                amulet.raise_status(amulet.FAIL, msg=ret)
 
     def test_412_ceph_glance_image_create_delete(self):
         """Create and confirm a ceph-backed glance image, and inspect

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import os
 import re
 import sys
 import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             self.d.configure(service, config)
 
     def _auto_wait_for_status(self, message=None, exclude_services=None,
-                              include_only=None, timeout=1800):
+                              include_only=None, timeout=None):
         """Wait for all units to have a specific extended status, except
         for any defined as excluded.  Unless specified via message, any
         status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         :param timeout: Maximum time in seconds to wait for status match
         :returns: None.  Raises if timeout is hit.
         """
-        self.log.info('Waiting for extended status on units...')
+        if not timeout:
+            timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
+        self.log.info('Waiting for extended status on units for {}s...'
+                      ''.format(timeout))
 
         all_services = self.d.services.keys()
 
@@ -252,9 +256,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         service_messages = {service: message for service in services}
 
         # Check for idleness
-        self.d.sentry.wait()
+        self.d.sentry.wait(timeout=timeout)
         # Check for error states and bail early
-        self.d.sentry.wait_for_status(self.d.juju_env, services)
+        self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
         # Check for ready messages
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
 

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -858,9 +858,12 @@ class OpenStackAmuletUtils(AmuletUtils):
         :returns: List of pool name, object count, kb disk space used
         """
         df = self.get_ceph_df(sentry_unit)
-        pool_name = df['pools'][pool_id]['name']
-        obj_count = df['pools'][pool_id]['stats']['objects']
-        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        for pool in df['pools']:
+            if pool['id'] == pool_id:
+                pool_name = pool['name']
+                obj_count = pool['stats']['objects']
+                kb_used = pool['stats']['kb_used']
+
         self.log.debug('Ceph {} pool (ID {}): {} objects, '
                        '{} kb used'.format(pool_name, pool_id,
                                            obj_count, kb_used))

tests/charmhelpers/core/host.py

@@ -549,6 +549,8 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
         with open(path, 'wb') as target:
             os.fchown(target.fileno(), uid, gid)
             os.fchmod(target.fileno(), perms)
+            if six.PY3 and isinstance(content, six.string_types):
+                content = content.encode('UTF-8')
             target.write(content)
         return
     # the contents were the same, but we might still need to change the

tests/charmhelpers/core/host_factory/ubuntu.py

@@ -20,6 +20,7 @@ UBUNTU_RELEASES = (
     'yakkety',
     'zesty',
     'artful',
+    'bionic',
 )
 
 

tests/gate-basic-artful-pike

@@ -21,3 +21,6 @@ from basic_deployment import CephBasicDeployment
 if __name__ == '__main__':
     deployment = CephBasicDeployment(series='artful')
     deployment.run_tests()
+
+# NOTE(beisner):  Artful target disabled, pending bug:
+# https://bugs.launchpad.net/charm-percona-cluster/+bug/1728132

tox.ini

@@ -60,7 +60,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test