Enable Bionic as a gate test

Change bionic test from dev to gate for 18.05.

Change-Id: Iaff111e3ac3802481448ac95b936cf043a441fc0

hooks/charmhelpers/contrib/openstack/utils.py

@@ -306,7 +306,7 @@ def get_os_codename_install_source(src):
 
     if src.startswith('cloud:'):
         ca_rel = src.split(':')[1]
-        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        ca_rel = ca_rel.split('-')[1].split('/')[0]
         return ca_rel
 
     # Best guess match based on deb string provided

hooks/charmhelpers/contrib/openstack/vaultlocker.py

@@ -21,6 +21,9 @@ import charmhelpers.contrib.openstack.context as context
 import charmhelpers.core.hookenv as hookenv
 import charmhelpers.core.host as host
 import charmhelpers.core.templating as templating
+import charmhelpers.core.unitdata as unitdata
+
+VAULTLOCKER_BACKEND = 'charm-vaultlocker'
 
 
 class VaultKVContext(context.OSContextGenerator):
@@ -34,30 +37,39 @@ class VaultKVContext(context.OSContextGenerator):
         )
 
     def __call__(self):
+        db = unitdata.kv()
+        last_token = db.get('last-token')
+        secret_id = db.get('secret-id')
         for relation_id in hookenv.relation_ids(self.interfaces[0]):
             for unit in hookenv.related_units(relation_id):
-                vault_url = hookenv.relation_get(
+                data = hookenv.relation_get(unit=unit,
-                    'vault_url',
+                                            rid=relation_id)
-                    unit=unit,
+                vault_url = data.get('vault_url')
-                    rid=relation_id
+                role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
-                )
+                token = data.get('{}_token'.format(hookenv.local_unit()))
-                role_id = hookenv.relation_get(
+
-                    '{}_role_id'.format(hookenv.local_unit()),
+                if all([vault_url, role_id, token]):
-                    unit=unit,
+                    token = json.loads(token)
-                    rid=relation_id
+                    vault_url = json.loads(vault_url)
-                )
+
+                    # Tokens may change when secret_id's are being
+                    # reissued - if so use token to get new secret_id
+                    if token != last_token:
+                        secret_id = retrieve_secret_id(
+                            url=vault_url,
+                            token=token
+                        )
+                        db.set('secret-id', secret_id)
+                        db.set('last-token', token)
+                        db.flush()
 
-                if vault_url and role_id:
                     ctxt = {
-                        'vault_url': json.loads(vault_url),
+                        'vault_url': vault_url,
                         'role_id': json.loads(role_id),
+                        'secret_id': secret_id,
                         'secret_backend': self.secret_backend,
                     }
-                    vault_ca = hookenv.relation_get(
+                    vault_ca = data.get('vault_ca')
-                        'vault_ca',
-                        unit=unit,
-                        rid=relation_id
-                    )
                     if vault_ca:
                         ctxt['vault_ca'] = json.loads(vault_ca)
                     self.complete = True
@@ -82,3 +94,33 @@ def write_vaultlocker_conf(context, priority=100):
     alternatives.install_alternative('vaultlocker.conf',
                                      '/etc/vaultlocker/vaultlocker.conf',
                                      charm_vl_path, priority)
+
+
+def vault_relation_complete(backend=None):
+    """Determine whether vault relation is complete
+
+    :param backend: Name of secrets backend requested
+    :ptype backend: string
+    :returns: whether the relation to vault is complete
+    :rtype: bool"""
+    vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND)
+    vault_kv()
+    return vault_kv.complete
+
+
+# TODO: contrib a high level unwrap method to hvac that works
+def retrieve_secret_id(url, token):
+    """Retrieve a response-wrapped secret_id from Vault
+
+    :param url: URL to Vault Server
+    :ptype url: str
+    :param token: One shot Token to use
+    :ptype token: str
+    :returns: secret_id to use for Vault Access
+    :rtype: str"""
+    import hvac
+    client = hvac.Client(url=url, token=token)
+    response = client._post('/v1/sys/wrapping/unwrap')
+    if response.status_code == 200:
+        data = response.json()
+        return data['data']['secret_id']

hooks/charmhelpers/contrib/storage/linux/utils.py

@@ -67,3 +67,19 @@ def is_device_mounted(device):
     except Exception:
         return False
     return bool(re.search(r'MOUNTPOINT=".+"', out))
+
+
+def mkfs_xfs(device, force=False):
+    """Format device with XFS filesystem.
+
+    By default this should fail if the device already has a filesystem on it.
+    :param device: Full path to device to format
+    :ptype device: tr
+    :param force: Force operation
+    :ptype: force: boolean"""
+    cmd = ['mkfs.xfs']
+    if force:
+        cmd.append("-f")
+
+    cmd += ['-i', 'size=1024', device]
+    check_call(cmd)

hooks/charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
-            self._prev_dict = json.load(f)
+            try:
+                self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v

hooks/charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
-    try:
+    if type(sysctl_dict) is not dict:
-        sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+        try:
-    except yaml.YAMLError:
+            sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
-        log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+        except yaml.YAMLError:
-            level=ERROR)
+            log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
-        return
+                level=ERROR)
+            return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

tests/basic_deployment.py

@@ -19,6 +19,10 @@ import re
 import time
 import json
 
+import keystoneclient
+from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import client as nova_client
+
 from charmhelpers.contrib.openstack.amulet.deployment import (
     OpenStackAmuletDeployment
 )
@@ -150,47 +154,97 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
             self._get_openstack_release_string()))
 
         # Authenticate admin with keystone
-        self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
+        self.keystone_session, self.keystone = u.get_default_keystone_session(
-                                                      user='admin',
+            self.keystone_sentry,
-                                                      password='openstack',
+            openstack_release=self._get_openstack_release())
-                                                      tenant='admin')
+
         # Authenticate admin with cinder endpoint
         self.cinder = u.authenticate_cinder_admin(self.keystone)
         # Authenticate admin with glance endpoint
         self.glance = u.authenticate_glance_admin(self.keystone)
 
         # Authenticate admin with nova endpoint
-        self.nova = u.authenticate_nova_user(self.keystone,
+        self.nova = nova_client.Client(2, session=self.keystone_session)
-                                             user='admin',
+
-                                             password='openstack',
+        keystone_ip = self.keystone_sentry.info['public-address']
-                                             tenant='admin')
 
         # Create a demo tenant/role/user
         self.demo_tenant = 'demoTenant'
         self.demo_role = 'demoRole'
         self.demo_user = 'demoUser'
+        self.demo_project = 'demoProject'
+        self.demo_domain = 'demoDomain'
+        if self._get_openstack_release() >= self.xenial_queens:
+            self.create_users_v3()
+            self.demo_user_session, auth = u.get_keystone_session(
+                keystone_ip,
+                self.demo_user,
+                'password',
+                api_version=3,
+                user_domain_name=self.demo_domain,
+                project_domain_name=self.demo_domain,
+                project_name=self.demo_project
+            )
+            self.keystone_demo = keystone_client_v3.Client(
+                session=self.demo_user_session)
+            self.nova_demo = nova_client.Client(
+                2,
+                session=self.demo_user_session)
+        else:
+            self.create_users_v2()
+            # Authenticate demo user with keystone
+            self.keystone_demo = \
+                u.authenticate_keystone_user(
+                    self.keystone, user=self.demo_user,
+                    password='password',
+                    tenant=self.demo_tenant)
+            # Authenticate demo user with nova-api
+            self.nova_demo = u.authenticate_nova_user(self.keystone,
+                                                      user=self.demo_user,
+                                                      password='password',
+                                                      tenant=self.demo_tenant)
+
+    def create_users_v3(self):
+        try:
+            self.keystone.projects.find(name=self.demo_project)
+        except keystoneclient.exceptions.NotFound:
+            domain = self.keystone.domains.create(
+                self.demo_domain,
+                description='Demo Domain',
+                enabled=True
+            )
+            project = self.keystone.projects.create(
+                self.demo_project,
+                domain,
+                description='Demo Project',
+                enabled=True,
+            )
+            user = self.keystone.users.create(
+                self.demo_user,
+                domain=domain.id,
+                project=self.demo_project,
+                password='password',
+                email='demov3@demo.com',
+                description='Demo',
+                enabled=True)
+            role = self.keystone.roles.find(name='Admin')
+            self.keystone.roles.grant(
+                role.id,
+                user=user.id,
+                project=project.id)
+
+    def create_users_v2(self):
         if not u.tenant_exists(self.keystone, self.demo_tenant):
             tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant,
                                                   description='demo tenant',
                                                   enabled=True)
+
             self.keystone.roles.create(name=self.demo_role)
             self.keystone.users.create(name=self.demo_user,
                                        password='password',
                                        tenant_id=tenant.id,
                                        email='demo@demo.com')
 
-        # Authenticate demo user with keystone
-        self.keystone_demo = u.authenticate_keystone_user(self.keystone,
-                                                          self.demo_user,
-                                                          'password',
-                                                          self.demo_tenant)
-
-        # Authenticate demo user with nova-api
-        self.nova_demo = u.authenticate_nova_user(self.keystone,
-                                                  self.demo_user,
-                                                  'password',
-                                                  self.demo_tenant)
-
     def test_100_ceph_processes(self):
         """Verify that the expected service processes are running
         on each ceph unit."""

tests/charmhelpers/core/hookenv.py

@@ -290,7 +290,7 @@ class Config(dict):
         self.implicit_save = True
         self._prev_dict = None
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
-        if os.path.exists(self.path):
+        if os.path.exists(self.path) and os.stat(self.path).st_size:
             self.load_previous()
         atexit(self._implicit_save)
 
@@ -310,7 +310,11 @@ class Config(dict):
         """
         self.path = path or self.path
         with open(self.path) as f:
-            self._prev_dict = json.load(f)
+            try:
+                self._prev_dict = json.load(f)
+            except ValueError as e:
+                log('Unable to parse previous config data - {}'.format(str(e)),
+                    level=ERROR)
         for k, v in copy.deepcopy(self._prev_dict).items():
             if k not in self:
                 self[k] = v

tests/charmhelpers/core/sysctl.py

@@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 def create(sysctl_dict, sysctl_file):
     """Creates a sysctl.conf file from a YAML associative array
 
-    :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
+    :param sysctl_dict: a dict or YAML-formatted string of sysctl
+                        options eg "{ 'kernel.max_pid': 1337 }"
     :type sysctl_dict: str
     :param sysctl_file: path to the sysctl file to be saved
     :type sysctl_file: str or unicode
     :returns: None
     """
-    try:
+    if type(sysctl_dict) is not dict:
-        sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
+        try:
-    except yaml.YAMLError:
+            sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
-        log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
+        except yaml.YAMLError:
-            level=ERROR)
+            log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
-        return
+                level=ERROR)
+            return
+    else:
+        sysctl_dict_parsed = sysctl_dict
 
     with open(sysctl_file, "w") as fd:
         for key, value in sysctl_dict_parsed.items():

tests/gate-basic-xenial-queens

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic ceph deployment on xenial-queens."""
+
+from basic_deployment import CephBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephBasicDeployment(series='xenial',
+                                     openstack='cloud:xenial-queens',
+                                     source='cloud:xenial-updates/queens')
+    deployment.run_tests()

tox.ini

@@ -65,7 +65,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test