Switch to using charm-store for amulet tests

All OpenStack charms are now directly published to the charm store
on landing; switch Amulet helper to resolve charms using the
charm store rather than bzr branches, removing the lag between
charm changes landing and being available for other charms to
use for testing.

This is also important for new layered charms where the charm must
be build and published prior to being consumable.

This patch also fixes a potential restart of apache2 when the unit
is in paused state during the execution of config-changed; this is
the root cause of the intermittent amulet test failures.

Change-Id: I90e61f5c217ae709bc30ffe16b1569b9fd2c5719

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.openstack = openstack
         self.source = source
         self.stable = stable
-        # Note(coreycb): this needs to be changed when new next branches come
-        # out.
-        self.current_next = "trusty"
 
     def get_logger(self, name="deployment-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         self.log.info('OpenStackAmuletDeployment:  determine branch locations')
 
-        # Charms outside the lp:~openstack-charmers namespace
+        # Charms outside the ~openstack-charmers
-        base_charms = ['mysql', 'mongodb', 'nrpe']
+        base_charms = {
-
+            'mysql': ['precise', 'trusty'],
-        # Force these charms to current series even when using an older series.
+            'mongodb': ['precise', 'trusty'],
-        # ie. Use trusty/nrpe even when series is precise, as the P charm
+            'nrpe': ['precise', 'trusty'],
-        # does not possess the necessary external master config and hooks.
+        }
-        force_series_current = ['nrpe']
-
-        if self.series in ['precise', 'trusty']:
-            base_series = self.series
-        else:
-            base_series = self.current_next
 
         for svc in other_services:
-            if svc['name'] in force_series_current:
-                base_series = self.current_next
             # If a location has been explicitly set, use it
             if svc.get('location'):
                 continue
-            if self.stable:
+            if svc['name'] in base_charms:
-                temp = 'lp:charms/{}/{}'
+                # NOTE: not all charms have support for all series we
-                svc['location'] = temp.format(base_series,
+                #       want/need to test against, so fix to most recent
-                                              svc['name'])
+                #       that each base charm supports
+                target_series = self.series
+                if self.series not in base_charms[svc['name']]:
+                    target_series = base_charms[svc['name']][-1]
+                svc['location'] = 'cs:{}/{}'.format(target_series,
+                                                    svc['name'])
+            elif self.stable:
+                svc['location'] = 'cs:{}/{}'.format(self.series,
+                                                    svc['name'])
             else:
-                if svc['name'] in base_charms:
+                svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
-                    temp = 'lp:charms/{}/{}'
+                    self.series,
-                    svc['location'] = temp.format(base_series,
+                    svc['name']
-                                                  svc['name'])
+                )
-                else:
-                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
-                    svc['location'] = temp.format(self.current_next,
-                                                  svc['name'])
 
         return other_services
 

charmhelpers/contrib/storage/linux/ceph.py

@@ -1231,7 +1231,7 @@ class CephConfContext(object):
 
         permitted = self.permitted_sections
         if permitted:
-            diff = set(conf.keys()).symmetric_difference(set(permitted))
+            diff = set(conf.keys()).difference(set(permitted))
             if diff:
                 log("Config-flags contains invalid keys '%s' - they will be "
                     "ignored" % (', '.join(diff)), level=WARNING)

charmhelpers/core/host.py

@@ -176,7 +176,7 @@ def init_is_systemd():
 
 
 def adduser(username, password=None, shell='/bin/bash', system_user=False,
-            primary_group=None, secondary_groups=None):
+            primary_group=None, secondary_groups=None, uid=None):
     """Add a user to the system.
 
     Will log but otherwise succeed if the user already exists.
@@ -187,15 +187,21 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False,
     :param bool system_user: Whether to create a login or system user
     :param str primary_group: Primary group for user; defaults to username
     :param list secondary_groups: Optional list of additional groups
+    :param int uid: UID for user being created
 
     :returns: The password database entry struct, as returned by `pwd.getpwnam`
     """
     try:
         user_info = pwd.getpwnam(username)
         log('user {0} already exists!'.format(username))
+        if uid:
+            user_info = pwd.getpwuid(int(uid))
+            log('user with uid {0} already exists!'.format(uid))
     except KeyError:
         log('creating user {0}'.format(username))
         cmd = ['useradd']
+        if uid:
+            cmd.extend(['--uid', str(uid)])
         if system_user or password is None:
             cmd.append('--system')
         else:
@@ -230,14 +236,58 @@ def user_exists(username):
     return user_exists
 
 
-def add_group(group_name, system_group=False):
+def uid_exists(uid):
-    """Add a group to the system"""
+    """Check if a uid exists"""
+    try:
+        pwd.getpwuid(uid)
+        uid_exists = True
+    except KeyError:
+        uid_exists = False
+    return uid_exists
+
+
+def group_exists(groupname):
+    """Check if a group exists"""
+    try:
+        grp.getgrnam(groupname)
+        group_exists = True
+    except KeyError:
+        group_exists = False
+    return group_exists
+
+
+def gid_exists(gid):
+    """Check if a gid exists"""
+    try:
+        grp.getgrgid(gid)
+        gid_exists = True
+    except KeyError:
+        gid_exists = False
+    return gid_exists
+
+
+def add_group(group_name, system_group=False, gid=None):
+    """Add a group to the system
+
+    Will log but otherwise succeed if the group already exists.
+
+    :param str group_name: group to create
+    :param bool system_group: Create system group
+    :param int gid: GID for user being created
+
+    :returns: The password database entry struct, as returned by `grp.getgrnam`
+    """
     try:
         group_info = grp.getgrnam(group_name)
         log('group {0} already exists!'.format(group_name))
+        if gid:
+            group_info = grp.getgrgid(gid)
+            log('group with gid {0} already exists!'.format(gid))
     except KeyError:
         log('creating group {0}'.format(group_name))
         cmd = ['addgroup']
+        if gid:
+            cmd.extend(['--gid', str(gid)])
         if system_group:
             cmd.append('--system')
         else:

hooks/keystone_hooks.py

@@ -51,6 +51,7 @@ from charmhelpers.contrib.openstack.utils import (
     sync_db_with_multi_ipv6_addresses,
     os_release,
     pausable_restart_on_change as restart_on_change,
+    is_unit_paused_set,
 )
 
 from keystone_utils import (
@@ -205,7 +206,8 @@ def config_changed_postupgrade():
         apt_install(filter_installed_packages(determine_packages()))
         service_pause('keystone')
         CONFIGS.write(WSGI_KEYSTONE_CONF)
-        restart_pid_check('apache2')
+        if not is_unit_paused_set():
+            restart_pid_check('apache2')
     configure_https()
     open_port(config('service-port'))
 

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.openstack = openstack
         self.source = source
         self.stable = stable
-        # Note(coreycb): this needs to be changed when new next branches come
-        # out.
-        self.current_next = "trusty"
 
     def get_logger(self, name="deployment-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         self.log.info('OpenStackAmuletDeployment:  determine branch locations')
 
-        # Charms outside the lp:~openstack-charmers namespace
+        # Charms outside the ~openstack-charmers
-        base_charms = ['mysql', 'mongodb', 'nrpe']
+        base_charms = {
-
+            'mysql': ['precise', 'trusty'],
-        # Force these charms to current series even when using an older series.
+            'mongodb': ['precise', 'trusty'],
-        # ie. Use trusty/nrpe even when series is precise, as the P charm
+            'nrpe': ['precise', 'trusty'],
-        # does not possess the necessary external master config and hooks.
+        }
-        force_series_current = ['nrpe']
-
-        if self.series in ['precise', 'trusty']:
-            base_series = self.series
-        else:
-            base_series = self.current_next
 
         for svc in other_services:
-            if svc['name'] in force_series_current:
-                base_series = self.current_next
             # If a location has been explicitly set, use it
             if svc.get('location'):
                 continue
-            if self.stable:
+            if svc['name'] in base_charms:
-                temp = 'lp:charms/{}/{}'
+                # NOTE: not all charms have support for all series we
-                svc['location'] = temp.format(base_series,
+                #       want/need to test against, so fix to most recent
-                                              svc['name'])
+                #       that each base charm supports
+                target_series = self.series
+                if self.series not in base_charms[svc['name']]:
+                    target_series = base_charms[svc['name']][-1]
+                svc['location'] = 'cs:{}/{}'.format(target_series,
+                                                    svc['name'])
+            elif self.stable:
+                svc['location'] = 'cs:{}/{}'.format(self.series,
+                                                    svc['name'])
             else:
-                if svc['name'] in base_charms:
+                svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
-                    temp = 'lp:charms/{}/{}'
+                    self.series,
-                    svc['location'] = temp.format(base_series,
+                    svc['name']
-                                                  svc['name'])
+                )
-                else:
-                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
-                    svc['location'] = temp.format(self.current_next,
-                                                  svc['name'])
 
         return other_services