diff --git a/charmhelpers/contrib/hahelpers/cluster.py b/charmhelpers/contrib/hahelpers/cluster.py index 4a737e24..ba34fba0 100644 --- a/charmhelpers/contrib/hahelpers/cluster.py +++ b/charmhelpers/contrib/hahelpers/cluster.py @@ -25,6 +25,7 @@ Helpers for clustering and determining "cluster leadership" and other clustering-related helpers. """ +import functools import subprocess import os import time @@ -281,6 +282,10 @@ def determine_apache_port(public_port, singlenode_mode=False): return public_port - (i * 10) +determine_apache_port_single = functools.partial( + determine_apache_port, singlenode_mode=True) + + def get_hacluster_config(exclude_keys=None): ''' Obtains all relevant configuration from charm configuration required @@ -404,3 +409,43 @@ def distributed_wait(modulo=None, wait=None, operation_name='operation'): log(msg, DEBUG) status_set('maintenance', msg) time.sleep(calculated_wait) + + +def get_managed_services_and_ports(services, external_ports, + external_services=None, + port_conv_f=determine_apache_port_single): + """Get the services and ports managed by this charm. + + Return only the services and corresponding ports that are managed by this + charm. This excludes haproxy when there is a relation with hacluster. This + is because this charm passes responsability for stopping and starting + haproxy to hacluster. + + Similarly, if a relation with hacluster exists then the ports returned by + this method correspond to those managed by the apache server rather than + haproxy. + + :param services: List of services. + :type services: List[str] + :param external_ports: List of ports managed by external services. + :type external_ports: List[int] + :param external_services: List of services to be removed if ha relation is + present. + :type external_services: List[str] + :param port_conv_f: Function to apply to ports to calculate the ports + managed by services controlled by this charm. + :type port_convert_func: f() + :returns: A tuple containing a list of services first followed by a list of + ports. + :rtype: Tuple[List[str], List[int]] + """ + if external_services is None: + external_services = ['haproxy'] + if relation_ids('ha'): + for svc in external_services: + try: + services.remove(svc) + except ValueError: + pass + external_ports = [port_conv_f(p) for p in external_ports] + return services, external_ports diff --git a/charmhelpers/contrib/hardening/audits/apt.py b/charmhelpers/contrib/hardening/audits/apt.py index 67521e17..cad7bf73 100644 --- a/charmhelpers/contrib/hardening/audits/apt.py +++ b/charmhelpers/contrib/hardening/audits/apt.py @@ -52,7 +52,7 @@ class RestrictedPackages(BaseAudit): def __init__(self, pkgs, **kwargs): super(RestrictedPackages, self).__init__(**kwargs) if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'): - self.pkgs = [pkgs] + self.pkgs = pkgs.split() else: self.pkgs = pkgs @@ -100,4 +100,5 @@ class RestrictedPackages(BaseAudit): apt_purge(pkg.name) def is_virtual_package(self, pkg): - return pkg.has_provides and not pkg.has_versions + return (pkg.get('has_provides', False) and + not pkg.get('has_versions', False)) diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 566404a0..161199c4 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import ( INFO, ERROR, related_units, + relation_get, relation_ids, relation_set, status_set, @@ -331,6 +332,10 @@ PACKAGE_CODENAMES = { DEFAULT_LOOPBACK_SIZE = '5G' +DB_SERIES_UPGRADING_KEY = 'cluster-series-upgrading' + +DB_MAINTENANCE_KEYS = [DB_SERIES_UPGRADING_KEY] + class CompareOpenStackReleases(BasicStringComparator): """Provide comparisons of OpenStack releases. @@ -1912,3 +1917,33 @@ def set_db_initialised(): """ juju_log('Setting db-initialised to True', 'DEBUG') leader_set({'db-initialised': True}) + + +def is_db_maintenance_mode(relid=None): + """Check relation data from notifications of db in maintenance mode. + + :returns: Whether db has notified it is in maintenance mode. + :rtype: bool + """ + juju_log('Checking for maintenance notifications', 'DEBUG') + if relid: + r_ids = [relid] + else: + r_ids = relation_ids('shared-db') + rids_units = [(r, u) for r in r_ids for u in related_units(r)] + notifications = [] + for r_id, unit in rids_units: + settings = relation_get(unit=unit, rid=r_id) + for key, value in settings.items(): + if value and key in DB_MAINTENANCE_KEYS: + juju_log( + 'Unit: {}, Key: {}, Value: {}'.format(unit, key, value), + 'DEBUG') + try: + notifications.append(bool_from_string(value)) + except ValueError: + juju_log( + 'Could not discern bool from {}'.format(value), + 'WARN') + pass + return True in notifications diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index 104f91f1..929a75d7 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -38,6 +38,7 @@ so with this we get rid of the dependency. import locale import os import subprocess +import sys class _container(dict): @@ -59,6 +60,13 @@ class Cache(object): def __init__(self, progress=None): pass + def __contains__(self, package): + try: + pkg = self.__getitem__(package) + return pkg is not None + except KeyError: + return False + def __getitem__(self, package): """Get information about a package from apt and dpkg databases. @@ -178,6 +186,28 @@ class Cache(object): return pkgs +class Config(_container): + def __init__(self): + super(Config, self).__init__(self._populate()) + + def _populate(self): + cfgs = {} + cmd = ['apt-config', 'dump'] + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True) + for line in output.splitlines(): + if not line.startswith("CommandLine"): + k, v = line.split(" ", 1) + cfgs[k] = v.strip(";").strip("\"") + + return cfgs + + +# Backwards compatibility with old apt_pkg module +sys.modules[__name__].config = Config() + + def init(): """Compability shim that does nothing.""" pass diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 8cfdd145..edd69e4e 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -33,6 +33,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( determine_api_port, https, get_hacluster_config, + get_managed_services_and_ports, ) from charmhelpers.contrib.openstack import context, templating @@ -2054,34 +2055,6 @@ def assess_status(configs): os_application_version_set(VERSION_PACKAGE) -def get_managed_services_and_ports(): - """Get the services and ports managed by this charm. - - Return only the services and corresponding ports that are managed by this - charm. This excludes haproxy when there is a relation with hacluster. This - is because this charm passes responsability for stopping and starting - haproxy to hacluster. - - Similarly, if a relation with hacluster exists then the ports returned by - this method correspond to those managed by the apache server rather than - haproxy. - - :returns: A tuple containing a list of services first followed by a list of - ports. - :rtype: Tuple[List[str], List[int]] - """ - _services = services() - _ports = determine_ports() - if relation_ids('ha'): - try: - _services.remove('haproxy') - except ValueError: - pass - _ports = [determine_api_port(api_port(space), singlenode_mode=True) - for space in ('keystone-admin', 'keystone-public')] - return _services, _ports - - def assess_status_func(configs, exclude_ha_resource=False): """Helper function to create the function that will assess_status() for the unit. @@ -2099,7 +2072,9 @@ def assess_status_func(configs, exclude_ha_resource=False): """ required_interfaces = REQUIRED_INTERFACES.copy() required_interfaces.update(get_optional_interfaces()) - _services, _ports = get_managed_services_and_ports() + _services, _ports = get_managed_services_and_ports( + services(), + determine_ports()) return make_assess_status_func( configs, required_interfaces, charm_func=check_optional_relations, @@ -2147,7 +2122,9 @@ def _pause_resume_helper(f, configs): @param f: the function to be used with the assess_status(...) function @returns None - this function is executed for its side-effect """ - _services, _ports = get_managed_services_and_ports() + _services, _ports = get_managed_services_and_ports( + services(), + determine_ports()) f(assess_status_func(configs), services=_services, ports=_ports) diff --git a/tests/tests.yaml b/tests/tests.yaml index 6ffefdd8..05b0f8a6 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -3,7 +3,6 @@ smoke_bundles: - bionic-train gate_bundles: - bionic-train -- disco-stein - bionic-stein - bionic-rocky - bionic-queens diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index e24ab7da..5cecd09b 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -871,6 +871,8 @@ class TestKeystoneUtils(CharmTestCase): utils.VERSION_PACKAGE ) + @patch.object(utils, 'determine_ports') + @patch.object(utils, 'services') @patch.object(utils, 'get_optional_interfaces') @patch.object(utils, 'REQUIRED_INTERFACES') @patch.object(utils, 'check_optional_relations') @@ -881,7 +883,11 @@ class TestKeystoneUtils(CharmTestCase): get_managed_services_and_ports, check_optional_relations, REQUIRED_INTERFACES, - get_optional_interfaces): + get_optional_interfaces, + services, + determine_ports): + services.return_value = ['s1'] + determine_ports.return_value = [200] get_managed_services_and_ports.return_value = (['s1'], [200]) REQUIRED_INTERFACES.copy.return_value = {'int': ['test 1']} get_optional_interfaces.return_value = {'opt': ['test 2']} @@ -899,8 +905,11 @@ class TestKeystoneUtils(CharmTestCase): utils.resume_unit_helper('random-config') prh.assert_called_once_with(utils.resume_unit, 'random-config') + @patch.object(utils, 'determine_ports') + @patch.object(utils, 'services') @patch.object(utils, 'get_managed_services_and_ports') - def test_pause_resume_helper(self, get_managed_services_and_ports): + def test_pause_resume_helper(self, get_managed_services_and_ports, + services, determine_ports): f = MagicMock() get_managed_services_and_ports.return_value = (['s1'], [200]) with patch.object(utils, 'assess_status_func') as asf: @@ -909,41 +918,6 @@ class TestKeystoneUtils(CharmTestCase): asf.assert_called_once_with('some-config') f.assert_called_once_with('assessor', services=['s1'], ports=[200]) - @patch.object(utils, 'services') - @patch.object(utils, 'determine_ports') - @patch.object(utils, 'relation_ids') - @patch.object(utils, 'determine_api_port') - def test_get_managed_services_and_ports_no_ha(self, determine_api_port, - relation_ids, - determine_ports, - services): - services.return_value = ['apache2', 'haproxy'] - determine_ports.return_value = [80, 8080] - relation_ids.return_value = None - self.assertEqual( - utils.get_managed_services_and_ports(), - (['apache2', 'haproxy'], [80, 8080])) - - @patch.object(utils, 'services') - @patch.object(utils, 'determine_ports') - @patch.object(utils, 'relation_ids') - @patch.object(utils, 'determine_api_port') - def test_get_managed_services_and_ports(self, determine_api_port, - relation_ids, determine_ports, - services): - ports = { - 'keystone-admin': 5000, - 'keystone-public': 3535} - services.return_value = ['apache2', 'haproxy'] - determine_ports.return_value = [80, 8080] - self.api_port.return_value = 100 - determine_api_port.side_effect = lambda x, singlenode_mode: x-10 - self.api_port.side_effect = lambda x: ports.get(x) - relation_ids.return_value = ['rel:1'] - self.assertEqual( - utils.get_managed_services_and_ports(), - (['apache2'], [4990, 3525])) - @patch.object(utils, 'run_in_apache') @patch.object(utils, 'restart_keystone') def test_restart_function_map(self, restart_keystone, run_in_apache):