From 7873b20126627e03f30214234841232d77a8642e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 3 Dec 2018 13:04:34 +0000 Subject: [PATCH] Use chelper generate_ha_relation_data for ha rel Use the generate_ha_relation_data helper from charmhelpers to generate the data to send down the relation to the hacluster charm. This results in a few changes in behaviour: 1) The charm will no longer specify a nic name to bind the vip. This is because Pacemaker VIP resources are able to automatically detect and configure correct iface and netmask parameters based on local configuration of the unit. 2) The original iface named VIP resource will be stopped and deleted prior to the creation of the new short hash named VIP resource. Change-Id: I50bddc990cb0182c8ba3a3d473fba2d60186322a --- .../charmhelpers/contrib/charmsupport/nrpe.py | 23 +-- .../contrib/openstack/amulet/deployment.py | 3 +- .../contrib/openstack/ha/utils.py | 97 +++++++++--- hooks/charmhelpers/contrib/openstack/utils.py | 99 +++++++++---- .../contrib/storage/linux/loopback.py | 6 +- hooks/cinder_hooks.py | 75 +--------- unit_tests/test_cinder_hooks.py | 42 +----- unit_tests/test_cluster_hooks.py | 138 ------------------ 8 files changed, 173 insertions(+), 310 deletions(-) diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index e3d10c1c..10d86ac0 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -416,15 +416,20 @@ def copy_nrpe_checks(nrpe_files_dir=None): """ NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' - default_nrpe_files_dir = os.path.join( - os.getenv('CHARM_DIR'), - 'hooks', - 'charmhelpers', - 'contrib', - 'openstack', - 'files') - if not nrpe_files_dir: - nrpe_files_dir = default_nrpe_files_dir + if nrpe_files_dir is None: + # determine if "charmhelpers" is in CHARMDIR or CHARMDIR/hooks + for segment in ['.', 'hooks']: + nrpe_files_dir = os.path.abspath(os.path.join( + os.getenv('CHARM_DIR'), + segment, + 'charmhelpers', + 'contrib', + 'openstack', + 'files')) + if os.path.isdir(nrpe_files_dir): + break + else: + raise RuntimeError("Couldn't find charmhelpers directory") if not os.path.exists(NAGIOS_PLUGINS): os.makedirs(NAGIOS_PLUGINS) for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 1c96752a..5b7e3cfb 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -168,7 +168,8 @@ class OpenStackAmuletDeployment(AmuletDeployment): 'nrpe', 'openvswitch-odl', 'neutron-api-odl', 'odl-controller', 'cinder-backup', 'nexentaedge-data', 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw', - 'cinder-nexentaedge', 'nexentaedge-mgmt'])) + 'cinder-nexentaedge', 'nexentaedge-mgmt', + 'ceilometer-agent'])) if self.openstack: for svc in services: diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py index add8eb9a..bbdee78d 100644 --- a/hooks/charmhelpers/contrib/openstack/ha/utils.py +++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py @@ -23,6 +23,7 @@ Helpers for high availability. """ +import hashlib import json import re @@ -35,7 +36,6 @@ from charmhelpers.core.hookenv import ( config, status_set, DEBUG, - WARNING, ) from charmhelpers.core.host import ( @@ -124,13 +124,29 @@ def expect_ha(): return len(ha_related_units) > 0 or config('vip') or config('dns-ha') -def generate_ha_relation_data(service): +def generate_ha_relation_data(service, extra_settings=None): """ Generate relation data for ha relation Based on configuration options and unit interfaces, generate a json encoded dict of relation data items for the hacluster relation, providing configuration for DNS HA or VIP's + haproxy clone sets. + Example of supplying additional settings:: + + COLO_CONSOLEAUTH = 'inf: res_nova_consoleauth grp_nova_vips' + AGENT_CONSOLEAUTH = 'ocf:openstack:nova-consoleauth' + AGENT_CA_PARAMS = 'op monitor interval="5s"' + + ha_console_settings = { + 'colocations': {'vip_consoleauth': COLO_CONSOLEAUTH}, + 'init_services': {'res_nova_consoleauth': 'nova-consoleauth'}, + 'resources': {'res_nova_consoleauth': AGENT_CONSOLEAUTH}, + 'resource_params': {'res_nova_consoleauth': AGENT_CA_PARAMS}) + generate_ha_relation_data('nova', extra_settings=ha_console_settings) + + + @param service: Name of the service being configured + @param extra_settings: Dict of additional resource data @returns dict: json encoded data for use with relation_set """ _haproxy_res = 'res_{}_haproxy'.format(service) @@ -149,6 +165,13 @@ def generate_ha_relation_data(service): }, } + if extra_settings: + for k, v in extra_settings.items(): + if _relation_data.get(k): + _relation_data[k].update(v) + else: + _relation_data[k] = v + if config('dns-ha'): update_hacluster_dns_ha(service, _relation_data) else: @@ -232,39 +255,67 @@ def update_hacluster_vip(service, relation_data): """ cluster_config = get_hacluster_config() vip_group = [] + vips_to_delete = [] for vip in cluster_config['vip'].split(): if is_ipv6(vip): - res_neutron_vip = 'ocf:heartbeat:IPv6addr' + res_vip = 'ocf:heartbeat:IPv6addr' vip_params = 'ipv6addr' else: - res_neutron_vip = 'ocf:heartbeat:IPaddr2' + res_vip = 'ocf:heartbeat:IPaddr2' vip_params = 'ip' - iface = (get_iface_for_address(vip) or - config('vip_iface')) - netmask = (get_netmask_for_address(vip) or - config('vip_cidr')) + iface = get_iface_for_address(vip) + netmask = get_netmask_for_address(vip) + + fallback_params = False + if iface is None: + iface = config('vip_iface') + fallback_params = True + if netmask is None: + netmask = config('vip_cidr') + fallback_params = True if iface is not None: + # NOTE(jamespage): Delete old VIP resources + # Old style naming encoding iface in name + # does not work well in environments where + # interface/subnet wiring is not consistent vip_key = 'res_{}_{}_vip'.format(service, iface) - if vip_key in vip_group: - if vip not in relation_data['resource_params'][vip_key]: - vip_key = '{}_{}'.format(vip_key, vip_params) - else: - log("Resource '%s' (vip='%s') already exists in " - "vip group - skipping" % (vip_key, vip), WARNING) - continue + if vip_key in vips_to_delete: + vip_key = '{}_{}'.format(vip_key, vip_params) + vips_to_delete.append(vip_key) + + vip_key = 'res_{}_{}_vip'.format( + service, + hashlib.sha1(vip.encode('UTF-8')).hexdigest()[:7]) + + relation_data['resources'][vip_key] = res_vip + # NOTE(jamespage): + # Use option provided vip params if these where used + # instead of auto-detected values + if fallback_params: + relation_data['resource_params'][vip_key] = ( + 'params {ip}="{vip}" cidr_netmask="{netmask}" ' + 'nic="{iface}"'.format(ip=vip_params, + vip=vip, + iface=iface, + netmask=netmask) + ) + else: + # NOTE(jamespage): + # let heartbeat figure out which interface and + # netmask to configure, which works nicely + # when network interface naming is not + # consistent across units. + relation_data['resource_params'][vip_key] = ( + 'params {ip}="{vip}"'.format(ip=vip_params, + vip=vip)) - relation_data['resources'][vip_key] = res_neutron_vip - relation_data['resource_params'][vip_key] = ( - 'params {ip}="{vip}" cidr_netmask="{netmask}" ' - 'nic="{iface}"'.format(ip=vip_params, - vip=vip, - iface=iface, - netmask=netmask) - ) vip_group.append(vip_key) + if vips_to_delete: + relation_data['delete_resources'] = vips_to_delete + if len(vip_group) >= 1: relation_data['groups'] = { 'grp_{}_vips'.format(service): ' '.join(vip_group) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29cad083..59312fcf 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -73,6 +73,8 @@ from charmhelpers.core.host import ( service_running, service_pause, service_resume, + service_stop, + service_start, restart_on_change_helper, ) from charmhelpers.fetch import ( @@ -299,7 +301,7 @@ def get_os_codename_install_source(src): rel = '' if src is None: return rel - if src in ['distro', 'distro-proposed']: + if src in ['distro', 'distro-proposed', 'proposed']: try: rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] except KeyError: @@ -1303,6 +1305,65 @@ def is_unit_paused_set(): return False +def manage_payload_services(action, services=None, charm_func=None): + """Run an action against all services. + + An optional charm_func() can be called. It should raise an Exception to + indicate that the function failed. If it was succesfull it should return + None or an optional message. + + The signature for charm_func is: + charm_func() -> message: str + + charm_func() is executed after any services are stopped, if supplied. + + The services object can either be: + - None : no services were passed (an empty dict is returned) + - a list of strings + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} + - An array of [{'service': service_name, ...}, ...] + + :param action: Action to run: pause, resume, start or stop. + :type action: str + :param services: See above + :type services: See above + :param charm_func: function to run for custom charm pausing. + :type charm_func: f() + :returns: Status boolean and list of messages + :rtype: (bool, []) + :raises: RuntimeError + """ + actions = { + 'pause': service_pause, + 'resume': service_resume, + 'start': service_start, + 'stop': service_stop} + action = action.lower() + if action not in actions.keys(): + raise RuntimeError( + "action: {} must be one of: {}".format(action, + ', '.join(actions.keys()))) + services = _extract_services_list_helper(services) + messages = [] + success = True + if services: + for service in services.keys(): + rc = actions[action](service) + if not rc: + success = False + messages.append("{} didn't {} cleanly.".format(service, + action)) + if charm_func: + try: + message = charm_func() + if message: + messages.append(message) + except Exception as e: + success = False + messages.append(str(e)) + return success, messages + + def pause_unit(assess_status_func, services=None, ports=None, charm_func=None): """Pause a unit by stopping the services and setting 'unit-paused' @@ -1333,20 +1394,10 @@ def pause_unit(assess_status_func, services=None, ports=None, @returns None @raises Exception(message) on an error for action_fail(). """ - services = _extract_services_list_helper(services) - messages = [] - if services: - for service in services.keys(): - stopped = service_pause(service) - if not stopped: - messages.append("{} didn't stop cleanly.".format(service)) - if charm_func: - try: - message = charm_func() - if message: - messages.append(message) - except Exception as e: - message.append(str(e)) + _, messages = manage_payload_services( + 'pause', + services=services, + charm_func=charm_func) set_unit_paused() if assess_status_func: message = assess_status_func() @@ -1385,20 +1436,10 @@ def resume_unit(assess_status_func, services=None, ports=None, @returns None @raises Exception(message) on an error for action_fail(). """ - services = _extract_services_list_helper(services) - messages = [] - if services: - for service in services.keys(): - started = service_resume(service) - if not started: - messages.append("{} didn't start cleanly.".format(service)) - if charm_func: - try: - message = charm_func() - if message: - messages.append(message) - except Exception as e: - message.append(str(e)) + _, messages = manage_payload_services( + 'resume', + services=services, + charm_func=charm_func) clear_unit_paused() if assess_status_func: message = assess_status_func() diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py index 0dfdae52..82472ff1 100644 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py @@ -36,8 +36,10 @@ def loopback_devices(): ''' loopbacks = {} cmd = ['losetup', '-a'] - devs = [d.strip().split(' ') for d in - check_output(cmd).splitlines() if d != ''] + output = check_output(cmd) + if six.PY3: + output = output.decode('utf-8') + devs = [d.strip().split(' ') for d in output.splitlines() if d != ''] for dev, _, f in devs: loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0] return loopbacks diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index cf6bccdc..821ba84b 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -64,7 +64,6 @@ from charmhelpers.core.hookenv import ( service_name, log, DEBUG, - WARNING, status_set, open_port, ) @@ -106,18 +105,14 @@ from charmhelpers.contrib.storage.linux.ceph import ( from charmhelpers.contrib.hahelpers.cluster import ( is_clustered, is_elected_leader, - get_hacluster_config, ) from charmhelpers.contrib.openstack.ha.utils import ( - update_dns_ha_resource_params, + generate_ha_relation_data, ) from charmhelpers.payload.execd import execd_preinstall from charmhelpers.contrib.network.ip import ( - get_iface_for_address, - get_netmask_for_address, - is_ipv6, get_relation_ip, ) from charmhelpers.contrib.openstack.ip import ( @@ -471,72 +466,8 @@ def cluster_changed(): @hooks.hook('ha-relation-joined') def ha_joined(relation_id=None): - cluster_config = get_hacluster_config() - - resources = { - 'res_cinder_haproxy': 'lsb:haproxy' - } - - resource_params = { - 'res_cinder_haproxy': 'op monitor interval="5s"' - } - - if config('dns-ha'): - update_dns_ha_resource_params(relation_id=relation_id, - resources=resources, - resource_params=resource_params) - else: - vip_group = [] - for vip in cluster_config['vip'].split(): - if is_ipv6(vip): - res_cinder_vip = 'ocf:heartbeat:IPv6addr' - vip_params = 'ipv6addr' - else: - res_cinder_vip = 'ocf:heartbeat:IPaddr2' - vip_params = 'ip' - - iface = (get_iface_for_address(vip) or - config('vip_iface')) - netmask = (get_netmask_for_address(vip) or - config('vip_cidr')) - - if iface is not None: - vip_key = 'res_cinder_{}_vip'.format(iface) - if vip_key in vip_group: - if vip not in resource_params[vip_key]: - vip_key = '{}_{}'.format(vip_key, vip_params) - else: - log("Resource '%s' (vip='%s') already exists in " - "vip group - skipping" % (vip_key, vip), WARNING) - continue - - resources[vip_key] = res_cinder_vip - resource_params[vip_key] = ( - 'params {ip}="{vip}" cidr_netmask="{netmask}"' - ' nic="{iface}"'.format(ip=vip_params, - vip=vip, - iface=iface, - netmask=netmask) - ) - vip_group.append(vip_key) - - if len(vip_group) >= 1: - relation_set(relation_id=relation_id, - groups={'grp_cinder_vips': ' '.join(vip_group)}) - - init_services = { - 'res_cinder_haproxy': 'haproxy' - } - clones = { - 'cl_cinder_haproxy': 'res_cinder_haproxy' - } - relation_set(relation_id=relation_id, - init_services=init_services, - corosync_bindiface=cluster_config['ha-bindiface'], - corosync_mcastport=cluster_config['ha-mcastport'], - resources=resources, - resource_params=resource_params, - clones=clones) + settings = generate_ha_relation_data('cinder') + relation_set(relation_id=relation_id, **settings) @hooks.hook('ha-relation-changed') diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index f6d8100d..5ef9328c 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -82,10 +82,9 @@ TO_PATCH = [ 'os_release', 'run_in_apache', # charmhelpers.contrib.openstack.openstack.ha.utils - 'update_dns_ha_resource_params', + 'generate_ha_relation_data', # charmhelpers.contrib.hahelpers.cluster_utils 'is_elected_leader', - 'get_hacluster_config', 'execd_preinstall', 'sync_db_with_multi_ipv6_addresses', 'delete_keyring', @@ -593,40 +592,11 @@ class TestJoinedHooks(CharmTestCase): self.ensure_ceph_keyring.return_value = True hooks.hooks.execute(['hooks/ceph-relation-changed']) - def test_ha_joined_dns_ha(self): - def _fake_update(resources, resource_params, relation_id=None): - resources.update({'res_cinder_public_hostname': 'ocf:maas:dns'}) - resource_params.update({'res_cinder_public_hostname': - 'params fqdn="keystone.maas" ' - 'ip_address="10.0.0.1"'}) - - self.test_config.set('dns-ha', True) - self.get_hacluster_config.return_value = { - 'vip': None, - 'ha-bindiface': 'em0', - 'ha-mcastport': '8080', - 'os-admin-hostname': None, - 'os-internal-hostname': None, - 'os-public-hostname': 'keystone.maas', - } - args = { - 'relation_id': None, - 'corosync_bindiface': 'em0', - 'corosync_mcastport': '8080', - 'init_services': {'res_cinder_haproxy': 'haproxy'}, - 'resources': {'res_cinder_public_hostname': 'ocf:maas:dns', - 'res_cinder_haproxy': 'lsb:haproxy'}, - 'resource_params': { - 'res_cinder_public_hostname': 'params fqdn="keystone.maas" ' - 'ip_address="10.0.0.1"', - 'res_cinder_haproxy': 'op monitor interval="5s"'}, - 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'} - } - self.update_dns_ha_resource_params.side_effect = _fake_update - - hooks.ha_joined() - self.assertTrue(self.update_dns_ha_resource_params.called) - self.relation_set.assert_called_with(**args) + def test_ha_joined(self): + self.generate_ha_relation_data.return_value = {'rel_data': 'data'} + hooks.hooks.execute(['hooks/ha-relation-joined']) + self.relation_set.assert_called_once_with( + relation_id=None, rel_data='data') class TestDepartedHooks(CharmTestCase): diff --git a/unit_tests/test_cluster_hooks.py b/unit_tests/test_cluster_hooks.py index 5bde6e5e..3a365f48 100644 --- a/unit_tests/test_cluster_hooks.py +++ b/unit_tests/test_cluster_hooks.py @@ -62,10 +62,7 @@ TO_PATCH = [ 'configure_installation_source', # charmhelpers.contrib.hahelpers.cluster_utils 'is_elected_leader', - 'get_hacluster_config', # charmhelpers.contrib.network.ip - 'get_iface_for_address', - 'get_netmask_for_address', 'get_relation_ip', ] @@ -104,141 +101,6 @@ class TestClusterHooks(CharmTestCase): ] self.assertEqual(ex, service.call_args_list) - @patch.object(hooks, 'check_local_db_actions_complete', - lambda *args, **kwargs: None) - def test_cluster_joined_hook(self): - self.config.side_effect = self.test_config.get - hooks.hooks.execute(['hooks/cluster-relation-joined']) - self.assertTrue(self.relation_set.called) - - @patch.object(hooks, 'check_local_db_actions_complete', - lambda *args, **kwargs: None) - def test_cluster_joined_hook_multinet(self): - self.config.side_effect = self.test_config.get - self.get_relation_ip.return_value = '10.1.1.1' - hooks.hooks.execute(['hooks/cluster-relation-joined']) - self.relation_set.assert_has_calls([ - call(relation_id=None, relation_settings={ - 'private-address': '10.1.1.1', - 'admin-address': '10.1.1.1', - 'internal-address': '10.1.1.1', - 'public-address': '10.1.1.1'}) - ]) - - def test_ha_joined_complete_config(self): - 'Ensure hacluster subordinate receives all relevant config' - conf = { - 'ha-bindiface': 'eth100', - 'ha-mcastport': '37373', - 'vip': '192.168.25.163', - 'vip_iface': 'eth101', - 'vip_cidr': '19', - } - - self.test_config.set('prefer-ipv6', 'False') - self.get_hacluster_config.return_value = conf - self.get_iface_for_address.return_value = 'eth101' - self.get_netmask_for_address.return_value = '255.255.224.0' - hooks.hooks.execute(['hooks/ha-relation-joined']) - ex_args = { - 'corosync_mcastport': '37373', - 'init_services': {'res_cinder_haproxy': 'haproxy'}, - 'resource_params': { - 'res_cinder_eth101_vip': - 'params ip="192.168.25.163" cidr_netmask="255.255.224.0"' - ' nic="eth101"', - 'res_cinder_haproxy': 'op monitor interval="5s"' - }, - 'corosync_bindiface': 'eth100', - 'relation_id': None, - 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'}, - 'resources': { - 'res_cinder_eth101_vip': 'ocf:heartbeat:IPaddr2', - 'res_cinder_haproxy': 'lsb:haproxy' - } - } - self.relation_set.assert_has_calls([ - call(relation_id=None, - groups={'grp_cinder_vips': 'res_cinder_eth101_vip'}), - call(**ex_args) - ]) - - def test_ha_joined_complete_config_with_ipv6(self): - 'Ensure hacluster subordinate receives all relevant config' - conf = { - 'ha-bindiface': 'eth100', - 'ha-mcastport': '37373', - 'vip': '2001:db8:1::1', - 'vip_iface': 'eth101', - 'vip_cidr': '64', - } - - self.test_config.set('prefer-ipv6', 'True') - self.get_hacluster_config.return_value = conf - self.get_iface_for_address.return_value = 'eth101' - self.get_netmask_for_address.return_value = 'ffff.ffff.ffff.ffff' - hooks.hooks.execute(['hooks/ha-relation-joined']) - ex_args = { - 'relation_id': None, - 'corosync_mcastport': '37373', - 'init_services': {'res_cinder_haproxy': 'haproxy'}, - 'resource_params': { - 'res_cinder_eth101_vip': - 'params ipv6addr="2001:db8:1::1" ' - 'cidr_netmask="ffff.ffff.ffff.ffff" ' - 'nic="eth101"', - 'res_cinder_haproxy': 'op monitor interval="5s"' - }, - 'corosync_bindiface': 'eth100', - 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'}, - 'resources': { - 'res_cinder_eth101_vip': 'ocf:heartbeat:IPv6addr', - 'res_cinder_haproxy': 'lsb:haproxy' - } - } - self.relation_set.assert_called_with(**ex_args) - - def test_ha_joined_no_bound_ip(self): - ''' - Ensure fallback configuration options are used if network - interface cannot be auto-detected - ''' - conf = { - 'ha-bindiface': 'eth100', - 'ha-mcastport': '37373', - 'vip': '192.168.25.163', - } - - self.test_config.set('prefer-ipv6', 'False') - self.test_config.set('vip_iface', 'eth120') - self.test_config.set('vip_cidr', '21') - self.get_hacluster_config.return_value = conf - self.get_iface_for_address.return_value = None - self.get_netmask_for_address.return_value = None - hooks.hooks.execute(['hooks/ha-relation-joined']) - ex_args = { - 'relation_id': None, - 'corosync_mcastport': '37373', - 'init_services': {'res_cinder_haproxy': 'haproxy'}, - 'resource_params': { - 'res_cinder_eth120_vip': - 'params ip="192.168.25.163" cidr_netmask="21"' - ' nic="eth120"', - 'res_cinder_haproxy': 'op monitor interval="5s"' - }, - 'corosync_bindiface': 'eth100', - 'clones': {'cl_cinder_haproxy': 'res_cinder_haproxy'}, - 'resources': { - 'res_cinder_eth120_vip': 'ocf:heartbeat:IPaddr2', - 'res_cinder_haproxy': 'lsb:haproxy' - } - } - self.relation_set.assert_has_calls([ - call(relation_id=None, - groups={'grp_cinder_vips': 'res_cinder_eth120_vip'}), - call(**ex_args) - ]) - @patch.object(hooks, 'identity_joined') def test_ha_changed_clustered(self, joined): self.relation_get.return_value = True