openstack
/
charm-rabbitmq-server
Code Issues Proposed changes

Wait until clustered before running client hooks 

RabbitMQ takes some time to fully cluster. The charm was previously
running amqp-relation-changed hooks whenever they were queued even
if the cluster was not yet complete. This led to split brain
scenarios. Client authentication to one or more nodes could fail.

This change confirms the entire cluster is ready before running
client amqp-relation-changed hooks.

min-cluster-size can now be used to attempt to guarantee the cluster
is ready with the expected number of nodes. If min-cluster-size is
not set the charm will still determine based on the information
available if all the cluster nodes are ready. Single node
deployments are still possible.

Partial-Bug: #1657245
Closes-Bug: #1657176
Change-Id: I870df71869c979e65a3a8764efdf35a746278507
This commit is contained in:
David Ames 2017-01-13 20:36:12 -08:00
parent 5b0d68c2fd
commit 2472e1ca9f
6 changed files with 433 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -15,11 +15,32 @@ To deploy this charm:
deploying multiple units will form a native RabbitMQ cluster:
juju deploy -n 3 rabbitmq-server
juju config rabbitmq-server min-cluster-size=3
To make use of AMQP services, simply relate other charms that support the rabbitmq interface:
juju add-relation rabbitmq-server nova-cloud-controller
# Clustering
When more than one unit of the charm is deployed the charm will bring up a
native RabbitMQ cluster. The process of clustering the units together takes
some time. Due to the nature of asynchronous hook execution it is possible
client relationship hooks may be executed before the cluster is complete.
In some cases, this can lead to client charm errors.
To guarantee client relation hooks will not be executed until clustering is
completed use the min-cluster-size configuration setting:
juju deploy -n 3 rabbitmq-server
juju config rabbitmq-server min-cluster-size=3
When min-cluster-size is not set the charm will still cluster, however,
there are no guarantees client relation hooks will not execute before it is
complete.
Single unit deployments behave as expected.
# Configuration: SSL
Generate an unencrypted RSA private key for the servers and a certificate:

146
hooks/rabbit_utils.py
View File

@ -48,15 +48,19 @@ from charmhelpers.core.hookenv import (
relation_ids,
related_units,
log, ERROR,
WARNING,
INFO, DEBUG,
service_name,
status_set,
cached,
unit_get,
relation_set,
relation_get,
application_version_set,
config,
network_get_primary_address,
is_leader,
leader_get,
)
from charmhelpers.core.host import (
@ -258,7 +262,7 @@ def set_ha_mode(vhost, mode, params=None, sync_mode='automatic'):
if caching_cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
log(("Mirroring queues cannot be enabled, only supported "
"in rabbitmq-server >= 3.0"), level='WARN')
"in rabbitmq-server >= 3.0"), level=WARNING)
log(("More information at http://www.rabbitmq.com/blog/"
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
return
@ -285,7 +289,7 @@ def clear_ha_mode(vhost, name='HA', force=False):
"""
if cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
log(("Mirroring queues not supported "
"in rabbitmq-server >= 3.0"), level='WARN')
"in rabbitmq-server >= 3.0"), level=WARNING)
log(("More information at http://www.rabbitmq.com/blog/"
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
return
@ -305,7 +309,7 @@ def set_all_mirroring_queues(enable):
"""
if cmp_pkgrevno('rabbitmq-server', '3.0.0') < 0:
log(("Mirroring queues not supported "
"in rabbitmq-server >= 3.0"), level='WARN')
"in rabbitmq-server >= 3.0"), level=WARNING)
log(("More information at http://www.rabbitmq.com/blog/"
"2012/11/19/breaking-things-with-rabbitmq-3-0"), level='INFO')
return
@ -410,7 +414,7 @@ def cluster_with():
join_cluster(node)
# NOTE: toggle the cluster relation to ensure that any peers
# already clustered re-assess status correctly
relation_set(clustered=get_unit_hostname())
relation_set(clustered=get_unit_hostname(), timestamp=time.time())
return True
except subprocess.CalledProcessError as e:
status_set('blocked', 'Failed to cluster with %s. Exception: %s'
@ -701,9 +705,9 @@ def get_node_hostname(ip_addr):
nodename = get_hostname(ip_addr, fqdn=False)
except:
log('Cannot resolve hostname for %s using DNS servers' % ip_addr,
level='WARNING')
level=WARNING)
log('Falling back to use socket.gethostname()',
level='WARNING')
level=WARNING)
# If the private-address is not resolvable using DNS
# then use the current hostname
nodename = socket.gethostname()
@ -734,8 +738,11 @@ def assess_cluster_status(*args):
''' Assess the status for the current running unit '''
# NOTE: ensure rabbitmq is actually installed before doing
# any checks
if os.path.exists(RABBITMQ_CTL):
if rabbitmq_is_installed():
# Clustering Check
if not is_sufficient_peers():
return 'waiting', ("Waiting for all {} peers to complete the "
"cluster.".format(config('min-cluster-size')))
peer_ids = relation_ids('cluster')
if peer_ids and len(related_units(peer_ids[0])):
if not clustered():
@ -912,3 +919,128 @@ def get_unit_hostname():
@returns hostname
"""
return socket.gethostname()
def is_sufficient_peers():
"""Sufficient number of expected peers to build a complete cluster
If min-cluster-size has been provided, check that we have sufficient
number of peers as expected for a complete cluster.
If not defined assume a single unit.
@returns boolean
"""
min_size = config('min-cluster-size')
if min_size:
log("Checking for minimum of {} peer units".format(min_size),
level=DEBUG)
# Include this unit
units = 1
for rid in relation_ids('cluster'):
units += len(related_units(rid))
if units < min_size:
log("Insufficient number of peer units to form cluster "
"(expected=%s, got=%s)" % (min_size, units), level=INFO)
return False
else:
log("Sufficient number of peer units to form cluster {}"
"".format(min_size, level=DEBUG))
return True
else:
log("min-cluster-size is not defined, race conditions may occur if "
"this is not a single unit deployment.", level=WARNING)
return True
def rabbitmq_is_installed():
"""Determine if rabbitmq is installed
@returns boolean
"""
return os.path.exists(RABBITMQ_CTL)
def cluster_ready():
"""Determine if each node in the cluster is ready and the cluster is
complete with the expected number of peers.
Once cluster_ready returns True it is safe to execute client relation
hooks. Having min-cluster-size set will guarantee cluster_ready will not
return True until the expected number of peers are clustered and ready.
If min-cluster-size is not set it must assume the cluster is ready in order
to allow for single unit deployments.
@returns boolean
"""
min_size = config('min-cluster-size')
units = 1
for relation_id in relation_ids('cluster'):
units += len(related_units(relation_id))
if not min_size:
min_size = units
if not is_sufficient_peers():
return False
elif min_size > 1:
if not clustered():
return False
clustered_units = 1
for relation_id in relation_ids('cluster'):
for remote_unit in related_units(relation_id):
if not relation_get(attribute='clustered',
rid=relation_id,
unit=remote_unit):
log("{} is not yet clustered".format(remote_unit),
DEBUG)
return False
else:
clustered_units += 1
if clustered_units < min_size:
log("Fewer than minimum cluster size:{} rabbit units reporting "
"clustered".format(min_size),
DEBUG)
return False
else:
log("All {} rabbit units reporting clustered"
"".format(min_size),
DEBUG)
return True
log("Must assume this is a single unit returning 'cluster' ready", DEBUG)
return True
def client_node_is_ready():
"""Determine if the leader node has set amqp client data
@returns boolean
"""
# Bail if this unit is paused
if is_unit_paused_set():
return False
for rid in relation_ids('amqp'):
if leader_get(attribute='{}_password'.format(rid)):
return True
return False
def leader_node_is_ready():
"""Determine if the leader node is ready to handle client relationship
hooks.
IFF rabbit is not paused, is installed, this is the leader node and the
cluster is complete.
@returns boolean
"""
# Paused check must run before other checks
# Bail if this unit is paused
if is_unit_paused_set():
return False
return (rabbitmq_is_installed() and
is_leader() and
cluster_ready())

128
hooks/rabbitmq_server_relations.py
View File

@ -135,11 +135,24 @@ def configure_amqp(username, vhost, admin=False):
return password
def update_clients():
"""Update amqp client relation hooks
IFF leader node is ready. Client nodes are considered ready once the leader
has already run amqp_changed.
"""
if rabbit.leader_node_is_ready() or rabbit.client_node_is_ready():
for rid in relation_ids('amqp'):
for unit in related_units(rid):
amqp_changed(relation_id=rid, remote_unit=unit)
@hooks.hook('amqp-relation-changed')
def amqp_changed(relation_id=None, remote_unit=None):
host_addr = rabbit.get_unit_ip()
if not is_elected_leader('res_rabbitmq_vip'):
# TODO: Simplify what the non-leader needs to do
if not is_leader() and rabbit.client_node_is_ready():
# NOTE(jamespage) clear relation to deal with data being
# removed from peer storage
relation_clear(relation_id)
@ -155,7 +168,7 @@ def amqp_changed(relation_id=None, remote_unit=None):
relation_set(relation_id=rel_id, **peerdb_settings)
log('amqp_changed(): Deferring amqp_changed'
' to is_elected_leader.')
' to the leader.')
# NOTE: active/active case
if config('prefer-ipv6'):
@ -165,6 +178,10 @@ def amqp_changed(relation_id=None, remote_unit=None):
return
# Bail if not completely ready
if not rabbit.leader_node_is_ready():
return
relation_settings = {}
settings = relation_get(rid=relation_id, unit=remote_unit)
@ -217,45 +234,6 @@ def amqp_changed(relation_id=None, remote_unit=None):
relation_settings=relation_settings)
def is_sufficient_peers():
"""If min-cluster-size has been provided, check that we have sufficient
number of peers to proceed with creating rabbitmq cluster.
"""
min_size = config('min-cluster-size')
leader_election_available = True
try:
is_leader()
except NotImplementedError:
leader_election_available = False
if min_size:
# Use min-cluster-size if we don't have Juju leader election.
if not leader_election_available:
log("Waiting for minimum of %d peer units since there's no Juju "
"leader election" % (min_size))
size = 0
for rid in relation_ids('cluster'):
size = len(related_units(rid))
# Include this unit
size += 1
if size < min_size:
log("Insufficient number of peer units to form cluster "
"(expected=%s, got=%s)" % (min_size, size), level=INFO)
return False
else:
return True
else:
log("Ignoring min-cluster-size in favour of Juju leader election")
return True
elif leader_election_available:
log("min-cluster-size is not defined, using juju leader-election.")
else:
log("min-cluster-size is not defined and juju leader election is not "
"available!", level="WARNING")
return True
@hooks.hook('cluster-relation-joined')
def cluster_joined(relation_id=None):
relation_settings = {
@ -286,10 +264,7 @@ def cluster_joined(relation_id=None):
level=ERROR)
return
if not is_sufficient_peers():
return
if is_elected_leader('res_rabbitmq_vip'):
if is_leader():
log('Leader peer_storing cookie', level=INFO)
cookie = open(rabbit.COOKIE_PATH, 'r').read().strip()
peer_store('cookie', cookie)
@ -298,9 +273,10 @@ def cluster_joined(relation_id=None):
@hooks.hook('cluster-relation-changed')
def cluster_changed():
def cluster_changed(relation_id=None, remote_unit=None):
# Future travelers beware ordering is significant
rdata = relation_get()
rdata = relation_get(rid=relation_id, unit=remote_unit)
# sync passwords
blacklist = ['hostname', 'private-address', 'public-address']
whitelist = [a for a in rdata.keys() if a not in blacklist]
@ -308,10 +284,9 @@ def cluster_changed():
cookie = peer_retrieve('cookie')
if not cookie:
log('cluster_joined: cookie not yet set.', level=INFO)
log('cluster_changed: cookie not yet set.', level=INFO)
return
rdata = relation_get()
if rdata:
hostname = rdata.get('hostname', None)
private_address = rdata.get('private-address', None)
@ -319,11 +294,6 @@ def cluster_changed():
if hostname and private_address:
rabbit.update_hosts_file({private_address: hostname})
if not is_sufficient_peers():
log('Not enough peers, waiting until leader is configured',
level=INFO)
return
# sync the cookie with peers if necessary
update_cookie()
@ -334,20 +304,9 @@ def cluster_changed():
return
# cluster with node?
try:
if not is_leader():
rabbit.cluster_with()
update_nrpe_checks()
except NotImplementedError:
if is_newer():
rabbit.cluster_with()
update_nrpe_checks()
# If cluster has changed peer db may have changed so run amqp_changed
# to sync any changes
for rid in relation_ids('amqp'):
for unit in related_units(rid):
amqp_changed(relation_id=rid, remote_unit=unit)
if not is_leader():
rabbit.cluster_with()
update_nrpe_checks()
def update_cookie(leaders_cookie=None):
@ -465,10 +424,6 @@ def ha_changed():
log('ha_changed(): We are now HA clustered. '
'Advertising our VIP (%s) to all AMQP clients.' %
vip)
# need to re-authenticate all clients since node-name changed.
for rid in relation_ids('amqp'):
for unit in related_units(rid):
amqp_changed(relation_id=rid, remote_unit=unit)
@hooks.hook('ceph-relation-joined')
@ -640,10 +595,15 @@ def config_changed():
rabbit.disable_plugin(MAN_PLUGIN)
close_port(55672)
rabbit.set_all_mirroring_queues(config('mirroring-queues'))
rabbit.ConfigRenderer(
rabbit.CONFIG_FILES).write_all()
# Only set values if this is the leader
if not is_leader():
return
rabbit.set_all_mirroring_queues(config('mirroring-queues'))
if is_relation_made("ha"):
ha_is_active_active = config("ha-vip-only")
@ -658,12 +618,16 @@ def config_changed():
else:
update_nrpe_checks()
# NOTE(jamespage)
# trigger amqp_changed to pickup and changes to network
# configuration via the access-network config option.
for rid in relation_ids('amqp'):
# Update cluster in case min-cluster-size has changed
for rid in relation_ids('cluster'):
for unit in related_units(rid):
amqp_changed(relation_id=rid, remote_unit=unit)
cluster_changed(relation_id=rid, remote_unit=unit)
@hooks.hook('leader-elected')
def leader_elected():
status_set("maintenance", "{} is the elected leader".format(local_unit()))
@hooks.hook('leader-settings-changed')
def leader_settings_changed():
@ -682,12 +646,6 @@ def leader_settings_changed():
for rid in relation_ids('cluster'):
relation_set(relation_id=rid, relation_settings={'cookie': cookie})
# If leader has changed and access credentials, ripple these
# out from all units
for rid in relation_ids('amqp'):
for unit in related_units(rid):
amqp_changed(relation_id=rid, remote_unit=unit)
def pre_install_hooks():
for f in glob.glob('exec.d/*/charm-pre-install'):
@ -706,4 +664,6 @@ if __name__ == '__main__':
hooks.execute(sys.argv)
except UnregisteredHookError as e:
log('Unknown hook {} - skipping.'.format(e))
# Gated client updates
update_clients()
rabbit.assess_status(rabbit.ConfigRenderer(rabbit.CONFIG_FILES))

0
tests/gate-basic-yakkety-newton Executable file → Normal file
View File

180
unit_tests/test_rabbit_utils.py
View File

@ -14,12 +14,12 @@
import mock
import os
import unittest
import tempfile
import sys
import collections
from functools import wraps
from test_utils import CharmTestCase
with mock.patch('charmhelpers.core.hookenv.cached') as cached:
def passthrough(func):
@ -33,8 +33,20 @@ with mock.patch('charmhelpers.core.hookenv.cached') as cached:
sys.modules['MySQLdb'] = mock.Mock()
TO_PATCH = [
# charmhelpers.core.hookenv
'is_leader',
'related_units',
'relation_ids',
'relation_get',
'relation_set',
'leader_get',
'config',
'is_unit_paused_set',
]
class ConfigRendererTests(unittest.TestCase):
class ConfigRendererTests(CharmTestCase):
class FakeContext(object):
def __call__(self, *a, **k):
@ -49,7 +61,8 @@ class ConfigRendererTests(unittest.TestCase):
)
def setUp(self):
super(ConfigRendererTests, self).setUp()
super(ConfigRendererTests, self).setUp(rabbit_utils,
TO_PATCH)
self.renderer = rabbit_utils.ConfigRenderer(
self.config_map)
@ -83,9 +96,10 @@ RABBITMQCTL_CLUSTERSTATUS_SOLO = """Cluster status of node 'rabbit@juju-devel3-m
"""
class UtilsTests(unittest.TestCase):
class UtilsTests(CharmTestCase):
def setUp(self):
super(UtilsTests, self).setUp()
super(UtilsTests, self).setUp(rabbit_utils,
TO_PATCH)
@mock.patch("rabbit_utils.log")
def test_update_empty_hosts_file(self, mock_log):
@ -346,3 +360,159 @@ class UtilsTests(unittest.TestCase):
check_call.return_value = 0
local_nodename.return_value = 'rabbitmq-server-0'
self.assertTrue(rabbit_utils.wait_app())
@mock.patch.object(rabbit_utils, 'is_leader')
@mock.patch.object(rabbit_utils, 'related_units')
@mock.patch.object(rabbit_utils, 'relation_ids')
@mock.patch.object(rabbit_utils, 'config')
def test_is_sufficient_peers(self, mock_config, mock_relation_ids,
mock_related_units, mock_is_leader):
# With leadership Election
mock_is_leader.return_value = False
_config = {'min-cluster-size': None}
mock_config.side_effect = lambda key: _config.get(key)
self.assertTrue(rabbit_utils.is_sufficient_peers())
mock_is_leader.return_value = False
mock_relation_ids.return_value = ['cluster:0']
mock_related_units.return_value = ['test/0']
_config = {'min-cluster-size': 3}
mock_config.side_effect = lambda key: _config.get(key)
self.assertFalse(rabbit_utils.is_sufficient_peers())
mock_is_leader.return_value = False
mock_related_units.return_value = ['test/0', 'test/1']
_config = {'min-cluster-size': 3}
mock_config.side_effect = lambda key: _config.get(key)
self.assertTrue(rabbit_utils.is_sufficient_peers())
@mock.patch.object(rabbit_utils.os.path, 'exists')
def test_rabbitmq_is_installed(self, mock_os_exists):
mock_os_exists.return_value = True
self.assertTrue(rabbit_utils.rabbitmq_is_installed())
mock_os_exists.return_value = False
self.assertFalse(rabbit_utils.rabbitmq_is_installed())
@mock.patch.object(rabbit_utils, 'clustered')
@mock.patch.object(rabbit_utils, 'rabbitmq_is_installed')
@mock.patch.object(rabbit_utils, 'is_sufficient_peers')
def test_cluster_ready(self, mock_is_sufficient_peers,
mock_rabbitmq_is_installed, mock_clustered):
# Not sufficient number of peers
mock_is_sufficient_peers.return_value = False
self.assertFalse(rabbit_utils.cluster_ready())
# This unit not yet clustered
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = ['cluster:0']
self.related_units.return_value = ['test/0', 'test/1']
self.relation_get.return_value = 'teset/0'
_config = {'min-cluster-size': 3}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = False
self.assertFalse(rabbit_utils.cluster_ready())
# Not all cluster ready
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = ['cluster:0']
self.related_units.return_value = ['test/0', 'test/1']
self.relation_get.return_value = False
_config = {'min-cluster-size': 3}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = True
self.assertFalse(rabbit_utils.cluster_ready())
# All cluster ready
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = ['cluster:0']
self.related_units.return_value = ['test/0', 'test/1']
self.relation_get.return_value = 'teset/0'
_config = {'min-cluster-size': 3}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = True
self.assertTrue(rabbit_utils.cluster_ready())
# Not all cluster ready no min-cluster-size
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = ['cluster:0']
self.related_units.return_value = ['test/0', 'test/1']
self.relation_get.return_value = False
_config = {'min-cluster-size': None}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = True
self.assertFalse(rabbit_utils.cluster_ready())
# All cluster ready no min-cluster-size
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = ['cluster:0']
self.related_units.return_value = ['test/0', 'test/1']
self.relation_get.return_value = 'teset/0'
_config = {'min-cluster-size': None}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = True
self.assertTrue(rabbit_utils.cluster_ready())
# Assume single unit no-min-cluster-size
mock_is_sufficient_peers.return_value = True
self.relation_ids.return_value = []
self.related_units.return_value = []
self.relation_get.return_value = None
_config = {'min-cluster-size': None}
self.config.side_effect = lambda key: _config.get(key)
mock_clustered.return_value = True
self.assertTrue(rabbit_utils.cluster_ready())
def test_client_node_is_ready(self):
# Paused
self.is_unit_paused_set.return_value = True
self.assertFalse(rabbit_utils.client_node_is_ready())
# Not ready
self.is_unit_paused_set.return_value = False
self.relation_ids.return_value = ['amqp:0']
self.leader_get.return_value = {}
self.assertFalse(rabbit_utils.client_node_is_ready())
# Ready
self.is_unit_paused_set.return_value = False
self.relation_ids.return_value = ['amqp:0']
self.leader_get.return_value = {'amqp:0_password': 'password'}
self.assertTrue(rabbit_utils.client_node_is_ready())
@mock.patch.object(rabbit_utils, 'cluster_ready')
@mock.patch.object(rabbit_utils, 'rabbitmq_is_installed')
def test_leader_node_is_ready(self, mock_rabbitmq_is_installed,
mock_cluster_ready):
# Paused
self.is_unit_paused_set.return_value = True
self.assertFalse(rabbit_utils.leader_node_is_ready())
# Not installed
self.is_unit_paused_set.return_value = False
mock_rabbitmq_is_installed.return_value = False
self.is_leader.return_value = True
mock_cluster_ready.return_value = True
self.assertFalse(rabbit_utils.leader_node_is_ready())
# Not leader
self.is_unit_paused_set.return_value = False
mock_rabbitmq_is_installed.return_value = True
self.is_leader.return_value = False
mock_cluster_ready.return_value = True
self.assertFalse(rabbit_utils.leader_node_is_ready())
# Not clustered
self.is_unit_paused_set.return_value = False
mock_rabbitmq_is_installed.return_value = True
self.is_leader.return_value = True
mock_cluster_ready.return_value = False
self.assertFalse(rabbit_utils.leader_node_is_ready())
# Leader ready
self.is_unit_paused_set.return_value = False
mock_rabbitmq_is_installed.return_value = True
self.is_leader.return_value = True
mock_cluster_ready.return_value = True
self.assertTrue(rabbit_utils.leader_node_is_ready())

93
unit_tests/test_rabbitmq_server_relations.py
View File

@ -15,7 +15,7 @@
import os
import sys
from testtools import TestCase
from test_utils import CharmTestCase
from mock import patch, MagicMock
os.environ['JUJU_UNIT_NAME'] = 'UNIT_TEST/0' # noqa - needed for import
@ -31,12 +31,21 @@ with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
lambda *args, **kwargs: f(*args, **kwargs))
import rabbitmq_server_relations
TO_PATCH = [
# charmhelpers.core.hookenv
'is_leader',
'relation_ids',
'related_units',
]
class RelationUtil(TestCase):
class RelationUtil(CharmTestCase):
def setUp(self):
self.fake_repo = {}
super(RelationUtil, self).setUp()
super(RelationUtil, self).setUp(rabbitmq_server_relations,
TO_PATCH)
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
@patch('rabbitmq_server_relations.peer_store_and_set')
@patch('rabbitmq_server_relations.config')
@patch('rabbitmq_server_relations.relation_set')
@ -56,7 +65,8 @@ class RelationUtil(TestCase):
cmp_pkgrevno,
relation_set,
mock_config,
mock_peer_store_and_set):
mock_peer_store_and_set,
mock_leader_node_is_ready):
"""
Compare version above and below 3.0.1.
Make sure ha_queues is set correctly on each side.
@ -68,6 +78,7 @@ class RelationUtil(TestCase):
return None
mock_leader_node_is_ready.return_value = True
mock_config.side_effect = config
host_addr = "10.1.2.3"
get_unit_ip.return_value = host_addr
@ -90,6 +101,7 @@ class RelationUtil(TestCase):
'hostname': host_addr},
relation_id=None)
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
@patch('rabbitmq_server_relations.peer_store_and_set')
@patch('rabbitmq_server_relations.config')
@patch('rabbitmq_server_relations.relation_set')
@ -109,7 +121,8 @@ class RelationUtil(TestCase):
cmp_pkgrevno,
relation_set,
mock_config,
mock_peer_store_and_set):
mock_peer_store_and_set,
mock_leader_node_is_ready):
"""
Compare version above and below 3.0.1.
Make sure ha_queues is set correctly on each side.
@ -121,6 +134,7 @@ class RelationUtil(TestCase):
return None
mock_leader_node_is_ready.return_value = True
mock_config.side_effect = config
ipv6_addr = "2001:db8:1:0:f816:3eff:fed6:c140"
get_unit_ip.return_value = ipv6_addr
@ -143,40 +157,41 @@ class RelationUtil(TestCase):
'hostname': ipv6_addr},
relation_id=None)
@patch.object(rabbitmq_server_relations, 'is_leader')
@patch.object(rabbitmq_server_relations, 'related_units')
@patch.object(rabbitmq_server_relations, 'relation_ids')
@patch.object(rabbitmq_server_relations, 'config')
def test_is_sufficient_peers(self, mock_config, mock_relation_ids,
mock_related_units, mock_is_leader):
# With leadership Election
mock_is_leader.return_value = False
_config = {'min-cluster-size': None}
mock_config.side_effect = lambda key: _config.get(key)
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
@patch('rabbitmq_server_relations.amqp_changed')
@patch('rabbitmq_server_relations.rabbit.client_node_is_ready')
@patch('rabbitmq_server_relations.rabbit.leader_node_is_ready')
def test_update_clients(self, mock_leader_node_is_ready,
mock_client_node_is_ready,
mock_amqp_changed):
# Not ready
mock_client_node_is_ready.return_value = False
mock_leader_node_is_ready.return_value = False
rabbitmq_server_relations.update_clients()
self.assertFalse(mock_amqp_changed.called)
mock_is_leader.return_value = False
mock_relation_ids.return_value = ['cluster:0']
mock_related_units.return_value = ['test/0']
_config = {'min-cluster-size': 3}
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
# Leader Ready
self.relation_ids.return_value = ['amqp:0']
self.related_units.return_value = ['client/0']
mock_leader_node_is_ready.return_value = True
mock_client_node_is_ready.return_value = False
rabbitmq_server_relations.update_clients()
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
remote_unit='client/0')
mock_is_leader.return_value = False
mock_related_units.return_value = ['test/0', 'test/1']
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
# Client Ready
self.relation_ids.return_value = ['amqp:0']
self.related_units.return_value = ['client/0']
mock_leader_node_is_ready.return_value = False
mock_client_node_is_ready.return_value = True
rabbitmq_server_relations.update_clients()
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
remote_unit='client/0')
# Without leadership Election
mock_is_leader.side_effect = NotImplementedError
_config = {'min-cluster-size': None}
mock_config.side_effect = lambda key: _config.get(key)
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
mock_is_leader.side_effect = NotImplementedError
mock_relation_ids.return_value = ['cluster:0']
mock_related_units.return_value = ['test/0']
_config = {'min-cluster-size': 3}
self.assertFalse(rabbitmq_server_relations.is_sufficient_peers())
mock_is_leader.side_effect = NotImplementedError
mock_related_units.return_value = ['test/0', 'test/1']
self.assertTrue(rabbitmq_server_relations.is_sufficient_peers())
# Both Ready
self.relation_ids.return_value = ['amqp:0']
self.related_units.return_value = ['client/0']
mock_leader_node_is_ready.return_value = True
mock_client_node_is_ready.return_value = True
rabbitmq_server_relations.update_clients()
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
remote_unit='client/0')