Merge "Workaround notification topic filling with OpenStack"
This commit is contained in:
commit
3cf1b0f1b3
|
@ -285,3 +285,11 @@ options:
|
|||
collisions. Known wait is the amount of time between one node executing
|
||||
an operation and another. On slower hardware this value may need to be
|
||||
larger than the default of 30 seconds.
|
||||
notification-ttl:
|
||||
type: int
|
||||
default: 3600000
|
||||
description: |
|
||||
TTL in MS for notification queues in the openstack vhost. Defaults to
|
||||
1 hour, but can be tuned up or down depending on deployment requirements.
|
||||
This ensures that any un-consumed notifications don't build up over
|
||||
time, causing disk capacity issues.
|
|
@ -348,6 +348,20 @@ def rabbitmqctl(action, *args):
|
|||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def configure_notification_ttl(vhost, ttl=3600000):
|
||||
''' Configure 1h minute TTL for notfication topics in the provided vhost
|
||||
This is a workaround for filling notification queues in OpenStack
|
||||
until a more general service discovery mechanism exists so that
|
||||
notifications can be enabled/disabled on each individual service.
|
||||
'''
|
||||
rabbitmqctl('set_policy',
|
||||
'TTL', '^(versioned_)?notifications.*',
|
||||
'{{"message-ttl":{ttl}}}'.format(ttl=ttl),
|
||||
'--priority', '1',
|
||||
'--apply-to', 'queues',
|
||||
'-p', vhost)
|
||||
|
||||
|
||||
def rabbitmqctl_normalized_output(*args):
|
||||
''' Run rabbitmqctl with args. Normalize output by removing
|
||||
whitespace and return it to caller for further processing.
|
||||
|
|
|
@ -228,6 +228,13 @@ def configure_amqp(username, vhost, relation_id, admin=False):
|
|||
|
||||
# update vhost
|
||||
rabbit.create_vhost(vhost)
|
||||
# NOTE(jamespage): Workaround until we have a good way
|
||||
# of generally disabling notifications
|
||||
# based on which services are deployed.
|
||||
if vhost == 'openstack':
|
||||
rabbit.configure_notification_ttl(vhost,
|
||||
config('notification-ttl'))
|
||||
|
||||
if admin:
|
||||
rabbit.create_user(username, password, ['administrator'])
|
||||
else:
|
||||
|
@ -851,6 +858,13 @@ def config_changed():
|
|||
for unit in related_units(rid):
|
||||
cluster_changed(relation_id=rid, remote_unit=unit)
|
||||
|
||||
# NOTE(jamespage): Workaround until we have a good way
|
||||
# of generally disabling notifications
|
||||
# based on which services are deployed.
|
||||
if 'openstack' in rabbit.list_vhosts():
|
||||
rabbit.configure_notification_ttl('openstack',
|
||||
config('notification-ttl'))
|
||||
|
||||
|
||||
@hooks.hook('leader-elected')
|
||||
def leader_elected():
|
||||
|
|
|
@ -742,3 +742,16 @@ class UtilsTests(CharmTestCase):
|
|||
self.config.side_effect = lambda key: _config.get(key)
|
||||
rabbit_utils.cluster_wait()
|
||||
mock_distributed_wait.assert_called_with(modulo=10, wait=60)
|
||||
|
||||
@mock.patch.object(rabbit_utils, 'rabbitmqctl')
|
||||
def test_configure_notification_ttl(self, rabbitmqctl):
|
||||
rabbit_utils.configure_notification_ttl('test',
|
||||
23000)
|
||||
rabbitmqctl.assert_called_once_with(
|
||||
'set_policy',
|
||||
'TTL', '^(versioned_)?notifications.*',
|
||||
'{"message-ttl":23000}',
|
||||
'--priority', '1',
|
||||
'--apply-to', 'queues',
|
||||
'-p', 'test'
|
||||
)
|
||||
|
|
|
@ -207,17 +207,26 @@ class RelationUtil(CharmTestCase):
|
|||
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
|
||||
remote_unit='client/0')
|
||||
|
||||
@patch.object(rabbitmq_server_relations.rabbit,
|
||||
'configure_notification_ttl')
|
||||
@patch.object(rabbitmq_server_relations, 'is_leader')
|
||||
@patch.object(rabbitmq_server_relations.rabbit, 'set_ha_mode')
|
||||
@patch.object(rabbitmq_server_relations.rabbit, 'get_rabbit_password')
|
||||
@patch.object(rabbitmq_server_relations.rabbit, 'create_vhost')
|
||||
@patch.object(rabbitmq_server_relations.rabbit, 'create_user')
|
||||
@patch.object(rabbitmq_server_relations.rabbit, 'grant_permissions')
|
||||
@patch.object(rabbitmq_server_relations, 'config', lambda *args: True)
|
||||
def test_configure_amqp(self, mock_grant_permissions, mock_create_vhost,
|
||||
@patch.object(rabbitmq_server_relations, 'config')
|
||||
def test_configure_amqp(self, mock_config,
|
||||
mock_grant_permissions, mock_create_vhost,
|
||||
mock_create_user, mock_get_rabbit_password,
|
||||
mock_set_ha_mode, mock_is_leader):
|
||||
mock_set_ha_mode, mock_is_leader,
|
||||
mock_configure_notification_ttl):
|
||||
config_data = {
|
||||
'notification-ttl': 450000,
|
||||
'mirroring-queues': True,
|
||||
}
|
||||
mock_is_leader.return_value = True
|
||||
mock_config.side_effect = lambda attribute: config_data.get(attribute)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
db_path = '{}/kv.db'.format(tmpdir)
|
||||
|
@ -282,6 +291,17 @@ class RelationUtil(CharmTestCase):
|
|||
del d[rid]['stale']
|
||||
self.assertEqual(store.get('amqp_config_tracker'), d)
|
||||
d[rid]['stale'] = True
|
||||
|
||||
mock_configure_notification_ttl.assert_not_called()
|
||||
|
||||
# Test openstack notification workaround
|
||||
d = {}
|
||||
for rid, user in [('amqp:1', 'userA')]:
|
||||
rabbitmq_server_relations.configure_amqp(user,
|
||||
'openstack', rid)
|
||||
(mock_configure_notification_ttl.
|
||||
assert_called_once_with('openstack', 450000))
|
||||
|
||||
finally:
|
||||
if os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir)
|
||||
|
|
Loading…
Reference in New Issue