Merge "Workaround notification topic filling with OpenStack"

This commit is contained in:
Zuul 2019-01-09 12:13:59 +00:00 committed by Gerrit Code Review
commit 3cf1b0f1b3
5 changed files with 72 additions and 3 deletions

View File

@ -285,3 +285,11 @@ options:
collisions. Known wait is the amount of time between one node executing
an operation and another. On slower hardware this value may need to be
larger than the default of 30 seconds.
notification-ttl:
type: int
default: 3600000
description: |
TTL in MS for notification queues in the openstack vhost. Defaults to
1 hour, but can be tuned up or down depending on deployment requirements.
This ensures that any un-consumed notifications don't build up over
time, causing disk capacity issues.

View File

@ -348,6 +348,20 @@ def rabbitmqctl(action, *args):
subprocess.check_call(cmd)
def configure_notification_ttl(vhost, ttl=3600000):
''' Configure 1h minute TTL for notfication topics in the provided vhost
This is a workaround for filling notification queues in OpenStack
until a more general service discovery mechanism exists so that
notifications can be enabled/disabled on each individual service.
'''
rabbitmqctl('set_policy',
'TTL', '^(versioned_)?notifications.*',
'{{"message-ttl":{ttl}}}'.format(ttl=ttl),
'--priority', '1',
'--apply-to', 'queues',
'-p', vhost)
def rabbitmqctl_normalized_output(*args):
''' Run rabbitmqctl with args. Normalize output by removing
whitespace and return it to caller for further processing.

View File

@ -228,6 +228,13 @@ def configure_amqp(username, vhost, relation_id, admin=False):
# update vhost
rabbit.create_vhost(vhost)
# NOTE(jamespage): Workaround until we have a good way
# of generally disabling notifications
# based on which services are deployed.
if vhost == 'openstack':
rabbit.configure_notification_ttl(vhost,
config('notification-ttl'))
if admin:
rabbit.create_user(username, password, ['administrator'])
else:
@ -851,6 +858,13 @@ def config_changed():
for unit in related_units(rid):
cluster_changed(relation_id=rid, remote_unit=unit)
# NOTE(jamespage): Workaround until we have a good way
# of generally disabling notifications
# based on which services are deployed.
if 'openstack' in rabbit.list_vhosts():
rabbit.configure_notification_ttl('openstack',
config('notification-ttl'))
@hooks.hook('leader-elected')
def leader_elected():

View File

@ -742,3 +742,16 @@ class UtilsTests(CharmTestCase):
self.config.side_effect = lambda key: _config.get(key)
rabbit_utils.cluster_wait()
mock_distributed_wait.assert_called_with(modulo=10, wait=60)
@mock.patch.object(rabbit_utils, 'rabbitmqctl')
def test_configure_notification_ttl(self, rabbitmqctl):
rabbit_utils.configure_notification_ttl('test',
23000)
rabbitmqctl.assert_called_once_with(
'set_policy',
'TTL', '^(versioned_)?notifications.*',
'{"message-ttl":23000}',
'--priority', '1',
'--apply-to', 'queues',
'-p', 'test'
)

View File

@ -207,17 +207,26 @@ class RelationUtil(CharmTestCase):
mock_amqp_changed.assert_called_with(relation_id='amqp:0',
remote_unit='client/0')
@patch.object(rabbitmq_server_relations.rabbit,
'configure_notification_ttl')
@patch.object(rabbitmq_server_relations, 'is_leader')
@patch.object(rabbitmq_server_relations.rabbit, 'set_ha_mode')
@patch.object(rabbitmq_server_relations.rabbit, 'get_rabbit_password')
@patch.object(rabbitmq_server_relations.rabbit, 'create_vhost')
@patch.object(rabbitmq_server_relations.rabbit, 'create_user')
@patch.object(rabbitmq_server_relations.rabbit, 'grant_permissions')
@patch.object(rabbitmq_server_relations, 'config', lambda *args: True)
def test_configure_amqp(self, mock_grant_permissions, mock_create_vhost,
@patch.object(rabbitmq_server_relations, 'config')
def test_configure_amqp(self, mock_config,
mock_grant_permissions, mock_create_vhost,
mock_create_user, mock_get_rabbit_password,
mock_set_ha_mode, mock_is_leader):
mock_set_ha_mode, mock_is_leader,
mock_configure_notification_ttl):
config_data = {
'notification-ttl': 450000,
'mirroring-queues': True,
}
mock_is_leader.return_value = True
mock_config.side_effect = lambda attribute: config_data.get(attribute)
tmpdir = tempfile.mkdtemp()
try:
db_path = '{}/kv.db'.format(tmpdir)
@ -282,6 +291,17 @@ class RelationUtil(CharmTestCase):
del d[rid]['stale']
self.assertEqual(store.get('amqp_config_tracker'), d)
d[rid]['stale'] = True
mock_configure_notification_ttl.assert_not_called()
# Test openstack notification workaround
d = {}
for rid, user in [('amqp:1', 'userA')]:
rabbitmq_server_relations.configure_amqp(user,
'openstack', rid)
(mock_configure_notification_ttl.
assert_called_once_with('openstack', 450000))
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)