openstack
/
blazar
Code Issues Proposed changes

Merge "Retry event actions"

This commit is contained in:
Zuul 2018-02-04 15:56:48 +00:00 committed by Gerrit Code Review
parent 328a8d902b 799a976545
commit 0fe88e62ae
2 changed files with 134 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
blazar/manager/service.py
View File

@ -40,7 +40,11 @@ manager_opts = [
default=60,
help='Minutes prior to the end of a lease in which actions '
'like notification and snapshot are taken. If this is '
'set to 0, then these actions are not taken.')
'set to 0, then these actions are not taken.'),
cfg.IntOpt('event_max_retries',
default=1,
max=50,
help='Number of times to retry an event action.')
]
CONF = cfg.CONF
@ -49,6 +53,8 @@ LOG = logging.getLogger(__name__)
LEASE_DATE_FORMAT = "%Y-%m-%d %H:%M"
EVENT_INTERVAL = 10
class ManagerService(service_utils.RPCServer):
"""Service class for the blazar-manager service.
@ -66,7 +72,7 @@ class ManagerService(service_utils.RPCServer):
def start(self):
super(ManagerService, self).start()
self.tg.add_timer(10, self._event)
self.tg.add_timer(EVENT_INTERVAL, self._event)
for m in self.monitors:
m.start_monitoring()
@ -146,25 +152,45 @@ class ManagerService(service_utils.RPCServer):
if event['time'] < datetime.datetime.utcnow():
db_api.event_update(event['id'],
{'status': status.event.IN_PROGRESS})
event_type = event['event_type']
event_fn = getattr(self, event_type, None)
if event_fn is None:
raise exceptions.EventError(error='Event type %s is not '
'supported' % event_type)
try:
eventlet.spawn_n(service_utils.with_empty_context(event_fn),
lease_id=event['lease_id'],
event_id=event['id'])
lease = db_api.lease_get(event['lease_id'])
with trusts.create_ctx_from_trust(lease['trust_id']) as ctx:
self._send_notification(lease,
ctx,
events=['event.%s' % event_type])
eventlet.spawn_n(
service_utils.with_empty_context(self._exec_event),
event)
except Exception:
db_api.event_update(event['id'],
{'status': status.event.ERROR})
LOG.exception('Error occurred while event handling.')
def _exec_event(self, event):
"""Execute an event function"""
event_fn = getattr(self, event['event_type'], None)
if event_fn is None:
raise exceptions.EventError(
error='Event type %s is not supported'
% event['event_type'])
try:
event_fn(lease_id=event['lease_id'], event_id=event['id'])
except common_ex.InvalidStatus:
now = datetime.datetime.utcnow()
if now < event['time'] + datetime.timedelta(
seconds=CONF.manager.event_max_retries * 10):
# Set the event status UNDONE for retrying the event
db_api.event_update(event['id'],
{'status': status.event.UNDONE})
else:
db_api.event_update(event['id'],
{'status': status.event.ERROR})
LOG.exception('Error occurred while handling event.')
except Exception:
db_api.event_update(event['id'],
{'status': status.event.ERROR})
LOG.exception('Error occurred while handling event.')
else:
lease = db_api.lease_get(event['lease_id'])
with trusts.create_ctx_from_trust(lease['trust_id']) as ctx:
self._send_notification(
lease, ctx, events=['event.%s' % event['event_type']])
def _date_from_string(self, date_string, date_format=LEASE_DATE_FORMAT):
try:
date = datetime.datetime.strptime(date_string, date_format)

117
blazar/tests/manager/test_service.py
View File

@ -226,46 +226,115 @@ class ServiceTestCase(tests.TestCase):
self.assertFalse(event_update.called)
def test_event_all_okay(self):
def test_event_success(self):
events = self.patch(self.db_api, 'event_get_first_sorted_by_filters')
event_update = self.patch(self.db_api, 'event_update')
events.return_value = {'id': '111-222-333', 'time': self.good_date,
'event_type': 'end_lease',
'lease_id': self.lease_id}
events.return_value = {'id': '111-222-333',
'time': self.good_date}
self.patch(eventlet, 'spawn_n')
self.manager._event()
event_update.assert_called_once_with('111-222-333',
{'status': 'IN_PROGRESS'})
event_update.assert_called_once_with(
'111-222-333', {'status': status.event.IN_PROGRESS})
def test_event_spawn_fail(self):
events = self.patch(self.db_api, 'event_get_first_sorted_by_filters')
event_update = self.patch(self.db_api, 'event_update')
self.patch(eventlet, 'spawn_n').side_effect = Exception
events.return_value = {'id': '111-222-333',
'time': self.good_date}
self.manager._event()
event_update.assert_has_calls([
mock.call('111-222-333', {'status': status.event.IN_PROGRESS}),
mock.call('111-222-333', {'status': status.event.ERROR})])
def test_exec_event_success(self):
event = {'id': '111-222-333',
'event_type': 'start_lease',
'lease_id': self.lease_id}
start_lease = self.patch(self.manager, 'start_lease')
self.manager._exec_event(event)
start_lease.assert_called_once_with(lease_id=event['lease_id'],
event_id=event['id'])
self.lease_get.assert_called_once_with(event['lease_id'])
expected_context = self.trust_ctx.return_value
self.fake_notifier.assert_called_once_with(
expected_context.__enter__.return_value,
notifier_api.format_lease_payload(self.lease),
'lease.event.end_lease')
'lease.event.start_lease')
def test_event_wrong_event_status(self):
events = self.patch(self.db_api, 'event_get_first_sorted_by_filters')
self.patch(self.db_api, 'event_update')
events.return_value = {'id': '111-222-333', 'time': self.good_date,
'event_type': 'wrong_type',
'lease_id': self.lease_id}
def test_exec_event_invalid_event_type(self):
event = {'id': '111-222-333',
'event_type': 'invalid',
'lease_id': self.lease_id}
self.assertRaises(manager_ex.EventError,
self.manager._event)
self.manager._exec_event,
event)
def test_event_wrong_eventlet_fail(self):
events = self.patch(self.db_api, 'event_get_first_sorted_by_filters')
def test_exec_event_retry(self):
event = {'id': '111-222-333',
'event_type': 'start_lease',
'lease_id': self.lease_id,
'time': self.good_date}
start_lease = self.patch(self.manager, 'start_lease')
start_lease.side_effect = exceptions.InvalidStatus
event_update = self.patch(self.db_api, 'event_update')
calls = [mock.call('111-222-333', {'status': 'IN_PROGRESS'}),
mock.call('111-222-333', {'status': 'ERROR'})]
self.patch(eventlet, 'spawn_n').side_effect = Exception
events.return_value = {'id': '111-222-333', 'time': self.good_date,
'event_type': 'end_lease',
'lease_id': self.lease_id}
self.manager._event()
with mock.patch.object(datetime, 'datetime',
mock.Mock(wraps=datetime.datetime)) as patched:
patched.utcnow.return_value = (self.good_date
+ datetime.timedelta(seconds=1))
self.manager._exec_event(event)
event_update.assert_has_calls(calls)
start_lease.assert_called_once_with(lease_id=event['lease_id'],
event_id=event['id'])
event_update.assert_called_once_with(
event['id'], {'status': status.event.UNDONE})
self.lease_get.assert_not_called()
def test_exec_event_no_more_retry(self):
event = {'id': '111-222-333',
'event_type': 'start_lease',
'lease_id': self.lease_id,
'time': self.good_date}
start_lease = self.patch(self.manager, 'start_lease')
start_lease.side_effect = exceptions.InvalidStatus
event_update = self.patch(self.db_api, 'event_update')
with mock.patch.object(datetime, 'datetime',
mock.Mock(wraps=datetime.datetime)) as patched:
patched.utcnow.return_value = (self.good_date
+ datetime.timedelta(days=1))
self.manager._exec_event(event)
start_lease.assert_called_once_with(lease_id=event['lease_id'],
event_id=event['id'])
event_update.assert_called_once_with(
event['id'], {'status': status.event.ERROR})
self.lease_get.assert_not_called()
def test_exec_event_handle_exception(self):
event = {'id': '111-222-333',
'event_type': 'start_lease',
'lease_id': self.lease_id,
'time': self.good_date}
start_lease = self.patch(self.manager, 'start_lease')
start_lease.side_effect = Exception
event_update = self.patch(self.db_api, 'event_update')
self.manager._exec_event(event)
start_lease.assert_called_once_with(lease_id=event['lease_id'],
event_id=event['id'])
event_update.assert_called_once_with(
event['id'], {'status': status.event.ERROR})
self.lease_get.assert_not_called()
def test_get_lease(self):
lease = self.manager.get_lease(self.lease_id)