Do not process events for instances without host
In some cases Neutron might send events such as 'VIF unplugged'
for instances which are either being deleted or shelved. When
that happens there will be a failure in dispatching the event
to the appropriate compute node - as there is no host for the
instance.
As multiple neutron events can be stashed in a single call
it is important to avoid that this kind of errors will prevent
processing of other events in the same call.
This patch does not process events for instances without a host,
marking them as failed.
When the above condition occurs, the create event request will
return a 207 response code. For specific events, a 422
unprocessable entity code will be set.
This patch also preserve the characteristic that events are
returned in the response in the same order they were found in
the request.
Change-Id: I18062b81e50c722ec96b4296ac39384493683ae3
Closes-Bug: #1333654
(cherry picked from commit 4f8ccd7b95)
This commit is contained in:
Salvatore Orlando
Attila Fazekas
parent
8bc955375c
commit
811cab7f3d
|
|
@ -70,7 +70,7 @@ class ServerExternalEventsController(wsgi.Controller):
|
|||
context = req.environ['nova.context']
|
||||
authorize(context, action='create')
|
||||
|
||||
events = []
|
||||
response_events = []
|
||||
accepted = []
|
||||
instances = {}
|
||||
result = 200
|
||||
|
|
@ -101,8 +101,8 @@ class ServerExternalEventsController(wsgi.Controller):
|
|||
raise webob.exc.HTTPBadRequest(
|
||||
_('Invalid event status `%s\'') % event.status)
|
||||
|
||||
events.append(_event)
|
||||
if event.instance_uuid not in instances:
|
||||
instance = instances.get(event.instance_uuid)
|
||||
if not instance:
|
||||
try:
|
||||
instance = instance_obj.Instance.get_by_uuid(
|
||||
context, event.instance_uuid)
|
||||
|
|
@ -115,24 +115,40 @@ class ServerExternalEventsController(wsgi.Controller):
|
|||
_event['code'] = 404
|
||||
result = 207
|
||||
|
||||
if event.instance_uuid in instances:
|
||||
accepted.append(event)
|
||||
_event['code'] = 200
|
||||
LOG.audit(_('Create event %(name)s:%(tag)s for instance '
|
||||
'%(instance_uuid)s'),
|
||||
dict(event.iteritems()))
|
||||
# NOTE: before accepting the event, make sure the instance
|
||||
# for which the event is sent is assigned to a host; otherwise
|
||||
# it will not be possible to dispatch the event
|
||||
if instance:
|
||||
if instance.host:
|
||||
accepted.append(event)
|
||||
LOG.audit(_('Creating event %(name)s:%(tag)s for instance '
|
||||
'%(instance_uuid)s'),
|
||||
dict(event.iteritems()))
|
||||
# NOTE: as the event is processed asynchronously verify
|
||||
# whether 202 is a more suitable response code than 200
|
||||
_event['status'] = 'completed'
|
||||
_event['code'] = 200
|
||||
else:
|
||||
LOG.debug("Unable to find a host for instance "
|
||||
"%(instance)s. Dropping event %(event)s",
|
||||
{'instance': event.instance_uuid,
|
||||
'event': event.name})
|
||||
_event['status'] = 'failed'
|
||||
_event['code'] = 422
|
||||
result = 207
|
||||
|
||||
response_events.append(_event)
|
||||
|
||||
if accepted:
|
||||
self.compute_api.external_instance_event(context,
|
||||
instances.values(),
|
||||
accepted)
|
||||
self.compute_api.external_instance_event(
|
||||
context, instances.values(), accepted)
|
||||
else:
|
||||
msg = _('No instances found for any event')
|
||||
raise webob.exc.HTTPNotFound(explanation=msg)
|
||||
|
||||
# FIXME(cyeoh): This needs some infrastructure support so that
|
||||
# we have a general way to do this
|
||||
robj = wsgi.ResponseObject({'events': events})
|
||||
robj = wsgi.ResponseObject({'events': response_events})
|
||||
robj._code = result
|
||||
return robj
|
||||
|
||||
|
|
|
|||
|
|
@ -3105,6 +3105,9 @@ class API(base.Base):
|
|||
events_by_host[host] = events_on_host
|
||||
|
||||
for host in instances_by_host:
|
||||
# TODO(salv-orlando): Handle exceptions raised by the rpc api layer
|
||||
# in order to ensure that a failure in processing events on a host
|
||||
# will not prevent processing events on other hosts
|
||||
self.compute_rpcapi.external_instance_event(
|
||||
context, instances_by_host[host], events_by_host[host])
|
||||
|
||||
|
|
|
|||
|
|
@ -30,9 +30,11 @@ fake_instances = {
|
|||
uuid='00000000-0000-0000-0000-000000000002', host='host1'),
|
||||
'00000000-0000-0000-0000-000000000003': instance_obj.Instance(
|
||||
uuid='00000000-0000-0000-0000-000000000003', host='host2'),
|
||||
'00000000-0000-0000-0000-000000000004': instance_obj.Instance(
|
||||
uuid='00000000-0000-0000-0000-000000000004', host=None),
|
||||
}
|
||||
fake_instance_uuids = sorted(fake_instances.keys())
|
||||
MISSING_UUID = '00000000-0000-0000-0000-000000000004'
|
||||
MISSING_UUID = '00000000-0000-0000-0000-000000000005'
|
||||
|
||||
|
||||
@classmethod
|
||||
|
|
@ -49,16 +51,20 @@ class ServerExternalEventsTest(test.NoDBTestCase):
|
|||
super(ServerExternalEventsTest, self).setUp()
|
||||
self.api = server_external_events.ServerExternalEventsController()
|
||||
self.context = context.get_admin_context()
|
||||
self.default_body = {
|
||||
'events': [
|
||||
{'name': 'network-vif-plugged',
|
||||
'tag': 'foo',
|
||||
'status': 'completed',
|
||||
'server_uuid': fake_instance_uuids[0]},
|
||||
{'name': 'network-changed',
|
||||
'server_uuid': fake_instance_uuids[1]},
|
||||
]
|
||||
}
|
||||
self.event_1 = {'name': 'network-vif-plugged',
|
||||
'tag': 'foo',
|
||||
'server_uuid': fake_instance_uuids[0]}
|
||||
self.event_2 = {'name': 'network-changed',
|
||||
'server_uuid': fake_instance_uuids[1]}
|
||||
self.default_body = {'events': [self.event_1, self.event_2]}
|
||||
self.resp_event_1 = dict(self.event_1)
|
||||
self.resp_event_1['code'] = 200
|
||||
self.resp_event_1['status'] = 'completed'
|
||||
self.resp_event_2 = dict(self.event_2)
|
||||
self.resp_event_2['code'] = 200
|
||||
self.resp_event_2['status'] = 'completed'
|
||||
self.default_resp_body = {'events': [self.resp_event_1,
|
||||
self.resp_event_2]}
|
||||
|
||||
def _create_req(self, body):
|
||||
req = webob.Request.blank('/v2/fake/os-server-external-events')
|
||||
|
|
@ -91,7 +97,7 @@ class ServerExternalEventsTest(test.NoDBTestCase):
|
|||
fake_instance_uuids[:2],
|
||||
['network-vif-plugged',
|
||||
'network-changed'])
|
||||
self.assertEqual(self.default_body, result)
|
||||
self.assertEqual(self.default_resp_body, result)
|
||||
self.assertEqual(200, code)
|
||||
|
||||
def test_create_one_bad_instance(self):
|
||||
|
|
@ -105,6 +111,19 @@ class ServerExternalEventsTest(test.NoDBTestCase):
|
|||
self.assertEqual(404, result['events'][1]['code'])
|
||||
self.assertEqual(207, code)
|
||||
|
||||
def test_create_event_instance_has_no_host(self):
|
||||
body = self.default_body
|
||||
body['events'][0]['server_uuid'] = fake_instance_uuids[-1]
|
||||
req = self._create_req(body)
|
||||
result, code = self._assert_call(req, body,
|
||||
[fake_instance_uuids[1],
|
||||
fake_instance_uuids[-1]],
|
||||
['network-changed'])
|
||||
self.assertEqual(422, result['events'][0]['code'])
|
||||
self.assertEqual('failed', result['events'][0]['status'])
|
||||
self.assertEqual(200, result['events'][1]['code'])
|
||||
self.assertEqual(207, code)
|
||||
|
||||
def test_create_no_good_instances(self):
|
||||
body = self.default_body
|
||||
body['events'][0]['server_uuid'] = MISSING_UUID
|
||||
|
|
|
|||
Loading…
Reference in New Issue