Merge "Transform missing delete notifications"

This commit is contained in:
Zuul 2018-09-26 20:51:26 +00:00 committed by Gerrit Code Review
commit 00dc4483b7
14 changed files with 276 additions and 39 deletions

View File

@ -0,0 +1,15 @@
{
"event_type":"instance.delete.end",
"payload":{
"$ref":"common_payloads/InstanceActionPayload.json#",
"nova_object.data":{
"block_devices":[],
"deleted_at":"2012-10-29T13:42:11Z",
"ip_addresses":[],
"state":"deleted",
"terminated_at":"2012-10-29T13:42:11Z"
}
},
"priority":"INFO",
"publisher_id":"nova-api:fake-mini"
}

View File

@ -0,0 +1,19 @@
{
"event_type":"instance.delete.end",
"payload":{
"$ref":"common_payloads/InstanceActionPayload.json#",
"nova_object.data":{
"block_devices":[],
"deleted_at":"2012-10-29T13:42:11Z",
"host":null,
"ip_addresses":[],
"launched_at":null,
"node":null,
"power_state":"pending",
"state":"deleted",
"terminated_at":"2012-10-29T13:42:11Z"
}
},
"priority":"INFO",
"publisher_id":"nova-api:fake-mini"
}

View File

@ -0,0 +1,11 @@
{
"event_type":"instance.delete.start",
"payload":{
"$ref":"common_payloads/InstanceActionPayload.json#",
"nova_object.data":{
"task_state":"deleting"
}
},
"priority":"INFO",
"publisher_id":"nova-api:fake-mini"
}

View File

@ -0,0 +1,18 @@
{
"event_type":"instance.delete.start",
"payload":{
"$ref":"common_payloads/InstanceActionPayload.json#",
"nova_object.data":{
"block_devices":[],
"host":null,
"ip_addresses":[],
"launched_at":null,
"node":null,
"power_state":"pending",
"state":"error",
"task_state":"deleting"
}
},
"priority":"INFO",
"publisher_id":"nova-api:fake-mini"
}

View File

@ -2820,7 +2820,8 @@ class ComputeManager(manager.Manager):
def soft_delete_instance(self, context, instance):
"""Soft delete an instance on this host."""
with compute_utils.notify_about_instance_delete(
self.notifier, context, instance, 'soft_delete'):
self.notifier, context, instance, 'soft_delete',
fields.NotificationSource.COMPUTE):
compute_utils.notify_about_instance_action(context, instance,
self.host, action=fields.NotificationAction.SOFT_DELETE,
phase=fields.NotificationPhase.START)

View File

@ -1181,11 +1181,31 @@ class UnlimitedSemaphore(object):
@contextlib.contextmanager
def notify_about_instance_delete(notifier, context, instance,
delete_type='delete'):
delete_type='delete',
source=fields.NotificationSource.API):
try:
notify_about_instance_usage(notifier, context, instance,
"%s.start" % delete_type)
# Note(gibi): soft_delete and force_delete types will be handled in a
# subsequent patch
if delete_type == 'delete':
notify_about_instance_action(
context,
instance,
host=CONF.host,
source=source,
action=fields.NotificationAction.DELETE,
phase=fields.NotificationPhase.START)
yield
finally:
notify_about_instance_usage(notifier, context, instance,
"%s.end" % delete_type)
if delete_type == 'delete':
notify_about_instance_action(
context,
instance,
host=CONF.host,
source=source,
action=fields.NotificationAction.DELETE,
phase=fields.NotificationPhase.END)

View File

@ -1422,7 +1422,8 @@ class ComputeTaskManager(base.Base):
# bdm, tags and instance record.
with obj_target_cell(instance, cell) as cctxt:
with compute_utils.notify_about_instance_delete(
self.notifier, cctxt, instance):
self.notifier, cctxt, instance,
source=fields.NotificationSource.CONDUCTOR):
try:
instance.destroy()
except exception.InstanceNotFound:

View File

@ -506,3 +506,15 @@ class TestOpenStackClient(object):
def get_hypervisor_stats(self):
return self.api_get(
'/os-hypervisors/statistics').body['hypervisor_statistics']
def get_service_id(self, binary_name):
for service in self.get_services():
if service['binary'] == binary_name:
return service['id']
raise OpenStackApiNotFoundException('Service cannot be found.')
def put_service_force_down(self, service_id, forced_down):
req = {
'forced_down': forced_down
}
return self.api_put('os-services/%s' % service_id, req).body['service']

View File

@ -455,6 +455,25 @@ class TestInstanceNotificationSample(
'fault.traceback': self.ANY},
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
fake_notifier.reset()
self.api.delete_server(server['id'])
self._wait_until_deleted(server)
self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
self._verify_notification(
'instance-delete-start_not_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
self._verify_notification(
'instance-delete-end_not_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
def test_instance_exists_usage_audit(self):
# TODO(xavvior): Should create a functional test for the
# "instance_usage_audit" periodic task. We didn't find usable
@ -495,6 +514,36 @@ class TestInstanceNotificationSample(
},
actual=notifications[0])
def test_delete_server_while_compute_is_down(self):
server = self._boot_a_server(
expected_status='ACTIVE',
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
service_id = self.api.get_service_id('nova-compute')
self.admin_api.put_service_force_down(service_id, True)
fake_notifier.reset()
self.api.delete_server(server['id'])
self._wait_until_deleted(server)
self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
self._verify_notification(
'instance-delete-start_compute_down',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
self._verify_notification(
'instance-delete-end_compute_down',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
self.admin_api.put_service_force_down(service_id, False)
def _verify_instance_update_steps(self, steps, notifications,
initial=None):
replacements = {}

View File

@ -8233,6 +8233,7 @@ class ComputeTestCase(BaseTestCase,
block_device_mapping=[])
self.assertEqual('Preserve this', instance.fault.message)
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.context.target_cell')
@mock.patch('nova.objects.InstanceMapping.get_by_instance_uuid')
@ -8240,8 +8241,8 @@ class ComputeTestCase(BaseTestCase,
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.objects.BuildRequest.get_by_instance_uuid')
def test_delete_while_booting_instance_not_in_cell_db_cellsv2(
self, br_get_by_instance, notify, minimum_server_version,
im_get_by_instance, target_cell, instance_destroy):
self, br_get_by_instance, legacy_notify, minimum_server_version,
im_get_by_instance, target_cell, instance_destroy, notify):
minimum_server_version.return_value = 15
im_get_by_instance.return_value = mock.Mock()
@ -8257,15 +8258,17 @@ class ComputeTestCase(BaseTestCase,
# the instance is updated during the delete so we only match by uuid
test_utils.assert_instance_delete_notification_by_uuid(
notify, instance.uuid, self.compute_api.notifier, self.context)
legacy_notify, notify, instance.uuid, self.compute_api.notifier,
self.context)
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.objects.Service.get_minimum_version')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.objects.BuildRequest.get_by_instance_uuid')
def test_delete_while_booting_instance_not_in_cell_db_cellsv1(
self, br_get_by_instance, notify, minimum_server_version,
instance_destroy):
self, br_get_by_instance, legacy_notify, minimum_server_version,
instance_destroy, notify):
minimum_server_version.return_value = 14
@ -8276,15 +8279,17 @@ class ComputeTestCase(BaseTestCase,
self.compute_api._delete_instance(self.context, instance)
test_utils.assert_instance_delete_notification_by_uuid(
notify, instance.uuid, self.compute_api.notifier, self.context)
legacy_notify, notify, instance.uuid, self.compute_api.notifier,
self.context)
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.objects.InstanceMapping.get_by_instance_uuid')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.objects.BuildRequest.get_by_instance_uuid')
def test_delete_while_booting_instance_not_scheduled_cellv1(
self, br_get_by_instance, notify, im_get_by_instance,
instance_destroy):
self, br_get_by_instance, legacy_notify, im_get_by_instance,
instance_destroy, notify):
instance = self._create_fake_instance_obj()
instance.host = None
@ -8300,16 +8305,20 @@ class ComputeTestCase(BaseTestCase,
self.compute_api._delete_instance(self.context, instance)
test_utils.assert_instance_delete_notification_by_uuid(
notify, instance.uuid, self.compute_api.notifier, self.context)
legacy_notify, notify, instance.uuid, self.compute_api.notifier,
self.context)
instance_destroy.assert_called_once_with()
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.context.target_cell')
@mock.patch('nova.objects.InstanceMapping.get_by_instance_uuid')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.objects.BuildRequest.get_by_instance_uuid')
def test_delete_while_booting_instance_not_scheduled_cellv2(
self, br_get_by_instance, notify, im_get_by_instance, target_cell,
instance_destroy):
self, br_get_by_instance, legacy_notify, im_get_by_instance,
target_cell, instance_destroy, notify):
target_cell.return_value.__enter__.return_value = self.context
instance = self._create_fake_instance_obj()
@ -8327,7 +8336,8 @@ class ComputeTestCase(BaseTestCase,
instance_destroy.assert_called_once_with()
test_utils.assert_instance_delete_notification_by_uuid(
notify, instance.uuid, self.compute_api.notifier, self.context)
legacy_notify, notify, instance.uuid, self.compute_api.notifier,
self.context)
@ddt.ddt

View File

@ -1005,6 +1005,8 @@ class _ComputeAPIUnitTestMixIn(object):
return snapshot_id
@mock.patch.object(compute_utils,
'notify_about_instance_action')
@mock.patch.object(objects.Migration, 'get_by_instance_and_status')
@mock.patch.object(image_api.API, 'delete')
@mock.patch.object(objects.InstanceMapping, 'save')
@ -1027,8 +1029,9 @@ class _ComputeAPIUnitTestMixIn(object):
def _test_delete(self, delete_type, mock_save, mock_bdm_get, mock_elevated,
mock_get_cn, mock_up, mock_record, mock_inst_update,
mock_deallocate, mock_inst_meta, mock_inst_destroy,
mock_notify, mock_del_token, mock_get_inst, mock_save_im,
mock_image_delete, mock_mig_get, **attrs):
mock_notify_legacy, mock_del_token, mock_get_inst,
mock_save_im, mock_image_delete, mock_mig_get,
mock_notify, **attrs):
expected_save_calls = [mock.call()]
expected_record_calls = []
expected_elevated_calls = []
@ -1171,13 +1174,13 @@ class _ComputeAPIUnitTestMixIn(object):
test.MatchType(objects.Service))
if is_downed_host:
if 'soft' in delete_type:
mock_notify.assert_has_calls([
mock_notify_legacy.assert_has_calls([
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.start'),
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.end')])
else:
mock_notify.assert_has_calls([
mock_notify_legacy.assert_has_calls([
mock.call(self.compute_api.notifier, self.context,
inst, '%s.start' % delete_type),
mock.call(self.compute_api.notifier, self.context,
@ -1203,6 +1206,14 @@ class _ComputeAPIUnitTestMixIn(object):
if is_shelved:
mock_image_delete.assert_called_once_with(self.context,
snapshot_id)
if not cast and delete_type == 'delete':
mock_notify.assert_has_calls([
mock.call(self.context, inst, host='fake-mini',
source='nova-api',
action=delete_type, phase='start'),
mock.call(self.context, inst, host='fake-mini',
source='nova-api',
action=delete_type, phase='end')])
def test_delete(self):
self._test_delete('delete')
@ -1345,6 +1356,7 @@ class _ComputeAPIUnitTestMixIn(object):
self._test_delete('force_delete', vm_state=vm_state,
task_state=task_states.RESIZE_MIGRATING)
@mock.patch.object(compute_utils, 'notify_about_instance_action')
@mock.patch.object(compute_utils, 'notify_about_instance_usage')
@mock.patch.object(db, 'instance_destroy')
@mock.patch.object(db, 'constraint')
@ -1353,7 +1365,8 @@ class _ComputeAPIUnitTestMixIn(object):
@mock.patch.object(objects.BuildRequest, 'get_by_instance_uuid')
def test_delete_fast_if_host_not_set(self, mock_br_get, mock_save,
mock_bdm_get, mock_cons,
mock_inst_destroy, mock_notify):
mock_inst_destroy,
mock_notify_legacy, mock_notify):
self.useFixture(nova_fixtures.AllServicesCurrent())
inst = self._create_instance_obj()
inst.host = ''
@ -1402,11 +1415,17 @@ class _ComputeAPIUnitTestMixIn(object):
test.MatchType(objects.BlockDeviceMappingList),
delete_type='delete')
else:
mock_notify.assert_has_calls([
mock_notify_legacy.assert_has_calls([
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.start'),
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.end')])
mock_notify.assert_has_calls([
mock.call(self.context, inst, host='fake-mini',
source='nova-api', action='delete', phase='start'),
mock.call(self.context, inst, host='fake-mini',
source='nova-api', action='delete', phase='end')])
mock_cons.assert_called_once_with(host=mock.ANY)
mock_inst_destroy.assert_called_once_with(
self.context, instance_uuid, constraint='constraint')
@ -1415,6 +1434,7 @@ class _ComputeAPIUnitTestMixIn(object):
rservations=None, local=False):
pass
@mock.patch.object(compute_utils, 'notify_about_instance_action')
@mock.patch.object(objects.BlockDeviceMapping, 'destroy')
@mock.patch.object(cinder.API, 'detach')
@mock.patch.object(compute_utils, 'notify_about_instance_usage')
@ -1423,7 +1443,7 @@ class _ComputeAPIUnitTestMixIn(object):
@mock.patch.object(objects.Instance, 'destroy')
def test_local_delete_with_deleted_volume(
self, mock_inst_destroy, mock_elevated, mock_dealloc,
mock_notify, mock_detach, mock_bdm_destroy):
mock_notify_legacy, mock_detach, mock_bdm_destroy, mock_notify):
bdms = [objects.BlockDeviceMapping(
**fake_block_device.FakeDbBlockDeviceDict(
{'id': 42, 'volume_id': 'volume_id',
@ -1439,11 +1459,17 @@ class _ComputeAPIUnitTestMixIn(object):
'delete',
self._fake_do_delete)
mock_notify.assert_has_calls([
mock_notify_legacy.assert_has_calls([
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.start'),
mock.call(self.compute_api.notifier, self.context,
inst, 'delete.end')])
mock_notify.assert_has_calls([
mock.call(self.context, inst, host='fake-mini', source='nova-api',
action='delete', phase='start'),
mock.call(self.context, inst, host='fake-mini', source='nova-api',
action='delete', phase='end')])
mock_elevated.assert_has_calls([mock.call(), mock.call()])
mock_detach.assert_called_once_with(mock.ANY, 'volume_id', inst.uuid)
mock_bdm_destroy.assert_called_once_with()
@ -6314,6 +6340,7 @@ class ComputeAPIUnitTestCase(_ComputeAPIUnitTestMixIn, test.NoDBTestCase):
fields=['device_id'])
self.assertEqual([], instances.objects)
@mock.patch.object(compute_utils, 'notify_about_instance_action')
@mock.patch('nova.compute.api.API._delete_while_booting',
return_value=False)
@mock.patch('nova.compute.api.API._lookup_instance')
@ -6326,7 +6353,8 @@ class ComputeAPIUnitTestCase(_ComputeAPIUnitTestMixIn, test.NoDBTestCase):
def _test_delete_volume_backed_instance(
self, vm_state, mock_instance_destroy, bdm_destroy,
notify_about_instance_usage, mock_save, mock_elevated,
bdm_get_by_instance_uuid, mock_lookup, _mock_del_booting):
bdm_get_by_instance_uuid, mock_lookup, _mock_del_booting,
notify_about_instance_action):
volume_id = uuidutils.generate_uuid()
conn_info = {'connector': {'host': 'orig-host'}}
bdms = [objects.BlockDeviceMapping(

View File

@ -1098,10 +1098,12 @@ class ComputeUtilsTestCase(test.NoDBTestCase):
self.assertEqual([], addresses)
mock_ifaddresses.assert_called_once_with(iface)
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.objects.Instance.destroy')
def test_notify_about_instance_delete(self, mock_instance_destroy,
mock_notify_usage):
mock_notify_usage,
mock_notify_action):
instance = fake_instance.fake_instance_obj(
self.context, expected_attrs=('system_metadata',))
with compute_utils.notify_about_instance_delete(
@ -1114,6 +1116,14 @@ class ComputeUtilsTestCase(test.NoDBTestCase):
'delete.end')
]
mock_notify_usage.assert_has_calls(expected_notify_calls)
mock_notify_action.assert_has_calls([
mock.call(self.context, instance,
host='fake-mini', source='nova-api',
action='delete', phase='start'),
mock.call(self.context, instance,
host='fake-mini', source='nova-api',
action='delete', phase='end'),
])
def test_get_stashed_volume_connector_none(self):
inst = fake_instance.fake_instance_obj(self.context)

View File

@ -1748,6 +1748,7 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
@mock.patch('nova.objects.TagList.destroy')
@mock.patch('nova.objects.TagList.create')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.compute.rpcapi.ComputeAPI.build_and_run_instance')
@mock.patch('nova.scheduler.rpcapi.SchedulerAPI.select_destinations')
@ -1758,6 +1759,7 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
br_destroy,
select_destinations,
build_and_run,
legacy_notify,
notify,
taglist_create,
taglist_destroy):
@ -1777,10 +1779,13 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
self.assertIsNotNone(taglist_destroy.call_args[0][0].db_connection)
test_utils.assert_instance_delete_notification_by_uuid(
notify, self.params['build_requests'][0].instance_uuid,
legacy_notify, notify,
self.params['build_requests'][0].instance_uuid,
self.conductor.notifier, test.MatchType(context.RequestContext),
expect_targeted_context=True)
expect_targeted_context=True, expected_source='nova-conductor',
expected_host='host1')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.compute.rpcapi.ComputeAPI.build_and_run_instance')
@ -1789,7 +1794,7 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
@mock.patch('nova.conductor.manager.ComputeTaskManager._bury_in_cell0')
def test_schedule_and_build_delete_during_scheduling_host_changed(
self, bury, br_destroy, select_destinations,
build_and_run, notify, instance_destroy):
build_and_run, legacy_notify, instance_destroy, notify):
br_destroy.side_effect = exc.BuildRequestNotFound(uuid='foo')
instance_destroy.side_effect = [
@ -1805,11 +1810,15 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
self.assertFalse(bury.called)
self.assertTrue(br_destroy.called)
self.assertEqual(2, instance_destroy.call_count)
test_utils.assert_instance_delete_notification_by_uuid(
notify, self.params['build_requests'][0].instance_uuid,
self.conductor.notifier, test.MatchType(context.RequestContext),
expect_targeted_context=True)
test_utils.assert_instance_delete_notification_by_uuid(
legacy_notify, notify,
self.params['build_requests'][0].instance_uuid,
self.conductor.notifier, test.MatchType(context.RequestContext),
expect_targeted_context=True, expected_source='nova-conductor',
expected_host='host1')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch('nova.objects.Instance.destroy')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.compute.rpcapi.ComputeAPI.build_and_run_instance')
@ -1818,7 +1827,7 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
@mock.patch('nova.conductor.manager.ComputeTaskManager._bury_in_cell0')
def test_schedule_and_build_delete_during_scheduling_instance_not_found(
self, bury, br_destroy, select_destinations,
build_and_run, notify, instance_destroy):
build_and_run, legacy_notify, instance_destroy, notify):
br_destroy.side_effect = exc.BuildRequestNotFound(uuid='foo')
instance_destroy.side_effect = [
@ -1834,9 +1843,11 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
self.assertTrue(br_destroy.called)
self.assertEqual(1, instance_destroy.call_count)
test_utils.assert_instance_delete_notification_by_uuid(
notify, self.params['build_requests'][0].instance_uuid,
legacy_notify, notify,
self.params['build_requests'][0].instance_uuid,
self.conductor.notifier, test.MatchType(context.RequestContext),
expect_targeted_context=True)
expect_targeted_context=True, expected_source='nova-conductor',
expected_host='host1')
@mock.patch('nova.compute.rpcapi.ComputeAPI.build_and_run_instance')
@mock.patch('nova.scheduler.rpcapi.SchedulerAPI.select_destinations')

View File

@ -299,21 +299,53 @@ class CustomMockCallMatcher(object):
def assert_instance_delete_notification_by_uuid(
mock_notify, expected_instance_uuid, expected_notifier,
expected_context, expect_targeted_context=False):
mock_legacy_notify, mock_notify, expected_instance_uuid,
expected_notifier, expected_context, expect_targeted_context=False,
expected_source='nova-api', expected_host='fake-mini'):
match_by_instance_uuid = CustomMockCallMatcher(
lambda instance:
instance.uuid == expected_instance_uuid)
assert_legacy_instance_delete_notification_by_uuid(
mock_legacy_notify, match_by_instance_uuid, expected_notifier,
expected_context, expect_targeted_context)
assert_versioned_instance_delete_notification_by_uuid(
mock_notify, match_by_instance_uuid,
expected_context, expected_source, expected_host=expected_host)
def assert_versioned_instance_delete_notification_by_uuid(
mock_notify, instance_matcher,
expected_context, expected_source, expected_host):
mock_notify.assert_has_calls([
mock.call(expected_context,
instance_matcher,
host=expected_host,
source=expected_source,
action='delete',
phase='start'),
mock.call(expected_context,
instance_matcher,
host=expected_host,
source=expected_source,
action='delete',
phase='end')])
def assert_legacy_instance_delete_notification_by_uuid(
mock_notify, instance_matcher, expected_notifier,
expected_context, expect_targeted_context):
mock_notify.assert_has_calls([
mock.call(expected_notifier,
expected_context,
match_by_instance_uuid,
instance_matcher,
'delete.start'),
mock.call(expected_notifier,
expected_context,
match_by_instance_uuid,
instance_matcher,
'delete.end')])
for call in mock_notify.call_args_list: