Merge "compute: Cleans up allocations after failed resize" into stable/pike

This commit is contained in:
Zuul 2018-03-31 00:53:19 +00:00 committed by Gerrit Code Review
commit 708342fe8d
4 changed files with 83 additions and 17 deletions

View File

@ -3941,6 +3941,18 @@ class ComputeManager(manager.Manager):
reservations, migration, instance_type,
clean_shutdown):
"""Starts the migration of a running instance to another host."""
try:
self._resize_instance(context, instance, image, migration,
instance_type, clean_shutdown)
except Exception:
with excutils.save_and_reraise_exception():
rt = self._get_resource_tracker()
node = self.driver.get_available_nodes(refresh=True)[0]
rt.delete_allocation_for_failed_resize(
instance, node, instance_type)
def _resize_instance(self, context, instance, image,
migration, instance_type, clean_shutdown):
with self._error_out_instance_on_exception(context, instance):
# TODO(chaochin) Remove this until v5 RPC API
# Code downstream may expect extra_specs to be populated since it
@ -4121,6 +4133,23 @@ class ComputeManager(manager.Manager):
Sets up the newly transferred disk and turns on the instance at its
new host machine.
"""
try:
self._finish_resize_helper(context, disk_info, image, instance,
migration)
except Exception:
with excutils.save_and_reraise_exception():
rt = self._get_resource_tracker()
node = self.driver.get_available_nodes(refresh=True)[0]
rt.delete_allocation_for_failed_resize(
instance, node, instance.new_flavor)
def _finish_resize_helper(self, context, disk_info, image, instance,
migration):
"""Completes the migration process.
The caller must revert the instance's allocations if the migration
process failed.
"""
with self._error_out_instance_on_exception(context, instance):
image_meta = objects.ImageMeta.from_dict(image)

View File

@ -2505,9 +2505,10 @@ class ServerMovingTests(ProviderUsageBaseTestCase):
source_usages = self._get_provider_usages(source_rp_uuid)
self.assertFlavorMatchesAllocation(self.flavor1, source_usages)
def test_resize_to_same_host_prep_resize_fails(self):
def _test_resize_to_same_host_instance_fails(self, failing_method,
event_name):
"""Tests that when we resize to the same host and resize fails in
the prep_resize method, we cleanup the allocations before rescheduling.
the given method, we cleanup the allocations before rescheduling.
"""
# make sure that the test only uses a single host
compute2_service_id = self.admin_api.get_services(
@ -2519,16 +2520,17 @@ class ServerMovingTests(ProviderUsageBaseTestCase):
server = self._boot_and_check_allocations(self.flavor1, hostname)
def fake_prep_resize(*args, **kwargs):
def fake_resize_method(*args, **kwargs):
# Ensure the allocations are doubled now before we fail.
usages = self._get_provider_usages(rp_uuid)
self.assertFlavorsMatchAllocation(
self.flavor1, self.flavor2, usages)
raise test.TestingException('Simulated _prep_resize failure.')
raise test.TestingException('Simulated resize failure.')
# Yes this isn't great in a functional test, but it's simple.
self.stub_out('nova.compute.manager.ComputeManager._prep_resize',
fake_prep_resize)
self.stub_out(
'nova.compute.manager.ComputeManager.%s' % failing_method,
fake_resize_method)
self.flags(allow_resize_to_same_host=True)
resize_req = {
@ -2539,7 +2541,7 @@ class ServerMovingTests(ProviderUsageBaseTestCase):
self.api.post_server_action(server['id'], resize_req)
self._wait_for_action_fail_completion(
server, instance_actions.RESIZE, 'compute_prep_resize')
server, instance_actions.RESIZE, event_name)
# Ensure the allocation records still exist on the host.
source_rp_uuid = self._get_provider_uuid_by_host(hostname)
@ -2548,6 +2550,18 @@ class ServerMovingTests(ProviderUsageBaseTestCase):
# allocation which just leaves us with the original flavor.
self.assertFlavorMatchesAllocation(self.flavor1, source_usages)
def test_resize_to_same_host_prep_resize_fails(self):
self._test_resize_to_same_host_instance_fails(
'_prep_resize', 'compute_prep_resize')
def test_resize_instance_fails_allocation_cleanup(self):
self._test_resize_to_same_host_instance_fails(
'_resize_instance', 'compute_resize_instance')
def test_finish_resize_fails_allocation_cleanup(self):
self._test_resize_to_same_host_instance_fails(
'_finish_resize', 'compute_finish_resize')
def _mock_live_migration(self, context, instance, dest,
post_method, recover_method,
block_migration=False, migrate_data=None):

View File

@ -4470,8 +4470,15 @@ class ComputeTestCase(BaseTestCase,
func = getattr(self.compute, operation)
self.assertRaises(test.TestingException,
func, self.context, instance=instance, **kwargs)
with mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize') as delete_alloc:
self.assertRaises(test.TestingException,
func, self.context, instance=instance, **kwargs)
if operation == 'resize_instance':
delete_alloc.assert_called_once_with(
instance, 'fakenode1', kwargs['instance_type'])
else:
delete_alloc.assert_not_called()
# self.context.elevated() is called in tearDown()
self.stub_out('nova.context.RequestContext.elevated', orig_elevated)
self.stub_out('nova.compute.manager.ComputeManager.'
@ -4487,6 +4494,7 @@ class ComputeTestCase(BaseTestCase,
# ensure that task_state is reverted after a failed operation.
migration = objects.Migration(context=self.context.elevated())
migration.instance_uuid = 'b48316c5-71e8-45e4-9884-6c78055b9b13'
migration.uuid = mock.sentinel.uuid
migration.new_instance_type_id = '1'
instance_type = objects.Flavor()
@ -5176,7 +5184,9 @@ class ComputeTestCase(BaseTestCase,
clean_shutdown=True)
self.compute.terminate_instance(self.context, instance, [], [])
def test_resize_instance_driver_error(self):
@mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize')
def test_resize_instance_driver_error(self, delete_alloc):
# Ensure instance status set to Error on resize error.
def throw_up(*args, **kwargs):
@ -5216,7 +5226,9 @@ class ComputeTestCase(BaseTestCase,
self.assertEqual(instance.vm_state, vm_states.ERROR)
self.compute.terminate_instance(self.context, instance, [], [])
def test_resize_instance_driver_rollback(self):
@mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize')
def test_resize_instance_driver_rollback(self, delete_alloc):
# Ensure instance status set to Running after rollback.
def throw_up(*args, **kwargs):
@ -5839,7 +5851,9 @@ class ComputeTestCase(BaseTestCase,
flavor_type = flavors.get_flavor_by_flavor_id(1)
self.assertEqual(flavor_type['name'], 'm1.tiny')
def test_resize_instance_handles_migration_error(self):
@mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize')
def test_resize_instance_handles_migration_error(self, delete_alloc):
# Ensure vm_state is ERROR when error occurs.
def raise_migration_failure(*args):
raise test.TestingException()

View File

@ -5628,7 +5628,8 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
vm_state=vm_states.ACTIVE,
expected_attrs=['metadata', 'system_metadata', 'info_cache'])
self.migration = objects.Migration(context=self.context.elevated(),
new_instance_type_id=7)
new_instance_type_id=7,
uuid=mock.sentinel.uuid)
self.migration.status = 'migrating'
self.useFixture(fixtures.SpawnIsSynchronousFixture())
self.useFixture(fixtures.EventReporterStub())
@ -5642,9 +5643,11 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
mock.patch.object(self.instance, 'save'),
mock.patch.object(self.migration, 'save'),
mock.patch.object(self.migration, 'obj_as_admin',
return_value=mock.MagicMock())
return_value=mock.MagicMock()),
mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize')
) as (meth, fault_create, instance_update, instance_save,
migration_save, migration_obj_as_admin):
migration_save, migration_obj_as_admin, delete_alloc):
fault_create.return_value = (
test_instance_fault.fake_faults['fake-uuid'][0])
self.assertRaises(
@ -5656,6 +5659,8 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
self.assertEqual("error", self.migration.status)
migration_save.assert_called_once_with()
migration_obj_as_admin.assert_called_once_with()
delete_alloc.assert_called_once_with(
self.instance, 'fake-mini', self.instance.new_flavor)
def test_resize_instance_failure(self):
self.migration.dest_host = None
@ -5680,10 +5685,12 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
return_value=None),
mock.patch.object(objects.Flavor,
'get_by_id',
return_value=None)
return_value=None),
mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_failed_resize')
) as (meth, fault_create, instance_update,
migration_save, migration_obj_as_admin, nw_info, save_inst,
notify, vol_block_info, bdm, flavor):
notify, vol_block_info, bdm, flavor, delete_alloc):
fault_create.return_value = (
test_instance_fault.fake_faults['fake-uuid'][0])
self.assertRaises(
@ -5696,6 +5703,8 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
migration_save.mock_calls)
self.assertEqual([mock.call(), mock.call()],
migration_obj_as_admin.mock_calls)
delete_alloc.assert_called_once_with(
self.instance, 'fake-mini', 'type')
def _test_revert_resize_instance_destroy_disks(self, is_shared=False):