unquiesce instance after quiesce failure
If the call to compute_rpcapi.quisece_instance() raises an exception,
any uncaught exception will break out of the function
snapshot_volume_backed(). This can leave the instance in frozen state.
This patch adds a blanket Exception catch to the try block and calls
compute_rpcapi.unquiesce_instance() before reraising.
This has been seen in the wild with RPC timeouts, but this is not the
only possible genesis for an unknown error from quiesce_instance.
Conflicts:
nova/tests/unit/compute/test_compute_api.py
NOTE(mriedem): The conflict is due to not having change
I4e7b46deb43c0c2430b480f1a498a52fc4a9daf0, and its dependencies,
in Pike.
Change-Id: Idca5998da8bb42b29a8fffdf52b4af3a043c6326
Closes-Bug: #1754360
(cherry picked from commit 1e77faaa41)
(cherry picked from commit bcae081c46)
(cherry picked from commit 1487ea7abb)
This commit is contained in:
Eric M Gonzalez
Matt Riedemann
parent
4246d5779e
commit
1535d42ea1
|
|
@ -2942,6 +2942,17 @@ class API(base.Base):
|
|||
LOG.info(_LI('Skipping quiescing instance: '
|
||||
'%(reason)s.'), {'reason': err},
|
||||
instance=instance)
|
||||
# NOTE(tasker): discovered that an uncaught exception could occur
|
||||
# after the instance has been frozen. catch and thaw.
|
||||
except Exception as ex:
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error(
|
||||
_LE("An error occurred during quiesce of instance. "
|
||||
"Unquiescing to ensure instance is thawed. "
|
||||
"Error: %s"), six.text_type(ex),
|
||||
instance=instance)
|
||||
self.compute_rpcapi.unquiesce_instance(context, instance,
|
||||
mapping=None)
|
||||
|
||||
bdms = objects.BlockDeviceMappingList.get_by_instance_uuid(
|
||||
context, instance.uuid)
|
||||
|
|
|
|||
|
|
@ -2757,7 +2757,9 @@ class _ComputeAPIUnitTestMixIn(object):
|
|||
mock_is_volume_backed.assert_called_once_with(self.context,
|
||||
instance)
|
||||
|
||||
def _test_snapshot_volume_backed(self, quiesce_required, quiesce_fails,
|
||||
def _test_snapshot_volume_backed(self, quiesce_required=False,
|
||||
quiesce_fails=False,
|
||||
quiesce_unsupported=False,
|
||||
vm_state=vm_states.ACTIVE,
|
||||
snapshot_fails=False):
|
||||
fake_sys_meta = {'image_min_ram': '11',
|
||||
|
|
@ -2791,7 +2793,8 @@ class _ComputeAPIUnitTestMixIn(object):
|
|||
expect_meta['properties']['os_require_quiesce'] = 'yes'
|
||||
|
||||
quiesced = [False, False]
|
||||
quiesce_expected = not quiesce_fails and vm_state == vm_states.ACTIVE
|
||||
quiesce_expected = not (quiesce_unsupported or quiesce_fails) \
|
||||
and vm_state == vm_states.ACTIVE
|
||||
|
||||
@classmethod
|
||||
def fake_bdm_list_get_by_instance_uuid(cls, context, instance_uuid):
|
||||
|
|
@ -2810,9 +2813,11 @@ class _ComputeAPIUnitTestMixIn(object):
|
|||
return {'id': '%s-snapshot' % volume_id}
|
||||
|
||||
def fake_quiesce_instance(context, instance):
|
||||
if quiesce_fails:
|
||||
if quiesce_unsupported:
|
||||
raise exception.InstanceQuiesceNotSupported(
|
||||
instance_id=instance['uuid'], reason='test')
|
||||
instance_id=instance['uuid'], reason='unsupported')
|
||||
if quiesce_fails:
|
||||
raise oslo_exceptions.MessagingTimeout('quiece timeout')
|
||||
quiesced[0] = True
|
||||
|
||||
def fake_unquiesce_instance(context, instance, mapping=None):
|
||||
|
|
@ -2912,33 +2917,46 @@ class _ComputeAPIUnitTestMixIn(object):
|
|||
self.assertEqual(quiesce_expected, quiesced[1])
|
||||
|
||||
def test_snapshot_volume_backed(self):
|
||||
self._test_snapshot_volume_backed(False, False)
|
||||
self._test_snapshot_volume_backed(quiesce_required=False,
|
||||
quiesce_unsupported=False)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce(self):
|
||||
self._test_snapshot_volume_backed(True, False)
|
||||
def test_snapshot_volume_backed_with_quiesce_unsupported(self):
|
||||
self._test_snapshot_volume_backed(quiesce_required=True,
|
||||
quiesce_unsupported=False)
|
||||
|
||||
def test_snaphost_volume_backed_with_quiesce_failure(self):
|
||||
self.assertRaises(oslo_exceptions.MessagingTimeout,
|
||||
self._test_snapshot_volume_backed,
|
||||
quiesce_required=True,
|
||||
quiesce_fails=True)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce_create_snap_fails(self):
|
||||
self._test_snapshot_volume_backed(quiesce_required=True,
|
||||
quiesce_fails=False,
|
||||
snapshot_fails=True)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce_skipped(self):
|
||||
self._test_snapshot_volume_backed(False, True)
|
||||
self._test_snapshot_volume_backed(quiesce_required=False,
|
||||
quiesce_unsupported=True)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce_exception(self):
|
||||
self.assertRaises(exception.NovaException,
|
||||
self._test_snapshot_volume_backed, True, True)
|
||||
self._test_snapshot_volume_backed,
|
||||
quiesce_required=True,
|
||||
quiesce_unsupported=True)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce_stopped(self):
|
||||
self._test_snapshot_volume_backed(True, True,
|
||||
self._test_snapshot_volume_backed(quiesce_required=True,
|
||||
quiesce_unsupported=True,
|
||||
vm_state=vm_states.STOPPED)
|
||||
|
||||
def test_snapshot_volume_backed_with_quiesce_suspended(self):
|
||||
self._test_snapshot_volume_backed(True, True,
|
||||
self._test_snapshot_volume_backed(quiesce_required=True,
|
||||
quiesce_unsupported=True,
|
||||
vm_state=vm_states.SUSPENDED)
|
||||
|
||||
def test_snapshot_volume_backed_with_suspended(self):
|
||||
self._test_snapshot_volume_backed(False, True,
|
||||
self._test_snapshot_volume_backed(quiesce_required=False,
|
||||
quiesce_unsupported=True,
|
||||
vm_state=vm_states.SUSPENDED)
|
||||
|
||||
def test_volume_snapshot_create(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue