From a37bc78ed57aeabbb87b26f77fd785db3ee6a9ba Mon Sep 17 00:00:00 2001 From: Joe Talerico Date: Mon, 30 Mar 2015 13:55:47 -0400 Subject: [PATCH] Fixes _cleanup_rbd code to capture ImageBusy exception This patch captures the rbd.ImageBusy exception and attempts to remove the image from the rbd volume. Co-Authored-By: Dan Smith Change-Id: I6a5ca4c68a39c2abf5f3f7d2d863fa2198d3c8e9 Closes-Bug: 1438331 --- nova/tests/unit/virt/libvirt/test_rbd.py | 9 ++++--- nova/virt/libvirt/rbd_utils.py | 31 +++++++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/nova/tests/unit/virt/libvirt/test_rbd.py b/nova/tests/unit/virt/libvirt/test_rbd.py index 516c7b9eae14..0d343146cd87 100644 --- a/nova/tests/unit/virt/libvirt/test_rbd.py +++ b/nova/tests/unit/virt/libvirt/test_rbd.py @@ -296,11 +296,14 @@ class RbdTestCase(test.NoDBTestCase): rbd.list.return_value = ['12345_test', '111_test'] client = mock_client.return_value - self.driver.cleanup_volumes(instance) - rbd.remove.assert_called_once_with(client.ioctx, '12345_test') + with mock.patch('eventlet.greenthread.sleep'): + self.driver.cleanup_volumes(instance) + rbd.remove.assert_any_call(client.ioctx, '12345_test') + # NOTE(danms): 10 retries + 1 final attempt to propagate = 11 + self.assertEqual(11, len(rbd.remove.call_args_list)) def test_cleanup_volumes_fail_not_found(self): - self._test_cleanup_exception('ImageNotFound') + self._test_cleanup_exception('ImageBusy') def test_cleanup_volumes_fail_snapshots(self): self._test_cleanup_exception('ImageHasSnapshots') diff --git a/nova/virt/libvirt/rbd_utils.py b/nova/virt/libvirt/rbd_utils.py index 8260a98491bc..715941b127bf 100644 --- a/nova/virt/libvirt/rbd_utils.py +++ b/nova/virt/libvirt/rbd_utils.py @@ -32,6 +32,7 @@ from nova import exception from nova.i18n import _ from nova.i18n import _LE from nova.i18n import _LW +from nova.openstack.common import loopingcall from nova import utils LOG = logging.getLogger(__name__) @@ -254,6 +255,18 @@ class RBDDriver(object): utils.execute('rbd', 'import', *args) def cleanup_volumes(self, instance): + def _cleanup_vol(ioctx, volume, retryctx): + try: + rbd.RBD().remove(client.ioctx, volume) + raise loopingcall.LoopingCallDone(retvalue=False) + except (rbd.ImageBusy, rbd.ImageHasSnapshots): + LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s ' + 'failed'), + {'volume': volume, 'pool': self.pool}) + retryctx['retries'] -= 1 + if retryctx['retries'] <= 0: + raise loopingcall.LoopingCallDone() + with RADOSClient(self, self.pool) as client: def belongs_to_instance(disk): @@ -261,12 +274,18 @@ class RBDDriver(object): volumes = rbd.RBD().list(client.ioctx) for volume in filter(belongs_to_instance, volumes): - try: - rbd.RBD().remove(client.ioctx, volume) - except (rbd.ImageNotFound, rbd.ImageHasSnapshots): - LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s ' - 'failed'), - {'volume': volume, 'pool': self.pool}) + # NOTE(danms): We let it go for ten seconds + retryctx = {'retries': 10} + timer = loopingcall.FixedIntervalLoopingCall( + _cleanup_vol, client.ioctx, volume, retryctx) + timed_out = timer.start(interval=1).wait() + if timed_out: + # NOTE(danms): Run this again to propagate the error, but + # if it succeeds, don't raise the loopingcall exception + try: + _cleanup_vol(client.ioctx, volume, retryctx) + except loopingcall.LoopingCallDone: + pass def get_pool_info(self): with RADOSClient(self) as client: