VMware: handle case when VM snapshot delete fails

Minesweeper fails on occasion with concurrent access to files.

If a snapshot deletion fails with the backend exception
TaskInProgress then we will retry.

A generic decorator has been added to do the retry operations.

Closes-bug: #1310817

Change-Id: I8ed7a24ccd34aeea49352ac98f34ec2960edbf97

nova/tests/virt/vmwareapi/test_driver_api.py

@@ -1294,6 +1294,37 @@ class VMwareAPIVMTestCase(test.NoDBTestCase):
 
         self._test_snapshot()
 
+    def _snapshot_delete_vm_snapshot_exception(self, exception, call_count=1):
+        self._create_vm()
+        fake_vm = vmwareapi_fake._get_objects("VirtualMachine").objects[0].obj
+        snapshot_ref = vmwareapi_fake.ManagedObjectReference(
+                               value="Snapshot-123",
+                               name="VirtualMachineSnapshot")
+
+        with contextlib.nested(
+            mock.patch.object(self.conn._session, '_wait_for_task',
+                              side_effect=exception),
+            mock.patch.object(time, 'sleep')
+        ) as (_fake_wait, _fake_sleep):
+            if exception != error_util.TaskInProgress:
+                self.assertRaises(exception,
+                                  self.conn._vmops._delete_vm_snapshot,
+                                  self.instance, fake_vm, snapshot_ref)
+                self.assertEqual(0, _fake_sleep.call_count)
+            else:
+                self.conn._vmops._delete_vm_snapshot(self.instance, fake_vm,
+                                                     snapshot_ref)
+                self.assertEqual(call_count - 1, _fake_sleep.call_count)
+            self.assertEqual(call_count, _fake_wait.call_count)
+
+    def test_snapshot_delete_vm_snapshot_exception(self):
+        self._snapshot_delete_vm_snapshot_exception(exception.NovaException)
+
+    def test_snapshot_delete_vm_snapshot_exception_retry(self):
+        self.flags(api_retry_count=5, group='vmware')
+        self._snapshot_delete_vm_snapshot_exception(error_util.TaskInProgress,
+                                                    5)
+
     def test_reboot(self):
         self._create_vm()
         info = self.conn.get_info({'name': 1, 'uuid': self.uuid,

nova/virt/vmwareapi/error_util.py

@@ -32,6 +32,7 @@ INVALID_POWER_STATE = 'InvalidPowerState'
 INVALID_PROPERTY = 'InvalidProperty'
 NO_PERMISSION = 'NoPermission'
 NOT_AUTHENTICATED = 'NotAuthenticated'
+TASK_IN_PROGRESS = 'TaskInProgress'
 
 
 class VimException(Exception):
@@ -203,6 +204,10 @@ class InvalidPowerStateException(VMwareDriverException):
     code = 409
 
 
+class TaskInProgress(VMwareDriverException):
+    msg_fmt = _("Virtual machine is busy.")
+
+
 # Populate the fault registry with the exceptions that have
 # special treatment.
 _fault_classes_registry = {
@@ -215,7 +220,8 @@ _fault_classes_registry = {
     INVALID_POWER_STATE: InvalidPowerStateException,
     INVALID_PROPERTY: InvalidPropertyException,
     NO_PERMISSION: NoPermissionException,
-    NOT_AUTHENTICATED: NotAuthenticatedException
+    NOT_AUTHENTICATED: NotAuthenticatedException,
+    TASK_IN_PROGRESS: TaskInProgress,
 }
 
 

nova/virt/vmwareapi/vim.py

@@ -19,8 +19,10 @@ Classes for making VMware VI SOAP calls.
 """
 
 import httplib
+import time
 import urllib2
 
+import decorator
 from oslo.config import cfg
 import suds
 
@@ -41,6 +43,21 @@ CONF = cfg.CONF
 CONF.register_opt(vmwareapi_wsdl_loc_opt, 'vmware')
 
 
+@decorator.decorator
+def retry_if_task_in_progress(f, *args, **kwargs):
+    retries = max(CONF.vmware.api_retry_count, 1)
+    delay = 1
+    for attempt in range(1, retries + 1):
+        if attempt != 1:
+            time.sleep(delay)
+            delay = min(2 * delay, 60)
+        try:
+            f(*args, **kwargs)
+            return
+        except error_util.TaskInProgress:
+            pass
+
+
 def get_moref(value, type):
     """Get managed object reference."""
     moref = suds.sudsobject.Property(value)

nova/virt/vmwareapi/vmops.py

@@ -698,6 +698,7 @@ class VMwareVMOps(object):
         snapshot = task_info.result
         return snapshot
 
+    @vim.retry_if_task_in_progress
     def _delete_vm_snapshot(self, instance, vm_ref, snapshot):
         LOG.debug("Deleting Snapshot of the VM instance", instance=instance)
         delete_snapshot_task = self._session._call_method(
@@ -800,7 +801,6 @@ class VMwareVMOps(object):
                       instance=instance)
 
         _copy_vmdk_content()
-        # Note(vui): handle snapshot cleanup on exceptions.
         self._delete_vm_snapshot(instance, vm_ref, snapshot)
 
         cookies = self._session._get_vim().client.options.transport.cookiejar