Add configurable timeout for in-use files

In some cases, Hyper-V locks on VHD images persist for a few seconds
even after the instances are deleted (not sure if that's due to
Hyper-V or the filesystem). In order to mitigate this, we're
already performing some retries when deleting instance files.

We've encountered similar issues with 3rd party backup tools
that kept open file handles.

This change adds a configurable timeout that will be used when waiting
for in-use fles. The default value is 15 seconds, slightly more than
the current hardcoded value. At the same time, this timeout will
also be used when deleting individual files or renaming folders or
files.

Change-Id: I49307146b79a20089190e1ce8bde73d656d37c6d

os_win/conf.py

@@ -39,7 +39,13 @@ os_win_opts = [
     cfg.IntOpt('connect_cluster_timeout',
                default=0,
                help='The amount of time to wait for the Failover Cluster '
-                    'service to be available.')
+                    'service to be available.'),
+    cfg.IntOpt('file_in_use_timeout',
+               default=15,
+               help='The amount of seconds to wait for in-use files when '
+                    'performing moves or deletions. This can help mitigate '
+                    'issues occurring due to Hyper-V locks or even 3rd party '
+                    'backup tools.'),
 ]
 
 CONF = cfg.CONF

os_win/tests/unit/utils/test_pathutils.py

@@ -120,16 +120,18 @@ class PathUtilsTestCase(test_base.OsWinBaseTestCase):
     @mock.patch.object(pathutils.shutil, 'rmtree')
     def _check_rmtree(self, mock_rmtree, mock_sleep, side_effect):
         mock_rmtree.side_effect = side_effect
-        self.assertRaises(exceptions.OSWinException, self._pathutils.rmtree,
+        self.assertRaises(exceptions.WindowsError, self._pathutils.rmtree,
                           mock.sentinel.FAKE_PATH)
 
     def test_rmtree_unexpected(self):
         self._check_rmtree(side_effect=exceptions.WindowsError)
 
-    def test_rmtree_exceeded(self):
+    @mock.patch('time.time')
+    def test_rmtree_exceeded(self, mock_time):
+        mock_time.side_effect = range(1, 100, 10)
         exc = exceptions.WindowsError()
         exc.winerror = w_const.ERROR_DIR_IS_NOT_EMPTY
-        self._check_rmtree(side_effect=[exc] * 6)
+        self._check_rmtree(side_effect=exc)
 
     @mock.patch.object(pathutils.PathUtils, 'makedirs')
     @mock.patch.object(pathutils.PathUtils, 'exists')

os_win/utils/pathutils.py

@@ -22,10 +22,10 @@ import tempfile
 
 from oslo_log import log as logging
 from oslo_utils import fileutils
-import six
 
 from os_win._i18n import _
 from os_win import _utils
+import os_win.conf
 from os_win import exceptions
 from os_win.utils import _acl_utils
 from os_win.utils.io import ioutils
@@ -38,8 +38,17 @@ from os_win.utils.winapi import wintypes
 
 kernel32 = w_lib.get_shared_lib_handle(w_lib.KERNEL32)
 
+CONF = os_win.conf.CONF
 LOG = logging.getLogger(__name__)
 
+file_in_use_retry_decorator = _utils.retry_decorator(
+    exceptions=exceptions.WindowsError,
+    extract_err_code_func=lambda x: x.winerror,
+    error_codes=[w_const.ERROR_SHARING_VIOLATION,
+                 w_const.ERROR_DIR_IS_NOT_EMPTY],
+    timeout=CONF.os_win.file_in_use_timeout,
+    max_retry_count=None)
+
 
 class PathUtils(object):
 
@@ -59,9 +68,11 @@ class PathUtils(object):
     def makedirs(self, path):
         os.makedirs(path)
 
+    @file_in_use_retry_decorator
     def remove(self, path):
         os.remove(path)
 
+    @file_in_use_retry_decorator
     def rename(self, src, dest):
         os.rename(src, dest)
 
@@ -127,16 +138,9 @@ class PathUtils(object):
             if os.path.isfile(src):
                 self.rename(src, os.path.join(dest_dir, fname))
 
-    @_utils.retry_decorator(exceptions=exceptions.OSWinException,
-                            error_codes=[w_const.ERROR_DIR_IS_NOT_EMPTY])
+    @file_in_use_retry_decorator
     def rmtree(self, path):
-        try:
-            shutil.rmtree(path)
-        except exceptions.WindowsError as ex:
-            # NOTE(claudiub): convert it to an OSWinException in order to use
-            # the retry_decorator.
-            raise exceptions.OSWinException(six.text_type(ex),
-                                            error_code=ex.winerror)
+        shutil.rmtree(path)
 
     def check_create_dir(self, path):
         if not self.exists(path):

os_win/utils/winapi/constants.py

@@ -20,6 +20,7 @@ from os_win.utils.winapi import wintypes
 # winerror.h
 ERROR_INVALID_HANDLE = 6
 ERROR_NOT_READY = 21
+ERROR_SHARING_VIOLATION = 32
 ERROR_SHARING_PAUSED = 70
 ERROR_INSUFFICIENT_BUFFER = 122
 ERROR_DIR_IS_NOT_EMPTY = 145