Merge "Add configurable timeout for in-use files"

os_win/conf.py

@@ -39,7 +39,13 @@ os_win_opts = [
     cfg.IntOpt('connect_cluster_timeout',
                default=0,
                help='The amount of time to wait for the Failover Cluster '
-                    'service to be available.')
+                    'service to be available.'),
+    cfg.IntOpt('file_in_use_timeout',
+               default=15,
+               help='The amount of seconds to wait for in-use files when '
+                    'performing moves or deletions. This can help mitigate '
+                    'issues occurring due to Hyper-V locks or even 3rd party '
+                    'backup tools.'),
 ]
 
 CONF = cfg.CONF

os_win/tests/unit/utils/test_pathutils.py

@@ -120,16 +120,18 @@ class PathUtilsTestCase(test_base.OsWinBaseTestCase):
     @mock.patch.object(pathutils.shutil, 'rmtree')
     def _check_rmtree(self, mock_rmtree, mock_sleep, side_effect):
         mock_rmtree.side_effect = side_effect
-        self.assertRaises(exceptions.OSWinException, self._pathutils.rmtree,
+        self.assertRaises(exceptions.WindowsError, self._pathutils.rmtree,
                           mock.sentinel.FAKE_PATH)
 
     def test_rmtree_unexpected(self):
         self._check_rmtree(side_effect=exceptions.WindowsError)
 
-    def test_rmtree_exceeded(self):
+    @mock.patch('time.time')
+    def test_rmtree_exceeded(self, mock_time):
+        mock_time.side_effect = range(1, 100, 10)
         exc = exceptions.WindowsError()
         exc.winerror = w_const.ERROR_DIR_IS_NOT_EMPTY
-        self._check_rmtree(side_effect=[exc] * 6)
+        self._check_rmtree(side_effect=exc)
 
     @mock.patch.object(pathutils.PathUtils, 'makedirs')
     @mock.patch.object(pathutils.PathUtils, 'exists')

os_win/utils/pathutils.py

@@ -22,10 +22,10 @@ import tempfile
 
 from oslo_log import log as logging
 from oslo_utils import fileutils
-import six
 
 from os_win._i18n import _
 from os_win import _utils
+import os_win.conf
 from os_win import exceptions
 from os_win.utils import _acl_utils
 from os_win.utils.io import ioutils
@@ -38,8 +38,17 @@ from os_win.utils.winapi import wintypes
 
 kernel32 = w_lib.get_shared_lib_handle(w_lib.KERNEL32)
 
+CONF = os_win.conf.CONF
 LOG = logging.getLogger(__name__)
 
+file_in_use_retry_decorator = _utils.retry_decorator(
+    exceptions=exceptions.WindowsError,
+    extract_err_code_func=lambda x: x.winerror,
+    error_codes=[w_const.ERROR_SHARING_VIOLATION,
+                 w_const.ERROR_DIR_IS_NOT_EMPTY],
+    timeout=CONF.os_win.file_in_use_timeout,
+    max_retry_count=None)
+
 
 class PathUtils(object):
 
@@ -59,9 +68,11 @@ class PathUtils(object):
     def makedirs(self, path):
         os.makedirs(path)
 
+    @file_in_use_retry_decorator
     def remove(self, path):
         os.remove(path)
 
+    @file_in_use_retry_decorator
     def rename(self, src, dest):
         os.rename(src, dest)
 
@@ -127,16 +138,9 @@ class PathUtils(object):
             if os.path.isfile(src):
                 self.rename(src, os.path.join(dest_dir, fname))
 
-    @_utils.retry_decorator(exceptions=exceptions.OSWinException,
-                            error_codes=[w_const.ERROR_DIR_IS_NOT_EMPTY])
+    @file_in_use_retry_decorator
     def rmtree(self, path):
-        try:
-            shutil.rmtree(path)
-        except exceptions.WindowsError as ex:
-            # NOTE(claudiub): convert it to an OSWinException in order to use
-            # the retry_decorator.
-            raise exceptions.OSWinException(six.text_type(ex),
-                                            error_code=ex.winerror)
+        shutil.rmtree(path)
 
     def check_create_dir(self, path):
         if not self.exists(path):

os_win/utils/winapi/constants.py

@@ -20,6 +20,7 @@ from os_win.utils.winapi import wintypes
 # winerror.h
 ERROR_INVALID_HANDLE = 6
 ERROR_NOT_READY = 21
+ERROR_SHARING_VIOLATION = 32
 ERROR_SHARING_PAUSED = 70
 ERROR_INSUFFICIENT_BUFFER = 122
 ERROR_DIR_IS_NOT_EMPTY = 145