Switch from retrying to tenacity

This switching our retry mechanism over from the retrying library to the
tenacity library. Retrying has been active for a few years now and
appears to be no longer maintained.

This has a small behavior change in that before we were applying an
exponential backoff to the first time a retry was needed. This no longer
happens, but retries will exponentially back off with each retry.

Also cleaned up some minor nits with unit test assert argument order.

Closes-bug: #1635397

Change-Id: I24cab206b16e63859d4886c55d40a03d398ce30d
Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>

lower-constraints.txt

@@ -80,6 +80,7 @@ sphinxcontrib-websupport==1.0.1
 stestr==1.0.0
 stevedore==1.20.0
 suds-jurko==0.6
+tenacity==6.0.0
 testscenarios==0.4
 testtools==2.2.0
 traceback2==1.4.0

os_brick/tests/base.py

@@ -59,6 +59,11 @@ class TestCase(testtools.TestCase):
             self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
                                                    format=log_format,
                                                    level=level))
+        # Protect against any case where someone doesn't remember to patch a
+        # retry decorated call
+        patcher = mock.patch('os_brick.utils._time_sleep')
+        patcher.start()
+        self.addCleanup(patcher.stop)
 
     def _common_cleanup(self):
         """Runs after each test method to tear down test environment."""

os_brick/tests/initiator/connectors/test_iscsi.py

@@ -1089,7 +1089,7 @@ Setting up iSCSI targets: unused
                        side_effect=(None, 'tgt2'))
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
     @mock.patch.object(iscsi.ISCSIConnector, '_cleanup_connection')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_single_volume(self, sleep_mock, cleanup_mock,
                                    connect_mock, get_wwn_mock):
         def my_connect(rescans, props, data):
@@ -1114,7 +1114,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'get_sysfs_wwn', return_value='')
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
     @mock.patch.object(iscsi.ISCSIConnector, '_cleanup_connection')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_single_volume_no_wwn(self, sleep_mock, cleanup_mock,
                                           connect_mock, get_wwn_mock):
         def my_connect(rescans, props, data):
@@ -1142,7 +1142,7 @@ Setting up iSCSI targets: unused
                        side_effect=(None, 'tgt2'))
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
     @mock.patch.object(iscsi.ISCSIConnector, '_cleanup_connection')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_single_volume_not_found(self, sleep_mock, cleanup_mock,
                                              connect_mock, get_wwn_mock):
 
@@ -1181,7 +1181,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_path')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_wwid')
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_multipath_volume_all_succeed(self, sleep_mock,
                                                   connect_mock, add_wwid_mock,
                                                   add_path_mock, get_wwn_mock,
@@ -1218,7 +1218,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_path')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_wwid')
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_multipath_volume_all_fail(self, sleep_mock, connect_mock,
                                                add_wwid_mock, add_path_mock,
                                                get_wwn_mock, find_dm_mock):
@@ -1245,7 +1245,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_path')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_wwid')
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_multipath_volume_some_fail_mp_found(self, sleep_mock,
                                                          connect_mock,
                                                          add_wwid_mock,
@@ -1287,7 +1287,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_wwid')
     @mock.patch.object(iscsi.time, 'time', side_effect=(0, 0, 11, 0))
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_multipath_volume_some_fail_mp_not_found(self, sleep_mock,
                                                              connect_mock,
                                                              time_mock,
@@ -1331,7 +1331,7 @@ Setting up iSCSI targets: unused
     @mock.patch.object(linuxscsi.LinuxSCSI, 'multipath_add_wwid')
     @mock.patch.object(iscsi.time, 'time', side_effect=(0, 0, 11, 0))
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_vol')
-    @mock.patch('time.sleep', mock.Mock())
+    @mock.patch('os_brick.utils._time_sleep', mock.Mock())
     def test_connect_multipath_volume_all_loging_not_found(self,
                                                            connect_mock,
                                                            time_mock,
@@ -1355,7 +1355,7 @@ Setting up iSCSI targets: unused
         find_dm_mock.assert_not_called()
         self.assertEqual(12, connect_mock.call_count)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'scan_iscsi')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'device_name_by_hctl',
                        return_value='sda')
@@ -1388,7 +1388,7 @@ Setting up iSCSI targets: unused
         dev_name_mock.assert_called_once_with(mock.sentinel.session, hctl)
         sleep_mock.assert_called_once_with(1)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'scan_iscsi')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'device_name_by_hctl',
                        side_effect=(None, None, None, None, 'sda'))
@@ -1419,7 +1419,7 @@ Setting up iSCSI targets: unused
         self.assertEqual(5, dev_name_mock.call_count)
         self.assertEqual(4, sleep_mock.call_count)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'scan_iscsi')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'device_name_by_hctl',
                        side_effect=(None, None, None, None, 'sda'))
@@ -1462,7 +1462,7 @@ Setting up iSCSI targets: unused
         expected.update(failed_logins=1, stopped_threads=1)
         self.assertDictEqual(expected, data)
 
-    @mock.patch('time.sleep', mock.Mock())
+    @mock.patch('os_brick.utils._time_sleep', mock.Mock())
     @mock.patch.object(linuxscsi.LinuxSCSI, 'scan_iscsi')
     @mock.patch.object(linuxscsi.LinuxSCSI, 'device_name_by_hctl',
                        return_value=None)
@@ -1493,7 +1493,7 @@ Setting up iSCSI targets: unused
             [mock.call(mock.sentinel.session, hctl),
              mock.call(mock.sentinel.session, hctl)])
 
-    @mock.patch('time.sleep', mock.Mock())
+    @mock.patch('os_brick.utils._time_sleep', mock.Mock())
     @mock.patch.object(linuxscsi.LinuxSCSI, 'scan_iscsi')
     @mock.patch.object(iscsi.ISCSIConnector, '_connect_to_iscsi_portal')
     def test_connect_vol_stop_connecting(self, connect_mock, scan_mock):
@@ -1583,7 +1583,7 @@ Setting up iSCSI targets: unused
             mock.call('/dev/disk/by-id/dm-...'),
             mock.call('/dev/disk/by-id/scsi-wwn')])
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch('os.path.realpath', return_value='/dev/sdz')
     @mock.patch('os.listdir', return_value=['dm-...', 'scsi-...'])
     def test__get_device_link_not_found(self, listdir_mock, realpath_mock,
@@ -1599,7 +1599,7 @@ Setting up iSCSI targets: unused
                  mock.call('/dev/disk/by-id/scsi-...')])
         self.assertEqual(9, realpath_mock.call_count)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch('os.path.realpath')
     @mock.patch('os.listdir', return_value=['dm-...', 'scsi-...'])
     def test__get_device_link_symlink_found_after_retry(self, mock_listdir,
@@ -1622,7 +1622,7 @@ Setting up iSCSI targets: unused
             + [mock.call('/dev/disk/by-id/scsi-wwn')])
         self.assertEqual(7, mock_realpath.call_count)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch('os.path.realpath')
     @mock.patch('os.listdir', return_value=['dm-...', 'scsi-...'])
     def test__get_device_link_symlink_found_after_retry_by_listdir(

os_brick/tests/initiator/connectors/test_nvmeof.py

@@ -135,7 +135,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
                        autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test__wait_for_blk(self, mock_sleep, mock_nvme_subsys,
                            mock_nvme_dev, expected, nvme,
                            list_subsys, nvme_list):
@@ -152,7 +152,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
                        autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test__wait_for_blk_raise(self, mock_sleep, mock_nvme_subsys,
                                  mock_nvme_dev, expected, nvme,
                                  list_subsys, nvme_list):
@@ -170,7 +170,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
                        autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test__wait_for_blk_retry_success(self, mock_sleep, mock_nvme_subsys,
                                          mock_nvme_dev, expected, nvme,
                                          list_subsys, nvme_list):
@@ -180,7 +180,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
         self.assertEqual(expected, actual)
 
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_get_nvme_devices_raise(self, mock_sleep, mock_execute):
         mock_execute.side_effect = putils.ProcessExecutionError
         self.assertRaises(exception.CommandExecutionFailed,
@@ -191,7 +191,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume(self, mock_sleep, mock_execute, mock_devices,
                             mock_blk):
         connection_properties = {'target_portal': 'portal',
@@ -226,7 +226,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume_hostnqn(
             self, mock_sleep, mock_execute, mock_devices,
             mock_blk):
@@ -260,7 +260,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
             run_as_root=True)
 
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume_raise(self, mock_sleep, mock_execute):
         connection_properties = {'target_portal': 'portal',
                                  'target_port': 1,
@@ -279,7 +279,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
                        autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume_wait_for_blk_raise(self, mock_sleep, mock_blk,
                                                mock_subsys, mock_devices,
                                                mock_execute):
@@ -298,7 +298,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume_max_retry(
             self, mock_sleep, mock_execute, mock_devices,
             mock_blk):
@@ -320,7 +320,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_volume_nvmelist_retry_success(
             self, mock_sleep, mock_execute, mock_devices,
             mock_blk):
@@ -344,7 +344,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_connect_nvme_retry_success(
             self, mock_sleep, mock_execute, mock_devices,
             mock_blk):
@@ -369,7 +369,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_disconnect_volume_nova(
             self, mock_sleep, mock_execute, mock_devices):
         connection_properties = {'target_portal': 'portal',
@@ -388,7 +388,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_disconnect_volume_cinder(
             self, mock_sleep, mock_execute, mock_devices):
         connection_properties = {'target_portal': 'portal',
@@ -410,7 +410,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
     @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
                        autospec=True)
     @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
-    @mock.patch('time.sleep', autospec=True)
+    @mock.patch('os_brick.utils._time_sleep')
     def test_disconnect_volume_raise(
             self, mock_sleep, mock_execute, mock_devices):
         mock_execute.side_effect = putils.ProcessExecutionError

os_brick/tests/initiator/connectors/test_scaleio.py

@@ -246,7 +246,7 @@ class ScaleIOConnectorTestCase(test_connector.ConnectorTestCase):
 
         self.assertRaises(exception.BrickException, self.test_connect_volume)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_error_path_not_found(self, sleep_mock):
         """Timeout waiting for volume to map to local file system"""
         self.mock_object(os, 'listdir', return_value=["emc-vol-no-volume"])

os_brick/tests/initiator/test_linuxscsi.py

@@ -15,7 +15,6 @@
 import os
 import os.path
 import textwrap
-import time
 from unittest import mock
 
 import ddt
@@ -103,7 +102,7 @@ class LinuxSCSITestCase(base.TestCase):
         expected_commands = [('tee -a /sys/block/sdc/device/delete')]
         self.assertEqual(expected_commands, self.cmds)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch('os.path.exists', return_value=True)
     def test_wait_for_volumes_removal_failure(self, exists_mock, sleep_mock):
         retries = 61
@@ -114,7 +113,7 @@ class LinuxSCSITestCase(base.TestCase):
                                       for name in names] * retries)
         self.assertEqual(retries - 1, sleep_mock.call_count)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch('os.path.exists', side_effect=(True, True, False, False))
     def test_wait_for_volumes_removal_retry(self, exists_mock, sleep_mock):
         names = ('sda', 'sdb')
@@ -195,7 +194,7 @@ class LinuxSCSITestCase(base.TestCase):
         expected_path = '/dev/disk/by-id/dm-uuid-mpath-%s' % fake_wwn
         self.assertEqual(expected_path, found_path)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     @mock.patch.object(os.path, 'exists')
     def test_find_multipath_device_path_mapper(self, exists_mock, sleep_mock):
         # the wait loop tries 3 times before it gives up
@@ -211,14 +210,14 @@ class LinuxSCSITestCase(base.TestCase):
         self.assertTrue(sleep_mock.called)
 
     @mock.patch.object(os.path, 'exists', return_value=False)
-    @mock.patch.object(time, 'sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_find_multipath_device_path_fail(self, exists_mock, sleep_mock):
         fake_wwn = '1234567890'
         found_path = self.linuxscsi.find_multipath_device_path(fake_wwn)
         self.assertIsNone(found_path)
 
     @mock.patch.object(os.path, 'exists', return_value=False)
-    @mock.patch.object(time, 'sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_wait_for_path_not_found(self, exists_mock, sleep_mock):
         path = "/dev/disk/by-id/dm-uuid-mpath-%s" % '1234567890'
         self.assertRaisesRegex(exception.VolumeDeviceNotFound,
@@ -490,7 +489,7 @@ class LinuxSCSITestCase(base.TestCase):
         self.assertEqual("1", info['devices'][1]['id'])
         self.assertEqual("9", info['devices'][1]['lun'])
 
-    @mock.patch.object(time, 'sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_wait_for_rw(self, mock_sleep):
         lsblk_output = """3624a93709a738ed78583fd1200143029 (dm-2)  0
 sdb                                       0
@@ -532,7 +531,7 @@ loop0                                     0"""
         self.linuxscsi.wait_for_rw(wwn, path)
         self.assertFalse(mock_sleep.called)
 
-    @mock.patch.object(time, 'sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_wait_for_rw_needs_retry(self, mock_sleep):
         lsblk_ro_output = """3624a93709a738ed78583fd1200143029 (dm-2)  0
 sdb                                       0
@@ -602,7 +601,7 @@ loop0                                     0"""
         self.linuxscsi.wait_for_rw(wwn, path)
         self.assertEqual(1, mock_sleep.call_count)
 
-    @mock.patch.object(time, 'sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_wait_for_rw_always_readonly(self, mock_sleep):
         lsblk_output = """3624a93709a738ed78583fd1200143029 (dm-2)  0
 sdb                                       0

os_brick/tests/local_dev/test_brick_lvm.py

@@ -380,7 +380,7 @@ class BrickLvmTestCase(base.TestCase):
             self.vg.create_volume('test', '1G')
             self.vg.deactivate_lv('test')
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_lv_deactivate_timeout(self, mock_sleep):
         with mock.patch.object(self.vg, '_execute'):
             is_active_mock = mock.Mock()

os_brick/tests/privileged/test_rootwrap.py

@@ -86,7 +86,7 @@ class PrivRootwrapTestCase(base.TestCase):
         proc = on_execute.call_args[0][0]
         on_completion.assert_called_once_with(proc)
 
-    @mock.patch('time.sleep')
+    @mock.patch('os_brick.utils._time_sleep')
     def test_custom_execute_timeout_raises_with_retries(self, sleep_mock):
         on_execute = mock.Mock()
         on_completion = mock.Mock()

os_brick/tests/test_utils.py

@@ -16,7 +16,6 @@ import functools
 import time
 from unittest import mock
 
-
 from os_brick import exception
 from os_brick.tests import base
 from os_brick import utils
@@ -31,7 +30,7 @@ class TestRetryDecorator(base.TestCase):
     def test_no_retry_required(self):
         self.counter = 0
 
-        with mock.patch.object(time, 'sleep') as mock_sleep:
+        with mock.patch.object(utils, '_time_sleep') as mock_sleep:
             @utils.retry(exceptions=exception.VolumeDeviceNotFound,
                          interval=2,
                          retries=3,
@@ -42,8 +41,8 @@ class TestRetryDecorator(base.TestCase):
 
             ret = succeeds()
             self.assertFalse(mock_sleep.called)
-            self.assertEqual(ret, 'success')
-            self.assertEqual(self.counter, 1)
+            self.assertEqual('success', ret)
+            self.assertEqual(1, self.counter)
 
     def test_retries_once(self):
         self.counter = 0
@@ -51,7 +50,7 @@ class TestRetryDecorator(base.TestCase):
         backoff_rate = 2
         retries = 3
 
-        with mock.patch.object(time, 'sleep') as mock_sleep:
+        with mock.patch.object(utils, '_time_sleep') as mock_sleep:
             @utils.retry(exception.VolumeDeviceNotFound,
                          interval,
                          retries,
@@ -64,10 +63,10 @@ class TestRetryDecorator(base.TestCase):
                     return 'success'
 
             ret = fails_once()
-            self.assertEqual(ret, 'success')
-            self.assertEqual(self.counter, 2)
-            self.assertEqual(mock_sleep.call_count, 1)
-            mock_sleep.assert_called_with(interval * backoff_rate)
+            self.assertEqual('success', ret)
+            self.assertEqual(2, self.counter)
+            self.assertEqual(1, mock_sleep.call_count)
+            mock_sleep.assert_called_with(interval)
 
     def test_limit_is_reached(self):
         self.counter = 0
@@ -75,7 +74,7 @@ class TestRetryDecorator(base.TestCase):
         interval = 2
         backoff_rate = 4
 
-        with mock.patch.object(time, 'sleep') as mock_sleep:
+        with mock.patch.object(utils, '_time_sleep') as mock_sleep:
             @utils.retry(exception.VolumeDeviceNotFound,
                          interval,
                          retries,
@@ -92,7 +91,7 @@ class TestRetryDecorator(base.TestCase):
 
             for i in range(retries):
                 if i > 0:
-                    interval *= backoff_rate
+                    interval *= (backoff_rate ** (i - 1))
                     expected_sleep_arg.append(float(interval))
 
             mock_sleep.assert_has_calls(
@@ -100,7 +99,7 @@ class TestRetryDecorator(base.TestCase):
 
     def test_wrong_exception_no_retry(self):
 
-        with mock.patch.object(time, 'sleep') as mock_sleep:
+        with mock.patch.object(utils, '_time_sleep') as mock_sleep:
             @utils.retry(exceptions=exception.VolumeDeviceNotFound)
             def raise_unexpected_error():
                 raise WrongException("wrong exception")

os_brick/utils.py

@@ -20,33 +20,35 @@ import time
 from oslo_log import log as logging
 from oslo_utils import encodeutils
 from oslo_utils import strutils
-import retrying
 import six
 
 from os_brick.i18n import _
 
+_time_sleep = time.sleep
+
+
+def _sleep(duration):
+    """Helper class to make it easier to work around tenacity's sleep calls.
+
+    Apparently we are all idiots for wanting to test our code here [0], so this
+    is a hack to be able to get retries to not actually sleep.
+
+    [0] https://github.com/jd/tenacity/issues/25
+    """
+    _time_sleep(duration)
+
+
+time.sleep = _sleep
+
+
+import tenacity  # noqa
+
 
 LOG = logging.getLogger(__name__)
 
 
 def retry(exceptions, interval=1, retries=3, backoff_rate=2):
 
-    def _retry_on_exception(e):
-        return isinstance(e, exceptions)
-
-    def _backoff_sleep(previous_attempt_number, delay_since_first_attempt_ms):
-        exp = backoff_rate ** previous_attempt_number
-        wait_for = max(0, interval * exp)
-        LOG.debug("Sleeping for %s seconds", wait_for)
-        return wait_for * 1000.0
-
-    def _print_stop(previous_attempt_number, delay_since_first_attempt_ms):
-        delay_since_first_attempt = delay_since_first_attempt_ms / 1000.0
-        LOG.debug("Failed attempt %s", previous_attempt_number)
-        LOG.debug("Have been at this for %s seconds",
-                  delay_since_first_attempt)
-        return previous_attempt_number == retries
-
     if retries < 1:
         raise ValueError(_('Retries must be greater than or '
                          'equal to 1 (received: %s). ') % retries)
@@ -55,9 +57,14 @@ def retry(exceptions, interval=1, retries=3, backoff_rate=2):
 
         @six.wraps(f)
         def _wrapper(*args, **kwargs):
-            r = retrying.Retrying(retry_on_exception=_retry_on_exception,
-                                  wait_func=_backoff_sleep,
-                                  stop_func=_print_stop)
+            r = tenacity.Retrying(
+                before_sleep=tenacity.before_sleep_log(LOG, logging.DEBUG),
+                after=tenacity.after_log(LOG, logging.DEBUG),
+                stop=tenacity.stop_after_attempt(retries),
+                reraise=True,
+                retry=tenacity.retry_if_exception_type(exceptions),
+                wait=tenacity.wait_exponential(
+                    multiplier=interval, min=0, exp_base=backoff_rate))
             return r.call(f, *args, **kwargs)
 
         return _wrapper

requirements.txt

@@ -11,6 +11,6 @@ oslo.privsep>=1.32.0 # Apache-2.0
 oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
 oslo.utils>=3.33.0 # Apache-2.0
 requests>=2.14.2 # Apache-2.0
-retrying!=1.3.0,>=1.2.3 # Apache-2.0
 six>=1.10.0 # MIT
+tenacity>=6.0.0 # Apache-2.0
 os-win>=3.0.0 # Apache-2.0