diff --git a/oslo_service/loopingcall.py b/oslo_service/loopingcall.py index cf3506a8..9dafb47d 100644 --- a/oslo_service/loopingcall.py +++ b/oslo_service/loopingcall.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import random import sys from eventlet import event @@ -47,6 +48,15 @@ class LoopingCallDone(Exception): self.retvalue = retvalue +class LoopingCallTimeOut(Exception): + """Exception for a timed out LoopingCall. + + The LoopingCall will raise this exception when a timeout is provided + and it is exceeded. + """ + pass + + def _safe_wrapper(f, kind, func_name): """Wrapper that calls into wrapped function and logs errors as needed.""" @@ -195,6 +205,92 @@ class DynamicLoopingCall(LoopingCallBase): stop_on_exception=stop_on_exception) +class BackOffLoopingCall(LoopingCallBase): + """Run a method in a loop with backoff on error. + + The passed in function should return True (no error, return to + initial_interval), + False (error, start backing off), or raise LoopingCallDone(retvalue=None) + (quit looping, return retvalue if set). + + When there is an error, the call will backoff on each failure. The + backoff will be equal to double the previous base interval times some + jitter. If a backoff would put it over the timeout, it halts immediately, + so the call will never take more than timeout, but may and likely will + take less time. + + When the function return value is True or False, the interval will be + multiplied by a random jitter. If min_jitter or max_jitter is None, + there will be no jitter (jitter=1). If min_jitter is below 0.5, the code + may not backoff and may increase its retry rate. + + If func constantly returns True, this function will not return. + + To run a func and wait for a call to finish (by raising a LoopingCallDone): + + timer = BackOffLoopingCall(func) + response = timer.start().wait() + + :param initial_delay: delay before first running of function + :param starting_interval: initial interval in seconds between calls to + function. When an error occurs and then a + success, the interval is returned to + starting_interval + :param timeout: time in seconds before a LoopingCallTimeout is raised. + The call will never take longer than timeout, but may quit + before timeout. + :param max_interval: The maximum interval between calls during errors + :param jitter: Used to vary when calls are actually run to avoid group of + calls all coming at the exact same time. Uses + random.gauss(jitter, 0.1), with jitter as the mean for the + distribution. If set below .5, it can cause the calls to + come more rapidly after each failure. + :raises: LoopingCallTimeout if time spent doing error retries would exceed + timeout. + """ + + _RNG = random.SystemRandom() + _KIND = 'Dynamic backoff interval looping call' + _RUN_ONLY_ONE_MESSAGE = ("A dynamic backoff interval looping call can" + " only run one function at a time") + + def __init__(self, f=None, *args, **kw): + super(BackOffLoopingCall, self).__init__(f=f, *args, **kw) + self._error_time = 0 + self._interval = 1 + + def start(self, initial_delay=None, starting_interval=1, timeout=300, + max_interval=300, jitter=0.75): + if self._thread is not None: + raise RuntimeError(self._RUN_ONLY_ONE_MESSAGE) + + # Reset any prior state. + self._error_time = 0 + self._interval = starting_interval + + def _idle_for(success, _elapsed): + random_jitter = self._RNG.gauss(jitter, 0.1) + if success: + # Reset error state now that it didn't error... + self._interval = starting_interval + self._error_time = 0 + return self._interval * random_jitter + else: + # Perform backoff + self._interval = idle = min( + self._interval * 2 * random_jitter, max_interval) + # Don't go over timeout, end early if necessary. If + # timeout is 0, keep going. + if timeout > 0 and self._error_time + idle > timeout: + raise LoopingCallTimeOut( + _('Looping call timed out after %.02f seconds') + % self._error_time) + self._error_time += idle + return idle + + return self._start(_idle_for, initial_delay=initial_delay) + + class RetryDecorator(object): """Decorator for retrying a function upon suggested exceptions. diff --git a/oslo_service/tests/test_loopingcall.py b/oslo_service/tests/test_loopingcall.py index 12b20d83..9d8fa198 100644 --- a/oslo_service/tests/test_loopingcall.py +++ b/oslo_service/tests/test_loopingcall.py @@ -235,6 +235,86 @@ class DynamicLoopingCallTestCase(test_base.BaseTestCase): sleep_mock.assert_has_calls([mock.call(3), mock.call(1)]) +class TestBackOffLoopingCall(test_base.BaseTestCase): + @mock.patch('random.SystemRandom.gauss') + @mock.patch('eventlet.greenthread.sleep') + def test_exponential_backoff(self, sleep_mock, random_mock): + def false(): + return False + + random_mock.return_value = .8 + + self.assertRaises(loopingcall.LoopingCallTimeOut, + loopingcall.BackOffLoopingCall(false).start() + .wait) + + expected_times = [mock.call(1.6000000000000001), + mock.call(2.5600000000000005), + mock.call(4.096000000000001), + mock.call(6.5536000000000021), + mock.call(10.485760000000004), + mock.call(16.777216000000006), + mock.call(26.843545600000013), + mock.call(42.949672960000022), + mock.call(68.719476736000033), + mock.call(109.95116277760006)] + self.assertEqual(expected_times, sleep_mock.call_args_list) + + @mock.patch('random.SystemRandom.gauss') + @mock.patch('eventlet.greenthread.sleep') + def test_no_backoff(self, sleep_mock, random_mock): + random_mock.return_value = 1 + func = mock.Mock() + # func.side_effect + func.side_effect = [True, True, True, loopingcall.LoopingCallDone( + retvalue='return value')] + + retvalue = loopingcall.BackOffLoopingCall(func).start().wait() + + expected_times = [mock.call(1), mock.call(1), mock.call(1)] + self.assertEqual(expected_times, sleep_mock.call_args_list) + self.assertTrue(retvalue, 'return value') + + @mock.patch('random.SystemRandom.gauss') + @mock.patch('eventlet.greenthread.sleep') + def test_no_sleep(self, sleep_mock, random_mock): + # Any call that executes properly the first time shouldn't sleep + random_mock.return_value = 1 + func = mock.Mock() + # func.side_effect + func.side_effect = loopingcall.LoopingCallDone(retvalue='return value') + + retvalue = loopingcall.BackOffLoopingCall(func).start().wait() + self.assertFalse(sleep_mock.called) + self.assertTrue(retvalue, 'return value') + + @mock.patch('random.SystemRandom.gauss') + @mock.patch('eventlet.greenthread.sleep') + def test_max_interval(self, sleep_mock, random_mock): + def false(): + return False + + random_mock.return_value = .8 + + self.assertRaises(loopingcall.LoopingCallTimeOut, + loopingcall.BackOffLoopingCall(false).start( + max_interval=60) + .wait) + + expected_times = [mock.call(1.6000000000000001), + mock.call(2.5600000000000005), + mock.call(4.096000000000001), + mock.call(6.5536000000000021), + mock.call(10.485760000000004), + mock.call(16.777216000000006), + mock.call(26.843545600000013), + mock.call(42.949672960000022), + mock.call(60), + mock.call(60), + mock.call(60)] + self.assertEqual(expected_times, sleep_mock.call_args_list) + + class AnException(Exception): pass