diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 0104f0a1..c2e602c3 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -1061,6 +1061,11 @@ JSON_ENCODE_OPTIONS = dict( ) +class RetryProxyManagerCall(Exception): + pass + + +@retry_on_exception(5, base_delay=3, exc_type=RetryProxyManagerCall) def _proxy_manager_call(path, api_version, args, kwargs): package = dict(path=path, api_version=api_version, @@ -1080,8 +1085,14 @@ def _proxy_manager_call(path, api_version, args, kwargs): "The call was: path={}, args={}, kwargs={}, api_version={}" .format(result['error'], path, args, kwargs, api_version)) log(s, level=ERROR) + if result.get('retry'): + stop_manager_instance() + raise RetryProxyManagerCall() raise RuntimeError(s) return json.loads(result_str)['result'] + except RetryProxyManagerCall: + # cause a retry + raise except RuntimeError as e: raise e except Exception as e: diff --git a/hooks/manager.py b/hooks/manager.py index 05abaaa6..6a215968 100755 --- a/hooks/manager.py +++ b/hooks/manager.py @@ -560,6 +560,12 @@ if __name__ == '__main__': _callable = getattr(_callable, attr) # now make the call and return the arguments result = {'result': _callable(*spec['args'], **spec['kwargs'])} + except exceptions.InternalServerError as e: + # we've hit a 500 error, which is bad, and really we want the + # parent process to restart us to try again. + print(str(e)) + result = {'error': str(e), + 'retry': True} except uds.UDSException as e: print(str(e)) import traceback