diff --git a/tripleo_common/image/exception.py b/tripleo_common/image/exception.py index a46fe350b..fde7ce8de 100644 --- a/tripleo_common/image/exception.py +++ b/tripleo_common/image/exception.py @@ -18,6 +18,10 @@ class ImageBuilderException(Exception): pass +class ImageRateLimitedException(Exception): + """Rate Limited request""" + + class ImageSpecificationException(Exception): pass diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 96097225c..e6469aa42 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -37,6 +37,7 @@ from oslo_log import log as logging from tripleo_common.actions import ansible from tripleo_common.image.base import BaseImageManager from tripleo_common.image.exception import ImageNotFoundException +from tripleo_common.image.exception import ImageRateLimitedException from tripleo_common.image.exception import ImageUploaderException from tripleo_common.image.exception import ImageUploaderThreadException from tripleo_common.image import image_export @@ -244,6 +245,10 @@ class RegistrySessionHelper(object): ) session.reauthenticate(**session.auth_args) + if status_code == 429: + raise ImageRateLimitedException('Rate Limited while requesting ' + '{}'.format(request.url)) + request.raise_for_status() @staticmethod @@ -296,6 +301,14 @@ class RegistrySessionHelper(object): return request_response @staticmethod + @tenacity.retry( # Retry up to 5 times with longer time for rate limit + reraise=True, + retry=tenacity.retry_if_exception_type( + ImageRateLimitedException + ), + wait=tenacity.wait_random_exponential(multiplier=1.5, max=60), + stop=tenacity.stop_after_attempt(5) + ) def _action(action, request_session, *args, **kwargs): """ Perform a session action and retry if auth fails @@ -1781,12 +1794,13 @@ class PythonImageUploader(BaseImageUploader): return r.headers['Location'] @classmethod - @tenacity.retry( # Retry up to 5 times with jittered exponential backoff + @tenacity.retry( # Retry up to 5 times with longer time reraise=True, retry=tenacity.retry_if_exception_type( - requests.exceptions.RequestException + (requests.exceptions.RequestException, + ImageRateLimitedException) ), - wait=tenacity.wait_random_exponential(multiplier=1, max=10), + wait=tenacity.wait_random_exponential(multiplier=1.5, max=60), stop=tenacity.stop_after_attempt(5) ) def _layer_stream_registry(cls, digest, source_url, calc_digest, diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index 76f23ceb3..09748eeb5 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -28,6 +28,7 @@ import zlib from oslo_concurrency import processutils from tripleo_common.image.exception import ImageNotFoundException +from tripleo_common.image.exception import ImageRateLimitedException from tripleo_common.image.exception import ImageUploaderException from tripleo_common.image import image_uploader from tripleo_common.tests import base @@ -76,6 +77,23 @@ class TestRegistrySessionHelper(base.TestCase): session_reauth_mock.assert_called_once_with() raise_for_status_mock.assert_called_once() + def test_check_status_ratelimit(self): + session = mock.Mock() + session_reauth_mock = mock.Mock() + session.headers = {} + session.auth_args = {} + session.reauthenticate = session_reauth_mock + raise_for_status_mock = mock.Mock() + request = mock.Mock() + request.headers = {'www-authenticate': 'foo'} + request.raise_for_status = raise_for_status_mock + request.status_code = 429 + + self.assertRaises(ImageRateLimitedException, + image_uploader.RegistrySessionHelper.check_status, + session, + request) + def test_check_redirect_trusted_no_redirect(self): get_mock = mock.Mock() session = mock.Mock()