From 67a3c4d8b31e6cac256012df65c36dcf3542e129 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Thu, 21 Feb 2019 09:24:05 +1300 Subject: [PATCH] Improve retry behaviour of _copy_layer_registry_to_registry There are failures occuring in CI which result in missing layers after a registry->registry transfer. A job.exception() check is added to ensure the whole image transfer fails if a single layer fails. _copy_layer_registry_to_registry now retries on IOError. This is the parent class of RequestException, and could plausably be raised during a layer transfer. Change-Id: Ibe9c046ed9172dd0bc276c6400496392b2c69dda Partial-Bug: #1815576 --- tripleo_common/image/image_uploader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index c7d4bbfaa..dad7a2efb 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -1099,7 +1099,7 @@ class PythonImageUploader(BaseImageUploader): @tenacity.retry( # Retry up to 5 times with jittered exponential backoff reraise=True, retry=tenacity.retry_if_exception_type( - requests.exceptions.RequestException + IOError ), wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(5) @@ -1181,6 +1181,9 @@ class PythonImageUploader(BaseImageUploader): target_session=target_session )) for job in copy_jobs: + e = job.exception() + if e: + raise e image = job.result() if image: LOG.debug('Upload complete for layer: %s' % image)