diff --git a/paunch/builder/compose1.py b/paunch/builder/compose1.py index 15b757d..9c75222 100644 --- a/paunch/builder/compose1.py +++ b/paunch/builder/compose1.py @@ -13,6 +13,7 @@ import json import logging +import tenacity LOG = logging.getLogger(__name__) @@ -215,24 +216,39 @@ class ComposeV1Builder(object): if self.runner.inspect(image, format='exists', type='image'): continue - cmd = [self.runner.docker_cmd, 'pull', image] - (cmd_stdout, cmd_stderr, rc) = self.runner.execute(cmd) + try: + (cmd_stdout, cmd_stderr) = self._pull(image) + except PullException as e: + returncode = e.rc + cmd_stdout = e.stdout + cmd_stderr = e.stderr + LOG.error("Error pulling %s. [%s]\n" % (image, returncode)) + LOG.error("stdout: %s" % e.stdout) + LOG.error("stderr: %s" % e.stderr) + else: + LOG.debug('Pulled %s' % image) + LOG.info("stdout: %s" % cmd_stdout) + LOG.info("stderr: %s" % cmd_stderr) + if cmd_stdout: stdout.append(cmd_stdout) if cmd_stderr: stderr.append(cmd_stderr) - if rc != 0: - returncode = rc - LOG.error("Error running %s. [%s]\n" % (cmd, returncode)) - LOG.error("stdout: %s" % cmd_stdout) - LOG.error("stderr: %s" % cmd_stderr) - else: - LOG.debug('Completed $ %s' % ' '.join(cmd)) - LOG.info("stdout: %s" % cmd_stdout) - LOG.info("stderr: %s" % cmd_stderr) return returncode + @tenacity.retry( # Retry up to 4 times with jittered exponential backoff + reraise=True, + wait=tenacity.wait_random_exponential(multiplier=1, max=10), + stop=tenacity.stop_after_attempt(4) + ) + def _pull(self, image): + cmd = [self.runner.docker_cmd, 'pull', image] + (stdout, stderr, rc) = self.runner.execute(cmd) + if rc != 0: + raise PullException(stdout, stderr, rc) + return stdout, stderr + @staticmethod def command_argument(command): if not command: @@ -240,3 +256,11 @@ class ComposeV1Builder(object): if not isinstance(command, list): return command.split() return command + + +class PullException(Exception): + + def __init__(self, stdout, stderr, rc): + self.stdout = stdout + self.stderr = stderr + self.rc = rc diff --git a/paunch/tests/test_builder_compose1.py b/paunch/tests/test_builder_compose1.py index e8623ba..faf8ab3 100644 --- a/paunch/tests/test_builder_compose1.py +++ b/paunch/tests/test_builder_compose1.py @@ -23,7 +23,9 @@ from paunch.tests import base class TestComposeV1Builder(base.TestCase): - def test_apply(self): + @mock.patch('tenacity.wait.wait_random_exponential.__call__') + def test_apply(self, mock_wait): + mock_wait.return_value = 0 config = { 'one': { 'start_order': 0, @@ -53,7 +55,8 @@ class TestComposeV1Builder(base.TestCase): exe.side_effect = [ ('exists', '', 0), # inspect for image centos:6 ('', '', 1), # inspect for missing image centos:7 - ('Pulled centos:7', '', 0), # pull centos:6 + ('Pulled centos:7', 'ouch', 1), # pull centos:6 fails + ('Pulled centos:7', '', 0), # pull centos:6 succeeds ('', '', 0), # ps for delete_missing_and_updated container_names ('', '', 0), # ps for after delete_missing_and_updated renames ('', '', 0), # ps to only create containers which don't exist @@ -91,6 +94,11 @@ class TestComposeV1Builder(base.TestCase): ['docker', 'inspect', '--type', 'image', '--format', 'exists', 'centos:7'] ), + # first pull attempt fails + mock.call( + ['docker', 'pull', 'centos:7'] + ), + # second pull attempt succeeds mock.call( ['docker', 'pull', 'centos:7'] ), @@ -301,7 +309,9 @@ three-12345678 three''', '', 0), ), ]) - def test_apply_failed_pull(self): + @mock.patch('tenacity.wait.wait_random_exponential.__call__') + def test_apply_failed_pull(self, mock_wait): + mock_wait.return_value = 0 config = { 'one': { 'start_order': 0, @@ -332,6 +342,9 @@ three-12345678 three''', '', 0), ('exists', '', 0), # inspect for image centos:6 ('', '', 1), # inspect for missing image centos:7 ('Pulling centos:7', 'ouch', 1), # pull centos:7 failure + ('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 2 + ('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 3 + ('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 4 ] r.execute = exe diff --git a/requirements.txt b/requirements.txt index bf4e10e..44a9f90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pbr>=2.0.0,!=2.1.0 # Apache-2.0 cliff>=2.6.0 # Apache-2.0 +tenacity>=3.2.1 # Apache-2.0