Add retry to execution commands
There are quite a few instances where the "exec_command" function fails due unrelated issues. This change adds a retry with a backoff. The exec function will now retry 3 times in the event of an exception with a cool down of 2 seconds. The play context for retries will also be set if undefined making it possible to recover from transient UNREACHABLE errors. Change-Id: Ic7c7af49e1c5eba21216b33814b92e276e3fba3e Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
parent
db07e45945
commit
621c552e23
|
@ -244,8 +244,11 @@ DOCUMENTATION = '''
|
|||
version_added: "2.6"
|
||||
'''
|
||||
|
||||
import functools
|
||||
import imp
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
# NOTICE(cloudnull): The connection plugin imported using the full path to the
|
||||
# file because the ssh connection plugin is not importable.
|
||||
|
@ -264,6 +267,36 @@ if not hasattr(SSH, 'shlex_quote'):
|
|||
setattr(SSH, 'shlex_quote', shlex_quote)
|
||||
|
||||
|
||||
def retry(ExceptionToCheck, tries=3, delay=1, backoff=2):
|
||||
"""Retry calling the decorated function using an exponential backoff.
|
||||
|
||||
:param ExceptionToCheck: the exception to check. may be a tuple of
|
||||
exceptions to check
|
||||
:type ExceptionToCheck: Exception or tuple
|
||||
:param tries: number of times to try (not retry) before giving up
|
||||
:type tries: int
|
||||
:param delay: initial delay between retries in seconds
|
||||
:type delay: int
|
||||
:param backoff: backoff multiplier e.g. value of 2 will double the delay
|
||||
each retry
|
||||
:type backoff: int
|
||||
"""
|
||||
def deco_retry(f):
|
||||
@functools.wraps(f)
|
||||
def f_retry(*args, **kwargs):
|
||||
mtries, mdelay = tries, delay
|
||||
while mtries > 1:
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except ExceptionToCheck:
|
||||
time.sleep(mdelay)
|
||||
mtries -= 1
|
||||
mdelay *= backoff
|
||||
return f(*args, **kwargs)
|
||||
return f_retry
|
||||
return deco_retry
|
||||
|
||||
|
||||
class Connection(SSH.Connection):
|
||||
"""Transport options for containers.
|
||||
|
||||
|
@ -327,6 +360,9 @@ class Connection(SSH.Connection):
|
|||
# revise this in the future.
|
||||
self.container_tech = 'lxc'
|
||||
|
||||
if not hasattr(self._play_context, 'retries'):
|
||||
self._play_context.retries = 3
|
||||
|
||||
# Store the container pid for multi-use
|
||||
self.container_pid = None
|
||||
|
||||
|
@ -360,6 +396,7 @@ class Connection(SSH.Connection):
|
|||
self.physical_host)
|
||||
self.host = self._play_context.remote_addr = physical_host_addr
|
||||
|
||||
@retry(ExceptionToCheck=Exception)
|
||||
def exec_command(self, cmd, in_data=None, sudoable=True):
|
||||
"""run a command on the remote host."""
|
||||
|
||||
|
|
Loading…
Reference in New Issue