openstack
/
charm-hacluster
Code Issues Proposed changes

Merge "Add retry handling to MaaS OCF DNS API calls"

This commit is contained in:
Zuul 2019-03-11 19:10:26 +00:00 committed by Gerrit Code Review
parent 6f3d0d39a6 175f1e2d2e
commit 94a35649c4
4 changed files with 241 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

208
ocf/maas/maas_dns.py
View File

@ -18,10 +18,156 @@ import argparse
import requests_oauthlib
import logging
import sys
import time
import maasclient
# Default MaaS API options
NUM_RETRIES = 5
RETRY_BASE_DELAY = 10
RETRY_CODES = [500]
# the global options that is parsed from the arguments
options = None
class RetriesException(Exception):
pass
def retry_on_request_error(retries=3, base_delay=0, codes=None):
"""Retry a function that retures a requests response.
If the response from the target function has an error code in the
:param:`codes` list then retry the function up to :param:`retries`. The
:param:`base_delay`, if not zero, will progressively back off at
`base_delay`, `base_delay * 2`, `base_delay * 3` ...
If the decorated function raises an exception, then the decorator DOESN'T
catch it, and this will bypass any retries.
In order to enable the decorator to access command line arguments, each of
the arguments can optionally be a Callable that returns the value, which
will be evaluated when the function is called.
:param retries: Number of attempts to run the decorated function.
:type retries: Option[int, Callable[..., int]]
:param base_delay: Back off time, which linearly increases by the number of
retries for each failed request.
:type base_delay: Option[int, Callable[..., int]]
:param codes: The codes to detect that force a retry that
response.status_code may contain.
:type codes: Option[List[int], Callable(..., List[int]]
:returns: decorated target function
:rtype: Callable
:raises: Exception, if the decorated function raises an exception
"""
if codes is None:
codes = [500]
def inner1(f):
def inner2(*args, **kwargs):
if callable(retries):
_retries = retries()
else:
_retries = retries
num_retries = _retries
if callable(base_delay):
_base_delay = base_delay()
else:
_base_delay = base_delay
if callable(codes):
_codes = codes()
else:
_codes = codes
multiplier = 1
while True:
response = f(*args, **kwargs)
if response.status_code not in _codes:
return response
if _retries <= 0:
raise RetriesException(
"Command {} failed after {} retries"
.format(f.__name__, num_retries))
delay = _base_delay * multiplier
multiplier += 1
logging.debug(
"Retrying '{}' {} more times (delay={})"
.format(f.__name__, _retries, delay))
_retries -= 1
if delay:
time.sleep(delay)
return inner2
return inner1
def options_retries():
"""Returns options.maas_api_retries value
It's used as a callable in the retry_on_request_error as follows:
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def some_function_that_needs_retries_that_returns_Response(...):
pass
:returns: options.maas_api_retries
:rtype: int
"""
global options
if options is not None:
return options.maas_api_retries
else:
return NUM_RETRIES
def options_base_delay():
"""Returns options.maas_base_delay
It's used as a callable in the retry_on_request_error as follows:
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def some_function_that_needs_retries_that_returns_Response(...):
pass
:returns: options.maas_base_delay
:rtype: int
"""
global options
if options is not None:
return options.maas_base_delay
else:
return RETRY_BASE_DELAY
def options_codes():
"""Returns options.maas_retry_codes
It's used as a callable in the retry_on_request_error as follows:
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def some_function_that_needs_retries_that_returns_Response(...):
pass
:returns: options.maas_retry_codes
:rtype: List[int]
"""
global options
if options is not None:
return options.maas_retry_codes
else:
return RETRY_CODES
class MAASDNS(object):
def __init__(self, options):
self.maas = maasclient.MAASClient(options.maas_server,
@ -51,6 +197,9 @@ class MAASDNS(object):
""" Get a dnsresource ID """
return self.dnsresource['id']
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def update_resource(self):
""" Update a dnsresource record with an IP """
return self.maas.update_dnsresource(self.dnsresource['id'],
@ -82,7 +231,14 @@ class MAASDNS(object):
'address_ttl': self.ttl,
'ip_addresses': self.ip,
}
return maas_session.post(dns_url, data=payload)
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def inner_maas_session_post(session, dns_url, payload):
return session.post(dns_url, data=payload)
return inner_maas_session_post(maas_session, dns_url, payload)
class MAASIP(object):
@ -104,6 +260,9 @@ class MAASIP(object):
self.ipaddress = ipaddress
return self.ipaddress
@retry_on_request_error(retries=options_retries,
base_delay=options_base_delay,
codes=options_codes)
def create_ipaddress(self, hostname=None):
""" Create an ipaddresses object
Due to https://bugs.launchpad.net/maas/+bug/1555393
@ -159,6 +318,28 @@ def dns_ha():
''.format(sys.argv[0]
.split('/')[-1]
.split('.')[0]))
parser.add_argument('--maas_api_retries', '-r',
help='The number of times to retry a MaaS API call',
type=int,
default=3)
parser.add_argument('--maas_base_delay', '-b',
help='The base delay after a failed MaaS API call',
type=int,
default=10)
def read_int_list(s):
try:
return [int(x.strip()) for x in s.split(',')]
except TypeError:
msg = "Can't convert '{}' into a list of integers".format(s)
return argparse.ArgumentTypeError(msg)
parser.add_argument('--maas_retry_codes', '-x',
help=('The codes to detect to auto-retry, as a '
'comma-separated list.'),
type=read_int_list,
default=[500])
global options
options = parser.parse_args()
setup_logging(options.logfile)
@ -183,5 +364,28 @@ def dns_ha():
dns_obj.update_resource()
def main():
"""Entry point for the script.
Runs dns_ha(), but wraps it with exception handling so that retries using
the MaaS API return 2 from the script, and all other errors return 1.
Otherwise the script returns 0 to indicate that it thinks it succeeded.
:returns: return code for script
:rtype: int
"""
try:
dns_ha()
except RetriesException as e:
logging.error("'{}' failed retries: {}".format(sys.argv[0], str(e)))
return 1
except Exception as e:
logging.error("'{}' failed due to: {}".format(sys.argv[0], str(e)))
import traceback
logging.error("Traceback:\n{}".format(traceback.format_exc()))
return 2
return 0
if __name__ == '__main__':
dns_ha()
sys.exit(main())

29
ocf/maas/maasclient/__init__.py
View File

@ -57,7 +57,6 @@ class MAASClient(object):
"""
Get a listing of DNS resources which are currently defined.
:returns: a list of DNS objects
DNS object is a dictionary of the form:
{'fqdn': 'keystone.maas',
'resource_records': [],
@ -65,6 +64,9 @@ class MAASClient(object):
'resource_uri': '/MAAS/api/2.0/dnsresources/1/',
'ip_addresses': [],
'id': 1}
:returns: a list of DNS objects
:rtype: List[Dict[str, Any]]
"""
resp = self.driver.get_dnsresources()
if resp.ok:
@ -79,12 +81,10 @@ class MAASClient(object):
/api/2.0/dnsresources/{dnsresource_id}/
:param fqdn: The fqdn address to update
:param ip_address: The ip address to update the A record to point to
:returns: True if the DNS object was updated, False otherwise.
:returns: the response from the requests method
:rtype: maasclient.driver.Response
"""
resp = self.driver.update_dnsresource(rid, fqdn, ip_address)
if resp.ok:
return True
return False
return self.driver.update_dnsresource(rid, fqdn, ip_address)
def create_dnsresource(self, fqdn, ip_address, address_ttl=None):
"""
@ -93,12 +93,10 @@ class MAASClient(object):
:param fqdn: The fqdn address to update
:param ip_address: The ip address to update the A record to point to
:param adress_ttl: DNS time to live
:returns: True if the DNS object was updated, False otherwise.
:returns: the response from the requests method
:rtype: maasclient.driver.Response
"""
resp = self.driver.create_dnsresource(fqdn, ip_address, address_ttl)
if resp.ok:
return True
return False
return self.driver.create_dnsresource(fqdn, ip_address, address_ttl)
###########################################################################
# IP API - http://maas.ubuntu.com/docs2.0/api.html#ip-address
@ -108,6 +106,7 @@ class MAASClient(object):
Get a list of ip addresses
:returns: a list of ip address dictionaries
:rtype: List[str]
"""
resp = self.driver.get_ipaddresses()
if resp.ok:
@ -120,9 +119,7 @@ class MAASClient(object):
:param ip_address: The ip address to register
:param hostname: the hostname to register at the same time
:returns: True if the DNS object was updated, False otherwise.
:returns: the response from the requests method
:rtype: maasclient.driver.Response
"""
resp = self.driver.create_ipaddress(ip_address, hostname)
if resp.ok:
return True
return False
return self.driver.create_ipaddress(ip_address, hostname)

29
ocf/maas/maasclient/apidriver.py
View File

@ -89,10 +89,11 @@ class APIDriver(MAASDriver):
log.debug("Request %s results: [%s] %s", path, response.getcode(),
payload)
if response.getcode() == OK:
return Response(True, yaml.load(payload))
code = response.getcode()
if code == OK:
return Response(True, yaml.load(payload), code)
else:
return Response(False, payload)
return Response(False, payload, code)
def _post(self, path, op='update', **kwargs):
"""
@ -104,17 +105,18 @@ class APIDriver(MAASDriver):
log.debug("Request %s results: [%s] %s", path, response.getcode(),
payload)
if response.getcode() == OK:
return Response(True, yaml.load(payload))
code = response.getcode()
if code == OK:
return Response(True, yaml.load(payload), code)
else:
return Response(False, payload)
return Response(False, payload, code)
except HTTPError as e:
log.error("Error encountered: %s for %s with params %s",
str(e), path, str(kwargs))
return Response(False, None)
return Response(False, None, None)
except Exception as e:
log.error("Post request raised exception: %s", e)
return Response(False, None)
return Response(False, None, None)
def _put(self, path, **kwargs):
"""
@ -125,17 +127,18 @@ class APIDriver(MAASDriver):
payload = response.read()
log.debug("Request %s results: [%s] %s", path, response.getcode(),
payload)
if response.getcode() == OK:
return Response(True, payload)
code = response.getcode()
if code == OK:
return Response(True, payload, code)
else:
return Response(False, payload)
return Response(False, payload, code)
except HTTPError as e:
log.error("Error encountered: %s with details: %s for %s with "
"params %s", e, e.read(), path, str(kwargs))
return Response(False, None)
return Response(False, None, None)
except Exception as e:
log.error("Put request raised exception: %s", e)
return Response(False, None)
return Response(False, None, None)
###########################################################################
# DNS API - http://maas.ubuntu.com/docs2.0/api.html#dnsresource

9
ocf/maas/maasclient/driver.py
View File

@ -18,12 +18,15 @@ log = logging.getLogger('vmaas.main')
class Response(object):
"""Response for the API calls to use internally
The status_code member is to make it look a bit like a requests Response
object so that it can be used in the retry decorator.
"""
Response for the API calls to use internally
"""
def __init__(self, ok=False, data=None):
def __init__(self, ok=False, data=None, status_code=None):
self.ok = ok
self.data = data
self.status_code = status_code
def __nonzero__(self):
"""Allow boolean comparison"""