Make client HTTP connections resilient

- Make Keystone session use a timeout to prevent hangs
- Support retries
- Make the above configurable

Change-Id: I7123bd2fdcd329eae5b8b40f09168a1d599fa0f7
This commit is contained in:
Scott Hussey 2018-06-26 14:41:24 -05:00
parent 7e7739ce2c
commit 06d6747b50
3 changed files with 67 additions and 14 deletions

View File

@ -103,6 +103,23 @@ class DrydockConfig(object):
'report_url',
default='http://localhost:9000/api/v1.0/bootactions/')
]
# Options for network traffic
network_options = [
cfg.IntOpt(
'http_client_connect_timeout',
default=16,
help='Timeout for initial read of outgoing HTTP calls from Drydock in seconds.'),
cfg.IntOpt(
'http_client_read_timeout',
default=300,
help='Timeout for initial read of outgoing HTTP calls from Drydock in seconds.'),
cfg.IntOpt(
'http_client_retries',
default=3,
help='Number of retries for transient errors of outgoing HTTP calls from Drydock.'),
]
# Enabled plugins
plugin_options = [
cfg.StrOpt(
@ -184,6 +201,7 @@ class DrydockConfig(object):
DrydockConfig.bootactions_options, group='bootactions')
self.conf.register_opts(DrydockConfig.logging_options, group='logging')
self.conf.register_opts(DrydockConfig.plugin_options, group='plugins')
self.conf.register_opts(DrydockConfig.network_options, group='network')
self.conf.register_opts(
DrydockConfig.database_options, group='database')
self.conf.register_opts(
@ -204,6 +222,7 @@ def list_opts():
'plugins': DrydockConfig.plugin_options,
'timeouts': DrydockConfig.timeout_options,
'database': DrydockConfig.database_options,
'network': DrydockConfig.network_options,
}
package_path = os.path.dirname(os.path.abspath(__file__))

View File

@ -15,6 +15,7 @@
import urllib.parse
import re
import time
import logging
import requests
@ -23,6 +24,7 @@ from beaker.util import parse_cache_config_options
from drydock_provisioner import error as errors
from drydock_provisioner.util import KeystoneUtils
from drydock_provisioner.config import config_mgr
cache_opts = {
'cache.type': 'memory',
@ -30,6 +32,7 @@ cache_opts = {
}
cache = CacheManager(**parse_cache_config_options(cache_opts))
LOG = logging.getLogger(__name__)
class ReferenceResolver(object):
"""Class for handling different data references to resolve them data."""
@ -54,8 +57,16 @@ class ReferenceResolver(object):
"Invalid reference scheme %s: no handler." %
design_uri.scheme)
else:
# Have to do a little magic to call the classmethod as a pointer
return handler.__get__(None, cls)(design_uri)
tries = 0
while tries < config_mgr.conf.network.http_client_retries:
try:
# Have to do a little magic to call the classmethod as a pointer
return handler.__get__(None, cls)(design_uri)
except Exception as ex:
tries = tries + 1
if tries < config_mgr.conf.network.http_client_retries:
LOG.debug("Retrying reference after failure: %s" % str(ex))
time.sleep(5 ** tries)
except ValueError:
raise errors.InvalidDesignReference(
"Cannot resolve design reference %s: unable to parse as valid URI."
@ -74,9 +85,9 @@ class ReferenceResolver(object):
response = requests.get(
design_uri.geturl(),
auth=(design_uri.username, design_uri.password),
timeout=30)
timeout=get_client_timeouts())
else:
response = requests.get(design_uri.geturl(), timeout=30)
response = requests.get(design_uri.geturl(), timeout=get_client_timeouts())
return response.content
@ -107,9 +118,8 @@ class ReferenceResolver(object):
url = urllib.parse.urlunparse(
(new_scheme, design_uri.netloc, design_uri.path, design_uri.params,
design_uri.query, design_uri.fragment))
logger = logging.getLogger(__name__)
logger.debug("Calling Keystone session for url %s" % str(url))
resp = ks_sess.get(url)
LOG.debug("Calling Keystone session for url %s" % str(url))
resp = ks_sess.get(url, timeout=get_client_timeouts())
if resp.status_code >= 400:
raise errors.InvalidDesignReference(
"Received error code for reference %s: %s - %s" %
@ -123,3 +133,8 @@ class ReferenceResolver(object):
'deckhand+http': resolve_reference_ucp,
'promenade+http': resolve_reference_ucp,
}
def get_client_timeouts():
"""Return a tuple of timeouts for the request library."""
return (config_mgr.conf.network.http_client_connect_timeout,
config_mgr.conf.network.http_client_read_timeout)

View File

@ -43,11 +43,11 @@
#domain_name = <None>
# Project ID to scope to (string value)
# Deprecated group/name - [keystone_authtoken]/tenant-id
# Deprecated group/name - [keystone_authtoken]/tenant_id
#project_id = <None>
# Project name to scope to (string value)
# Deprecated group/name - [keystone_authtoken]/tenant-name
# Deprecated group/name - [keystone_authtoken]/tenant_name
#project_name = <None>
# Domain ID containing project (string value)
@ -73,7 +73,7 @@
#user_id = <None>
# Username (string value)
# Deprecated group/name - [keystone_authtoken]/user-name
# Deprecated group/name - [keystone_authtoken]/user_name
#username = <None>
# User's domain id (string value)
@ -159,7 +159,10 @@
# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
# cache. If the value is not one of these options or empty, auth_token will
# raise an exception on initialization. (string value)
# Allowed values: None, MAC, ENCRYPT
# Possible values:
# None - <No description provided>
# MAC - <No description provided>
# ENCRYPT - <No description provided>
#memcache_security_strategy = None
# (Optional, mandatory if memcache_security_strategy is defined) This string is
@ -274,6 +277,25 @@
#poll_interval = 10
[network]
#
# From drydock_provisioner
#
# Timeout for initial read of outgoing HTTP calls from Drydock in seconds.
# (integer value)
#http_client_connect_timeout = 16
# Timeout for initial read of outgoing HTTP calls from Drydock in seconds.
# (integer value)
#http_client_read_timeout = 300
# Number of retries for transient errors of outgoing HTTP calls from Drydock.
# (integer value)
#http_client_retries = 3
[oslo_policy]
#
@ -281,11 +303,9 @@
#
# The file that defines policies. (string value)
# Deprecated group/name - [DEFAULT]/policy_file
#policy_file = policy.json
# Default rule. Enforced when a requested rule is not found. (string value)
# Deprecated group/name - [DEFAULT]/policy_default_rule
#policy_default_rule = default
# Directories where policy configuration files are stored. They can be relative
@ -293,7 +313,6 @@
# absolute paths. The file defined by policy_file must exist for these
# directories to be searched. Missing or empty directories are ignored. (multi
# valued)
# Deprecated group/name - [DEFAULT]/policy_dirs
#policy_dirs = policy.d