Migration to requests lib
Implements: bp migrating-to-requests Change-Id: I62786e642dd54fa2602af5c0d30c6d9a17c0563d
This commit is contained in:
Andrei V. Ostapenko
parent
695368cf07
commit
63eb829b3d
|
|
@ -20,19 +20,17 @@ except ImportError:
|
|||
import simplejson as json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
|
||||
import httplib2
|
||||
|
||||
from magnetodbclient.common import exceptions
|
||||
from magnetodbclient.common import utils
|
||||
from magnetodbclient.openstack.common.gettextutils import _
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# httplib2 retries requests on socket.timeout which
|
||||
# is not idempotent and can lead to orhan objects.
|
||||
# See: https://code.google.com/p/httplib2/issues/detail?id=124
|
||||
httplib2.RETRIES = 1
|
||||
requests_log = logging.getLogger("requests")
|
||||
requests_log.setLevel(logging.WARNING)
|
||||
|
||||
if os.environ.get('MAGNETODBCLIENT_DEBUG'):
|
||||
ch = logging.StreamHandler()
|
||||
|
|
@ -92,11 +90,11 @@ class ServiceCatalog(object):
|
|||
class ServiceCatalogV3(object):
|
||||
|
||||
def __init__(self, resp, resource_dict):
|
||||
self.resp = resp
|
||||
self.headers = resp.headers
|
||||
self.catalog = resource_dict
|
||||
|
||||
def get_token(self):
|
||||
return self.resp['x-subject-token']
|
||||
return self.headers['x-subject-token']
|
||||
|
||||
def url_for(self, attr=None, filter_value=None,
|
||||
service_type='kv-storage', endpoint_type='public'):
|
||||
|
|
@ -126,7 +124,7 @@ class ServiceCatalogV3(object):
|
|||
return matching_endpoints[0]['url']
|
||||
|
||||
|
||||
class HTTPClient(httplib2.Http):
|
||||
class HTTPClient(object):
|
||||
"""Handles the REST calls and responses, include authn."""
|
||||
|
||||
USER_AGENT = 'python-magnetodbclient'
|
||||
|
|
@ -139,7 +137,6 @@ class HTTPClient(httplib2.Http):
|
|||
service_type='kv-storage', domain_id='default',
|
||||
domain_name='Default',
|
||||
**kwargs):
|
||||
super(HTTPClient, self).__init__(timeout=timeout, ca_certs=ca_cert)
|
||||
|
||||
self.username = username
|
||||
self.tenant_name = tenant_name
|
||||
|
|
@ -161,53 +158,84 @@ class HTTPClient(httplib2.Http):
|
|||
self.project_name = self.tenant_name
|
||||
self.project_id = self.tenant_id
|
||||
|
||||
# httplib2 overrides
|
||||
self.disable_ssl_certificate_validation = insecure
|
||||
self.times = []
|
||||
|
||||
def _cs_request(self, *args, **kwargs):
|
||||
kargs = {}
|
||||
kargs.setdefault('headers', kwargs.get('headers', {}))
|
||||
kargs['headers']['User-Agent'] = self.USER_AGENT
|
||||
|
||||
if 'content_type' in kwargs:
|
||||
kargs['headers']['Content-Type'] = kwargs['content_type']
|
||||
kargs['headers']['Accept'] = kwargs['content_type']
|
||||
if insecure:
|
||||
self.verify_cert = False
|
||||
else:
|
||||
kargs['headers']['Content-Type'] = self.content_type
|
||||
kargs['headers']['Accept'] = self.content_type
|
||||
self.verify_cert = ca_cert if ca_cert else True
|
||||
|
||||
def request(self, url, method, **kwargs):
|
||||
kwargs.setdefault('headers', kwargs.get('headers', {}))
|
||||
kwargs['headers']['User-Agent'] = self.USER_AGENT
|
||||
kwargs['headers']['Accept'] = 'application/json'
|
||||
|
||||
utils.http_log_req(_logger, (url, method), kwargs)
|
||||
|
||||
if 'body' in kwargs:
|
||||
kargs['body'] = kwargs['body']
|
||||
args = utils.safe_encode_list(args)
|
||||
kargs = utils.safe_encode_dict(kargs)
|
||||
kwargs['headers']['Content-Type'] = 'application/json'
|
||||
kwargs['data'] = kwargs['body']
|
||||
del kwargs['body']
|
||||
|
||||
if self.log_credentials:
|
||||
log_kargs = kargs
|
||||
resp = requests.request(
|
||||
method,
|
||||
url,
|
||||
verify=self.verify_cert,
|
||||
**kwargs)
|
||||
utils.http_log_resp(_logger, resp)
|
||||
|
||||
if resp.text:
|
||||
# TODO(dtroyer): verify the note below in a requests context
|
||||
# NOTE(alaski): Because force_exceptions_to_status_code=True
|
||||
# httplib2 returns a connection refused event as a 400 response.
|
||||
# To determine if it is a bad request or refused connection we need
|
||||
# to check the body. httplib2 tests check for 'Connection refused'
|
||||
# or 'actively refused' in the body, so that's what we'll do.
|
||||
if resp.status_code == 400:
|
||||
if ('Connection refused' in resp.text or
|
||||
'actively refused' in resp.text):
|
||||
raise exceptions.ConnectionRefused(resp.text)
|
||||
try:
|
||||
body = json.loads(resp.text)
|
||||
except ValueError:
|
||||
pass
|
||||
body = None
|
||||
else:
|
||||
log_kargs = self._strip_credentials(kargs)
|
||||
body = None
|
||||
|
||||
if resp.status_code >= 400:
|
||||
raise exceptions.from_response(resp, body, url)
|
||||
|
||||
utils.http_log_req(_logger, args, log_kargs)
|
||||
try:
|
||||
resp, body = self.request(*args, **kargs)
|
||||
except httplib2.SSLHandshakeError as e:
|
||||
raise exceptions.SslCertificateValidationError(reason=e)
|
||||
except Exception as e:
|
||||
# Wrap the low-level connection error (socket timeout, redirect
|
||||
# limit, decompression error, etc) into our custom high-level
|
||||
# connection exception (it is excepted in the upper layers of code)
|
||||
_logger.debug("throwing ConnectionFailed : %s", e)
|
||||
raise exceptions.ConnectionFailed(reason=e)
|
||||
finally:
|
||||
# Temporary Fix for gate failures. RPC calls and HTTP requests
|
||||
# seem to be stepping on each other resulting in bogus fd's being
|
||||
# picked up for making http requests
|
||||
self.connections.clear()
|
||||
utils.http_log_resp(_logger, resp, body)
|
||||
status_code = self.get_status_code(resp)
|
||||
if status_code == 401:
|
||||
raise exceptions.Unauthorized(message=body)
|
||||
return resp, body
|
||||
|
||||
def _time_request(self, url, method, **kwargs):
|
||||
start_time = time.time()
|
||||
resp, body = self.request(url, method, **kwargs)
|
||||
self.times.append(("%s %s" % (method, url),
|
||||
start_time, time.time()))
|
||||
return resp, body
|
||||
|
||||
def _cs_request(self, url, method, **kwargs):
|
||||
# Perform the request once. If we get a 401 back then it
|
||||
# might be because the auth token expired, so try to
|
||||
# re-authenticate and try again. If it still fails, bail.
|
||||
try:
|
||||
kwargs.setdefault('headers', {})['X-Auth-Token'] = self.auth_token
|
||||
if self.project_id:
|
||||
kwargs['headers']['X-Auth-Project-Id'] = self.project_id
|
||||
|
||||
resp, body = self._time_request(url, method, **kwargs)
|
||||
return resp, body
|
||||
except exceptions.Unauthorized as ex:
|
||||
try:
|
||||
self.authenticate()
|
||||
kwargs['headers']['X-Auth-Token'] = self.auth_token
|
||||
resp, body = self._time_request(self.management_url + url,
|
||||
method, **kwargs)
|
||||
return resp, body
|
||||
except exceptions.Unauthorized:
|
||||
raise ex
|
||||
|
||||
def _strip_credentials(self, kwargs):
|
||||
if kwargs.get('body') and self.password:
|
||||
log_kwargs = kwargs.copy()
|
||||
|
|
@ -305,11 +333,6 @@ class HTTPClient(httplib2.Http):
|
|||
resp, body = self._cs_request(req_url, 'POST',
|
||||
headers=headers, body=body)
|
||||
|
||||
if body:
|
||||
try:
|
||||
body = json.loads(body)
|
||||
except ValueError:
|
||||
pass
|
||||
self._extract_service_catalog_v3(resp, body)
|
||||
|
||||
def _authenticate_keystone(self):
|
||||
|
|
@ -330,24 +353,11 @@ class HTTPClient(httplib2.Http):
|
|||
token_url = self.auth_url + "/tokens"
|
||||
|
||||
# Make sure we follow redirects when trying to reach Keystone
|
||||
tmp_follow_all_redirects = self.follow_all_redirects
|
||||
self.follow_all_redirects = True
|
||||
try:
|
||||
resp, resp_body = self._cs_request(token_url, "POST",
|
||||
body=json.dumps(body),
|
||||
content_type="application/json")
|
||||
finally:
|
||||
self.follow_all_redirects = tmp_follow_all_redirects
|
||||
resp, resp_body = self._cs_request(token_url, "POST",
|
||||
body=json.dumps(body))
|
||||
status_code = self.get_status_code(resp)
|
||||
if status_code != 200:
|
||||
raise exceptions.Unauthorized(message=resp_body)
|
||||
if resp_body:
|
||||
try:
|
||||
resp_body = json.loads(resp_body)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
resp_body = None
|
||||
self._extract_service_catalog(resp_body)
|
||||
|
||||
def _authenticate_noauth(self):
|
||||
|
|
@ -410,4 +420,4 @@ class HTTPClient(httplib2.Http):
|
|||
if hasattr(response, 'status_int'):
|
||||
return response.status_int
|
||||
else:
|
||||
return response.status
|
||||
return response.status_code
|
||||
|
|
|
|||
|
|
@ -179,3 +179,40 @@ class UnsupportedVersion(MagnetoDBCLIError):
|
|||
class MagnetoDBClientNoUniqueMatch(MagnetoDBCLIError):
|
||||
message = _("Multiple %(resource)s matches found for name '%(name)s',"
|
||||
" use an ID to be more specific.")
|
||||
|
||||
|
||||
def from_response(response, body, url, method=None):
|
||||
"""Return an instance of an ClientException or subclass
|
||||
based on an requests response.
|
||||
|
||||
Usage::
|
||||
|
||||
resp, body = requests.request(...)
|
||||
if resp.status_code != 200:
|
||||
raise from_response(resp, rest.text)
|
||||
"""
|
||||
kwargs = {
|
||||
'http_status': response.status_code,
|
||||
'method': method,
|
||||
'url': url,
|
||||
'request_id': None,
|
||||
}
|
||||
|
||||
if response.headers:
|
||||
kwargs['request_id'] = response.headers.get('x-compute-request-id')
|
||||
|
||||
if 'retry-after' in response.headers:
|
||||
kwargs['retry_after'] = response.headers.get('retry-after')
|
||||
|
||||
if body:
|
||||
message = "n/a"
|
||||
|
||||
if hasattr(body, 'keys'):
|
||||
error = body['error']
|
||||
message = error.get('message')
|
||||
|
||||
kwargs['message'] = message
|
||||
|
||||
cls = HTTP_EXCEPTION_MAP.get(response.status_code,
|
||||
MagnetoDBClientException)
|
||||
return cls(**kwargs)
|
||||
|
|
|
|||
|
|
@ -176,10 +176,12 @@ def http_log_req(_logger, args, kwargs):
|
|||
_logger.debug(_("\nREQ: %s\n"), "".join(string_parts))
|
||||
|
||||
|
||||
def http_log_resp(_logger, resp, body):
|
||||
if not _logger.isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
_logger.debug(_("RESP:%(resp)s %(body)s\n"), {'resp': resp, 'body': body})
|
||||
def http_log_resp(_logger, resp):
|
||||
_logger.debug(
|
||||
"RESP: [%s] %s\nRESP BODY: %s\n",
|
||||
resp.status_code,
|
||||
resp.headers,
|
||||
resp.text)
|
||||
|
||||
|
||||
def _safe_encode_without_obj(data):
|
||||
|
|
|
|||
|
|
@ -196,15 +196,7 @@ class Client(object):
|
|||
def _handle_fault_response(self, status_code, response_body):
|
||||
# Create exception with HTTP status code and message
|
||||
_logger.debug(_("Error message: %s"), response_body)
|
||||
# Add deserialized error message to exception arguments
|
||||
try:
|
||||
des_error_body = self.deserialize(response_body, status_code)
|
||||
except Exception:
|
||||
# If unable to deserialized body it is probably not a
|
||||
# MagnetoDB error
|
||||
des_error_body = {'message': response_body}
|
||||
# Raise the appropriate exception
|
||||
exception_handler_v1(status_code, des_error_body)
|
||||
exception_handler_v1(status_code, response_body)
|
||||
|
||||
def _check_uri_length(self, action):
|
||||
uri_len = len(self.httpclient.endpoint_url) + len(action)
|
||||
|
|
@ -229,7 +221,7 @@ class Client(object):
|
|||
httplib.CREATED,
|
||||
httplib.ACCEPTED,
|
||||
httplib.NO_CONTENT):
|
||||
return self.deserialize(replybody, status_code)
|
||||
return replybody
|
||||
else:
|
||||
if not replybody:
|
||||
replybody = resp.reason
|
||||
|
|
@ -247,7 +239,7 @@ class Client(object):
|
|||
if hasattr(response, 'status_int'):
|
||||
return response.status_int
|
||||
else:
|
||||
return response.status
|
||||
return response.status_code
|
||||
|
||||
def serialize(self, data):
|
||||
"""Serializes a dictionary into either xml or json.
|
||||
|
|
@ -263,13 +255,6 @@ class Client(object):
|
|||
raise Exception(_("Unable to serialize object of type = '%s'") %
|
||||
type(data))
|
||||
|
||||
def deserialize(self, data, status_code):
|
||||
"""Deserializes an xml or json string into a dictionary."""
|
||||
if status_code == 204:
|
||||
return data
|
||||
return serializer.Serializer().deserialize(
|
||||
data, self.content_type)['body']
|
||||
|
||||
def retry_request(self, method, action, body=None,
|
||||
headers=None, params=None):
|
||||
"""Call do_request with the default retry configuration.
|
||||
|
|
|
|||
Loading…
Reference in New Issue