Merge "Swift object client: use urllib3 builtin support for chunked transfer"
This commit is contained in:
commit
b02fa114b8
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
features:
|
||||
- The RestClient (in tempest.lib.common.rest_client) now supports POSTing
|
||||
and PUTing data with chunked transfer encoding. Just pass an `iterable`
|
||||
object as the `body` argument and set the `chunked` argument to `True`.
|
||||
- A new generator called `chunkify` is added in
|
||||
tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices)
|
||||
from a Python sequence.
|
||||
|
|
@ -20,7 +20,6 @@ import time
|
|||
import zlib
|
||||
|
||||
import six
|
||||
from six import moves
|
||||
|
||||
from tempest.api.object_storage import base
|
||||
from tempest.common import custom_matchers
|
||||
|
@ -201,8 +200,8 @@ class ObjectTest(base.BaseObjectTest):
|
|||
status, _, resp_headers = self.object_client.put_object_with_chunk(
|
||||
container=self.container_name,
|
||||
name=object_name,
|
||||
contents=moves.cStringIO(data),
|
||||
chunk_size=512)
|
||||
contents=data_utils.chunkify(data, 512)
|
||||
)
|
||||
self.assertHeaders(resp_headers, 'Object', 'PUT')
|
||||
|
||||
# check uploaded content
|
||||
|
|
|
@ -243,7 +243,8 @@ class RestClient(object):
|
|||
details = pattern.format(read_code, expected_code)
|
||||
raise exceptions.InvalidHttpSuccessCode(details)
|
||||
|
||||
def post(self, url, body, headers=None, extra_headers=False):
|
||||
def post(self, url, body, headers=None, extra_headers=False,
|
||||
chunked=False):
|
||||
"""Send a HTTP POST request using keystone auth
|
||||
|
||||
:param str url: the relative url to send the post request to
|
||||
|
@ -253,11 +254,12 @@ class RestClient(object):
|
|||
returned by the get_headers() method are to
|
||||
be used but additional headers are needed in
|
||||
the request pass them in as a dict.
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
:rtype: tuple
|
||||
"""
|
||||
return self.request('POST', url, extra_headers, headers, body)
|
||||
return self.request('POST', url, extra_headers, headers, body, chunked)
|
||||
|
||||
def get(self, url, headers=None, extra_headers=False):
|
||||
"""Send a HTTP GET request using keystone service catalog and auth
|
||||
|
@ -306,7 +308,7 @@ class RestClient(object):
|
|||
"""
|
||||
return self.request('PATCH', url, extra_headers, headers, body)
|
||||
|
||||
def put(self, url, body, headers=None, extra_headers=False):
|
||||
def put(self, url, body, headers=None, extra_headers=False, chunked=False):
|
||||
"""Send a HTTP PUT request using keystone service catalog and auth
|
||||
|
||||
:param str url: the relative url to send the post request to
|
||||
|
@ -316,11 +318,12 @@ class RestClient(object):
|
|||
returned by the get_headers() method are to
|
||||
be used but additional headers are needed in
|
||||
the request pass them in as a dict.
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
:rtype: tuple
|
||||
"""
|
||||
return self.request('PUT', url, extra_headers, headers, body)
|
||||
return self.request('PUT', url, extra_headers, headers, body, chunked)
|
||||
|
||||
def head(self, url, headers=None, extra_headers=False):
|
||||
"""Send a HTTP HEAD request using keystone service catalog and auth
|
||||
|
@ -520,7 +523,7 @@ class RestClient(object):
|
|||
if method != 'HEAD' and not resp_body and resp.status >= 400:
|
||||
self.LOG.warning("status >= 400 response with empty body")
|
||||
|
||||
def _request(self, method, url, headers=None, body=None):
|
||||
def _request(self, method, url, headers=None, body=None, chunked=False):
|
||||
"""A simple HTTP request interface."""
|
||||
# Authenticate the request with the auth provider
|
||||
req_url, req_headers, req_body = self.auth_provider.auth_request(
|
||||
|
@ -530,7 +533,9 @@ class RestClient(object):
|
|||
start = time.time()
|
||||
self._log_request_start(method, req_url)
|
||||
resp, resp_body = self.raw_request(
|
||||
req_url, method, headers=req_headers, body=req_body)
|
||||
req_url, method, headers=req_headers, body=req_body,
|
||||
chunked=chunked
|
||||
)
|
||||
end = time.time()
|
||||
self._log_request(method, req_url, resp, secs=(end - start),
|
||||
req_headers=req_headers, req_body=req_body,
|
||||
|
@ -541,7 +546,7 @@ class RestClient(object):
|
|||
|
||||
return resp, resp_body
|
||||
|
||||
def raw_request(self, url, method, headers=None, body=None):
|
||||
def raw_request(self, url, method, headers=None, body=None, chunked=False):
|
||||
"""Send a raw HTTP request without the keystone catalog or auth
|
||||
|
||||
This method sends a HTTP request in the same manner as the request()
|
||||
|
@ -554,17 +559,18 @@ class RestClient(object):
|
|||
:param str headers: Headers to use for the request if none are specifed
|
||||
the headers
|
||||
:param str body: Body to send with the request
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:rtype: tuple
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers()
|
||||
return self.http_obj.request(url, method,
|
||||
headers=headers, body=body)
|
||||
return self.http_obj.request(url, method, headers=headers,
|
||||
body=body, chunked=chunked)
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
body=None, chunked=False):
|
||||
"""Send a HTTP request with keystone auth and using the catalog
|
||||
|
||||
This method will send an HTTP request using keystone auth in the
|
||||
|
@ -590,6 +596,7 @@ class RestClient(object):
|
|||
get_headers() method are used. If the request
|
||||
explicitly requires no headers use an empty dict.
|
||||
:param str body: Body to send with the request
|
||||
:param bool chunked: sends the body with chunked encoding
|
||||
:rtype: tuple
|
||||
:return: a tuple with the first entry containing the response headers
|
||||
and the second the response body
|
||||
|
@ -629,8 +636,8 @@ class RestClient(object):
|
|||
except (ValueError, TypeError):
|
||||
headers = self.get_headers()
|
||||
|
||||
resp, resp_body = self._request(method, url,
|
||||
headers=headers, body=body)
|
||||
resp, resp_body = self._request(method, url, headers=headers,
|
||||
body=body, chunked=chunked)
|
||||
|
||||
while (resp.status == 413 and
|
||||
'retry-after' in resp and
|
||||
|
|
|
@ -19,6 +19,8 @@ import random
|
|||
import string
|
||||
import uuid
|
||||
|
||||
import six.moves
|
||||
|
||||
|
||||
def rand_uuid():
|
||||
"""Generate a random UUID string
|
||||
|
@ -196,3 +198,10 @@ def get_ipv6_addr_by_EUI64(cidr, mac):
|
|||
except TypeError:
|
||||
raise TypeError('Bad prefix type for generate IPv6 address by '
|
||||
'EUI-64: %s' % cidr)
|
||||
|
||||
|
||||
# Courtesy of http://stackoverflow.com/a/312464
|
||||
def chunkify(sequence, chunksize):
|
||||
"""Yield successive chunks from `sequence`."""
|
||||
for i in six.moves.xrange(0, len(sequence), chunksize):
|
||||
yield sequence[i:i + chunksize]
|
||||
|
|
|
@ -48,9 +48,9 @@ class BaseComputeClient(rest_client.RestClient):
|
|||
return headers
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
body=None, chunked=False):
|
||||
resp, resp_body = super(BaseComputeClient, self).request(
|
||||
method, url, extra_headers, headers, body)
|
||||
method, url, extra_headers, headers, body, chunked)
|
||||
if (COMPUTE_MICROVERSION and
|
||||
COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION):
|
||||
api_version_utils.assert_version_header_matches_request(
|
||||
|
|
|
@ -75,8 +75,12 @@ class TokenClient(rest_client.RestClient):
|
|||
return rest_client.ResponseBody(resp, body['access'])
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
"""A simple HTTP request interface."""
|
||||
body=None, chunked=False):
|
||||
"""A simple HTTP request interface.
|
||||
|
||||
Note: this overloads the `request` method from the parent class and
|
||||
thus must implement the same method signature.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.get_headers(accept_type="json")
|
||||
elif extra_headers:
|
||||
|
|
|
@ -122,8 +122,12 @@ class V3TokenClient(rest_client.RestClient):
|
|||
return rest_client.ResponseBody(resp, body)
|
||||
|
||||
def request(self, method, url, extra_headers=False, headers=None,
|
||||
body=None):
|
||||
"""A simple HTTP request interface."""
|
||||
body=None, chunked=False):
|
||||
"""A simple HTTP request interface.
|
||||
|
||||
Note: this overloads the `request` method from the parent class and
|
||||
thus must implement the same method signature.
|
||||
"""
|
||||
if headers is None:
|
||||
# Always accept 'json', for xml token client too.
|
||||
# Because XML response is not easily
|
||||
|
|
|
@ -149,25 +149,30 @@ class ObjectClient(rest_client.RestClient):
|
|||
self.expected_success(201, resp.status)
|
||||
return resp, body
|
||||
|
||||
def put_object_with_chunk(self, container, name, contents, chunk_size):
|
||||
"""Put an object with Transfer-Encoding header"""
|
||||
def put_object_with_chunk(self, container, name, contents):
|
||||
"""Put an object with Transfer-Encoding header
|
||||
|
||||
:param container: name of the container
|
||||
:type container: string
|
||||
:param name: name of the object
|
||||
:type name: string
|
||||
:param contents: object data
|
||||
:type contents: iterable
|
||||
"""
|
||||
headers = {'Transfer-Encoding': 'chunked'}
|
||||
if self.token:
|
||||
headers['X-Auth-Token'] = self.token
|
||||
|
||||
conn = put_object_connection(self.base_url, container, name, contents,
|
||||
chunk_size, headers)
|
||||
|
||||
resp = conn.getresponse()
|
||||
body = resp.read()
|
||||
|
||||
resp_headers = {}
|
||||
for header, value in resp.getheaders():
|
||||
resp_headers[header.lower()] = value
|
||||
url = "%s/%s" % (container, name)
|
||||
resp, body = self.put(
|
||||
url, headers=headers,
|
||||
body=contents,
|
||||
chunked=True
|
||||
)
|
||||
|
||||
self._error_checker('PUT', None, headers, contents, resp, body)
|
||||
self.expected_success(201, resp.status)
|
||||
return resp.status, resp.reason, resp_headers
|
||||
return resp.status, resp.reason, resp
|
||||
|
||||
def create_object_continue(self, container, object_name,
|
||||
data, metadata=None):
|
||||
|
@ -262,30 +267,7 @@ def put_object_connection(base_url, container, name, contents=None,
|
|||
headers = dict(headers)
|
||||
else:
|
||||
headers = {}
|
||||
if hasattr(contents, 'read'):
|
||||
conn.putrequest('PUT', path)
|
||||
for header, value in six.iteritems(headers):
|
||||
conn.putheader(header, value)
|
||||
if 'Content-Length' not in headers:
|
||||
if 'Transfer-Encoding' not in headers:
|
||||
conn.putheader('Transfer-Encoding', 'chunked')
|
||||
conn.endheaders()
|
||||
chunk = contents.read(chunk_size)
|
||||
while chunk:
|
||||
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
|
||||
chunk = contents.read(chunk_size)
|
||||
conn.send('0\r\n\r\n')
|
||||
else:
|
||||
conn.endheaders()
|
||||
left = headers['Content-Length']
|
||||
while left > 0:
|
||||
size = chunk_size
|
||||
if size > left:
|
||||
size = left
|
||||
chunk = contents.read(size)
|
||||
conn.send(chunk)
|
||||
left -= len(chunk)
|
||||
else:
|
||||
conn.request('PUT', path, contents, headers)
|
||||
|
||||
conn.request('PUT', path, contents, headers)
|
||||
|
||||
return conn
|
||||
|
|
|
@ -169,3 +169,10 @@ class TestDataUtils(base.TestCase):
|
|||
bad_mac = 99999999999999999999
|
||||
self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64,
|
||||
cidr, bad_mac)
|
||||
|
||||
def test_chunkify(self):
|
||||
data = "aaa"
|
||||
chunks = data_utils.chunkify(data, 2)
|
||||
self.assertEqual("aa", next(chunks))
|
||||
self.assertEqual("a", next(chunks))
|
||||
self.assertRaises(StopIteration, next, chunks)
|
||||
|
|
|
@ -21,7 +21,7 @@ class fake_httplib2(object):
|
|||
self.return_type = return_type
|
||||
|
||||
def request(self, uri, method="GET", body=None, headers=None,
|
||||
redirections=5, connection_type=None):
|
||||
redirections=5, connection_type=None, chunked=False):
|
||||
if not self.return_type:
|
||||
fake_headers = fake_http_response(headers)
|
||||
return_obj = {
|
||||
|
|
Loading…
Reference in New Issue