From 4408c4a5fe5a862a7e05d0df8b7990fd1c6053e1 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Fri, 29 Apr 2016 15:05:09 +0200 Subject: [PATCH] Swift object client: use urllib3 builtin support for chunked transfer Urllib3 has native support for chunked encoding, so let's use this instead of rolling our own. Less code to maintain, additional logging and timing (thanks to our common RestClient). Yeah \O/. Change-Id: I4a253a5cec0fc35009af25872239363625d417e3 --- ...ort-chunked-encoding-d71f53225f68edf3.yaml | 9 +++ .../object_storage/test_object_services.py | 5 +- tempest/lib/common/rest_client.py | 31 ++++++---- tempest/lib/common/utils/data_utils.py | 9 +++ .../services/compute/base_compute_client.py | 4 +- .../lib/services/identity/v2/token_client.py | 8 ++- .../lib/services/identity/v3/token_client.py | 8 ++- .../services/object_storage/object_client.py | 56 +++++++------------ .../tests/lib/common/utils/test_data_utils.py | 7 +++ tempest/tests/lib/fake_http.py | 2 +- 10 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml diff --git a/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml new file mode 100644 index 0000000000..eb45523fee --- /dev/null +++ b/releasenotes/notes/support-chunked-encoding-d71f53225f68edf3.yaml @@ -0,0 +1,9 @@ +--- +features: + - The RestClient (in tempest.lib.common.rest_client) now supports POSTing + and PUTing data with chunked transfer encoding. Just pass an `iterable` + object as the `body` argument and set the `chunked` argument to `True`. + - A new generator called `chunkify` is added in + tempest.lib.common.utils.data_utils that yields fixed-size chunks (slices) + from a Python sequence. + diff --git a/tempest/api/object_storage/test_object_services.py b/tempest/api/object_storage/test_object_services.py index dc926e0f41..a88e4f41a4 100644 --- a/tempest/api/object_storage/test_object_services.py +++ b/tempest/api/object_storage/test_object_services.py @@ -20,7 +20,6 @@ import time import zlib import six -from six import moves from tempest.api.object_storage import base from tempest.common import custom_matchers @@ -201,8 +200,8 @@ class ObjectTest(base.BaseObjectTest): status, _, resp_headers = self.object_client.put_object_with_chunk( container=self.container_name, name=object_name, - contents=moves.cStringIO(data), - chunk_size=512) + contents=data_utils.chunkify(data, 512) + ) self.assertHeaders(resp_headers, 'Object', 'PUT') # check uploaded content diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py index 30750dec17..179db17a58 100644 --- a/tempest/lib/common/rest_client.py +++ b/tempest/lib/common/rest_client.py @@ -243,7 +243,8 @@ class RestClient(object): details = pattern.format(read_code, expected_code) raise exceptions.InvalidHttpSuccessCode(details) - def post(self, url, body, headers=None, extra_headers=False): + def post(self, url, body, headers=None, extra_headers=False, + chunked=False): """Send a HTTP POST request using keystone auth :param str url: the relative url to send the post request to @@ -253,11 +254,12 @@ class RestClient(object): returned by the get_headers() method are to be used but additional headers are needed in the request pass them in as a dict. + :param bool chunked: sends the body with chunked encoding :return: a tuple with the first entry containing the response headers and the second the response body :rtype: tuple """ - return self.request('POST', url, extra_headers, headers, body) + return self.request('POST', url, extra_headers, headers, body, chunked) def get(self, url, headers=None, extra_headers=False): """Send a HTTP GET request using keystone service catalog and auth @@ -306,7 +308,7 @@ class RestClient(object): """ return self.request('PATCH', url, extra_headers, headers, body) - def put(self, url, body, headers=None, extra_headers=False): + def put(self, url, body, headers=None, extra_headers=False, chunked=False): """Send a HTTP PUT request using keystone service catalog and auth :param str url: the relative url to send the post request to @@ -316,11 +318,12 @@ class RestClient(object): returned by the get_headers() method are to be used but additional headers are needed in the request pass them in as a dict. + :param bool chunked: sends the body with chunked encoding :return: a tuple with the first entry containing the response headers and the second the response body :rtype: tuple """ - return self.request('PUT', url, extra_headers, headers, body) + return self.request('PUT', url, extra_headers, headers, body, chunked) def head(self, url, headers=None, extra_headers=False): """Send a HTTP HEAD request using keystone service catalog and auth @@ -520,7 +523,7 @@ class RestClient(object): if method != 'HEAD' and not resp_body and resp.status >= 400: self.LOG.warning("status >= 400 response with empty body") - def _request(self, method, url, headers=None, body=None): + def _request(self, method, url, headers=None, body=None, chunked=False): """A simple HTTP request interface.""" # Authenticate the request with the auth provider req_url, req_headers, req_body = self.auth_provider.auth_request( @@ -530,7 +533,9 @@ class RestClient(object): start = time.time() self._log_request_start(method, req_url) resp, resp_body = self.raw_request( - req_url, method, headers=req_headers, body=req_body) + req_url, method, headers=req_headers, body=req_body, + chunked=chunked + ) end = time.time() self._log_request(method, req_url, resp, secs=(end - start), req_headers=req_headers, req_body=req_body, @@ -541,7 +546,7 @@ class RestClient(object): return resp, resp_body - def raw_request(self, url, method, headers=None, body=None): + def raw_request(self, url, method, headers=None, body=None, chunked=False): """Send a raw HTTP request without the keystone catalog or auth This method sends a HTTP request in the same manner as the request() @@ -554,17 +559,18 @@ class RestClient(object): :param str headers: Headers to use for the request if none are specifed the headers :param str body: Body to send with the request + :param bool chunked: sends the body with chunked encoding :rtype: tuple :return: a tuple with the first entry containing the response headers and the second the response body """ if headers is None: headers = self.get_headers() - return self.http_obj.request(url, method, - headers=headers, body=body) + return self.http_obj.request(url, method, headers=headers, + body=body, chunked=chunked) def request(self, method, url, extra_headers=False, headers=None, - body=None): + body=None, chunked=False): """Send a HTTP request with keystone auth and using the catalog This method will send an HTTP request using keystone auth in the @@ -590,6 +596,7 @@ class RestClient(object): get_headers() method are used. If the request explicitly requires no headers use an empty dict. :param str body: Body to send with the request + :param bool chunked: sends the body with chunked encoding :rtype: tuple :return: a tuple with the first entry containing the response headers and the second the response body @@ -629,8 +636,8 @@ class RestClient(object): except (ValueError, TypeError): headers = self.get_headers() - resp, resp_body = self._request(method, url, - headers=headers, body=body) + resp, resp_body = self._request(method, url, headers=headers, + body=body, chunked=chunked) while (resp.status == 413 and 'retry-after' in resp and diff --git a/tempest/lib/common/utils/data_utils.py b/tempest/lib/common/utils/data_utils.py index 9605479623..45e50673d7 100644 --- a/tempest/lib/common/utils/data_utils.py +++ b/tempest/lib/common/utils/data_utils.py @@ -19,6 +19,8 @@ import random import string import uuid +import six.moves + def rand_uuid(): """Generate a random UUID string @@ -196,3 +198,10 @@ def get_ipv6_addr_by_EUI64(cidr, mac): except TypeError: raise TypeError('Bad prefix type for generate IPv6 address by ' 'EUI-64: %s' % cidr) + + +# Courtesy of http://stackoverflow.com/a/312464 +def chunkify(sequence, chunksize): + """Yield successive chunks from `sequence`.""" + for i in six.moves.xrange(0, len(sequence), chunksize): + yield sequence[i:i + chunksize] diff --git a/tempest/lib/services/compute/base_compute_client.py b/tempest/lib/services/compute/base_compute_client.py index 9161abbdec..a387b855f7 100644 --- a/tempest/lib/services/compute/base_compute_client.py +++ b/tempest/lib/services/compute/base_compute_client.py @@ -48,9 +48,9 @@ class BaseComputeClient(rest_client.RestClient): return headers def request(self, method, url, extra_headers=False, headers=None, - body=None): + body=None, chunked=False): resp, resp_body = super(BaseComputeClient, self).request( - method, url, extra_headers, headers, body) + method, url, extra_headers, headers, body, chunked) if (COMPUTE_MICROVERSION and COMPUTE_MICROVERSION != api_version_utils.LATEST_MICROVERSION): api_version_utils.assert_version_header_matches_request( diff --git a/tempest/lib/services/identity/v2/token_client.py b/tempest/lib/services/identity/v2/token_client.py index 03501757a7..571602735d 100644 --- a/tempest/lib/services/identity/v2/token_client.py +++ b/tempest/lib/services/identity/v2/token_client.py @@ -75,8 +75,12 @@ class TokenClient(rest_client.RestClient): return rest_client.ResponseBody(resp, body['access']) def request(self, method, url, extra_headers=False, headers=None, - body=None): - """A simple HTTP request interface.""" + body=None, chunked=False): + """A simple HTTP request interface. + + Note: this overloads the `request` method from the parent class and + thus must implement the same method signature. + """ if headers is None: headers = self.get_headers(accept_type="json") elif extra_headers: diff --git a/tempest/lib/services/identity/v3/token_client.py b/tempest/lib/services/identity/v3/token_client.py index f342a497e9..964d43f0b7 100644 --- a/tempest/lib/services/identity/v3/token_client.py +++ b/tempest/lib/services/identity/v3/token_client.py @@ -122,8 +122,12 @@ class V3TokenClient(rest_client.RestClient): return rest_client.ResponseBody(resp, body) def request(self, method, url, extra_headers=False, headers=None, - body=None): - """A simple HTTP request interface.""" + body=None, chunked=False): + """A simple HTTP request interface. + + Note: this overloads the `request` method from the parent class and + thus must implement the same method signature. + """ if headers is None: # Always accept 'json', for xml token client too. # Because XML response is not easily diff --git a/tempest/services/object_storage/object_client.py b/tempest/services/object_storage/object_client.py index fa43d947cd..33dba6e412 100644 --- a/tempest/services/object_storage/object_client.py +++ b/tempest/services/object_storage/object_client.py @@ -149,25 +149,30 @@ class ObjectClient(rest_client.RestClient): self.expected_success(201, resp.status) return resp, body - def put_object_with_chunk(self, container, name, contents, chunk_size): - """Put an object with Transfer-Encoding header""" + def put_object_with_chunk(self, container, name, contents): + """Put an object with Transfer-Encoding header + + :param container: name of the container + :type container: string + :param name: name of the object + :type name: string + :param contents: object data + :type contents: iterable + """ headers = {'Transfer-Encoding': 'chunked'} if self.token: headers['X-Auth-Token'] = self.token - conn = put_object_connection(self.base_url, container, name, contents, - chunk_size, headers) - - resp = conn.getresponse() - body = resp.read() - - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + url = "%s/%s" % (container, name) + resp, body = self.put( + url, headers=headers, + body=contents, + chunked=True + ) self._error_checker('PUT', None, headers, contents, resp, body) self.expected_success(201, resp.status) - return resp.status, resp.reason, resp_headers + return resp.status, resp.reason, resp def create_object_continue(self, container, object_name, data, metadata=None): @@ -262,30 +267,7 @@ def put_object_connection(base_url, container, name, contents=None, headers = dict(headers) else: headers = {} - if hasattr(contents, 'read'): - conn.putrequest('PUT', path) - for header, value in six.iteritems(headers): - conn.putheader(header, value) - if 'Content-Length' not in headers: - if 'Transfer-Encoding' not in headers: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = contents.read(chunk_size) - conn.send('0\r\n\r\n') - else: - conn.endheaders() - left = headers['Content-Length'] - while left > 0: - size = chunk_size - if size > left: - size = left - chunk = contents.read(size) - conn.send(chunk) - left -= len(chunk) - else: - conn.request('PUT', path, contents, headers) + + conn.request('PUT', path, contents, headers) return conn diff --git a/tempest/tests/lib/common/utils/test_data_utils.py b/tempest/tests/lib/common/utils/test_data_utils.py index f9e1f44492..399c4afc1c 100644 --- a/tempest/tests/lib/common/utils/test_data_utils.py +++ b/tempest/tests/lib/common/utils/test_data_utils.py @@ -169,3 +169,10 @@ class TestDataUtils(base.TestCase): bad_mac = 99999999999999999999 self.assertRaises(TypeError, data_utils.get_ipv6_addr_by_EUI64, cidr, bad_mac) + + def test_chunkify(self): + data = "aaa" + chunks = data_utils.chunkify(data, 2) + self.assertEqual("aa", next(chunks)) + self.assertEqual("a", next(chunks)) + self.assertRaises(StopIteration, next, chunks) diff --git a/tempest/tests/lib/fake_http.py b/tempest/tests/lib/fake_http.py index 397c856459..cfa4b93685 100644 --- a/tempest/tests/lib/fake_http.py +++ b/tempest/tests/lib/fake_http.py @@ -21,7 +21,7 @@ class fake_httplib2(object): self.return_type = return_type def request(self, uri, method="GET", body=None, headers=None, - redirections=5, connection_type=None): + redirections=5, connection_type=None, chunked=False): if not self.return_type: fake_headers = fake_http_response(headers) return_obj = {