openstack
/
python-swiftclient
Code Issues Proposed changes

Accept gzip-encoded API responses 

Previously, we would accept gzip-encoded responses, but only because we
were letting requests decode *all* responses (even object data). This
restores the previous capability, but with tighter controls about which
requests will accept gzipped responses and where the decoding happens.

Change-Id: I4fd8b97207b9ab01b1bcf825cc16efd8ad46344a
Related-Bug: 1282861
Related-Bug: 1338464
This commit is contained in:
Tim Burke 2015-05-21 22:44:36 -07:00
parent 20e0c515bf
commit f728027bed
4 changed files with 65 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swiftclient/client.py
View File

@ -732,7 +732,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
if end_marker:
qs += '&end_marker=%s' % quote(end_marker)
full_path = '%s?%s' % (parsed.path, qs)
headers = {'X-Auth-Token': token}
headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}
if service_token:
headers['X-Service-Token'] = service_token
method = 'GET'
@ -859,6 +859,7 @@ def get_container(url, token, container, marker=None, limit=None,
else:
headers = {}
headers['X-Auth-Token'] = token
headers['Accept-Encoding'] = 'gzip'
if full_listing:
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, end_marker, path, http_conn,
@ -1457,10 +1458,11 @@ def get_capabilities(http_conn):
:raises ClientException: HTTP Capabilities GET failed
"""
parsed, conn = http_conn
conn.request('GET', parsed.path, '')
headers = {'Accept-Encoding': 'gzip'}
conn.request('GET', parsed.path, '', headers)
resp = conn.getresponse()
body = resp.read()
http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body)
http_log((parsed.geturl(), 'GET',), {'headers': headers}, resp, body)
if resp.status < 200 or resp.status >= 300:
raise ClientException.from_response(
resp, 'Capabilities GET failed', body)

5
swiftclient/utils.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Miscellaneous utility functions for use with Swift."""
import gzip
import hashlib
import hmac
import json
@ -120,6 +121,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False):
def parse_api_response(headers, body):
if headers.get('content-encoding') == 'gzip':
with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz:
body = gz.read()
charset = 'utf-8'
# Swift *should* be speaking UTF-8, but check content-type just in case
content_type = headers.get('content-type', '')

21
tests/unit/test_swiftclient.py
View File

@ -581,6 +581,7 @@ class TestGetAccount(MockHttpTest):
self.assertEqual(value, [])
self.assertRequests([
('GET', '/v1/acct?format=json', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -591,6 +592,7 @@ class TestGetAccount(MockHttpTest):
c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker')
self.assertRequests([
('GET', '/v1/acct?format=json&marker=marker', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -601,6 +603,7 @@ class TestGetAccount(MockHttpTest):
c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10)
self.assertRequests([
('GET', '/v1/acct?format=json&limit=10', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -611,6 +614,7 @@ class TestGetAccount(MockHttpTest):
c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/')
self.assertRequests([
('GET', '/v1/acct?format=json&prefix=asdf/', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -622,6 +626,7 @@ class TestGetAccount(MockHttpTest):
end_marker='end_marker')
self.assertRequests([
('GET', '/v1/acct?format=json&end_marker=end_marker', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -700,6 +705,7 @@ class TestGetContainer(MockHttpTest):
self.assertEqual(value, [])
self.assertRequests([
('GET', '/v1/acct/container?format=json', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -711,6 +717,7 @@ class TestGetContainer(MockHttpTest):
marker='marker')
self.assertRequests([
('GET', '/v1/acct/container?format=json&marker=marker', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -722,6 +729,7 @@ class TestGetContainer(MockHttpTest):
limit=10)
self.assertRequests([
('GET', '/v1/acct/container?format=json&limit=10', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -733,6 +741,7 @@ class TestGetContainer(MockHttpTest):
prefix='asdf/')
self.assertRequests([
('GET', '/v1/acct/container?format=json&prefix=asdf/', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -744,6 +753,7 @@ class TestGetContainer(MockHttpTest):
delimiter='/')
self.assertRequests([
('GET', '/v1/acct/container?format=json&delimiter=/', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -755,7 +765,7 @@ class TestGetContainer(MockHttpTest):
end_marker='end_marker')
self.assertRequests([
('GET', '/v1/acct/container?format=json&end_marker=end_marker',
'', {'x-auth-token': 'token'}),
'', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}),
])
def test_param_path(self):
@ -766,6 +776,7 @@ class TestGetContainer(MockHttpTest):
path='asdf')
self.assertRequests([
('GET', '/v1/acct/container?format=json&path=asdf', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'token'}),
])
@ -780,6 +791,7 @@ class TestGetContainer(MockHttpTest):
('GET', '/container?format=json', '', {
'x-auth-token': 'TOKEN',
'x-client-key': 'client key',
'accept-encoding': 'gzip',
}),
])
@ -790,6 +802,7 @@ class TestGetContainer(MockHttpTest):
query_string="hello=20")
self.assertRequests([
('GET', '/asdf?format=json&hello=20', '', {
'accept-encoding': 'gzip',
'x-auth-token': 'asdf'}),
])
@ -1583,7 +1596,7 @@ class TestGetCapabilities(MockHttpTest):
http_conn = conn('http://www.test.com/info')
info = c.get_capabilities(http_conn)
self.assertRequests([
('GET', '/info', '', {}),
('GET', '/info', '', {'Accept-Encoding': 'gzip'}),
])
self.assertEqual(info, {})
self.assertTrue(http_conn[1].resp.has_been_read)
@ -1619,7 +1632,8 @@ class TestGetCapabilities(MockHttpTest):
('GET', '/auth/v1.0', '', {
'x-auth-user': 'user',
'x-auth-key': 'key'}),
('GET', 'http://storage.example.com/info', '', {}),
('GET', 'http://storage.example.com/info', '', {
'accept-encoding': 'gzip'}),
])
def test_conn_get_capabilities_with_os_auth(self):
@ -2341,6 +2355,7 @@ class TestConnection(MockHttpTest):
('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', {
'x-auth-token': 'token',
'X-Favourite-Pet': 'Aardvark',
'accept-encoding': 'gzip',
}),
])
self.assertEqual(conn.attempts, 1)

37
tests/unit/test_utils.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import gzip
import unittest
import mock
import six
@ -394,3 +395,39 @@ class TestGroupers(unittest.TestCase):
result = list(u.n_groups(range(100), 12))
self.assertEqual([9] * 11 + [1], list(map(len, result)))
class TestApiResponeParser(unittest.TestCase):
def test_utf8_default(self):
result = u.parse_api_response(
{}, u'{"test": "\u2603"}'.encode('utf8'))
self.assertEqual({'test': u'\u2603'}, result)
result = u.parse_api_response(
{}, u'{"test": "\\u2603"}'.encode('utf8'))
self.assertEqual({'test': u'\u2603'}, result)
def test_bad_json(self):
self.assertRaises(ValueError, u.parse_api_response,
{}, b'{"foo": "bar}')
def test_bad_utf8(self):
self.assertRaises(UnicodeDecodeError, u.parse_api_response,
{}, b'{"foo": "b\xffr"}')
def test_latin_1(self):
result = u.parse_api_response(
{'content-type': 'application/json; charset=iso8859-1'},
b'{"t\xe9st": "\xff"}')
self.assertEqual({u't\xe9st': u'\xff'}, result)
def test_gzipped_utf8(self):
buf = six.BytesIO()
gz = gzip.GzipFile(fileobj=buf, mode='w')
gz.write(u'{"test": "\u2603"}'.encode('utf8'))
gz.close()
result = u.parse_api_response(
{'content-encoding': 'gzip'},
buf.getvalue())
self.assertEqual({'test': u'\u2603'}, result)