py3: port dlo

Change-Id: I7236ddea0acde93d0789ad8affa76df0097a86aa
This commit is contained in:
Pete Zaitcev 2019-02-20 17:09:08 -06:00
parent 5090a15f52
commit 893acffbc0
3 changed files with 113 additions and 66 deletions

View File

@ -121,14 +121,14 @@ Here's an example using ``curl`` with tiny 1-byte segments::
import json
import six
from six.moves.urllib.parse import unquote
from hashlib import md5
from swift.common import constraints
from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success
from swift.common.swob import Request, Response, \
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict
HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict, \
str_to_wsgi, wsgi_to_str, wsgi_quote, wsgi_unquote
from swift.common.utils import get_logger, \
RateLimitedIterator, quote, close_if_possible, closing_if_possible
from swift.common.request_helpers import SegmentedIterable
@ -143,9 +143,18 @@ class GetContext(WSGIContext):
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=quote('/'.join(['', version, account, container])),
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
@ -156,14 +165,24 @@ class GetContext(WSGIContext):
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = ''
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(''.join(con_resp.app_iter))
return None, json.loads(b''.join(con_resp.app_iter))
def _segment_listing_iterator(self, req, version, account, container,
prefix, segments, first_byte=None,
last_byte=None):
'''
:param req: upstream request
:param version: native
:param account: native
:param container: native
:param prefix: native
:param segments: array of dicts, with native strings
:param first_byte: number
:param last_byte: number
'''
# It's sort of hokey that this thing takes in the first page of
# segments as an argument, but we need to compute the etag and content
# length from the first page, and it's better to have a hokey
@ -173,7 +192,6 @@ class GetContext(WSGIContext):
if last_byte is None:
last_byte = float("inf")
marker = ''
while True:
for segment in segments:
seg_length = int(segment['bytes'])
@ -188,7 +206,7 @@ class GetContext(WSGIContext):
break
seg_name = segment['name']
if isinstance(seg_name, six.text_type):
if six.PY2:
seg_name = seg_name.encode("utf-8")
# We deliberately omit the etag and size here;
@ -227,16 +245,18 @@ class GetContext(WSGIContext):
"Got status %d listing container /%s/%s" %
(error_response.status_int, account, container))
def get_or_head_response(self, req, x_object_manifest,
response_headers=None):
if response_headers is None:
def get_or_head_response(self, req, x_object_manifest):
'''
:param req: user's request
:param x_object_manifest: as unquoted, native string
'''
response_headers = self._response_headers
container, obj_prefix = x_object_manifest.split('/', 1)
container = unquote(container)
obj_prefix = unquote(obj_prefix)
version, account, _junk = req.split_path(2, 3, True)
version = wsgi_to_str(version)
account = wsgi_to_str(account)
error_response, segments = self._get_container_listing(
req, version, account, container, obj_prefix)
if error_response:
@ -311,7 +331,7 @@ class GetContext(WSGIContext):
if h.lower() != "etag"]
etag = md5()
for seg_dict in segments:
etag.update(seg_dict['hash'].strip('"'))
etag.update(seg_dict['hash'].strip('"').encode('utf8'))
response_headers.append(('Etag', '"%s"' % etag.hexdigest()))
app_iter = None
@ -353,7 +373,8 @@ class GetContext(WSGIContext):
for header, value in self._response_headers:
if (header.lower() == 'x-object-manifest'):
close_if_possible(resp_iter)
response = self.get_or_head_response(req, value)
response = self.get_or_head_response(
req, wsgi_to_str(wsgi_unquote(value)))
return response(req.environ, start_response)
# Not a dynamic large object manifest; just pass it through.
start_response(self._response_status,

View File

@ -23,6 +23,8 @@ from textwrap import dedent
import time
import unittest
import six
from swift.common import swob
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.middleware import dlo
@ -34,6 +36,8 @@ LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
def md5hex(s):
if isinstance(s, six.text_type):
s = s.encode('utf-8')
return hashlib.md5(s).hexdigest()
@ -52,7 +56,7 @@ class DloTestCase(unittest.TestCase):
headers[0] = h
body_iter = app(req.environ, start_response)
body = ''
body = b''
# appease the close-checker
with closing_if_possible(body_iter):
for chunk in body_iter:
@ -69,36 +73,36 @@ class DloTestCase(unittest.TestCase):
self.app.register(
'GET', '/v1/AUTH_test/c/seg_01',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")},
'aaaaa')
b'aaaaa')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")},
'bbbbb')
b'bbbbb')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")},
'ccccc')
b'ccccc')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_04',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")},
'ddddd')
b'ddddd')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_05',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")},
'eeeee')
b'eeeee')
# an unrelated object (not seg*) to test the prefix matching
self.app.register(
'GET', '/v1/AUTH_test/c/catpicture.jpg',
swob.HTTPOk, {'Content-Length': '9',
'Etag': md5hex("meow meow meow meow")},
'meow meow meow meow')
b'meow meow meow meow')
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest',
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'},
'manifest-contents')
b'manifest-contents')
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
@ -120,11 +124,11 @@ class DloTestCase(unittest.TestCase):
self.app.register(
'GET', '/v1/AUTH_test/c',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing))
json.dumps(full_container_listing).encode('ascii'))
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
json.dumps(segs).encode('ascii'))
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
@ -135,26 +139,26 @@ class DloTestCase(unittest.TestCase):
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'},
'manyseg')
b'manyseg')
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3]))
json.dumps(segs[:3]).encode('ascii'))
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:]))
json.dumps(segs[3:]).encode('ascii'))
# Here's a manifest with 0 segments
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'},
'noseg')
b'noseg')
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=noseg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps([]))
json.dumps([]).encode('ascii'))
class TestDloPutManifest(DloTestCase):
@ -284,7 +288,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], expected_etag)
self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
for _, _, hdrs in self.app.calls_with_headers[1:]:
ua = hdrs.get("User-Agent", "")
@ -302,7 +306,7 @@ class TestDloGetManifest(DloTestCase):
req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
self.assertEqual(body, "meow meow meow meow")
self.assertEqual(body, b"meow meow meow meow")
def test_get_non_object_passthrough(self):
self.app.register('GET', '/info', swob.HTTPOk,
@ -311,7 +315,7 @@ class TestDloGetManifest(DloTestCase):
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
self.assertEqual(status, '200 OK')
self.assertEqual(body, 'useful stuff here')
self.assertEqual(body, b'useful stuff here')
self.assertEqual(self.app.call_count, 1)
def test_get_manifest_passthrough(self):
@ -328,7 +332,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"], "manifest-etag")
self.assertEqual(body, "manifest-contents")
self.assertEqual(body, b'manifest-contents')
def test_error_passthrough(self):
self.app.register(
@ -347,7 +351,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, "bbcccccddd")
self.assertEqual(body, b'bbcccccddd')
expected_etag = '"%s"' % md5hex(
md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
md5hex("ddddd") + md5hex("eeeee"))
@ -361,7 +365,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "10")
self.assertEqual(body, "cccccddddd")
self.assertEqual(body, b'cccccddddd')
def test_get_range_first_byte(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -371,7 +375,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "1")
self.assertEqual(body, "a")
self.assertEqual(body, b'a')
def test_get_range_last_byte(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -381,7 +385,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "1")
self.assertEqual(body, "e")
self.assertEqual(body, b'e')
def test_get_range_overlapping_end(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -392,7 +396,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "7")
self.assertEqual(headers["Content-Range"], "bytes 18-24/25")
self.assertEqual(body, "ddeeeee")
self.assertEqual(body, b'ddeeeee')
def test_get_range_unsatisfiable(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -428,7 +432,7 @@ class TestDloGetManifest(DloTestCase):
#
# Since the truth is forbidden, we lie.
self.assertEqual(headers["Content-Range"], "bytes 3-12/15")
self.assertEqual(body, "aabbbbbccc")
self.assertEqual(body, b"aabbbbbccc")
self.assertEqual(
self.app.calls,
@ -449,7 +453,7 @@ class TestDloGetManifest(DloTestCase):
# this requires multiple pages of container listing, so we can't send
# a Content-Length header
self.assertIsNone(headers.get("Content-Length"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_suffix_range(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -459,7 +463,7 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "206 Partial Content")
self.assertEqual(headers["Content-Length"], "25")
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_suffix_range_many_segments(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
@ -471,7 +475,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_get_multi_range(self):
# DLO doesn't support multi-range GETs. The way that you express that
@ -485,7 +489,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, "200 OK")
self.assertIsNone(headers.get("Content-Length"))
self.assertIsNone(headers.get("Content-Range"))
self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee")
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_if_match_matches(self):
manifest_etag = '"%s"' % md5hex(
@ -500,7 +504,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_if_match_does_not_match(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -512,7 +516,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '412 Precondition Failed')
self.assertEqual(headers['Content-Length'], '0')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def test_if_none_match_matches(self):
manifest_etag = '"%s"' % md5hex(
@ -527,7 +531,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '304 Not Modified')
self.assertEqual(headers['Content-Length'], '0')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def test_if_none_match_does_not_match(self):
req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
@ -539,7 +543,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK')
self.assertEqual(headers['Content-Length'], '25')
self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee')
self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def test_get_with_if_modified_since(self):
# It's important not to pass the If-[Un]Modified-Since header to the
@ -581,7 +585,11 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaa") # first segment made it out
# first segment made it out
if six.PY2:
self.assertEqual(''.join(body), "aaaaa")
else:
self.assertEqual(body, b'aaaaa')
self.assertEqual(self.dlo.logger.get_lines_for_level('error'), [
'While processing manifest /v1/AUTH_test/mancon/manifest, '
'got 403 while retrieving /v1/AUTH_test/c/seg_02',
@ -610,7 +618,7 @@ class TestDloGetManifest(DloTestCase):
with mock.patch(LIMIT, 3):
status, headers, body = self.call_dlo(req)
self.assertEqual(status, "200 OK")
self.assertEqual(body, "aaaaabbbbbccccc")
self.assertEqual(body, b'aaaaabbbbbccccc')
def test_error_listing_container_HEAD(self):
self.app.register(
@ -639,7 +647,11 @@ class TestDloGetManifest(DloTestCase):
headers = HeaderKeyDict(headers)
self.assertEqual(status, "200 OK")
self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
if six.PY2:
# stop after error
self.assertEqual(''.join(body), "aaaaabbWRONGbb")
else:
self.assertEqual(body, b"aaaaabbWRONGbb")
def test_etag_comparison_ignores_quotes(self):
# a little future-proofing here in case we ever fix this in swob
@ -662,7 +674,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req)
headers = HeaderKeyDict(headers)
self.assertEqual(headers["Etag"],
'"' + hashlib.md5("abcdef").hexdigest() + '"')
'"' + hashlib.md5(b"abcdef").hexdigest() + '"')
def test_object_prefix_quoting(self):
self.app.register(
@ -675,22 +687,26 @@ class TestDloGetManifest(DloTestCase):
self.app.register(
'GET', '/v1/AUTH_test/c?prefix=%C3%A9',
swob.HTTPOk, {'Content-Type': 'application/json'},
json.dumps(segs))
json.dumps(segs).encode('ascii'))
if six.PY2:
path = b'/v1/AUTH_test/c/\xC3\xa9'
else:
path = u'/v1/AUTH_test/c/\xc3\xa9'
self.app.register(
'GET', '/v1/AUTH_test/c/\xC3\xa91',
'GET', path + '1',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")},
"AAAAA")
b"AAAAA")
self.app.register(
'GET', '/v1/AUTH_test/c/\xC3\xA92',
'GET', path + '2',
swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")},
"BBBBB")
b"BBBBB")
req = swob.Request.blank('/v1/AUTH_test/man/accent',
environ={'REQUEST_METHOD': 'GET'})
status, headers, body = self.call_dlo(req)
self.assertEqual(status, "200 OK")
self.assertEqual(body, "AAAAABBBBB")
self.assertEqual(body, b'AAAAABBBBB')
def test_get_taking_too_long(self):
the_time = [time.time()]
@ -715,7 +731,7 @@ class TestDloGetManifest(DloTestCase):
status, headers, body = self.call_dlo(req)
self.assertEqual(status, '200 OK')
self.assertEqual(body, 'aaaaabbbbbccccc')
self.assertEqual(body, b'aaaaabbbbbccccc')
def test_get_oversize_segment(self):
# If we send a Content-Length header to the client, it's based on the
@ -738,7 +754,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccccccccccccc')
self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
self.assertEqual(
self.app.calls,
[('GET', '/v1/AUTH_test/mancon/manifest'),
@ -770,7 +786,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '200 OK') # sanity check
self.assertEqual(headers.get('Content-Length'), '25') # sanity check
self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
self.assertEqual(body, b'aaaaabbbbbccccdddddeeeee')
def test_get_undersize_segment_range(self):
# Shrink it by a single byte
@ -788,7 +804,7 @@ class TestDloGetManifest(DloTestCase):
self.assertEqual(status, '206 Partial Content') # sanity check
self.assertEqual(headers.get('Content-Length'), '15') # sanity check
self.assertEqual(body, 'aaaaabbbbbcccc')
self.assertEqual(body, b'aaaaabbbbbcccc')
def test_get_with_auth_overridden(self):
auth_got_called = [0]
@ -840,7 +856,7 @@ class TestDloConfiguration(unittest.TestCase):
max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile()
conffile = tempfile.NamedTemporaryFile(mode='w')
conffile.write(proxy_conf)
conffile.flush()
@ -853,6 +869,8 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(10, mware.rate_limit_after_segment)
self.assertEqual(3600, mware.max_get_time)
conffile.close()
def test_finding_defaults_from_file(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
@ -875,7 +893,7 @@ class TestDloConfiguration(unittest.TestCase):
set max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile()
conffile = tempfile.NamedTemporaryFile(mode='w')
conffile.write(proxy_conf)
conffile.flush()
@ -887,6 +905,8 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
conffile.close()
def test_finding_defaults_from_dir(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
@ -913,11 +933,13 @@ class TestDloConfiguration(unittest.TestCase):
conf_dir = self.tmpdir
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
conffile1 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile1.write(proxy_conf1)
conffile1.flush()
conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
conffile2 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile2.write(proxy_conf2)
conffile2.flush()
@ -929,6 +951,9 @@ class TestDloConfiguration(unittest.TestCase):
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
conffile1.close()
conffile2.close()
if __name__ == '__main__':
unittest.main()

View File

@ -48,6 +48,7 @@ commands =
test/unit/common/middleware/test_container_sync.py \
test/unit/common/middleware/test_copy.py \
test/unit/common/middleware/test_crossdomain.py \
test/unit/common/middleware/test_dlo.py \
test/unit/common/middleware/test_domain_remap.py \
test/unit/common/middleware/test_formpost.py \
test/unit/common/middleware/test_gatekeeper.py \