diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index b46d3910b5..853a51c661 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -147,11 +147,12 @@ metadata which can be used for stats purposes. """ from six.moves import range -from six.moves import cStringIO as StringIO from datetime import datetime import mimetypes import re +import six +from six import BytesIO from hashlib import md5 from swift.common.exceptions import ListingIterError, SegmentError from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ @@ -681,8 +682,10 @@ class StaticLargeObject(object): env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) + if six.PY3: + json_data = json_data.encode('utf-8') env['CONTENT_LENGTH'] = str(len(json_data)) - env['wsgi.input'] = StringIO(json_data) + env['wsgi.input'] = BytesIO(json_data) slo_put_context = SloPutContext(self, slo_etag) return slo_put_context.handle_slo_put(req, start_response) diff --git a/swift/common/swob.py b/swift/common/swob.py index e4ac78d25d..70c1fc4415 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -48,6 +48,7 @@ import random import functools import inspect +from six import BytesIO from six import StringIO from swift.common.utils import reiterate, split_path, Timestamp, pairs, \ @@ -130,10 +131,10 @@ class _UTC(tzinfo): UTC = _UTC() -class WsgiStringIO(StringIO): +class WsgiStringIO(BytesIO): """ This class adds support for the additional wsgi.input methods defined on - eventlet.wsgi.Input to the StringIO class which would otherwise be a fine + eventlet.wsgi.Input to the BytesIO class which would otherwise be a fine stand-in for the file-like object in the WSGI environment. """ @@ -865,7 +866,7 @@ class Request(object): 'SERVER_PROTOCOL': 'HTTP/1.0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': parsed_path.scheme or 'http', - 'wsgi.errors': StringIO(''), + 'wsgi.errors': StringIO(), 'wsgi.multithread': False, 'wsgi.multiprocess': False } @@ -874,7 +875,7 @@ class Request(object): env['wsgi.input'] = WsgiStringIO(body) env['CONTENT_LENGTH'] = str(len(body)) elif 'wsgi.input' not in env: - env['wsgi.input'] = WsgiStringIO('') + env['wsgi.input'] = WsgiStringIO() req = Request(env) for key, val in headers.items(): req.headers[key] = val @@ -981,7 +982,7 @@ class Request(object): env.update({ 'REQUEST_METHOD': 'GET', 'CONTENT_LENGTH': '0', - 'wsgi.input': WsgiStringIO(''), + 'wsgi.input': WsgiStringIO(), }) return Request(env) diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 46789826f1..28810f6f8d 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -31,6 +31,7 @@ import eventlet.debug from eventlet import greenio, GreenPool, sleep, wsgi, listen, Timeout from paste.deploy import loadwsgi from eventlet.green import socket, ssl, os as green_os +from six import BytesIO from six import StringIO from urllib import unquote @@ -1102,7 +1103,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None, del newenv['HTTP_USER_AGENT'] if swift_source: newenv['swift.source'] = swift_source - newenv['wsgi.input'] = StringIO('') + newenv['wsgi.input'] = BytesIO() if 'SCRIPT_NAME' not in newenv: newenv['SCRIPT_NAME'] = '' return newenv diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 8bd8ea6d82..4ea16134ec 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -25,6 +25,7 @@ import itertools import random import simplejson +from six import BytesIO from six import StringIO import xml.dom.minidom @@ -1358,7 +1359,7 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 507) def test_through_call(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -1384,7 +1385,7 @@ class TestAccountController(unittest.TestCase): self.assertEqual(outbuf.getvalue()[:4], '404 ') def test_through_call_invalid_path(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -1410,7 +1411,7 @@ class TestAccountController(unittest.TestCase): self.assertEqual(outbuf.getvalue()[:4], '400 ') def test_through_call_invalid_path_utf8(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -1582,7 +1583,7 @@ class TestAccountController(unittest.TestCase): def test_correct_allowed_method(self): # Test correct work for allowed method using # swift.account.server.AccountController.__call__ - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = AccountController( @@ -1621,7 +1622,7 @@ class TestAccountController(unittest.TestCase): def test_not_allowed_method(self): # Test correct work for NOT allowed method using # swift.account.server.AccountController.__call__ - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = AccountController( @@ -1658,7 +1659,7 @@ class TestAccountController(unittest.TestCase): self.assertEqual(response, answer) def test_call_incorrect_replication_method(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = AccountController( diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index d912c1065a..e763424a32 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -15,13 +15,13 @@ # limitations under the License. import numbers -from six import StringIO import unittest import os import tarfile import urllib import zlib import mock +from six import BytesIO from shutil import rmtree from tempfile import mkdtemp from eventlet import sleep @@ -155,7 +155,7 @@ class TestUntarMetadata(unittest.TestCase): with open(os.path.join(self.testdir, "obj2"), "w") as fh2: fh2.write("obj2 contents\n") - tar_ball = StringIO() + tar_ball = BytesIO() tar_file = tarfile.TarFile.open(fileobj=tar_ball, mode="w", format=tarfile.PAX_FORMAT) @@ -678,7 +678,7 @@ class TestDelete(unittest.TestCase): headers={'Accept': 'application/json', 'Content-Type': 'text/xml'}) req.method = 'POST' - req.environ['wsgi.input'] = StringIO('/c/f\n/c/f404') + req.environ['wsgi.input'] = BytesIO(b'/c/f\n/c/f404') resp_body = self.handle_delete_and_iter(req) resp_data = utils.json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '406 Not Acceptable') @@ -691,7 +691,7 @@ class TestDelete(unittest.TestCase): req.method = 'POST' req.headers['Transfer-Encoding'] = 'chunked' req.headers['Accept'] = 'application/json' - req.environ['wsgi.input'] = StringIO('/c/f%20') + req.environ['wsgi.input'] = BytesIO(b'/c/f%20') list(self.bulk(req.environ, fake_start_response)) # iterate over resp self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f ']) @@ -706,7 +706,7 @@ class TestDelete(unittest.TestCase): with patch.object(self.bulk, 'max_path_length', 2): results = [] - req.environ['wsgi.input'] = StringIO('1\n2\n3') + req.environ['wsgi.input'] = BytesIO(b'1\n2\n3') results = self.bulk.get_objs_to_delete(req) self.assertEquals(results, [{'name': '1'}, {'name': '2'}, {'name': '3'}]) @@ -737,8 +737,8 @@ class TestDelete(unittest.TestCase): def test_bulk_delete_too_many_newlines(self): req = Request.blank('/delete_works/AUTH_Acc') req.method = 'POST' - data = '\n\n' * self.bulk.max_deletes_per_request - req.environ['wsgi.input'] = StringIO(data) + data = b'\n\n' * self.bulk.max_deletes_per_request + req.environ['wsgi.input'] = BytesIO(data) req.content_length = len(data) resp_body = self.handle_delete_and_iter(req) self.assertTrue('413 Request Entity Too Large' in resp_body) @@ -857,8 +857,8 @@ class TestDelete(unittest.TestCase): headers={'Accept': 'application/json'}) req.method = 'POST' bad_file = 'c/' + ('1' * self.bulk.max_path_length) - data = '/c/f\n' + bad_file + '\n/c/f' - req.environ['wsgi.input'] = StringIO(data) + data = b'/c/f\n' + bad_file.encode('ascii') + b'\n/c/f' + req.environ['wsgi.input'] = BytesIO(data) req.headers['Transfer-Encoding'] = 'chunked' resp_body = self.handle_delete_and_iter(req) resp_data = utils.json.loads(resp_body) diff --git a/test/unit/common/middleware/test_formpost.py b/test/unit/common/middleware/test_formpost.py index 2f4e3d0918..6b7ecead17 100644 --- a/test/unit/common/middleware/test_formpost.py +++ b/test/unit/common/middleware/test_formpost.py @@ -18,7 +18,8 @@ import unittest from hashlib import sha1 from time import time -from six import StringIO +import six +from six import BytesIO from swift.common.swob import Request, Response from swift.common.middleware import tempauth, formpost @@ -43,13 +44,13 @@ class FakeApp(object): if self.check_no_query_string and env.get('QUERY_STRING'): raise Exception('Query string %s should have been discarded!' % env['QUERY_STRING']) - body = '' + body = b'' while True: chunk = env['wsgi.input'].read() if not chunk: break body += chunk - env['wsgi.input'] = StringIO(body) + env['wsgi.input'] = BytesIO(body) self.requests.append(Request.blank('', environ=env)) if env.get('swift.authorize_override') and \ env.get('REMOTE_USER') != '.wsgi.pre_authed': @@ -73,39 +74,40 @@ class TestCappedFileLikeObject(unittest.TestCase): def test_whole(self): self.assertEquals( - formpost._CappedFileLikeObject(StringIO('abc'), 10).read(), 'abc') + formpost._CappedFileLikeObject(BytesIO(b'abc'), 10).read(), + b'abc') def test_exceeded(self): exc = None try: - formpost._CappedFileLikeObject(StringIO('abc'), 2).read() + formpost._CappedFileLikeObject(BytesIO(b'abc'), 2).read() except EOFError as err: exc = err self.assertEquals(str(exc), 'max_file_size exceeded') def test_whole_readline(self): - fp = formpost._CappedFileLikeObject(StringIO('abc\ndef'), 10) - self.assertEquals(fp.readline(), 'abc\n') - self.assertEquals(fp.readline(), 'def') - self.assertEquals(fp.readline(), '') + fp = formpost._CappedFileLikeObject(BytesIO(b'abc\ndef'), 10) + self.assertEquals(fp.readline(), b'abc\n') + self.assertEquals(fp.readline(), b'def') + self.assertEquals(fp.readline(), b'') def test_exceeded_readline(self): - fp = formpost._CappedFileLikeObject(StringIO('abc\ndef'), 5) - self.assertEquals(fp.readline(), 'abc\n') + fp = formpost._CappedFileLikeObject(BytesIO(b'abc\ndef'), 5) + self.assertEquals(fp.readline(), b'abc\n') exc = None try: - self.assertEquals(fp.readline(), 'def') + self.assertEquals(fp.readline(), b'def') except EOFError as err: exc = err self.assertEquals(str(exc), 'max_file_size exceeded') def test_read_sized(self): - fp = formpost._CappedFileLikeObject(StringIO('abcdefg'), 10) - self.assertEquals(fp.read(2), 'ab') - self.assertEquals(fp.read(2), 'cd') - self.assertEquals(fp.read(2), 'ef') - self.assertEquals(fp.read(2), 'g') - self.assertEquals(fp.read(2), '') + fp = formpost._CappedFileLikeObject(BytesIO(b'abcdefg'), 10) + self.assertEquals(fp.read(2), b'ab') + self.assertEquals(fp.read(2), b'cd') + self.assertEquals(fp.read(2), b'ef') + self.assertEquals(fp.read(2), b'g') + self.assertEquals(fp.read(2), b'') class TestFormPost(unittest.TestCase): @@ -196,7 +198,9 @@ class TestFormPost(unittest.TestCase): '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', ] - wsgi_errors = StringIO() + if six.PY3: + body = [line.encode('utf-8') for line in body] + wsgi_errors = six.StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryNcxTqxSlX7t4TDkR', @@ -242,7 +246,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -282,7 +286,7 @@ class TestFormPost(unittest.TestCase): '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() - wsgi_input = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', @@ -322,8 +326,11 @@ class TestFormPost(unittest.TestCase): '', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', - ])) - wsgi_errors = StringIO() + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + wsgi_input = BytesIO(wsgi_input) + wsgi_errors = six.StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryNcxTqxSlX7t4TDkR', @@ -396,7 +403,7 @@ class TestFormPost(unittest.TestCase): '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() - wsgi_input = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="redirect"', '', @@ -436,8 +443,11 @@ class TestFormPost(unittest.TestCase): '', '-----------------------------168072824752491622650073--', '' - ])) - wsgi_errors = StringIO() + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + wsgi_input = BytesIO(wsgi_input) + wsgi_errors = six.StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=---------------------------168072824752491622650073', @@ -509,7 +519,7 @@ class TestFormPost(unittest.TestCase): '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() - wsgi_input = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="redirect"', '', @@ -549,8 +559,11 @@ class TestFormPost(unittest.TestCase): '', '------WebKitFormBoundaryq3CFxUjfsDMu8XsA--', '' - ])) - wsgi_errors = StringIO() + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + wsgi_input = BytesIO(wsgi_input) + wsgi_errors = six.StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryq3CFxUjfsDMu8XsA', @@ -625,7 +638,7 @@ class TestFormPost(unittest.TestCase): '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() - wsgi_input = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="redirect"', '', @@ -665,8 +678,11 @@ class TestFormPost(unittest.TestCase): '', '-----------------------------7db20d93017c--', '' - ])) - wsgi_errors = StringIO() + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + wsgi_input = BytesIO(wsgi_input) + wsgi_errors = six.StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=---------------------------7db20d93017c', @@ -730,7 +746,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('XX' + '\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -766,7 +782,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -797,7 +813,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 1024, 1, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -838,7 +854,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['QUERY_STRING'] = 'this=should¬=get&passed' - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -873,7 +889,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -916,7 +932,7 @@ class TestFormPost(unittest.TestCase): # Tack on an extra char to redirect, but shouldn't matter since it # should get truncated off on read. redirect += 'b' - env['wsgi.input'] = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', @@ -956,7 +972,10 @@ class TestFormPost(unittest.TestCase): '', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', - ])) + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1001,7 +1020,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', redirect, max_file_size, max_file_count, expires, key) - env['wsgi.input'] = StringIO('\r\n'.join([ + wsgi_input = '\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', @@ -1024,7 +1043,10 @@ class TestFormPost(unittest.TestCase): sig, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', - ])) + ]) + if six.PY3: + wsgi_input = wsgi_input.encode('utf-8') + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1064,7 +1086,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1085,7 +1107,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1113,7 +1135,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) # Stick it in X-Account-Meta-Temp-URL-Key-2 and make sure we get it env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', ['bert', key]) @@ -1150,7 +1172,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env('AUTH_test') # Stick it in X-Container-Meta-Temp-URL-Key-2 and ensure we get it env['swift.container/AUTH_test/container'] = {'meta': meta} @@ -1176,7 +1198,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1214,7 +1236,7 @@ class TestFormPost(unittest.TestCase): sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect?one=two', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1252,7 +1274,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1289,7 +1311,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1322,7 +1344,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) # Change key to invalidate sig env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key + ' is bogus now']) @@ -1356,7 +1378,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('XX' + '\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'XX' + b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1389,7 +1411,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v2/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1422,7 +1444,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '//AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1455,7 +1477,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1//container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1488,7 +1510,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_tst/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([ @@ -1523,7 +1545,7 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1561,7 +1583,7 @@ class TestFormPost(unittest.TestCase): if v == str(expires): body[i] = 'badvalue' break - env['wsgi.input'] = StringIO('\r\n'.join(body)) + env['wsgi.input'] = BytesIO(b'\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1601,7 +1623,8 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) + wsgi_input = b'\r\n'.join(x_delete_body_part + body) + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1643,7 +1666,8 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) + wsgi_input = b'\r\n'.join(x_delete_body_part + body) + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), @@ -1677,7 +1701,8 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) + wsgi_input = b'\r\n'.join(x_delete_body_part + body) + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['swift.container/AUTH_test/container'] = {'meta': {}} @@ -1719,7 +1744,8 @@ class TestFormPost(unittest.TestCase): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) - env['wsgi.input'] = StringIO('\r\n'.join(x_delete_body_part + body)) + wsgi_input = b'\r\n'.join(x_delete_body_part + body) + env['wsgi.input'] = BytesIO(wsgi_input) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), diff --git a/test/unit/common/middleware/test_proxy_logging.py b/test/unit/common/middleware/test_proxy_logging.py index 85cd89163e..1b7eb65e04 100644 --- a/test/unit/common/middleware/test_proxy_logging.py +++ b/test/unit/common/middleware/test_proxy_logging.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from six import moves import unittest from urllib import unquote from logging.handlers import SysLogHandler + import mock +from six import BytesIO from test.unit import FakeLogger from swift.common.utils import get_logger @@ -194,7 +195,7 @@ class TestProxyLogging(unittest.TestCase): app.access_logger = FakeLogger() req = Request.blank(path, environ={ 'REQUEST_METHOD': 'GET', - 'wsgi.input': moves.cStringIO('4321')}) + 'wsgi.input': BytesIO(b'4321')}) stub_times = [18.0, 20.71828182846] iter_response = app(req.environ, lambda *_: None) self.assertEqual('7654321', ''.join(iter_response)) @@ -213,7 +214,7 @@ class TestProxyLogging(unittest.TestCase): req = Request.blank(path, environ={ 'REQUEST_METHOD': 'GET', 'swift.proxy_access_log_made': True, - 'wsgi.input': moves.cStringIO('4321')}) + 'wsgi.input': BytesIO(b'4321')}) stub_times = [18.0, 20.71828182846] iter_response = app(req.environ, lambda *_: None) self.assertEqual('7654321', ''.join(iter_response)) @@ -229,7 +230,7 @@ class TestProxyLogging(unittest.TestCase): app.access_logger = FakeLogger() req = Request.blank(path, environ={ 'REQUEST_METHOD': 'PUT', - 'wsgi.input': moves.cStringIO('654321')}) + 'wsgi.input': BytesIO(b'654321')}) # (it's not a GET, so time() doesn't have a 2nd call) stub_times = [58.2, 58.2 + 7.3321] iter_response = app(req.environ, lambda *_: None) @@ -377,7 +378,7 @@ class TestProxyLogging(unittest.TestCase): req = Request.blank( '/v1/a/c/o/foo', environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': moves.cStringIO('some stuff')}) + 'wsgi.input': BytesIO(b'some stuff')}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] @@ -395,8 +396,7 @@ class TestProxyLogging(unittest.TestCase): req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', - 'wsgi.input': moves.cStringIO( - 'some stuff\nsome other stuff\n')}) + 'wsgi.input': BytesIO(b'some stuff\nsome other stuff\n')}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] diff --git a/test/unit/common/middleware/test_xprofile.py b/test/unit/common/middleware/test_xprofile.py index 0a16a95473..dde6d4f834 100644 --- a/test/unit/common/middleware/test_xprofile.py +++ b/test/unit/common/middleware/test_xprofile.py @@ -20,7 +20,7 @@ import tempfile import unittest from nose import SkipTest -import six +from six import BytesIO from swift import gettext_ as _ from swift.common.swob import Request, Response @@ -110,9 +110,9 @@ class TestProfileMiddleware(unittest.TestCase): self.headers = headers def test_combine_body_qs(self): - body = "profile=all&sort=time&limit=-1&fulldirs=1&nfl_filter=__call__"\ - + "&query=query&metric=nc&format=default" - wsgi_input = six.StringIO(body) + body = (b"profile=all&sort=time&limit=-1&fulldirs=1" + b"&nfl_filter=__call__&query=query&metric=nc&format=default") + wsgi_input = BytesIO(body) environ = {'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'profile=all&format=json', 'wsgi.input': wsgi_input} @@ -128,9 +128,8 @@ class TestProfileMiddleware(unittest.TestCase): self.assertEqual(query_dict['format'], ['default']) def test_call(self): - body = "sort=time&limit=-1&fulldirs=1&nfl_filter="\ - + "&metric=nc" - wsgi_input = six.StringIO(body + '&query=query') + body = b"sort=time&limit=-1&fulldirs=1&nfl_filter=&metric=nc" + wsgi_input = BytesIO(body + b'&query=query') environ = {'HTTP_HOST': 'localhost:8080', 'PATH_INFO': '/__profile__', 'REQUEST_METHOD': 'GET', @@ -140,7 +139,7 @@ class TestProfileMiddleware(unittest.TestCase): self.assert_(resp[0].find('') > 0, resp) self.assertEqual(self.got_statuses, ['200 OK']) self.assertEqual(self.headers, [('content-type', 'text/html')]) - wsgi_input = six.StringIO(body + '&plot=plot') + wsgi_input = BytesIO(body + b'&plot=plot') environ['wsgi.input'] = wsgi_input if PLOTLIB_INSTALLED: resp = self.app(environ, self.start_response) @@ -149,12 +148,12 @@ class TestProfileMiddleware(unittest.TestCase): else: resp = self.app(environ, self.start_response) self.assertEqual(self.got_statuses, ['500 Internal Server Error']) - wsgi_input = six.StringIO(body + '&download=download&format=default') + wsgi_input = BytesIO(body + '&download=download&format=default') environ['wsgi.input'] = wsgi_input resp = self.app(environ, self.start_response) self.assertEqual(self.headers, [('content-type', HTMLViewer.format_dict['default'])]) - wsgi_input = six.StringIO(body + '&download=download&format=json') + wsgi_input = BytesIO(body + '&download=download&format=json') environ['wsgi.input'] = wsgi_input resp = self.app(environ, self.start_response) self.assert_(self.headers == [('content-type', @@ -165,12 +164,12 @@ class TestProfileMiddleware(unittest.TestCase): self.assertEqual(self.got_statuses, ['405 Method Not Allowed'], resp) # use a totally bogus profile identifier - wsgi_input = six.StringIO(body + '&profile=ABC&download=download') + wsgi_input = BytesIO(body + b'&profile=ABC&download=download') environ['wsgi.input'] = wsgi_input resp = self.app(environ, self.start_response) self.assertEqual(self.got_statuses, ['404 Not Found'], resp) - wsgi_input = six.StringIO(body + '&download=download&format=ods') + wsgi_input = BytesIO(body + b'&download=download&format=ods') environ['wsgi.input'] = wsgi_input resp = self.app(environ, self.start_response) if ODFLIB_INSTALLED: @@ -298,9 +297,9 @@ class Test_html_viewer(unittest.TestCase): self.log_files.append(self.profile_log.dump_profile(profiler, pid)) self.viewer = HTMLViewer('__profile__', 'eventlet.green.profile', self.profile_log) - body = "profile=123&profile=456&sort=time&sort=nc&limit=10"\ - + "&fulldirs=1&nfl_filter=getcwd&query=query&metric=nc" - wsgi_input = six.StringIO(body) + body = (b"profile=123&profile=456&sort=time&sort=nc&limit=10" + b"&fulldirs=1&nfl_filter=getcwd&query=query&metric=nc") + wsgi_input = BytesIO(body) environ = {'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'profile=all', 'wsgi.input': wsgi_input} diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index e2c141613f..9e71e9783e 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -21,7 +21,7 @@ import re import time from urllib import quote -from six import StringIO +from six import BytesIO import swift.common.swob from swift.common import utils, exceptions @@ -476,22 +476,22 @@ class TestRequest(unittest.TestCase): def test_blank_body_precedence(self): req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', - 'wsgi.input': StringIO('')}, + 'wsgi.input': BytesIO(b'')}, headers={'Content-Type': 'text/plain'}, body='hi') self.assertEquals(req.path_info, '/') self.assertEquals(req.body, 'hi') self.assertEquals(req.headers['Content-Type'], 'text/plain') self.assertEquals(req.method, 'POST') - body_file = StringIO('asdf') + body_file = BytesIO(b'asdf') req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', - 'wsgi.input': StringIO('')}, + 'wsgi.input': BytesIO(b'')}, headers={'Content-Type': 'text/plain'}, body='hi', body_file=body_file) self.assert_(req.body_file is body_file) req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', - 'wsgi.input': StringIO('')}, + 'wsgi.input': BytesIO(b'')}, headers={'Content-Type': 'text/plain'}, body='hi', content_length=3) self.assertEquals(req.content_length, 3) diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index 87e542b7e2..51c7dfac1e 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -27,6 +27,7 @@ from collections import defaultdict from urllib import quote from eventlet import listen +from six import BytesIO from six import StringIO import mock @@ -556,7 +557,7 @@ class TestWSGI(unittest.TestCase): self.assertTrue('wsgi.input' in newenv) self.assertEquals(newenv['wsgi.input'].read(), '') - oldenv = {'wsgi.input': StringIO('original wsgi.input')} + oldenv = {'wsgi.input': BytesIO(b'original wsgi.input')} newenv = wsgi.make_pre_authed_env(oldenv) self.assertTrue('wsgi.input' in newenv) self.assertEquals(newenv['wsgi.input'].read(), '') diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 620c49f1c8..ac75b05671 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -16,7 +16,6 @@ import operator import os import mock -from six import StringIO import unittest import itertools from contextlib import contextmanager @@ -30,6 +29,8 @@ import random from eventlet import spawn, Timeout, listen import simplejson +from six import BytesIO +from six import StringIO from swift import __version__ as swift_version from swift.common.swob import Request, HeaderKeyDict @@ -2126,7 +2127,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 507) def test_through_call(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -2152,7 +2153,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(outbuf.getvalue()[:4], '404 ') def test_through_call_invalid_path(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -2178,7 +2179,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(outbuf.getvalue()[:4], '400 ') def test_through_call_invalid_path_utf8(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() @@ -2466,7 +2467,7 @@ class TestContainerController(unittest.TestCase): def test_correct_allowed_method(self): # Test correct work for allowed method using # swift.container.server.ContainerController.__call__ - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = container_server.ContainerController( @@ -2503,7 +2504,7 @@ class TestContainerController(unittest.TestCase): def test_not_allowed_method(self): # Test correct work for NOT allowed method using # swift.container.server.ContainerController.__call__ - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = container_server.ContainerController( @@ -2539,7 +2540,7 @@ class TestContainerController(unittest.TestCase): self.assertEqual(response, answer) def test_call_incorrect_replication_method(self): - inbuf = StringIO() + inbuf = BytesIO() errbuf = StringIO() outbuf = StringIO() self.controller = container_server.ContainerController( diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index efe01f254e..5917bb7719 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -23,6 +23,7 @@ import errno import operator import os import mock +import six from six import StringIO import unittest import math @@ -1017,7 +1018,7 @@ class TestObjectController(unittest.TestCase): headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Length': '6'}) - req.environ['wsgi.input'] = WsgiStringIO('VERIFY') + req.environ['wsgi.input'] = WsgiStringIO(b'VERIFY') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 408) @@ -5326,6 +5327,8 @@ class TestObjectServer(unittest.TestCase): "potato potato potato potato potato potato potato", "--boundary123--" )) + if six.PY3: + test_doc = test_doc.encode('utf-8') # phase1 - PUT request with object metadata in footer and # multiphase commit conversation diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 54c790cf07..fcddd3f342 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -41,6 +41,7 @@ import random import mock from eventlet import sleep, spawn, wsgi, listen, Timeout +from six import BytesIO from six import StringIO from six.moves import range from swift.common.utils import hash_path, json, storage_directory, \ @@ -1300,7 +1301,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank( '/v1/a/c1/wrong-o', environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': StringIO("hello")}, + 'wsgi.input': BytesIO(b"hello")}, headers={'Content-Type': 'text/plain', 'Content-Length': '5', 'X-Backend-Storage-Policy-Index': '2'})