[Py34] Enable api.openstack.test_wsgi unit test

Issues:
- Method `webob.multidict.MultiDict.items` returns list object in py2 env[1]
  and listiterator in  py3 env[2]. ListIterator prevents changing values of
  headers in a loop(changes of item's order prevent skipping some headers).
- Statement `nova.utils.utf8(str(some_value))` did not work for bytes type in
  py3 env. Good example is `test_resource_headers_are_utf8` test. Int value 1
  became to b'b"b\'1\'"' after deserialization.
- Wrong checks for utf-8. (including check for encoding headers names, which
  is not encoded anywhere in the code)
- Wrong checks for `body` property of `webob.response.Response` cls. Method
  `_body__get` returns bytes[3]. There is no problem in python 2, since
  type(b"some") equals to type("some"), but it wrong for python 3.

[1] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/multidict.py#L266-L267
[2] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/multidict.py#L260-L264
[3] - https://github.com/Pylons/webob/blob/1.5.0b0/webob/response.py#L350

Related bp nova-python3-mitaka

Change-Id: I15150c8c8f204081b9b5d8bc28e622692d7d749a
This commit is contained in:
Andrey Kurilin 2015-10-05 18:18:48 +03:00 committed by Andrey Kurilin
parent 240df42859
commit cc1ff0051a
6 changed files with 116 additions and 71 deletions

View File

@ -297,7 +297,7 @@ class JSONDictSerializer(DictSerializer):
"""Default JSON request body serialization."""
def default(self, data):
return jsonutils.dumps(data)
return six.text_type(jsonutils.dumps(data))
def serializers(**serializers):
@ -457,14 +457,20 @@ class ResponseObject(object):
default_serializers)
serializer = _serializer()
response = webob.Response()
body = None
if self.obj is not None:
body = serializer.serialize(self.obj)
response = webob.Response(body=body)
if response.headers.get('Content-Length'):
# NOTE(andreykurilin): we need to encode 'Content-Length' header,
# since webob.Response auto sets it if "body" attr is presented.
# https://github.com/Pylons/webob/blob/1.5.0b0/webob/response.py#L147
response.headers['Content-Length'] = utils.utf8(
response.headers['Content-Length'])
response.status_int = self.code
for hdr, value in self._headers.items():
response.headers[hdr] = utils.utf8(str(value))
response.headers[hdr] = utils.utf8(value)
response.headers['Content-Type'] = utils.utf8(content_type)
if self.obj is not None:
response.body = serializer.serialize(self.obj)
return response
@property
@ -645,7 +651,7 @@ class Resource(wsgi.Application):
content_type = request.get_content_type()
except exception.InvalidContentType:
LOG.debug("Unrecognized Content-Type provided in request")
return None, ''
return None, b''
return content_type, request.body
@ -860,10 +866,9 @@ class Resource(wsgi.Application):
self.default_serializers)
if hasattr(response, 'headers'):
for hdr, val in response.headers.items():
for hdr, val in list(response.headers.items()):
# Headers must be utf-8 strings
response.headers[hdr] = utils.utf8(str(val))
response.headers[hdr] = utils.utf8(val)
if not request.api_version_request.is_null():
response.headers[API_VERSION_REQUEST_HEADER] = \
@ -1158,7 +1163,7 @@ class Fault(webob.exc.HTTPException):
def __init__(self, exception):
"""Create a Fault for the given webob.exc.exception."""
self.wrapped_exc = exception
for key, value in self.wrapped_exc.headers.items():
for key, value in list(self.wrapped_exc.headers.items()):
self.wrapped_exc.headers[key] = str(value)
self.status_int = exception.status_int
@ -1195,8 +1200,9 @@ class Fault(webob.exc.HTTPException):
'application/json': JSONDictSerializer(),
}[content_type]
self.wrapped_exc.body = serializer.serialize(fault_data)
self.wrapped_exc.content_type = content_type
self.wrapped_exc.charset = 'UTF-8'
self.wrapped_exc.text = serializer.serialize(fault_data)
return self.wrapped_exc
@ -1245,7 +1251,8 @@ class RateLimitFault(webob.exc.HTTPException):
}[content_type]
content = serializer.serialize(self.content)
self.wrapped_exc.body = content
self.wrapped_exc.charset = 'UTF-8'
self.wrapped_exc.content_type = content_type
self.wrapped_exc.text = content
return self.wrapped_exc

View File

@ -23,6 +23,7 @@ from nova import exception
from nova import i18n
from nova import test
from nova.tests.unit.api.openstack import fakes
from nova.tests.unit import matchers
from nova.tests.unit import utils
from oslo_serialization import jsonutils
@ -32,13 +33,13 @@ class RequestTest(test.NoDBTestCase):
def test_content_type_missing(self):
request = wsgi.Request.blank('/tests/123', method='POST')
request.body = "<body />"
request.body = b"<body />"
self.assertIsNone(request.get_content_type())
def test_content_type_unsupported(self):
request = wsgi.Request.blank('/tests/123', method='POST')
request.headers["Content-Type"] = "text/html"
request.body = "asdf<br />"
request.body = b"asdf<br />"
self.assertRaises(exception.InvalidContentType,
request.get_content_type)
@ -250,7 +251,7 @@ class JSONDeserializerTest(test.NoDBTestCase):
self.assertEqual(deserializer.deserialize(data), as_dict)
def test_json_valid_utf8(self):
data = """{"server": {"min_count": 1, "flavorRef": "1",
data = b"""{"server": {"min_count": 1, "flavorRef": "1",
"name": "\xe6\xa6\x82\xe5\xbf\xb5",
"imageRef": "10bab10c-1304-47d",
"max_count": 1}} """
@ -269,7 +270,7 @@ class JSONDeserializerTest(test.NoDBTestCase):
def test_json_invalid_utf8(self):
"""Send invalid utf-8 to JSONDeserializer."""
data = """{"server": {"min_count": 1, "flavorRef": "1",
data = b"""{"server": {"min_count": 1, "flavorRef": "1",
"name": "\xf0\x28\x8c\x28",
"imageRef": "10bab10c-1304-47d",
"max_count": 1}} """
@ -301,7 +302,7 @@ class ResourceTest(test.NoDBTestCase):
app = fakes.TestRouterV21(Controller())
req = webob.Request.blank('/tests')
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
@mock.patch("nova.api.openstack.api_version_request.max_api_version")
@ -320,7 +321,7 @@ class ResourceTest(test.NoDBTestCase):
req = webob.Request.blank('/tests')
req.headers = {self.header_name: version}
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
def test_resource_receives_api_version_request_invalid(self):
@ -345,15 +346,15 @@ class ResourceTest(test.NoDBTestCase):
# the default method is GET
req = webob.Request.blank('/tests')
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
req.body = '{"body": {"key": "value"}}'
req.body = b'{"body": {"key": "value"}}'
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
req.content_type = 'application/json'
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
def test_resource_call_with_method_post(self):
@ -368,20 +369,20 @@ class ResourceTest(test.NoDBTestCase):
app = fakes.TestRouter(Controller())
req = webob.Request.blank('/tests', method="POST",
content_type='application/json')
req.body = '{"body": {"key": "value"}}'
req.body = b'{"body": {"key": "value"}}'
expected_body = {'body': {
"key": "value"
}
}
response = req.get_response(app)
self.assertEqual(response.status_int, 200)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
# verify without body
expected_body = None
req.body = None
response = req.get_response(app)
self.assertEqual(response.status_int, 200)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
# the body is validated in the controller
expected_body = {'body': None}
response = req.get_response(app)
@ -402,13 +403,13 @@ class ResourceTest(test.NoDBTestCase):
app = fakes.TestRouter(Controller())
req = webob.Request.blank('/tests/test_id', method="PUT",
content_type='application/json')
req.body = '{"body": {"key": "value"}}'
req.body = b'{"body": {"key": "value"}}'
expected_body = {'body': {
"key": "value"
}
}
response = req.get_response(app)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
self.assertEqual(response.status_int, 200)
req.body = None
expected_body = None
@ -416,7 +417,7 @@ class ResourceTest(test.NoDBTestCase):
self.assertEqual(response.status_int, 200)
# verify no content_type is contained in the request
req.content_type = None
req.body = '{"body": {"key": "value"}}'
req.body = b'{"body": {"key": "value"}}'
response = req.get_response(app)
expected_unsupported_type_body = {'badRequest':
{'message': 'Unsupported Content-Type', 'code': 400}}
@ -434,12 +435,12 @@ class ResourceTest(test.NoDBTestCase):
req = webob.Request.blank('/tests/test_id', method="DELETE")
response = req.get_response(app)
self.assertEqual(response.status_int, 200)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
# ignore the body
req.body = '{"body": {"key": "value"}}'
req.body = b'{"body": {"key": "value"}}'
response = req.get_response(app)
self.assertEqual(response.status_int, 200)
self.assertEqual(response.body, 'success')
self.assertEqual(b'success', response.body)
def test_resource_not_authorized(self):
class Controller(object):
@ -552,11 +553,11 @@ class ResourceTest(test.NoDBTestCase):
request = wsgi.Request.blank('/', method='POST')
request.headers['Content-Type'] = 'application/none'
request.body = 'foo'
request.body = b'foo'
content_type, body = resource.get_body(request)
self.assertIsNone(content_type)
self.assertEqual(body, '')
self.assertEqual(b'', body)
def test_get_body_no_content_type(self):
class Controller(object):
@ -567,11 +568,11 @@ class ResourceTest(test.NoDBTestCase):
resource = wsgi.Resource(controller)
request = wsgi.Request.blank('/', method='POST')
request.body = 'foo'
request.body = b'foo'
content_type, body = resource.get_body(request)
self.assertIsNone(content_type)
self.assertEqual(body, 'foo')
self.assertEqual(b'foo', body)
def test_get_body_no_content_body(self):
class Controller(object):
@ -583,11 +584,11 @@ class ResourceTest(test.NoDBTestCase):
request = wsgi.Request.blank('/', method='POST')
request.headers['Content-Type'] = 'application/json'
request.body = ''
request.body = b''
content_type, body = resource.get_body(request)
self.assertEqual('application/json', content_type)
self.assertEqual(body, '')
self.assertEqual(b'', body)
def test_get_body(self):
class Controller(object):
@ -599,11 +600,11 @@ class ResourceTest(test.NoDBTestCase):
request = wsgi.Request.blank('/', method='POST')
request.headers['Content-Type'] = 'application/json'
request.body = 'foo'
request.body = b'foo'
content_type, body = resource.get_body(request)
self.assertEqual(content_type, 'application/json')
self.assertEqual(body, 'foo')
self.assertEqual(b'foo', body)
def test_get_request_id_with_dict_response_body(self):
class Controller(wsgi.Controller):
@ -614,7 +615,7 @@ class ResourceTest(test.NoDBTestCase):
app = fakes.TestRouter(Controller())
response = req.get_response(app)
self.assertIn('nova.context', req.environ)
self.assertEqual(response.body, '{"foo": "bar"}')
self.assertEqual(b'{"foo": "bar"}', response.body)
self.assertEqual(response.status_int, 200)
def test_no_request_id_with_str_response_body(self):
@ -630,7 +631,7 @@ class ResourceTest(test.NoDBTestCase):
# our wsgi setup, ideally it would be there.
expected_header = self.get_req_id_header_name(req)
self.assertFalse(hasattr(response.headers, expected_header))
self.assertEqual(response.body, 'foo')
self.assertEqual(b'foo', response.body)
self.assertEqual(response.status_int, 200)
def test_get_request_id_no_response_body(self):
@ -642,7 +643,7 @@ class ResourceTest(test.NoDBTestCase):
app = fakes.TestRouter(Controller())
response = req.get_response(app)
self.assertIn('nova.context', req.environ)
self.assertEqual(response.body, '')
self.assertEqual(b'', response.body)
self.assertEqual(response.status_int, 200)
def test_deserialize_badtype(self):
@ -1006,14 +1007,12 @@ class ResourceTest(test.NoDBTestCase):
req = webob.Request.blank('/tests')
app = fakes.TestRouter(Controller())
response = req.get_response(app)
for hdr, val in six.iteritems(response.headers):
for val in six.itervalues(response.headers):
# All headers must be utf8
self.assertIsInstance(hdr, str)
self.assertIsInstance(val, str)
self.assertEqual(response.headers['x-header1'], '1')
self.assertEqual(response.headers['x-header2'], 'header2')
self.assertEqual(response.headers['x-header3'], 'header3')
self.assertThat(val, matchers.EncodedByUTF8())
self.assertEqual(b'1', response.headers['x-header1'])
self.assertEqual(b'header2', response.headers['x-header2'])
self.assertEqual(b'header3', response.headers['x-header3'])
def test_resource_valid_utf8_body(self):
class Controller(object):
@ -1021,8 +1020,8 @@ class ResourceTest(test.NoDBTestCase):
return body
req = webob.Request.blank('/tests/test_id', method="PUT")
body = """ {"name": "\xe6\xa6\x82\xe5\xbf\xb5" } """
expected_body = '{"name": "\\u6982\\u5ff5"}'
body = b""" {"name": "\xe6\xa6\x82\xe5\xbf\xb5" } """
expected_body = b'{"name": "\\u6982\\u5ff5"}'
req.body = body
req.headers['Content-Type'] = 'application/json'
app = fakes.TestRouter(Controller())
@ -1036,7 +1035,7 @@ class ResourceTest(test.NoDBTestCase):
return body
req = webob.Request.blank('/tests/test_id', method="PUT")
body = """ {"name": "\xf0\x28\x8c\x28" } """
body = b""" {"name": "\xf0\x28\x8c\x28" } """
req.body = body
req.headers['Content-Type'] = 'application/json'
app = fakes.TestRouter(Controller())
@ -1130,17 +1129,16 @@ class ResponseObjectTest(test.NoDBTestCase):
for content_type, mtype in wsgi._MEDIA_TYPE_MAP.items():
request = wsgi.Request.blank('/tests/123')
response = robj.serialize(request, content_type)
self.assertEqual(response.headers['Content-Type'], content_type)
self.assertEqual(content_type.encode("utf-8"),
response.headers['Content-Type'])
for hdr, val in six.iteritems(response.headers):
# All headers must be utf8
self.assertIsInstance(hdr, str)
self.assertIsInstance(val, str)
self.assertEqual(response.headers['X-header1'], 'header1')
self.assertEqual(response.headers['X-header2'], 'header2')
self.assertEqual(response.headers['X-header3'], '3')
self.assertThat(val, matchers.EncodedByUTF8())
self.assertEqual(b'header1', response.headers['X-header1'])
self.assertEqual(b'header2', response.headers['X-header2'])
self.assertEqual(b'3', response.headers['X-header3'])
self.assertEqual(response.status_int, 202)
self.assertEqual(response.body, mtype)
self.assertEqual(mtype.encode("utf-8"), response.body)
class ValidBodyTest(test.NoDBTestCase):

View File

@ -21,6 +21,7 @@ import pprint
from lxml import etree
import six
from testtools import content
import testtools.matchers
class DictKeysMismatch(object):
@ -528,3 +529,23 @@ class XMLMatches(object):
actual_child_idx)
# The nodes match
return True
class EncodedByUTF8(object):
def match(self, obj):
if isinstance(obj, six.binary_type):
if hasattr(obj, "decode"):
try:
obj.decode("utf-8")
except UnicodeDecodeError:
return testtools.matchers.Mismatch(
"%s is not encoded in UTF-8." % obj)
else:
reason = ("Type of '%(obj)s' is '%(obj_type)s', "
"should be '%(correct_type)s'."
% {
"obj": obj,
"obj_type": type(obj).__name__,
"correct_type": six.binary_type.__name__
})
return testtools.matchers.Mismatch(reason)

View File

@ -1374,3 +1374,24 @@ class SpawnTestCase(SpawnNTestCase):
def setUp(self):
super(SpawnTestCase, self).setUp()
self.spawn_name = 'spawn'
class UT8TestCase(test.NoDBTestCase):
def test_none_value(self):
self.assertIsInstance(utils.utf8(None), type(None))
def test_bytes_value(self):
some_value = b"fake data"
return_value = utils.utf8(some_value)
# check that type of returned value doesn't changed
self.assertIsInstance(return_value, type(some_value))
self.assertEqual(some_value, return_value)
def test_not_text_type(self):
return_value = utils.utf8(1)
self.assertEqual(b"1", return_value)
self.assertIsInstance(return_value, six.binary_type)
def test_text_type_with_encoding(self):
some_value = 'test\u2026config'
self.assertEqual(some_value, utils.utf8(some_value).decode("utf-8"))

View File

@ -587,14 +587,17 @@ def xhtml_escape(value):
def utf8(value):
"""Try to turn a string into utf-8 if possible.
Code is directly from the utf8 function in
The original code was copied from the utf8 function in
http://github.com/facebook/tornado/blob/master/tornado/escape.py
"""
if isinstance(value, six.text_type):
return value.encode('utf-8')
assert isinstance(value, str)
return value
if value is None or isinstance(value, six.binary_type):
return value
if not isinstance(value, six.text_type):
value = six.text_type(value)
return value.encode('utf-8')
def check_isinstance(obj, cls):

View File

@ -140,11 +140,6 @@ nova.tests.unit.api.openstack.compute.test_volumes.VolumeApiTestV21
nova.tests.unit.api.openstack.compute.test_volumes.VolumeAttachTestsV2
nova.tests.unit.api.openstack.compute.test_volumes.VolumeAttachTestsV21
nova.tests.unit.api.openstack.test_faults.TestFaultWrapper
nova.tests.unit.api.openstack.test_faults.TestFaults
nova.tests.unit.api.openstack.test_wsgi.JSONDeserializerTest
nova.tests.unit.api.openstack.test_wsgi.RequestTest
nova.tests.unit.api.openstack.test_wsgi.ResourceTest
nova.tests.unit.api.openstack.test_wsgi.ResponseObjectTest
nova.tests.unit.api.test_compute_req_id.RequestIdTest
nova.tests.unit.api.test_validator.ValidatorTestCase
nova.tests.unit.api.test_wsgi.Test