Enforce unicode json output for jsonutils.load[s]()
simplejson module applies some optimizations on ASCII-only unicode
strings which result in non-unicode json output. See details at [1]
and [2]. To make sure we always return consistent json output no
matter which json implementation is used, we should explicitly convert
the input for json.load[s]() to unicode.
If user wants to pass a file object of non UTF-8 encoding to
json.load[s](), she must also specify this encoding as an argument. To
support this scenario too, we've added 'encoding' argument to
jsonutils.load[s]() implementation.
Made all present JSON tests to run on both supported json library
implementations.
Added explicit dependency for simplejson to be able to test different
implementations in unit tests. Distributors still running Python 2.6
are recommended but not required to install simplejson.
Related-Bug: 1314129
[1]: https://code.djangoproject.com/ticket/11742
[2]: https://code.google.com/p/simplejson/issues/detail?id=40
Conflicts:
test-requirements-py3.txt
tests/unit/test_jsonutils.py
Change-Id: Ic815ca3df94c33edec9104172048b2cd94b92e3f
(cherry picked from commit 18f2bc1bf0
)
This commit is contained in:
parent
f1cb775ede
commit
6a290f5ded
|
@ -33,6 +33,7 @@ This module provides a few things:
|
|||
'''
|
||||
|
||||
|
||||
import codecs
|
||||
import datetime
|
||||
import functools
|
||||
import inspect
|
||||
|
@ -48,6 +49,7 @@ import six
|
|||
|
||||
from openstack.common import gettextutils
|
||||
from openstack.common import importutils
|
||||
from openstack.common import strutils
|
||||
from openstack.common import timeutils
|
||||
|
||||
netaddr = importutils.try_import("netaddr")
|
||||
|
@ -162,12 +164,12 @@ def dumps(value, default=to_primitive, **kwargs):
|
|||
return json.dumps(value, default=default, **kwargs)
|
||||
|
||||
|
||||
def loads(s):
|
||||
return json.loads(s)
|
||||
def loads(s, encoding='utf-8'):
|
||||
return json.loads(strutils.safe_decode(s, encoding))
|
||||
|
||||
|
||||
def load(fp):
|
||||
return json.load(fp)
|
||||
def load(fp, encoding='utf-8'):
|
||||
return json.load(codecs.getreader(encoding)(fp))
|
||||
|
||||
|
||||
try:
|
||||
|
|
|
@ -11,6 +11,7 @@ pyflakes==0.7.2
|
|||
pylint==0.25.2
|
||||
pyzmq==2.2.0.1
|
||||
redis
|
||||
simplejson>=2.0.9
|
||||
sphinx>=1.1.2,<1.2
|
||||
testrepository>=0.0.17
|
||||
testtools>=0.9.32
|
||||
|
|
|
@ -16,12 +16,15 @@
|
|||
# under the License.
|
||||
|
||||
import datetime
|
||||
import json
|
||||
try:
|
||||
import xmlrpclib
|
||||
except ImportError:
|
||||
xmlrpclib = None
|
||||
|
||||
import mock
|
||||
import netaddr
|
||||
import simplejson
|
||||
import six
|
||||
import testtools
|
||||
|
||||
|
@ -30,17 +33,57 @@ from openstack.common import jsonutils
|
|||
from openstack.common import test
|
||||
|
||||
|
||||
class JSONUtilsTestCase(test.BaseTestCase):
|
||||
class JSONUtilsTestMixin(object):
|
||||
|
||||
json_impl = None
|
||||
|
||||
def setUp(self):
|
||||
super(JSONUtilsTestMixin, self).setUp()
|
||||
self.json_patcher = mock.patch.object(
|
||||
jsonutils, 'json', self.json_impl)
|
||||
self.json_impl_mock = self.json_patcher.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.json_patcher.stop()
|
||||
super(JSONUtilsTestMixin, self).tearDown()
|
||||
|
||||
def test_dumps(self):
|
||||
self.assertEqual(jsonutils.dumps({'a': 'b'}), '{"a": "b"}')
|
||||
self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'}))
|
||||
|
||||
def test_loads(self):
|
||||
self.assertEqual(jsonutils.loads('{"a": "b"}'), {'a': 'b'})
|
||||
self.assertEqual({'a': 'b'}, jsonutils.loads('{"a": "b"}'))
|
||||
|
||||
def test_loads_unicode(self):
|
||||
self.assertIsInstance(jsonutils.loads(b'"foo"'), six.text_type)
|
||||
self.assertIsInstance(jsonutils.loads(u'"foo"'), six.text_type)
|
||||
|
||||
# 'test' in Ukrainian
|
||||
i18n_str_unicode = u'"\u0442\u0435\u0441\u0442"'
|
||||
self.assertIsInstance(jsonutils.loads(i18n_str_unicode), six.text_type)
|
||||
|
||||
i18n_str = i18n_str_unicode.encode('utf-8')
|
||||
self.assertIsInstance(jsonutils.loads(i18n_str), six.text_type)
|
||||
|
||||
def test_load(self):
|
||||
x = six.StringIO('{"a": "b"}')
|
||||
self.assertEqual(jsonutils.load(x), {'a': 'b'})
|
||||
|
||||
jsontext = u'{"a": "\u0442\u044d\u0441\u0442"}'
|
||||
expected = {u'a': u'\u0442\u044d\u0441\u0442'}
|
||||
|
||||
for encoding in ('utf-8', 'cp1251'):
|
||||
fp = six.BytesIO(jsontext.encode(encoding))
|
||||
result = jsonutils.load(fp, encoding=encoding)
|
||||
self.assertEqual(expected, result)
|
||||
for key, val in result.items():
|
||||
self.assertIsInstance(key, six.text_type)
|
||||
self.assertIsInstance(val, six.text_type)
|
||||
|
||||
|
||||
class JSONUtilsTestJson(JSONUtilsTestMixin, test.BaseTestCase):
|
||||
json_impl = json
|
||||
|
||||
|
||||
class JSONUtilsTestSimpleJson(JSONUtilsTestMixin, test.BaseTestCase):
|
||||
json_impl = simplejson
|
||||
|
||||
|
||||
class ToPrimitiveTestCase(test.BaseTestCase):
|
||||
|
|
Loading…
Reference in New Issue