From 1b9018859fa3b420d01ed191b916476db6eb9182 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Wed, 30 Apr 2014 14:53:15 +0200 Subject: [PATCH] Enforce unicode json output for jsonutils.load[s]() simplejson module applies some optimizations on ASCII-only unicode strings which result in non-unicode json output. See details at [1] and [2]. To make sure we always return consistent json output no matter which json implementation is used, we should explicitly convert the input for json.load[s]() to unicode. If user wants to pass a file object of non UTF-8 encoding to json.load[s](), she must also specify this encoding as an argument. To support this scenario too, we've added 'encoding' argument to jsonutils.load[s]() implementation. Made all present JSON tests to run on both supported json library implementations. Added explicit dependency for simplejson to be able to test different implementations in unit tests. Distributors still running Python 2.6 are recommended but not required to install simplejson. Related-Bug: 1314129 [1]: https://code.djangoproject.com/ticket/11742 [2]: https://code.google.com/p/simplejson/issues/detail?id=40 Conflicts: tests/unit/test_jsonutils.py Change-Id: Ic815ca3df94c33edec9104172048b2cd94b92e3f (cherry picked from commit 18f2bc1bf080b41a22c70842e7c127da21c63b8b) --- openstack/common/jsonutils.py | 10 ++++--- test-requirements-py3.txt | 1 + test-requirements.txt | 1 + tests/unit/test_jsonutils.py | 53 +++++++++++++++++++++++++++++++---- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/openstack/common/jsonutils.py b/openstack/common/jsonutils.py index f00a801a1..1a35e0459 100644 --- a/openstack/common/jsonutils.py +++ b/openstack/common/jsonutils.py @@ -31,6 +31,7 @@ This module provides a few things: ''' +import codecs import datetime import functools import inspect @@ -52,6 +53,7 @@ import six.moves.xmlrpc_client as xmlrpclib from openstack.common import gettextutils from openstack.common import importutils +from openstack.common import strutils from openstack.common import timeutils netaddr = importutils.try_import("netaddr") @@ -166,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) -def loads(s): - return json.loads(s) +def loads(s, encoding='utf-8'): + return json.loads(strutils.safe_decode(s, encoding)) -def load(fp): - return json.load(fp) +def load(fp, encoding='utf-8'): + return json.load(codecs.getreader(encoding)(fp)) try: diff --git a/test-requirements-py3.txt b/test-requirements-py3.txt index 987b69338..75f6e7507 100644 --- a/test-requirements-py3.txt +++ b/test-requirements-py3.txt @@ -9,6 +9,7 @@ pep8==1.4.5 pyflakes>=0.7.2,<0.7.4 pyzmq==2.2.0.1 redis +simplejson>=2.0.9 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 testtools>=0.9.34 diff --git a/test-requirements.txt b/test-requirements.txt index 77350d2ff..7cc1e159d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,6 +11,7 @@ pyflakes>=0.7.2,<0.7.4 pylint==0.25.2 pyzmq==2.2.0.1 redis +simplejson>=2.0.9 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 testscenarios>=0.4 diff --git a/tests/unit/test_jsonutils.py b/tests/unit/test_jsonutils.py index 5ab04edab..cf939c4b0 100644 --- a/tests/unit/test_jsonutils.py +++ b/tests/unit/test_jsonutils.py @@ -14,8 +14,11 @@ # under the License. import datetime +import json +import mock import netaddr +import simplejson import six import six.moves.xmlrpc_client as xmlrpclib @@ -24,17 +27,57 @@ from openstack.common import jsonutils from openstack.common import test -class JSONUtilsTestCase(test.BaseTestCase): +class JSONUtilsTestMixin(object): + + json_impl = None + + def setUp(self): + super(JSONUtilsTestMixin, self).setUp() + self.json_patcher = mock.patch.object( + jsonutils, 'json', self.json_impl) + self.json_impl_mock = self.json_patcher.start() + + def tearDown(self): + self.json_patcher.stop() + super(JSONUtilsTestMixin, self).tearDown() def test_dumps(self): - self.assertEqual(jsonutils.dumps({'a': 'b'}), '{"a": "b"}') + self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'})) def test_loads(self): - self.assertEqual(jsonutils.loads('{"a": "b"}'), {'a': 'b'}) + self.assertEqual({'a': 'b'}, jsonutils.loads('{"a": "b"}')) + + def test_loads_unicode(self): + self.assertIsInstance(jsonutils.loads(b'"foo"'), six.text_type) + self.assertIsInstance(jsonutils.loads(u'"foo"'), six.text_type) + + # 'test' in Ukrainian + i18n_str_unicode = u'"\u0442\u0435\u0441\u0442"' + self.assertIsInstance(jsonutils.loads(i18n_str_unicode), six.text_type) + + i18n_str = i18n_str_unicode.encode('utf-8') + self.assertIsInstance(jsonutils.loads(i18n_str), six.text_type) def test_load(self): - x = six.StringIO('{"a": "b"}') - self.assertEqual(jsonutils.load(x), {'a': 'b'}) + + jsontext = u'{"a": "\u0442\u044d\u0441\u0442"}' + expected = {u'a': u'\u0442\u044d\u0441\u0442'} + + for encoding in ('utf-8', 'cp1251'): + fp = six.BytesIO(jsontext.encode(encoding)) + result = jsonutils.load(fp, encoding=encoding) + self.assertEqual(expected, result) + for key, val in result.items(): + self.assertIsInstance(key, six.text_type) + self.assertIsInstance(val, six.text_type) + + +class JSONUtilsTestJson(JSONUtilsTestMixin, test.BaseTestCase): + json_impl = json + + +class JSONUtilsTestSimpleJson(JSONUtilsTestMixin, test.BaseTestCase): + json_impl = simplejson class ToPrimitiveTestCase(test.BaseTestCase):