Enforce unicode json output for jsonutils.load[s]()

simplejson module applies some optimizations on ASCII-only unicode
strings which result in non-unicode json output. See details at [1]
and [2]. To make sure we always return consistent json output no
matter which json implementation is used, we should explicitly convert
the input for json.load[s]() to unicode.

If user wants to pass a file object of non UTF-8 encoding to
json.load[s](), she must also specify this encoding as an argument. To
support this scenario too, we've added 'encoding' argument to
jsonutils.load[s]() implementation.

Made all present JSON tests to run on both supported json library
implementations.

Added explicit dependency for simplejson to be able to test different
implementations in unit tests. Distributors still running Python 2.6
are recommended but not required to install simplejson.

Related-Bug: 1314129

[1]: https://code.djangoproject.com/ticket/11742
[2]: https://code.google.com/p/simplejson/issues/detail?id=40

Change-Id: Ic815ca3df94c33edec9104172048b2cd94b92e3f

openstack/common/jsonutils.py

@@ -31,6 +31,7 @@ This module provides a few things:
 '''
 
 
+import codecs
 import datetime
 import functools
 import inspect
@@ -52,6 +53,7 @@ import six.moves.xmlrpc_client as xmlrpclib
 
 from openstack.common import gettextutils
 from openstack.common import importutils
+from openstack.common import strutils
 from openstack.common import timeutils
 
 netaddr = importutils.try_import("netaddr")
@@ -166,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs):
     return json.dumps(value, default=default, **kwargs)
 
 
-def loads(s):
-    return json.loads(s)
+def loads(s, encoding='utf-8'):
+    return json.loads(strutils.safe_decode(s, encoding))
 
 
-def load(fp):
-    return json.load(fp)
+def load(fp, encoding='utf-8'):
+    return json.load(codecs.getreader(encoding)(fp))
 
 
 try:

test-requirements-py3.txt

@@ -6,6 +6,7 @@ mock>=1.0
 mox3>=0.7.0
 pyzmq==2.2.0.1
 redis
+simplejson>=2.0.9
 sphinx>=1.1.2,<1.2
 testrepository>=0.0.18
 testtools>=0.9.34

test-requirements.txt

@@ -8,6 +8,7 @@ MySQL-python
 pylint==0.25.2
 pyzmq==2.2.0.1
 redis
+simplejson>=2.0.9
 sphinx>=1.1.2,<1.2
 testrepository>=0.0.18
 testscenarios>=0.4

tests/unit/test_jsonutils.py

@@ -14,9 +14,12 @@
 #    under the License.
 
 import datetime
+import json
 
+import mock
 import netaddr
 from oslotest import base as test_base
+import simplejson
 import six
 import six.moves.xmlrpc_client as xmlrpclib
 
@@ -24,17 +27,57 @@ from openstack.common import gettextutils
 from openstack.common import jsonutils
 
 
-class JSONUtilsTestCase(test_base.BaseTestCase):
+class JSONUtilsTestMixin(object):
+
+    json_impl = None
+
+    def setUp(self):
+        super(JSONUtilsTestMixin, self).setUp()
+        self.json_patcher = mock.patch.object(
+            jsonutils, 'json', self.json_impl)
+        self.json_impl_mock = self.json_patcher.start()
+
+    def tearDown(self):
+        self.json_patcher.stop()
+        super(JSONUtilsTestMixin, self).tearDown()
 
     def test_dumps(self):
-        self.assertEqual(jsonutils.dumps({'a': 'b'}), '{"a": "b"}')
+        self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'}))
 
     def test_loads(self):
-        self.assertEqual(jsonutils.loads('{"a": "b"}'), {'a': 'b'})
+        self.assertEqual({'a': 'b'}, jsonutils.loads('{"a": "b"}'))
+
+    def test_loads_unicode(self):
+        self.assertIsInstance(jsonutils.loads(b'"foo"'), six.text_type)
+        self.assertIsInstance(jsonutils.loads(u'"foo"'), six.text_type)
+
+        # 'test' in Ukrainian
+        i18n_str_unicode = u'"\u0442\u0435\u0441\u0442"'
+        self.assertIsInstance(jsonutils.loads(i18n_str_unicode), six.text_type)
+
+        i18n_str = i18n_str_unicode.encode('utf-8')
+        self.assertIsInstance(jsonutils.loads(i18n_str), six.text_type)
 
     def test_load(self):
-        x = six.StringIO('{"a": "b"}')
-        self.assertEqual(jsonutils.load(x), {'a': 'b'})
+
+        jsontext = u'{"a": "\u0442\u044d\u0441\u0442"}'
+        expected = {u'a': u'\u0442\u044d\u0441\u0442'}
+
+        for encoding in ('utf-8', 'cp1251'):
+            fp = six.BytesIO(jsontext.encode(encoding))
+            result = jsonutils.load(fp, encoding=encoding)
+            self.assertEqual(expected, result)
+            for key, val in result.items():
+                self.assertIsInstance(key, six.text_type)
+                self.assertIsInstance(val, six.text_type)
+
+
+class JSONUtilsTestJson(JSONUtilsTestMixin, test_base.BaseTestCase):
+    json_impl = json
+
+
+class JSONUtilsTestSimpleJson(JSONUtilsTestMixin, test_base.BaseTestCase):
+    json_impl = simplejson
 
 
 class ToPrimitiveTestCase(test_base.BaseTestCase):