jsonutils.to_primitive(): add fallback parameter

For example, to_primitive(fallback=repr) can be used to prevent
serialialization error like "ValueError: Circular reference detected"
when using the JSONFormatter of oslo.log.

If fallback is set, it is also used to convert itertools.count(),
"nasty" objects like types, and to handle TypeError.

Use fallback=six.text_type to convert objects to text.

This patch doesn't change the default behaviour.

Related-Bug: #1593641
Change-Id: Ie0f7f2d09355c3d2a9f7c5ee8f7e02dfea3b073b
This commit is contained in:
Victor Stinner 2017-09-14 10:03:43 +02:00
parent 780a027dfd
commit cdb2f60d26
2 changed files with 83 additions and 5 deletions

View File

@ -58,7 +58,8 @@ _simple_types = ((six.text_type,) + six.integer_types
def to_primitive(value, convert_instances=False, convert_datetime=True,
level=0, max_depth=3, encoding='utf-8'):
level=0, max_depth=3, encoding='utf-8',
fallback=None):
"""Convert a complex object into primitives.
Handy for JSON serialization. We can optionally handle instances,
@ -71,12 +72,22 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
Therefore, ``convert_instances=True`` is lossy ... be aware.
If the object cannot be converted to primitive, it is returned unchanged
if fallback is not set, return fallback(value) otherwise.
.. versionchanged:: 2.22
Added *fallback* parameter.
.. versionchanged:: 1.3
Support UUID encoding.
.. versionchanged:: 1.6
Dictionary keys are now also encoded.
"""
orig_fallback = fallback
if fallback is None:
fallback = six.text_type
# handle obvious types first - order of basic types determined by running
# full tests on nova project, resulting in the following counts:
# 572754 <type 'NoneType'>
@ -124,10 +135,10 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
# value of itertools.count doesn't get caught by nasty_type_tests
# and results in infinite loop when list(value) is called.
if type(value) == itertools.count:
return six.text_type(value)
return fallback(value)
if any(test(value) for test in _nasty_type_tests):
return six.text_type(value)
return fallback(value)
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
# tests that raise an exception in a mocked method that
@ -148,7 +159,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
convert_datetime=convert_datetime,
level=level,
max_depth=max_depth,
encoding=encoding)
encoding=encoding,
fallback=orig_fallback)
if isinstance(value, dict):
return {recursive(k): recursive(v)
for k, v in value.items()}
@ -166,7 +178,10 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
except TypeError:
# Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list().
return six.text_type(value)
return fallback(value)
if orig_fallback is not None:
return orig_fallback(value)
# TODO(gcb) raise ValueError in version 3.0
warnings.warn("Cannot convert %r to primitive, will raise ValueError "

View File

@ -15,7 +15,9 @@
import collections
import datetime
import functools
import ipaddress
import itertools
import json
import mock
@ -28,6 +30,11 @@ import six.moves.xmlrpc_client as xmlrpclib
from oslo_serialization import jsonutils
class ReprObject(object):
def __repr__(self):
return 'repr'
class JSONUtilsTestMixin(object):
json_impl = None
@ -46,6 +53,11 @@ class JSONUtilsTestMixin(object):
def test_dumps(self):
self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'}))
def test_dumps_default(self):
args = [ReprObject()]
convert = functools.partial(jsonutils.to_primitive, fallback=repr)
self.assertEqual('["repr"]', jsonutils.dumps(args, default=convert))
def test_dump_as_bytes(self):
self.assertEqual(b'{"a": "b"}', jsonutils.dump_as_bytes({'a': 'b'}))
@ -338,3 +350,54 @@ class ToPrimitiveTestCase(test_base.BaseTestCase):
msg = msg % {'param': 'hello'}
ret = jsonutils.to_primitive(msg)
self.assertEqual(msg, ret)
def test_fallback(self):
obj = ReprObject()
ret = jsonutils.to_primitive(obj)
self.assertIs(obj, ret)
ret = jsonutils.to_primitive(obj, fallback=repr)
self.assertEqual('repr', ret)
def test_fallback_list(self):
obj = ReprObject()
obj_list = [obj]
ret = jsonutils.to_primitive(obj_list)
self.assertEqual([obj], ret)
ret = jsonutils.to_primitive(obj_list, fallback=repr)
self.assertEqual(['repr'], ret)
def test_fallback_itertools_count(self):
obj = itertools.count(1)
ret = jsonutils.to_primitive(obj)
self.assertEqual(six.text_type(obj), ret)
ret = jsonutils.to_primitive(obj, fallback=lambda _: 'itertools_count')
self.assertEqual('itertools_count', ret)
def test_fallback_nasty(self):
obj = int
ret = jsonutils.to_primitive(obj)
self.assertEqual(six.text_type(obj), ret)
def formatter(typeobj):
return 'type:%s' % typeobj.__name__
ret = jsonutils.to_primitive(obj, fallback=formatter)
self.assertEqual("type:int", ret)
def test_fallback_typeerror(self):
class NotIterable(object):
# __iter__ is not callable, cause a TypeError in to_primitive()
__iter__ = None
obj = NotIterable()
ret = jsonutils.to_primitive(obj)
self.assertEqual(six.text_type(obj), ret)
ret = jsonutils.to_primitive(obj, fallback=lambda _: 'fallback')
self.assertEqual('fallback', ret)