From 01177314a1eef8b9c3e1a2cfaa06854be73f9881 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 Dec 2017 17:39:51 +0100 Subject: [PATCH] jsonutils.to_primitive(): add fallback parameter For example, to_primitive(fallback=repr) can be used to prevent serialialization error like "ValueError: Circular reference detected" when using the JSONFormatter of oslo.log. If fallback is set, it is also used to convert itertools.count(), "nasty" objects like types, and to handle TypeError. Use fallback=six.text_type to convert objects to text. This patch doesn't change the default behaviour. Related-Bug: #1593641 Change-Id: Ie0f7f2d09355c3d2a9f7c5ee8f7e02dfea3b073b (cherry picked from commit cdb2f60d26e3b65b6370f87b2e9864045651c117) --- oslo_serialization/jsonutils.py | 25 +++++++-- oslo_serialization/tests/test_jsonutils.py | 63 ++++++++++++++++++++++ 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/oslo_serialization/jsonutils.py b/oslo_serialization/jsonutils.py index 9659563..b2b6f8d 100644 --- a/oslo_serialization/jsonutils.py +++ b/oslo_serialization/jsonutils.py @@ -57,7 +57,8 @@ _simple_types = ((six.text_type,) + six.integer_types def to_primitive(value, convert_instances=False, convert_datetime=True, - level=0, max_depth=3, encoding='utf-8'): + level=0, max_depth=3, encoding='utf-8', + fallback=None): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, @@ -70,12 +71,22 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, Therefore, ``convert_instances=True`` is lossy ... be aware. + If the object cannot be converted to primitive, it is returned unchanged + if fallback is not set, return fallback(value) otherwise. + + .. versionchanged:: 2.22 + Added *fallback* parameter. + .. versionchanged:: 1.3 Support UUID encoding. .. versionchanged:: 1.6 Dictionary keys are now also encoded. """ + orig_fallback = fallback + if fallback is None: + fallback = six.text_type + # handle obvious types first - order of basic types determined by running # full tests on nova project, resulting in the following counts: # 572754 @@ -123,10 +134,10 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, # value of itertools.count doesn't get caught by nasty_type_tests # and results in infinite loop when list(value) is called. if type(value) == itertools.count: - return six.text_type(value) + return fallback(value) if any(test(value) for test in _nasty_type_tests): - return six.text_type(value) + return fallback(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that @@ -147,7 +158,8 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, convert_datetime=convert_datetime, level=level, max_depth=max_depth, - encoding=encoding) + encoding=encoding, + fallback=orig_fallback) if isinstance(value, dict): return {recursive(k): recursive(v) for k, v in value.items()} @@ -165,7 +177,10 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). - return six.text_type(value) + return fallback(value) + + if orig_fallback is not None: + return orig_fallback(value) return value diff --git a/oslo_serialization/tests/test_jsonutils.py b/oslo_serialization/tests/test_jsonutils.py index 9eb3c25..b54a8ab 100644 --- a/oslo_serialization/tests/test_jsonutils.py +++ b/oslo_serialization/tests/test_jsonutils.py @@ -15,7 +15,9 @@ import collections import datetime +import functools import ipaddress +import itertools import json import mock @@ -28,6 +30,11 @@ import six.moves.xmlrpc_client as xmlrpclib from oslo_serialization import jsonutils +class ReprObject(object): + def __repr__(self): + return 'repr' + + class JSONUtilsTestMixin(object): json_impl = None @@ -46,6 +53,11 @@ class JSONUtilsTestMixin(object): def test_dumps(self): self.assertEqual('{"a": "b"}', jsonutils.dumps({'a': 'b'})) + def test_dumps_default(self): + args = [ReprObject()] + convert = functools.partial(jsonutils.to_primitive, fallback=repr) + self.assertEqual('["repr"]', jsonutils.dumps(args, default=convert)) + def test_dump_as_bytes(self): self.assertEqual(b'{"a": "b"}', jsonutils.dump_as_bytes({'a': 'b'})) @@ -336,3 +348,54 @@ class ToPrimitiveTestCase(test_base.BaseTestCase): msg = msg % {'param': 'hello'} ret = jsonutils.to_primitive(msg) self.assertEqual(msg, ret) + + def test_fallback(self): + obj = ReprObject() + + ret = jsonutils.to_primitive(obj) + self.assertIs(obj, ret) + + ret = jsonutils.to_primitive(obj, fallback=repr) + self.assertEqual('repr', ret) + + def test_fallback_list(self): + obj = ReprObject() + obj_list = [obj] + + ret = jsonutils.to_primitive(obj_list) + self.assertEqual([obj], ret) + + ret = jsonutils.to_primitive(obj_list, fallback=repr) + self.assertEqual(['repr'], ret) + + def test_fallback_itertools_count(self): + obj = itertools.count(1) + + ret = jsonutils.to_primitive(obj) + self.assertEqual(six.text_type(obj), ret) + + ret = jsonutils.to_primitive(obj, fallback=lambda _: 'itertools_count') + self.assertEqual('itertools_count', ret) + + def test_fallback_nasty(self): + obj = int + ret = jsonutils.to_primitive(obj) + self.assertEqual(six.text_type(obj), ret) + + def formatter(typeobj): + return 'type:%s' % typeobj.__name__ + ret = jsonutils.to_primitive(obj, fallback=formatter) + self.assertEqual("type:int", ret) + + def test_fallback_typeerror(self): + class NotIterable(object): + # __iter__ is not callable, cause a TypeError in to_primitive() + __iter__ = None + + obj = NotIterable() + + ret = jsonutils.to_primitive(obj) + self.assertEqual(six.text_type(obj), ret) + + ret = jsonutils.to_primitive(obj, fallback=lambda _: 'fallback') + self.assertEqual('fallback', ret)