Add a utility for JSON serialization

* This patch adds a utility that helps serialize data into a JSON
  string that might contain some non-standard data types like
  generators iterators and frozen dicts coming from YAQL. The
  utility uses oslo.serialization project that already takes care
  of iterators and any kinds of custom dicts. And in addition, it
  handles generators (assuming a generator represents an iterable
  similar to an iterator).
* Unit tests.
* Added YAQL into requirements and bumped the version of
  oslo.serialization to make sure to have the "fallback" parameter
  in "jsonutils.to_primitive"

Change-Id: I2fe891525bc86beb92aecf9ac2d8a490837c47d3
This commit is contained in:
Renat Akhmerov 2020-03-31 18:09:08 +07:00
parent 237a10d22e
commit 44a2738460
4 changed files with 126 additions and 2 deletions

View File

@ -28,7 +28,7 @@ openstackdocstheme==1.18.1
os-client-config==1.28.0
oslo.i18n==3.15.3
oslo.log==3.36.0
oslo.serialization==2.18.0
oslo.serialization==2.21.1
oslo.utils==3.33.0
oslotest==3.2.0
pbr==2.0.0
@ -54,3 +54,4 @@ testtools==2.2.0
traceback2==1.4.0
unittest2==1.1.0
wrapt==1.7.0
yaql==1.1.3 # Apache 2.0 License

View File

@ -14,10 +14,14 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
from yaql.language import utils as yaql_utils
from mistral_lib.tests import base as tests_base
from mistral_lib import utils
import testtools.matchers as ttm
@ -267,3 +271,70 @@ class TestUtils(tests_base.TestCase):
payload = ["adminPass", 'fooBarBaz']
expected = ["adminPass", 'fooBarBaz']
self.assertEqual(expected, utils.mask_data(payload))
def test_json_serialize_frozen_dict(self):
data = yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3]))
json_str = utils.to_json_str(data)
self.assertIsNotNone(json_str)
self.assertIn('"a": 1', json_str)
self.assertIn('"b": 2', json_str)
self.assertIn('"c": [1, 2, 3]', json_str)
def test_json_serialize_generator(self):
def _list_stream(_list):
for i in _list:
yield i
gen = _list_stream(
[1, yaql_utils.FrozenDict(a=1), _list_stream([12, 15])]
)
self.assertEqual('[1, {"a": 1}, [12, 15]]', utils.to_json_str(gen))
def test_json_serialize_dict_of_generators(self):
def _f(cnt):
for i in range(1, cnt + 1):
yield i
data = {'numbers': _f(3)}
self.assertEqual('{"numbers": [1, 2, 3]}', utils.to_json_str(data))
def test_json_serialize_range(self):
self.assertEqual("[1, 2, 3, 4]", utils.to_json_str(range(1, 5)))
def test_json_serialize_iterator_of_frozen_dicts(self):
data = iter(
[
yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3])),
yaql_utils.FrozenDict(
a=11,
b=yaql_utils.FrozenDict(b='222'),
c=iter(
[
1,
yaql_utils.FrozenDict(
a=iter([4, yaql_utils.FrozenDict(a=99)])
)
]
)
)
]
)
json_str = utils.to_json_str(data)
self.assertIsNotNone(json_str)
# Checking the first item.
self.assertIn('"a": 1', json_str)
self.assertIn('"b": 2', json_str)
self.assertIn('"c": [1, 2, 3]', json_str)
# Checking the first item.
self.assertIn('"a": 11', json_str)
self.assertIn('"b": {"b": "222"}', json_str)
self.assertIn('"c": [1, {"a": [4, {"a": 99}]}]', json_str)

View File

@ -17,6 +17,7 @@
import datetime
import functools
import inspect
import json
import os
from os import path
@ -28,6 +29,7 @@ import threading
import eventlet
from eventlet import corolocal
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils.strutils import mask_dict_password
from oslo_utils.strutils import mask_password
from oslo_utils import timeutils
@ -495,3 +497,52 @@ def mask_data(obj):
return [mask_data(i) for i in obj]
else:
return mask_password(obj)
def to_json_str(obj):
"""Serializes an object into a JSON string.
:param obj: Object to serialize.
:return: JSON string.
"""
if obj is None:
return None
def _fallback(value):
if inspect.isgenerator(value):
result = list(value)
# The result of the generator call may be again not primitive
# so we need to call "to_primitive" again with the same fallback
# function. Note that the endless recursion here is not a problem
# because "to_primitive" limits the depth for custom classes,
# if they are present in the object graph being traversed.
return jsonutils.to_primitive(
result,
convert_instances=True,
fallback=_fallback
)
return value
# We need to convert the root of the given object graph into
# a primitive by hand so that we also enable conversion of
# object of custom classes into primitives. Otherwise, they are
# ignored by the "json" lib.
return jsonutils.dumps(
jsonutils.to_primitive(obj, convert_instances=True, fallback=_fallback)
)
def from_json_str(json_str):
"""Reconstructs an object from a JSON string.
:param json_str: A JSON string.
:return: Deserialized object.
"""
if json_str is None:
return None
return jsonutils.loads(json_str)

View File

@ -5,4 +5,5 @@
eventlet!=0.20.1,>=0.20.0 # MIT
oslo.log>=3.36.0 # Apache-2.0
pbr!=2.1.0,>=2.0.0 # Apache-2.0
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
oslo.serialization>=2.21.1 # Apache-2.0
yaql>=1.1.3 # Apache 2.0 License