Merge "Utility functions for REST API JSON handling"

This commit is contained in:
Zuul 2020-11-18 07:10:20 +00:00 committed by Gerrit Code Review
commit 569db1063b
4 changed files with 746 additions and 0 deletions

View File

@ -24,6 +24,44 @@ def has_next(collection, limit):
return len(collection) and len(collection) == limit
def list_convert_with_links(items, item_name, limit, url=None, fields=None,
sanitize_func=None, key_field='uuid', **kwargs):
"""Build a collection dict including the next link for paging support.
:param items:
List of unsanitized items to include in the collection
:param item_name:
Name of dict key for items value
:param limit:
Paging limit
:param url:
Base URL for building next link
:param fields:
Optional fields to use for sanitize function
:param sanitize_func:
Optional sanitize function run on each item
:param key_field:
Key name for building next URL
:param kwargs:
other arguments passed to ``get_next``
:returns:
A dict containing ``item_name`` and ``next`` values
"""
items_dict = {
item_name: items
}
next_uuid = get_next(
items, limit, url=url, fields=fields, key_field=key_field, **kwargs)
if next_uuid:
items_dict['next'] = next_uuid
if sanitize_func:
for item in items:
sanitize_func(item, fields=fields)
return items_dict
def get_next(collection, limit, url=None, key_field='uuid', **kwargs):
"""Return a link to the next subset of the collection."""
if not has_next(collection, limit):

View File

@ -27,8 +27,10 @@ from oslo_utils import uuidutils
from pecan import rest
from ironic import api
from ironic.api.controllers import link
from ironic.api.controllers.v1 import versions
from ironic.api import types as atypes
from ironic.common import args
from ironic.common import exception
from ironic.common import faults
from ironic.common.i18n import _
@ -85,6 +87,244 @@ TRAITS_SCHEMA = {'anyOf': [
]}
def object_to_dict(obj, created_at=True, updated_at=True, uuid=True,
link_resource=None, link_resource_args=None, fields=None,
list_fields=None, date_fields=None, boolean_fields=None):
"""Helper function to convert RPC objects to REST API dicts.
:param obj:
RPC object to convert to a dict
:param created_at:
Whether to include standard base class attribute created_at
:param updated_at:
Whether to include standard base class attribute updated_at
:param uuid:
Whether to include standard base class attribute uuid
:param link_resource:
When specified, generate a ``links`` value with a ``self`` and
``bookmark`` using this resource name
:param link_resource_args:
Resource arguments to be added to generated links. When not specified,
the object ``uuid`` will be used.
:param fields:
Dict values to populate directly from object attributes
:param list_fields:
Dict values to populate from object attributes where an empty list is
the default for empty attributes
:param date_fields:
Dict values to populate from object attributes as ISO 8601 dates,
or None if the value is None
:param boolean_fields:
Dict values to populate from object attributes as boolean values
or False if the value is empty
:returns: A dict containing values from the object
"""
url = api.request.public_url
to_dict = {}
if uuid:
to_dict['uuid'] = obj.uuid
if created_at:
to_dict['created_at'] = (obj.created_at
and obj.created_at.isoformat() or None)
if updated_at:
to_dict['updated_at'] = (obj.updated_at
and obj.updated_at.isoformat() or None)
if fields:
for field in fields:
to_dict[field] = getattr(obj, field)
if list_fields:
for field in list_fields:
to_dict[field] = getattr(obj, field) or []
if date_fields:
for field in date_fields:
date = getattr(obj, field)
to_dict[field] = date and date.isoformat() or None
if boolean_fields:
for field in boolean_fields:
to_dict[field] = getattr(obj, field) or False
if link_resource:
if not link_resource_args:
link_resource_args = obj.uuid
to_dict['links'] = [
link.make_link('self', url, link_resource, link_resource_args),
link.make_link('bookmark', url, link_resource, link_resource_args,
bookmark=True)
]
return to_dict
def populate_node_uuid(obj, to_dict, raise_notfound=True):
"""Look up the node referenced in the object and populate a dict.
The node is fetched with the object ``node_id`` attribute and the
dict ``node_uuid`` value is populated with the node uuid
:param obj:
object to get the node_id attribute
:param to_dict:
dict to populate with a ``node_uuid`` value
:param raise_notfound:
If ``True`` raise a NodeNotFound exception if the node doesn't exist
otherwise set the dict ``node_uuid`` value to None.
:raises:
exception.NodeNotFound if raise_notfound and the node is not found
"""
if not obj.node_id:
to_dict['node_uuid'] = None
return
try:
to_dict['node_uuid'] = objects.Node.get_by_id(
api.request.context,
obj.node_id).uuid
except exception.NodeNotFound:
if raise_notfound:
raise
to_dict['node_uuid'] = None
def replace_node_uuid_with_id(to_dict):
"""Replace ``node_uuid`` dict value with ``node_id``
``node_id`` is found by fetching the node by uuid lookup.
:param to_dict: Dict to set ``node_id`` value on
:returns: The node object from the lookup
:raises: NodeNotFound with status_code set to 400 BAD_REQUEST
when node is not found.
"""
try:
node = objects.Node.get_by_uuid(api.request.context,
to_dict.pop('node_uuid'))
to_dict['node_id'] = node.id
except exception.NodeNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for requests acting on non-nodes
e.code = http_client.BAD_REQUEST # BadRequest
raise
return node
def replace_node_id_with_uuid(to_dict):
"""Replace ``node_id`` dict value with ``node_uuid``
``node_uuid`` is found by fetching the node by id lookup.
:param to_dict: Dict to set ``node_uuid`` value on
:returns: The node object from the lookup
:raises: NodeNotFound with status_code set to 400 BAD_REQUEST
when node is not found.
"""
try:
node = objects.Node.get_by_id(api.request.context,
to_dict.pop('node_id'))
to_dict['node_uuid'] = node.uuid
except exception.NodeNotFound as e:
# Change error code because 404 (NotFound) is inappropriate
# response for requests acting on non-nodes
e.code = http_client.BAD_REQUEST # BadRequest
raise
return node
def patch_update_changed_fields(from_dict, rpc_object, fields,
schema, id_map=None):
"""Update rpc object based on changed fields in a dict.
Only fields which have a corresponding schema field are updated when
changed. Other values can be updated using the id_map.
:param from_dict: Dict containing changed field values
:param rpc_object: Object to update changed fields on
:param fields: Field names on the rpc object
:param schema: jsonschema to get field names of the dict
:param id_map: Optional dict mapping object field names to
arbitrary values when there is no matching field in the schema
"""
schema_fields = schema['properties']
def _patch_val(field, patch_val):
if field in rpc_object and rpc_object[field] != patch_val:
rpc_object[field] = patch_val
for field in fields:
if id_map and field in id_map:
_patch_val(field, id_map[field])
elif field in schema_fields:
_patch_val(field, from_dict.get(field))
def patched_validate_with_schema(patched_dict, schema, validator=None):
"""Validate a patched dict object against a validator or schema.
This function has the side-effect of deleting any dict value which
is not in the schema. This allows database-loaded objects to be pruned
of their internal values before validation.
:param patched_dict: dict representation of the object with patch
updates applied
:param schema: Any dict key not in the schema will be deleted from the
dict. If no validator is specified then the resulting ``patched_dict``
will be validated agains the schema
:param validator: Optional validator to use if there is extra validation
required beyond the schema
:raises: exception.Invalid if validation fails
"""
schema_fields = schema['properties']
for field in set(patched_dict.keys()):
if field not in schema_fields:
patched_dict.pop(field, None)
if not validator:
validator = args.schema(schema)
validator('patch', patched_dict)
def patch_validate_allowed_fields(patch, allowed_fields):
"""Validate that a patch list only modifies allowed fields.
:param patch: List of patch dicts to validate
:param allowed_fields: List of fields which are allowed to be patched
:returns: The list of fields which will be patched
:raises: exception.Invalid if any patch changes a field not in
``allowed_fields``
"""
fields = set()
for p in patch:
path = p['path'].split('/')[1]
if path not in allowed_fields:
msg = _("Cannot patch %s. Only the following can be updated: %s")
raise exception.Invalid(
msg % (p['path'], ', '.join(allowed_fields)))
fields.add(path)
return fields
def sanitize_dict(to_sanitize, fields):
"""Removes sensitive and unrequested data.
Will only keep the fields specified in the ``fields`` parameter (plus
the ``links`` field).
:param to_sanitize: dict to sanitize
:param fields:
list of fields to preserve, or ``None`` to preserve them all
:type fields: list of str
"""
if fields is None:
return
for key in set(to_sanitize.keys()):
if key not in fields and key != 'links':
to_sanitize.pop(key, None)
def validate_limit(limit):
if limit is None:
return CONF.api.max_limit

View File

@ -0,0 +1,102 @@
# Copyright 2020 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
from oslo_utils import uuidutils
from ironic import api
from ironic.api.controllers.v1 import collection
from ironic.tests import base
class TestCollection(base.TestCase):
def setUp(self):
super(TestCollection, self).setUp()
p = mock.patch.object(api, 'request', autospec=False)
mock_req = p.start()
mock_req.public_url = 'http://192.0.2.1:5050'
self.addCleanup(p.stop)
def test_has_next(self):
self.assertFalse(collection.has_next([], 5))
self.assertFalse(collection.has_next([1, 2, 3], 5))
self.assertFalse(collection.has_next([1, 2, 3, 4], 5))
self.assertTrue(collection.has_next([1, 2, 3, 4, 5], 5))
def test_list_convert_with_links(self):
col = self._generate_collection(3)
# build with next link
result = collection.list_convert_with_links(
col, 'things', 3, url='thing')
self.assertEqual({
'things': col,
'next': 'http://192.0.2.1:5050/v1/thing?limit=3&'
'marker=%s' % col[2]['uuid']
}, result)
# build without next link
result = collection.list_convert_with_links(
col, 'things', 5, url='thing')
self.assertEqual({'things': col}, result)
# build with a custom sanitize function
def sanitize(item, fields):
item.pop('name')
result = collection.list_convert_with_links(
col, 'things', 5, url='thing', sanitize_func=sanitize)
self.assertEqual({
'things': [
{'uuid': col[0]['uuid']},
{'uuid': col[1]['uuid']},
{'uuid': col[2]['uuid']}
]
}, result)
# items in the original collection are also sanitized
self.assertEqual(col, result['things'])
def _generate_collection(self, length, key_field='uuid'):
return [{
key_field: uuidutils.generate_uuid(),
'name': 'thing-%s' % i}
for i in range(length)]
def test_get_next(self):
col = self._generate_collection(3)
# build next URL, marker is the last item uuid
self.assertEqual(
'http://192.0.2.1:5050/v1/foo?limit=3&marker=%s' % col[-1]['uuid'],
collection.get_next(col, 3, 'foo'))
# no next URL, return None
self.assertIsNone(collection.get_next(col, 4, 'foo'))
# build next URL, fields and other keyword args included in the url
self.assertEqual(
'http://192.0.2.1:5050/v1/foo?bar=baz&fields=uuid,one,two&'
'limit=3&marker=%s' % col[-1]['uuid'],
collection.get_next(col, 3, 'foo', fields=['uuid', 'one', 'two'],
bar='baz'))
# build next URL, use alternate sort key
col = self._generate_collection(3, key_field='identifier')
self.assertEqual(
'http://192.0.2.1:5050/v1/foo?limit=3&'
'marker=%s' % col[-1]['identifier'],
collection.get_next(col, 3, 'foo', key_field='identifier'))

View File

@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from http import client as http_client
import io
from unittest import mock
@ -33,6 +34,7 @@ from ironic.common import states
from ironic import objects
from ironic.tests import base
from ironic.tests.unit.api import utils as test_api_utils
from ironic.tests.unit.objects import utils as obj_utils
CONF = cfg.CONF
@ -221,6 +223,221 @@ class TestApiUtils(base.TestCase):
utils.check_for_invalid_fields,
requested, supported)
def test_patch_update_changed_fields(self):
schema = {
'properties': {
'one': {},
'two': {},
'three': {},
'four': {},
'five_uuid': {},
}
}
fields = [
'one',
'two',
'three',
'four',
'five_id'
]
def rpc_object():
obj = mock.MagicMock()
items = {
'one': 1,
'two': 'ii',
'three': None,
'four': [1, 2, 3, 4],
'five_id': 123
}
obj.__getitem__.side_effect = items.__getitem__
obj.__contains__.side_effect = items.__contains__
return obj
# test no change
o = rpc_object()
utils.patch_update_changed_fields({
'one': 1,
'two': 'ii',
'three': None,
'four': [1, 2, 3, 4],
}, o, fields, schema, id_map={'five_id': 123})
o.__setitem__.assert_not_called()
# test everything changes, and id_map values override from_dict values
o = rpc_object()
utils.patch_update_changed_fields({
'one': 2,
'two': 'iii',
'three': '',
'four': [2, 3],
}, o, fields, schema, id_map={'four': [4], 'five_id': 456})
o.__setitem__.assert_has_calls([
mock.call('one', 2),
mock.call('two', 'iii'),
mock.call('three', ''),
mock.call('four', [4]),
mock.call('five_id', 456)
])
# test None fields from None values and missing keys
# also five_id is untouched with no id_map
o = rpc_object()
utils.patch_update_changed_fields({
'two': None,
}, o, fields, schema)
o.__setitem__.assert_has_calls([
mock.call('two', None),
])
# test fields not in the schema are untouched
fields = [
'six',
'seven',
'eight'
]
o = rpc_object()
utils.patch_update_changed_fields({
'six': 2,
'seven': 'iii',
'eight': '',
}, o, fields, schema)
o.__setitem__.assert_not_called()
def test_patched_validate_with_schema(self):
schema = {
'properties': {
'one': {'type': 'string'},
'two': {'type': 'integer'},
'three': {'type': 'boolean'},
}
}
# test non-schema fields removed
pd = {
'one': 'one',
'two': 2,
'three': True,
'four': 4,
'five': 'five'
}
utils.patched_validate_with_schema(pd, schema)
self.assertEqual({
'one': 'one',
'two': 2,
'three': True,
}, pd)
# test fails schema validation
pd = {
'one': 1,
'two': 2,
'three': False
}
e = self.assertRaises(exception.InvalidParameterValue,
utils.patched_validate_with_schema, pd, schema)
self.assertIn("1 is not of type 'string'", str(e))
# test fails custom validation
def validate(name, value):
raise exception.InvalidParameterValue('big ouch')
pd = {
'one': 'one',
'two': 2,
'three': False
}
e = self.assertRaises(exception.InvalidParameterValue,
utils.patched_validate_with_schema, pd, schema,
validate)
self.assertIn("big ouch", str(e))
def test_patch_validate_allowed_fields(self):
allowed_fields = ['one', 'two', 'three']
# patch all
self.assertEqual(
{'one', 'two', 'three'},
utils.patch_validate_allowed_fields([
{'path': '/one'},
{'path': '/two'},
{'path': '/three/four'},
], allowed_fields))
# patch one
self.assertEqual(
{'one'},
utils.patch_validate_allowed_fields([
{'path': '/one'},
], allowed_fields))
# patch invalid field
e = self.assertRaises(
exception.Invalid,
utils.patch_validate_allowed_fields,
[{'path': '/four'}],
allowed_fields)
self.assertIn("Cannot patch /four. "
"Only the following can be updated: "
"one, two, three", str(e))
@mock.patch.object(api, 'request', autospec=False)
def test_sanitize_dict(self, mock_req):
mock_req.public_url = 'http://192.0.2.1:5050'
node = obj_utils.get_test_node(
self.context,
created_at=datetime.datetime(2000, 1, 1, 0, 0),
updated_at=datetime.datetime(2001, 1, 1, 0, 0),
inspection_started_at=datetime.datetime(2002, 1, 1, 0, 0),
console_enabled=True,
tags=['one', 'two', 'three'])
expected_links = [{
'href': 'http://192.0.2.1:5050/v1/node/'
'1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
'rel': 'self'
}, {
'href': 'http://192.0.2.1:5050/node/'
'1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
'rel': 'bookmark'
}]
# all fields
node_dict = utils.object_to_dict(
node,
link_resource='node',
)
utils.sanitize_dict(node_dict, None)
self.assertEqual({
'created_at': '2000-01-01T00:00:00+00:00',
'links': expected_links,
'updated_at': '2001-01-01T00:00:00+00:00',
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123'
}, node_dict)
# some fields
node_dict = utils.object_to_dict(
node,
link_resource='node',
)
utils.sanitize_dict(node_dict, ['uuid', 'created_at'])
self.assertEqual({
'created_at': '2000-01-01T00:00:00+00:00',
'links': expected_links,
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123'
}, node_dict)
# no fields
node_dict = utils.object_to_dict(
node,
link_resource='node',
)
utils.sanitize_dict(node_dict, [])
self.assertEqual({
'links': expected_links,
}, node_dict)
@mock.patch.object(api, 'request', spec_set=['version'])
class TestCheckAllowFields(base.TestCase):
@ -681,6 +898,69 @@ class TestNodeIdent(base.TestCase):
utils.get_rpc_node,
self.valid_name)
@mock.patch.object(objects.Node, 'get_by_id', autospec=True)
def test_populate_node_uuid(self, mock_gbi, mock_pr):
port = obj_utils.get_test_port(self.context)
node = obj_utils.get_test_node(self.context, id=port.node_id)
mock_gbi.return_value = node
# successful lookup
d = {}
utils.populate_node_uuid(port, d)
self.assertEqual({
'node_uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123'
}, d)
# not found, don't raise
mock_gbi.side_effect = exception.NodeNotFound(node=port.node_id)
d = {}
utils.populate_node_uuid(port, d, raise_notfound=False)
self.assertEqual({
'node_uuid': None
}, d)
# not found, raise exception
mock_gbi.side_effect = exception.NodeNotFound(node=port.node_id)
d = {}
self.assertRaises(exception.NodeNotFound,
utils.populate_node_uuid, port, d)
@mock.patch.object(objects.Node, 'get_by_uuid', autospec=True)
def test_replace_node_uuid_with_id(self, mock_gbu, mock_pr):
node = obj_utils.get_test_node(self.context, id=1)
mock_gbu.return_value = node
to_dict = {'node_uuid': self.valid_uuid}
self.assertEqual(node, utils.replace_node_uuid_with_id(to_dict))
self.assertEqual({'node_id': 1}, to_dict)
@mock.patch.object(objects.Node, 'get_by_uuid', autospec=True)
def test_replace_node_uuid_with_id_not_found(self, mock_gbu, mock_pr):
to_dict = {'node_uuid': self.valid_uuid}
mock_gbu.side_effect = exception.NodeNotFound(node=self.valid_uuid)
e = self.assertRaises(exception.NodeNotFound,
utils.replace_node_uuid_with_id, to_dict)
self.assertEqual(400, e.code)
@mock.patch.object(objects.Node, 'get_by_id', autospec=True)
def test_replace_node_id_with_uuid(self, mock_gbi, mock_pr):
node = obj_utils.get_test_node(self.context, uuid=self.valid_uuid)
mock_gbi.return_value = node
to_dict = {'node_id': 1}
self.assertEqual(node, utils.replace_node_id_with_uuid(to_dict))
self.assertEqual({'node_uuid': self.valid_uuid}, to_dict)
@mock.patch.object(objects.Node, 'get_by_id', autospec=True)
def test_replace_node_id_with_uuid_not_found(self, mock_gbi, mock_pr):
to_dict = {'node_id': 1}
mock_gbi.side_effect = exception.NodeNotFound(node=1)
e = self.assertRaises(exception.NodeNotFound,
utils.replace_node_id_with_uuid, to_dict)
self.assertEqual(400, e.code)
class TestVendorPassthru(base.TestCase):
@ -1366,3 +1646,89 @@ class TestCheckPortListPolicy(base.TestCase):
owner = utils.check_port_list_policy()
self.assertEqual(owner, '12345')
class TestObjectToDict(base.TestCase):
def setUp(self):
super(TestObjectToDict, self).setUp()
self.node = obj_utils.get_test_node(
self.context,
created_at=datetime.datetime(2000, 1, 1, 0, 0),
updated_at=datetime.datetime(2001, 1, 1, 0, 0),
inspection_started_at=datetime.datetime(2002, 1, 1, 0, 0),
console_enabled=True,
tags=['one', 'two', 'three'])
p = mock.patch.object(api, 'request', autospec=False)
mock_req = p.start()
mock_req.public_url = 'http://192.0.2.1:5050'
self.addCleanup(p.stop)
def test_no_args(self):
self.assertEqual({
'created_at': '2000-01-01T00:00:00+00:00',
'updated_at': '2001-01-01T00:00:00+00:00',
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123'
}, utils.object_to_dict(self.node))
def test_no_base_attributes(self):
self.assertEqual({}, utils.object_to_dict(
self.node,
created_at=False,
updated_at=False,
uuid=False)
)
def test_fields(self):
self.assertEqual({
'conductor_group': '',
'console_enabled': True,
'created_at': '2000-01-01T00:00:00+00:00',
'driver': 'fake-hardware',
'inspection_finished_at': None,
'inspection_started_at': '2002-01-01T00:00:00+00:00',
'maintenance': False,
'tags': ['one', 'two', 'three'],
'traits': [],
'updated_at': '2001-01-01T00:00:00+00:00',
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123'
}, utils.object_to_dict(
self.node,
fields=['conductor_group', 'driver'],
boolean_fields=['maintenance', 'console_enabled'],
date_fields=['inspection_started_at', 'inspection_finished_at'],
list_fields=['tags', 'traits'])
)
def test_links(self):
self.assertEqual({
'created_at': '2000-01-01T00:00:00+00:00',
'links': [{
'href': 'http://192.0.2.1:5050/v1/node/'
'1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
'rel': 'self'
}, {
'href': 'http://192.0.2.1:5050/node/'
'1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
'rel': 'bookmark'
}],
'updated_at': '2001-01-01T00:00:00+00:00',
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
}, utils.object_to_dict(self.node, link_resource='node'))
self.assertEqual({
'created_at': '2000-01-01T00:00:00+00:00',
'links': [{
'href': 'http://192.0.2.1:5050/v1/node/foo',
'rel': 'self'
}, {
'href': 'http://192.0.2.1:5050/node/foo',
'rel': 'bookmark'
}],
'updated_at': '2001-01-01T00:00:00+00:00',
'uuid': '1be26c0b-03f2-4d2e-ae87-c02d7f33c123',
}, utils.object_to_dict(
self.node,
link_resource='node',
link_resource_args='foo'))