From 76fdc1aba5c9574ecff7076d072917db01510836 Mon Sep 17 00:00:00 2001 From: Tomi Juvonen Date: Mon, 10 Feb 2020 12:03:38 +0200 Subject: [PATCH] API schema validation Story: 2007278 Task: #38717 Change-Id: I7a6fc62e8f2c0c3cc21560f9f889d0fe136ca33e Signed-off-by: Tomi Juvonen --- .zuul.yaml | 4 +- doc/source/api-ref/v1/maintenance.inc | 2 +- doc/source/api-ref/v1/parameters.yaml | 15 +- doc/source/api-ref/v1/project.inc | 8 +- .../samples/instance-group-constraints.json | 4 +- fenix/api/v1/controllers/maintenance.py | 95 +++++++- fenix/api/v1/schema.py | 202 ++++++++++++++++++ lower-constraints.txt | 21 +- requirements.txt | 2 +- setup.cfg | 8 +- test-requirements.txt | 9 +- tox.ini | 13 +- 12 files changed, 334 insertions(+), 49 deletions(-) create mode 100644 fenix/api/v1/schema.py diff --git a/.zuul.yaml b/.zuul.yaml index 4568184..4c95fd8 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,8 +1,6 @@ - project: templates: - - openstack-python-jobs - - openstack-python35-jobs - - openstack-python36-jobs + - openstack-python3-ussuri-jobs - check-requirements - openstack-lower-constraints-jobs - build-openstack-docs-pti diff --git a/doc/source/api-ref/v1/maintenance.inc b/doc/source/api-ref/v1/maintenance.inc index 96fb861..fecabe4 100644 --- a/doc/source/api-ref/v1/maintenance.inc +++ b/doc/source/api-ref/v1/maintenance.inc @@ -47,7 +47,7 @@ Response codes Update maintenance session (planned future functionality) ========================================================= -.. rest_method:: POST /v1/maintenance/{session_id}/ +.. rest_method:: PUT /v1/maintenance/{session_id}/ Update existing maintenance session. This can be used to continue a failed session. diff --git a/doc/source/api-ref/v1/parameters.yaml b/doc/source/api-ref/v1/parameters.yaml index 1f28ba1..208cef9 100644 --- a/doc/source/api-ref/v1/parameters.yaml +++ b/doc/source/api-ref/v1/parameters.yaml @@ -17,9 +17,8 @@ session_id: description: | Session ID in: path - required: false + required: true type: string - min_version: \> 1 uuid-path: description: | @@ -68,7 +67,7 @@ action-plugins: description: | List of action plug-ins. in: body - required: true + required: false type: list of dictionaries boolean: @@ -90,7 +89,7 @@ hosts: Hosts to be maintained. An empty list can indicate hosts are to be discovered. in: body - required: true + required: false type: list of strings instance-action: @@ -102,7 +101,8 @@ instance-action: instance-actions: description: | - instance ID : action string + instance ID : action string. This variable is not needed in reply to state + MAINTENANCE, SCALE_IN or MAINTENANCE_COMPLETE in: body required: true type: dictionary @@ -133,7 +133,10 @@ lead-time: How long lead time VNF needs for 'migration_type' operation. VNF needs to report back to Fenix as soon as it is ready, but at least within this time. Reporting as fast as can is crucial for optimizing - infrastructure upgrade/maintenance. + infrastructure upgrade/maintenance. Zero value means interaction with + VNFM is not used for this instance, but instance_group recovery_time + needs to be obeyed towards max_impacted_members. + in: body required: true type: integer diff --git a/doc/source/api-ref/v1/project.inc b/doc/source/api-ref/v1/project.inc index ab88cd9..ca14d95 100644 --- a/doc/source/api-ref/v1/project.inc +++ b/doc/source/api-ref/v1/project.inc @@ -127,7 +127,7 @@ Request - instance_id: uuid-path - instance_id: uuid - group_id: group-uuid - - name: instance-name + - instance_name: instance-name - migration_type: migration-type - max_interruption_time: max-interruption-time - resource_mitigation: resource-mitigation @@ -160,7 +160,7 @@ Request - instance_id: uuid-path - instance_id: uuid - group_id: group-uuid - - name: instance-name + - instance_name: instance-name - migration_type: migration-type - max_interruption_time: max-interruption-time - resource_mitigation: resource-mitigation @@ -218,7 +218,7 @@ Request - project_id: uuid - instance_id: uuid-path - instance_id: uuid - - name: instance-name + - group_name: instance-group - migration_type: migration-type - max_interruption_time: max-interruption-time - resource_mitigation: resource-mitigation @@ -250,7 +250,7 @@ Request - group_id: group-uuid-path - group_id: group-uuid - project_id: uuid - - name: instance-group + - group_name: instance-group - anti_affinity_group: boolean - max_instances_per_host: max-instances-per-host - max_impacted_members: max-impacted-members diff --git a/doc/source/api-ref/v1/samples/instance-group-constraints.json b/doc/source/api-ref/v1/samples/instance-group-constraints.json index 6297b42..403e382 100644 --- a/doc/source/api-ref/v1/samples/instance-group-constraints.json +++ b/doc/source/api-ref/v1/samples/instance-group-constraints.json @@ -1,8 +1,8 @@ { "project_id": "1ad1154137ac41799cefd5caebae379b", "group_id": "a01d192c-328e-4708-9b3c-9d716cd24a92", - "name": "vm_ha_group", - "anti_affinity_group": "True", + "group_name": "vm_ha_group", + "anti_affinity_group": True, "max_instances_per_host": 1, "max_impacted_members": 1, "recovery_time": 15, diff --git a/fenix/api/v1/controllers/maintenance.py b/fenix/api/v1/controllers/maintenance.py index 65b27a4..847bbeb 100644 --- a/fenix/api/v1/controllers/maintenance.py +++ b/fenix/api/v1/controllers/maintenance.py @@ -14,6 +14,7 @@ # under the License. import json +import jsonschema from pecan import abort from pecan import expose from pecan import request @@ -24,6 +25,7 @@ from oslo_log import log from oslo_serialization import jsonutils from fenix.api.v1 import maintenance +from fenix.api.v1 import schema from fenix import policy LOG = log.getLogger(__name__) @@ -43,6 +45,12 @@ class ProjectController(rest.RestController): if request.body: LOG.error("Unexpected data") abort(400) + try: + jsonschema.validate(session_id, schema.uid) + jsonschema.validate(project_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = self.engine_rpcapi.project_get_session(session_id, project_id) try: @@ -55,6 +63,13 @@ class ProjectController(rest.RestController): @expose(content_type='application/json') def put(self, session_id, project_id): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(session_id, schema.uid) + jsonschema.validate(project_id, schema.uid) + jsonschema.validate(data, schema.maintenance_session_project_put) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = self.engine_rpcapi.project_update_session(session_id, project_id, data) @@ -76,6 +91,16 @@ class ProjectInstanceController(rest.RestController): @expose(content_type='application/json') def put(self, session_id, project_id, instance_id): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(session_id, schema.uid) + jsonschema.validate(project_id, schema.uid) + jsonschema.validate(instance_id, schema.uid) + jsonschema.validate( + data, + schema.maintenance_session_project_instance_put) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = ( self.engine_rpcapi.project_update_session_instance(session_id, project_id, @@ -98,13 +123,18 @@ class SessionController(rest.RestController): @policy.authorize('maintenance:session', 'get') @expose(content_type='application/json') def get(self, session_id): + try: + jsonschema.validate(session_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) session = self.engine_rpcapi.admin_get_session(session_id) if session is None: - response.status = 404 - return {"error": "Invalid session"} + LOG.error("Invalid session") + abort(404) try: response.text = jsonutils.dumps(session) except TypeError: @@ -115,6 +145,13 @@ class SessionController(rest.RestController): @expose(content_type='application/json') def put(self, session_id): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(session_id, schema.uid) + # TBD implement this API + # jsonschema.validate(data, schema.maintenance_session_put) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = self.engine_rpcapi.admin_update_session(session_id, data) try: response.text = jsonutils.dumps(engine_data) @@ -125,6 +162,11 @@ class SessionController(rest.RestController): @policy.authorize('maintenance:session', 'delete') @expose(content_type='application/json') def delete(self, session_id): + try: + jsonschema.validate(session_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) @@ -160,10 +202,15 @@ class MaintenanceController(rest.RestController): @expose(content_type='application/json') def post(self): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(data, schema.maintenance_post) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) session = self.engine_rpcapi.admin_create_session(data) if session is None: - response.status = 509 - return {"error": "Too many sessions"} + LOG.error("Too many sessions") + abort(509) try: response.text = jsonutils.dumps(session) except TypeError: @@ -181,13 +228,18 @@ class InstanceController(rest.RestController): @policy.authorize('instance', 'get') @expose(content_type='application/json') def get(self, instance_id): + try: + jsonschema.validate(instance_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) session = self.engine_rpcapi.get_instance(instance_id) if session is None: - response.status = 404 - return {"error": "Invalid session"} + LOG.error("Invalid session") + abort(404) try: response.text = jsonutils.dumps(session) except TypeError: @@ -198,6 +250,12 @@ class InstanceController(rest.RestController): @expose(content_type='application/json') def put(self, instance_id): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(instance_id, schema.uid) + jsonschema.validate(data, schema.instance_put) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = self.engine_rpcapi.update_instance(instance_id, data) try: @@ -209,6 +267,11 @@ class InstanceController(rest.RestController): @policy.authorize('instance', 'delete') @expose(content_type='application/json') def delete(self, instance_id): + try: + jsonschema.validate(instance_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) @@ -230,13 +293,18 @@ class InstanceGroupController(rest.RestController): @policy.authorize('instance_group', 'get') @expose(content_type='application/json') def get(self, group_id): + try: + jsonschema.validate(group_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) session = self.engine_rpcapi.get_instance_group(group_id) if session is None: - response.status = 404 - return {"error": "Invalid session"} + LOG.error("Invalid session") + abort(404) try: response.text = jsonutils.dumps(session) except TypeError: @@ -247,6 +315,12 @@ class InstanceGroupController(rest.RestController): @expose(content_type='application/json') def put(self, group_id): data = json.loads(request.body.decode('utf8')) + try: + jsonschema.validate(group_id, schema.uid) + jsonschema.validate(data, schema.instance_group_put) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) engine_data = ( self.engine_rpcapi.update_instance_group(group_id, data)) try: @@ -258,6 +332,11 @@ class InstanceGroupController(rest.RestController): @policy.authorize('instance_group', 'delete') @expose(content_type='application/json') def delete(self, group_id): + try: + jsonschema.validate(group_id, schema.uid) + except jsonschema.exceptions.ValidationError as e: + LOG.error(str(e.message)) + abort(422) if request.body: LOG.error("Unexpected data") abort(400) diff --git a/fenix/api/v1/schema.py b/fenix/api/v1/schema.py new file mode 100644 index 0000000..d041ac8 --- /dev/null +++ b/fenix/api/v1/schema.py @@ -0,0 +1,202 @@ +# Copyright 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +uid = { + 'type': 'string', + 'minLength': 8, + 'maxLength': 36, +} + +states = ['MAINTENANCE', + 'SCALE_IN', + 'PREPARE_MAINTENANCE', + 'START_MAINTENANCE', + 'PLANNED_MAINTENANCE', + 'MAINTENANCE_COMPLETE', + 'MAINTENANCE_DONE', + 'MAINTENANCE_FAILED'] + +reply_states = ['ACK_MAINTENANCE', + 'ACK_SCALE_IN', + 'ACK_PREPARE_MAINTENANCE', + 'ACK_START_MAINTENANCE', + 'ACK_PLANNED_MAINTENANCE', + 'ACK_MAINTENANCE_COMPLETE', + 'NACK_MAINTENANCE', + 'NACK_SCALE_IN', + 'NACK_PREPARE_MAINTENANCE', + 'NACK_START_MAINTENANCE', + 'NACK_PLANNED_MAINTENANCE', + 'NACK_MAINTENANCE_COMPLETE'] + +allowed_actions = ['MIGRATE', 'LIVE_MIGRATE', 'OWN_ACTION'] + +maintenance_session_project_put = { + 'type': 'object', + 'properties': { + 'instance_actions': { + 'type': 'object' + }, + 'state': { + 'type': 'string', + 'enum': reply_states, + }, + }, + 'required': ['state'] +} + +maintenance_session_project_instance_put = { + 'type': 'object', + 'properties': { + 'instance_action': { + 'type': 'string', + 'enum': allowed_actions, + }, + 'state': { + 'type': 'string', + 'enum': reply_states, + } + }, + 'required': ['instance_action', 'state'] +} + +# TBD +# maintenance_session_put = { +# +# } + +maintenance_post = { + 'type': 'object', + 'properties': { + 'hosts': { + 'type': 'array', + 'minItems': 0, + 'maxItems': 1000, + 'items': { + 'type': 'string', + 'minLength': 2, + 'maxLength': 255, + }, + }, + 'state': { + 'type': 'string', + 'enum': states, + }, + 'maintenance_at': { + 'type': 'string', + 'format': 'date-time', + }, + 'metadata': {'type': 'object'}, + 'workflow': { + 'type': 'string', + 'minLength': 2, + 'maxLength': 255, + }, + 'download': { + 'type': 'array', + 'minItems': 5, + 'maxItems': 255, + 'items': { + 'type': 'string', + 'minLength': 2, + 'maxLength': 165, + }, + }, + 'actions': { + 'type': 'array', + 'minItems': 0, + 'maxItems': 255, + 'items': { + 'type': 'object', + 'properties': { + 'plugin': { + 'type': 'string', + 'minLength': 2, + 'maxLength': 255, + }, + 'type': { + 'type': 'string', + 'minLength': 2, + 'maxLength': 32, + }, + 'metadata': {'type': 'object'}, + }, + 'required': ['plugin', 'type'] + } + } + }, + 'required': ['state', 'maintenance_at', 'workflow', 'metadata'] +} + +instance_put = { + 'type': 'object', + 'properties': { + 'instance_id': uid, + 'project_id': uid, + 'group_id': uid, + 'instance_name': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 255, + }, + 'max_interruption_time': { + 'type': 'number', + 'maximum': 21600 + }, + 'migration_type': { + 'type': 'string', + 'enum': allowed_actions, + }, + 'resource_mitigation': {'type': 'boolean'}, + 'lead_time': { + 'type': 'number', + 'maximum': 21600 + }, + }, + 'required': ['instance_id', 'project_id', 'group_id', 'instance_name', + 'max_interruption_time', 'migration_type', + 'resource_mitigation', 'lead_time'] +} + +instance_group_put = { + 'type': 'object', + 'properties': { + 'project_id': uid, + 'group_id': uid, + 'group_name': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 255, + }, + 'anti_affinity_group': {'type': 'boolean'}, + 'max_instances_per_host': { + 'type': 'number', + 'maximum': 32000 + }, + 'max_impacted_members': { + 'type': 'number', + 'minimum': 1, + 'maximum': 32000 + }, + 'recovery_time': { + 'type': 'number', + 'maximum': 21600 + }, + 'resource_mitigation': {'type': 'boolean'}, + }, + 'required': ['project_id', 'group_id', 'group_name', + 'anti_affinity_group', 'max_instances_per_host', + 'max_impacted_members', 'recovery_time', + 'resource_mitigation'] +} diff --git a/lower-constraints.txt b/lower-constraints.txt index 85c8547..b3d93be 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -1,11 +1,16 @@ coverage==4.0 # Apache-2.0 -hacking==0.12.0 # Apache-2.0 -coverage==4.0 # Apache-2.0 -openstackdocstheme==1.18.1 # Apache-2.0 -oslotest==1.10.0 # Apache-2.0 +hacking==2.0 # Apache-2.0 +openstackdocstheme==1.31.2 # Apache-2.0 +oslotest==3.8.0 # Apache-2.0 pbr==2.0 # Apache-2.0 -python-subunit==0.0.18 # Apache-2.0/BSD -reno==2.5.0 # Apache-2.0 -sphinx==1.6.2 # BSD +python-subunit==1.3.0 # Apache-2.0/BSD +reno==2.11.3;python_version=='2.7' +reno==3.0.0;python_version=='3.5' +reno==3.0.0;python_version=='3.6' +reno==3.0.0;python_version=='3.7' +sphinx==1.8.5;python_version=='2.7' +sphinx==2.3.1;python_version=='3.5' +sphinx==2.3.1;python_version=='3.6' +sphinx==2.3.1;python_version=='3.7' stestr==1.0.0 # Apache-2.0 -testtools==1.4.0 # MIT +testtools==2.2.0 # MIT diff --git a/requirements.txt b/requirements.txt index 1d18dd3..5a8c6b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr>=2.0 # Apache-2.0 +pbr!=2.1.0,>=2.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 5c3b31f..ab8d293 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,8 @@ description-file = README.rst author = OpenStack author-email = openstack-discuss@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = https://wiki.openstack.org/wiki/Fenix +python-requires = >=3.6 classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -13,11 +14,10 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3 :: Only [global] setup-hooks = pbr.hooks.setup_hook diff --git a/test-requirements.txt b/test-requirements.txt index e2afc81..3433121 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,10 +2,9 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.12.0,<0.13 # Apache-2.0 - +hacking>=2.0,<2.1 # Apache-2.0 coverage>=4.0,!=4.4 # Apache-2.0 -python-subunit>=0.0.18 # Apache-2.0/BSD -oslotest>=1.10.0 # Apache-2.0 +python-subunit>=1.3.0 # Apache-2.0/BSD +oslotest>=3.8.0 # Apache-2.0 stestr>=1.0.0 # Apache-2.0 -testtools>=1.4.0 # MIT +testtools>=2.2.0 # MIT diff --git a/tox.ini b/tox.ini index d8f021e..71111cc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] -minversion = 2.0 -envlist = py36,py35,pep8,docs -skipsdist = True +minversion = 3.1.1 +envlist = py36,py37,pep8,docs +ignore_basepython_conflict = True [testenv] usedevelop = True @@ -13,7 +13,7 @@ setenv = OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} + -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} @@ -53,10 +53,9 @@ basepython = python3 commands = oslo_debug_helper {posargs} [flake8] -# E123, E125 skipped as they are invalid PEP-8. - show-source = True -ignore = E123,E125 +enable-extensions = H106,H203 +ignore = E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,E305,E402,H405,W503,W504,E731 builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build