From 5b652f9a1f0fb9ba327cead26eb30c697b57eca8 Mon Sep 17 00:00:00 2001 From: Fabio Verboso Date: Wed, 29 May 2019 18:43:26 +0200 Subject: [PATCH] New Request/Result System. Change-Id: I55c54bfa92eb339daf3f5b3683dcc33cbe0cee90 --- iotronic/api/controllers/v1/__init__.py | 10 + iotronic/api/controllers/v1/fleet.py | 8 +- iotronic/api/controllers/v1/request.py | 296 ++++++++++++++++++ iotronic/api/controllers/v1/result.py | 214 +++++++++++++ iotronic/common/policy.py | 20 ++ iotronic/conductor/endpoints.py | 138 +++++--- iotronic/db/api.py | 36 ++- .../76c628d60004_add_results_requests.py | 9 +- iotronic/db/sqlalchemy/api.py | 55 +++- iotronic/db/sqlalchemy/models.py | 14 +- iotronic/objects/request.py | 67 ++-- iotronic/objects/result.py | 51 +-- iotronic/wamp/agent.py | 6 +- iotronic/wamp/functions.py | 14 +- 14 files changed, 809 insertions(+), 129 deletions(-) create mode 100644 iotronic/api/controllers/v1/request.py create mode 100644 iotronic/api/controllers/v1/result.py diff --git a/iotronic/api/controllers/v1/__init__.py b/iotronic/api/controllers/v1/__init__.py index 76b5ce2..35a632d 100644 --- a/iotronic/api/controllers/v1/__init__.py +++ b/iotronic/api/controllers/v1/__init__.py @@ -30,6 +30,7 @@ from iotronic.api.controllers.v1 import enabledwebservice from iotronic.api.controllers.v1 import fleet from iotronic.api.controllers.v1 import plugin from iotronic.api.controllers.v1 import port +from iotronic.api.controllers.v1 import request from iotronic.api.controllers.v1 import service from iotronic.api.controllers.v1 import webservice @@ -145,6 +146,14 @@ class V1(base.APIBase): bookmark=True) ] + v1.requests = [link.Link.make_link('self', pecan.request.public_url, + 'requests', ''), + link.Link.make_link('bookmark', + pecan.request.public_url, + 'requests', '', + bookmark=True) + ] + return v1 @@ -158,6 +167,7 @@ class Controller(rest.RestController): ports = port.PortsController() fleets = fleet.FleetsController() webservices = webservice.WebservicesController() + requests = request.RequestsController() @expose.expose(V1) def get(self): diff --git a/iotronic/api/controllers/v1/fleet.py b/iotronic/api/controllers/v1/fleet.py index 9ae45df..1cd1d5e 100644 --- a/iotronic/api/controllers/v1/fleet.py +++ b/iotronic/api/controllers/v1/fleet.py @@ -167,6 +167,7 @@ class FleetsController(rest.RestController): def _get_fleets_collection(self, marker, limit, sort_key, sort_dir, + project=None, fields=None): limit = api_utils.validate_limit(limit) @@ -183,6 +184,10 @@ class FleetsController(rest.RestController): "sorting") % {'key': sort_key}) filters = {} + + if project: + if pecan.request.context.is_admin: + filters['project_id'] = project fleets = objects.Fleet.list(pecan.request.context, limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir, @@ -205,7 +210,7 @@ class FleetsController(rest.RestController): rpc_fleet = api_utils.get_rpc_fleet(fleet_ident) cdict = pecan.request.context.to_policy_values() - cdict['project'] = rpc_fleet.project + cdict['project_id'] = rpc_fleet.project policy.authorize('iot:fleet:get_one', cdict, cdict) return Fleet.convert_with_links(rpc_fleet, fields=fields) @@ -237,6 +242,7 @@ class FleetsController(rest.RestController): fields = _DEFAULT_RETURN_FIELDS return self._get_fleets_collection(marker, limit, sort_key, sort_dir, + project=cdict['project_id'], fields=fields) @expose.expose(Fleet, body=Fleet, status_code=201) diff --git a/iotronic/api/controllers/v1/request.py b/iotronic/api/controllers/v1/request.py new file mode 100644 index 0000000..dcfbaaa --- /dev/null +++ b/iotronic/api/controllers/v1/request.py @@ -0,0 +1,296 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes + +from iotronic.api.controllers import base +from iotronic.api.controllers import link +from iotronic.api.controllers.v1 import collection +from iotronic.api.controllers.v1.result import ResultCollection +from iotronic.api.controllers.v1 import types +from iotronic.api.controllers.v1 import utils as api_utils +from iotronic.api import expose +from iotronic.common import exception +from iotronic.common import policy +from iotronic import objects + +_DEFAULT_RETURN_FIELDS = ( + 'uuid', + 'destination_uuid', + 'main_request_uuid', + 'pending_requests', + 'status', + 'type', + 'action' +) + +_DEFAULT_RESULT_RETURN_FIELDS = ( + 'board_uuid', + 'request_uuid', + 'result', + 'message' +) + + +class Request(base.APIBase): + """API representation of a request. + + """ + + uuid = types.uuid + destination_uuid = types.uuid + main_request_uuid = types.uuid + pending_requests = wsme.types.IntegerType() + status = wsme.wsattr(wtypes.text) + project = types.uuid + type = wsme.types.IntegerType() + action = wsme.wsattr(wtypes.text) + + links = wsme.wsattr([link.Link], readonly=True) + + def __init__(self, **kwargs): + self.fields = [] + fields = list(objects.Request.fields) + for k in fields: + # Skip fields we do not expose. + if not hasattr(self, k): + continue + self.fields.append(k) + setattr(self, k, kwargs.get(k, wtypes.Unset)) + + @staticmethod + def _convert_with_links(request, url, fields=None): + request_uuid = request.uuid + if fields is not None: + request.unset_fields_except(fields) + + request.links = [link.Link.make_link('self', url, 'requests', + request_uuid), + link.Link.make_link('bookmark', url, 'requests', + request_uuid, bookmark=True) + ] + return request + + @classmethod + def convert_with_links(cls, rpc_request, fields=None): + request = Request(**rpc_request.as_dict()) + + if fields is not None: + api_utils.check_for_invalid_fields(fields, request.as_dict()) + + return cls._convert_with_links(request, pecan.request.public_url, + fields=fields) + + +class RequestCollection(collection.Collection): + """API representation of a collection of requests.""" + + requests = [Request] + """A list containing requests objects""" + + def __init__(self, **kwargs): + self._type = 'requests' + + @staticmethod + def convert_with_links(requests, limit, url=None, fields=None, **kwargs): + collection = RequestCollection() + collection.requests = [Request.convert_with_links(n, fields=fields) + for n in requests] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class ResultsRequestController(rest.RestController): + def __init__(self, request_ident): + self.request_ident = request_ident + + @expose.expose(ResultCollection, types.uuid, int, wtypes.text, + wtypes.text, types.listtype) + def get_all(self, marker=None, + limit=None, sort_key='id', sort_dir='asc', + fields=None): + """Retrieve a list of boards. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + This value cannot be larger than the value of max_limit + in the [api] section of the ironic configuration, or only + max_limit resources will be returned. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + cdict = pecan.request.context.to_policy_values() + policy.authorize('iot:result:get', cdict, cdict) + + if fields is None: + fields = _DEFAULT_RESULT_RETURN_FIELDS + + filters = {} + filters['request_uuid'] = self.request_ident + + results = objects.Result.list(pecan.request.context, limit, marker, + sort_key=sort_key, sort_dir=sort_dir, + filters=filters) + + parameters = {'sort_key': sort_key, 'sort_dir': sort_dir} + + return ResultCollection.convert_with_links(results, limit, + fields=fields, + **parameters) + + +class RequestsController(rest.RestController): + """REST controller for Requests.""" + + _subcontroller_map = { + 'results': ResultsRequestController, + } + + invalid_sort_key_list = ['extra', ] + + _custom_actions = { + 'detail': ['GET'], + } + + @pecan.expose() + def _lookup(self, ident, *remainder): + try: + ident = types.uuid_or_name.validate(ident) + except exception.InvalidUuidOrName as e: + pecan.abort('400', e.args[0]) + if not remainder: + return + + subcontroller = self._subcontroller_map.get(remainder[0]) + if subcontroller: + return subcontroller(request_ident=ident), remainder[1:] + + def _get_requests_collection(self, marker, limit, + sort_key, sort_dir, + project=None, + fields=None): + + limit = api_utils.validate_limit(limit) + sort_dir = api_utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Request.get_by_uuid(pecan.request.context, + marker) + + if sort_key in self.invalid_sort_key_list: + raise exception.InvalidParameterValue( + ("The sort_key value %(key)s is an invalid field for " + "sorting") % {'key': sort_key}) + + filters = {} + + if project: + if pecan.request.context.is_admin: + filters['project_id'] = project + requests = objects.Request.list(pecan.request.context, limit, + marker_obj, + sort_key=sort_key, sort_dir=sort_dir, + filters=filters) + + parameters = {'sort_key': sort_key, 'sort_dir': sort_dir} + + return RequestCollection.convert_with_links(requests, limit, + fields=fields, + **parameters) + + @expose.expose(Request, types.uuid_or_name, types.listtype) + def get_one(self, request_ident, fields=None): + """Retrieve information about the given request. + + :param request_ident: UUID or logical name of a request. + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + + cdict = pecan.request.context.to_policy_values() + policy.authorize('iot:request:get_one', cdict, cdict) + + rpc_request = objects.Request.get_by_uuid(pecan.request.context, + request_ident) + return Request.convert_with_links(rpc_request, fields=fields) + + @expose.expose(RequestCollection, types.uuid, int, wtypes.text, + wtypes.text, types.listtype, types.boolean, types.boolean) + def get_all(self, marker=None, + limit=None, sort_key='id', sort_dir='asc', + fields=None): + """Retrieve a list of requests. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + This value cannot be larger than the value of max_limit + in the [api] section of the ironic configuration, or only + max_limit resources will be returned. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + :param with_public: Optional boolean to get also public pluings. + :param all_requests: Optional boolean to get all the pluings. + Only for the admin + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + cdict = pecan.request.context.to_policy_values() + policy.authorize('iot:request:get', cdict, cdict) + + if fields is None: + fields = _DEFAULT_RETURN_FIELDS + return self._get_requests_collection(marker, + limit, sort_key, sort_dir, + project=cdict['project_id'], + fields=fields) + + @expose.expose(RequestCollection, types.uuid, int, wtypes.text, + wtypes.text, types.listtype, types.boolean, types.boolean) + def detail(self, marker=None, + limit=None, sort_key='id', sort_dir='asc', + fields=None, with_public=False, all_requests=False): + """Retrieve a list of requests. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + This value cannot be larger than the value of max_limit + in the [api] section of the ironic configuration, or only + max_limit resources will be returned. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + :param with_public: Optional boolean to get also public request. + :param all_requests: Optional boolean to get all the requests. + Only for the admin + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + + cdict = pecan.request.context.to_policy_values() + policy.authorize('iot:request:get', cdict, cdict) + + # /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "requests": + raise exception.HTTPNotFound() + + return self._get_requests_collection(marker, + limit, sort_key, sort_dir, + with_public=with_public, + all_requests=all_requests, + fields=fields) diff --git a/iotronic/api/controllers/v1/result.py b/iotronic/api/controllers/v1/result.py new file mode 100644 index 0000000..9282da0 --- /dev/null +++ b/iotronic/api/controllers/v1/result.py @@ -0,0 +1,214 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes + +from iotronic.api.controllers import base +from iotronic.api.controllers import link +from iotronic.api.controllers.v1 import collection +from iotronic.api.controllers.v1 import types +from iotronic.api.controllers.v1 import utils as api_utils +from iotronic.api import expose +from iotronic.common import exception +from iotronic.common import policy +from iotronic import objects + +_DEFAULT_RETURN_FIELDS = ( + 'board_uuid', + 'request_uuid', + 'result', +) + + +class Result(base.APIBase): + """API representation of a result. + + """ + board_uuid = types.uuid + request_uuid = types.uuid + result = wsme.wsattr(wtypes.text) + message = wsme.wsattr(wtypes.text) + + links = wsme.wsattr([link.Link], readonly=True) + + def __init__(self, **kwargs): + self.fields = [] + fields = list(objects.Result.fields) + for k in fields: + # Skip fields we do not expose. + if not hasattr(self, k): + continue + self.fields.append(k) + setattr(self, k, kwargs.get(k, wtypes.Unset)) + + @staticmethod + def _convert_with_links(result, fields=None): + if fields is not None: + result.unset_fields_except(fields) + return result + + @classmethod + def convert_with_links(cls, rpc_result, fields=None): + result = Result(**rpc_result.as_dict()) + + if fields is not None: + api_utils.check_for_invalid_fields(fields, result.as_dict()) + + return cls._convert_with_links(result, + fields=fields) + + +class ResultCollection(collection.Collection): + """API representation of a collection of results.""" + + results = [Result] + """A list containing results objects""" + + def __init__(self, **kwargs): + self._type = 'results' + + @staticmethod + def convert_with_links(results, limit, url=None, fields=None, **kwargs): + collection = ResultCollection() + collection.results = [Result.convert_with_links(n, fields=fields) + for n in results] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class ResultsController(rest.RestController): + """REST controller for Results.""" + + invalid_sort_key_list = ['extra', ] + + _custom_actions = { + 'detail': ['GET'], + } + + def _get_results_collection(self, marker, limit, + sort_key, sort_dir, + project=None, + fields=None): + + limit = api_utils.validate_limit(limit) + sort_dir = api_utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Result.get_by_uuid(pecan.result.context, + marker) + + if sort_key in self.invalid_sort_key_list: + raise exception.InvalidParameterValue( + ("The sort_key value %(key)s is an invalid field for " + "sorting") % {'key': sort_key}) + + filters = {} + + if project: + if pecan.result.context.is_admin: + filters['project_id'] = project + results = objects.Result.list(pecan.result.context, limit, + marker_obj, + sort_key=sort_key, sort_dir=sort_dir, + filters=filters) + + parameters = {'sort_key': sort_key, 'sort_dir': sort_dir} + + return ResultCollection.convert_with_links(results, limit, + fields=fields, + **parameters) + + @expose.expose(Result, types.uuid_or_name, types.listtype) + def get_one(self, result_ident, fields=None): + """Retrieve information about the given result. + + :param result_ident: UUID or logical name of a result. + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + + cdict = pecan.result.context.to_policy_values() + policy.authorize('iot:result:get_one', cdict, cdict) + + rpc_result = objects.Result.get_by_uuid(pecan.result.context, + result_ident) + return Result.convert_with_links(rpc_result, fields=fields) + + @expose.expose(ResultCollection, types.uuid, int, wtypes.text, + wtypes.text, types.listtype, types.boolean, types.boolean) + def get_all(self, request_uuid, marker=None, + limit=None, sort_key='id', sort_dir='asc', + fields=None): + """Retrieve a list of results. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + This value cannot be larger than the value of max_limit + in the [api] section of the ironic configuration, or only + max_limit resources will be returned. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + :param with_public: Optional boolean to get also public pluings. + :param all_results: Optional boolean to get all the pluings. + Only for the admin + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + cdict = pecan.result.context.to_policy_values() + policy.authorize('iot:result:get', cdict, cdict) + + if fields is None: + fields = _DEFAULT_RETURN_FIELDS + return self._get_results_collection(marker, + limit, sort_key, sort_dir, + request_uuid=request_uuid, + fields=fields) + + @expose.expose(ResultCollection, types.uuid, int, wtypes.text, + wtypes.text, types.listtype, types.boolean, types.boolean) + def detail(self, marker=None, + limit=None, sort_key='id', sort_dir='asc', + fields=None, with_public=False, all_results=False): + """Retrieve a list of results. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + This value cannot be larger than the value of max_limit + in the [api] section of the ironic configuration, or only + max_limit resources will be returned. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + :param with_public: Optional boolean to get also public result. + :param all_results: Optional boolean to get all the results. + Only for the admin + :param fields: Optional, a list with a specified set of fields + of the resource to be returned. + """ + + cdict = pecan.result.context.to_policy_values() + policy.authorize('iot:result:get', cdict, cdict) + + # /detail should only work against collections + parent = pecan.result.path.split('/')[:-1][-1] + if parent != "results": + raise exception.HTTPNotFound() + + return self._get_results_collection(marker, + limit, sort_key, sort_dir, + with_public=with_public, + all_results=all_results, + fields=fields) diff --git a/iotronic/common/policy.py b/iotronic/common/policy.py index cb361ba..a66ae6a 100644 --- a/iotronic/common/policy.py +++ b/iotronic/common/policy.py @@ -209,6 +209,24 @@ enabledwebservice_policies = [ description='Retrieve a EnabledWebservice record'), ] +request_policies = [ + policy.RuleDefault('iot:request:get', + 'rule:is_admin or rule:is_iot_member', + description='Retrieve Request records'), + policy.RuleDefault('iot:request:get_one', + 'rule:is_admin or rule:is_iot_member', + description='Retrieve Request record'), +] + +result_policies = [ + policy.RuleDefault('iot:result:get', + 'rule:is_admin or rule:is_iot_member', + description='Retrieve Request records'), + policy.RuleDefault('iot:result:get_one', + 'rule:is_admin or rule:is_iot_member', + description='Retrieve Request record'), +] + def list_policies(): policies = (default_policies @@ -221,6 +239,8 @@ def list_policies(): + fleet_policies + webservice_policies + enabledwebservice_policies + + request_policies + + result_policies ) return policies diff --git a/iotronic/conductor/endpoints.py b/iotronic/conductor/endpoints.py index 34f6f8e..9551265 100644 --- a/iotronic/conductor/endpoints.py +++ b/iotronic/conductor/endpoints.py @@ -21,18 +21,21 @@ except Exception: # allow iotronic api to run also with python2.7 import pickle as cpickle +import random +import socket + + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging + +from iotronic import objects from iotronic.common import exception, designate from iotronic.common import neutron from iotronic.common import states from iotronic.conductor.provisioner import Provisioner -from iotronic import objects from iotronic.objects import base as objects_base from iotronic.wamp import wampmessage as wm -from oslo_config import cfg -from oslo_log import log as logging -import oslo_messaging -import random -import socket LOG = logging.getLogger(__name__) @@ -41,11 +44,13 @@ serializer = objects_base.IotronicObjectSerializer() Port = list() -# Method to compare two versions. -# Return 1 if v2 is smaller, -# -1 if v1 is smaller, -# 0 if equal def versionCompare(v1, v2): + """Method to compare two versions. + Return 1 if v2 is smaller, + -1 if v1 is smaller, + 0 if equal + """ + v1_list = v1.split(".")[:3] v2_list = v2.split(".")[:3] i = 0 @@ -62,6 +67,35 @@ def versionCompare(v1, v2): return 0 +def new_req(ctx, board, type, action, main_req=None, pending_requests=0): + req_data = { + 'destination_uuid': board.uuid, + 'type': type, + 'status': objects.request.PENDING, + 'action': action, + 'project': board.project, + 'pending_requests': pending_requests + } + if main_req: + req_data['main_request_uuid'] = main_req + req = objects.Request(ctx, **req_data) + req.create() + return req + + +def new_res(ctx, board, req_uuid): + res_data = { + 'board_uuid': board.uuid, + 'request_uuid': req_uuid, + 'result': objects.result.RUNNING, + 'message': "" + } + + res = objects.Result(ctx, **res_data) + res.create() + return res + + def get_best_agent(ctx): agents = objects.WampAgent.list(ctx, filters={'online': True}) LOG.debug('found %d Agent(s).', len(agents)) @@ -238,9 +272,10 @@ class ConductorEndpoint(object): return serializer.serialize_entity(ctx, new_board) - def execute_on_board(self, ctx, board_uuid, wamp_rpc_call, wamp_rpc_args): - LOG.debug('Executing \"%s\" on the board: %s', - wamp_rpc_call, board_uuid) + def execute_on_board(self, ctx, board_uuid, wamp_rpc_call, wamp_rpc_args, + main_req=None): + LOG.debug('Executing \"%s\" on the board: %s (req %s)', + wamp_rpc_call, board_uuid, main_req) board = objects.Board.get_by_uuid(ctx, board_uuid) # session should be taken from board, not from the object @@ -250,28 +285,12 @@ class ConductorEndpoint(object): session.session_id + "." + \ board.uuid + "." + wamp_rpc_call - # check the session; it rise an excpetion if session miss if not board.is_online(): raise exception.BoardNotConnected(board=board.uuid) - req_data = { - 'destination_uuid': board_uuid, - 'type': objects.request.BOARD, - 'status': objects.request.PENDING, - 'action': wamp_rpc_call - } - req = objects.Request(ctx, **req_data) - req.create() - - res_data = { - 'board_uuid': board_uuid, - 'request_uuid': req.uuid, - 'result': objects.result.RUNNING, - 'message': "" - } - - res = objects.Result(ctx, **res_data) - res.create() + req = new_req(ctx, board, objects.request.BOARD, wamp_rpc_call, + main_req) + res = new_res(ctx, board, req.uuid) cctx = self.wamp_agent_client.prepare(server=board.agent) @@ -285,7 +304,7 @@ class ConductorEndpoint(object): response = cctx.call(ctx, 's4t_invoke_wamp', wamp_rpc_call=full_wamp_call, - req_uuid=req.uuid, + req=req, data=wamp_rpc_args) response = wm.deserialize(response) @@ -297,6 +316,13 @@ class ConductorEndpoint(object): req.status = objects.request.COMPLETED req.save() + if req.main_request_uuid: + mreq = objects.Request.get_by_uuid(ctx, req.main_request_uuid) + mreq.pending_requests = mreq.pending_requests - 1 + if mreq.pending_requests == 0: + mreq.status = objects.request.COMPLETED + mreq.save() + return response def action_board(self, ctx, board_uuid, action, params): @@ -514,6 +540,8 @@ class ConductorEndpoint(object): board_uuid) board = objects.Board.get_by_uuid(ctx, board_uuid) + if not board.is_online(): + raise exception.BoardNotConnected(board=board.uuid) port_iotronic = objects.Port(ctx) subnet_info = neutron.subnet_info(subnet_uuid) @@ -641,6 +669,10 @@ class ConductorEndpoint(object): LOG.debug('Creating webservice %s', newwbs.name) + board = objects.Board.get_by_uuid(ctx, newwbs.board_uuid) + if not board.is_online(): + raise exception.BoardNotConnected(board=board.uuid) + en_webservice = objects.enabledwebservice. \ EnabledWebservice.get_by_board_uuid(ctx, newwbs.board_uuid) @@ -648,8 +680,6 @@ class ConductorEndpoint(object): full_zone_domain = en_webservice.dns + "." + en_webservice.zone dns_domain = newwbs.name + "." + full_zone_domain - board = objects.Board.get_by_uuid(ctx, newwbs.board_uuid) - create_record_dns_webservice(ctx, board, newwbs.name, en_webservice.dns, en_webservice.zone) @@ -691,8 +721,13 @@ class ConductorEndpoint(object): def destroy_webservice(self, ctx, webservice_id): LOG.info('Destroying webservice with id %s', webservice_id) + wbsrv = objects.Webservice.get_by_uuid(ctx, webservice_id) + board = objects.Board.get_by_uuid(ctx, wbsrv.board_uuid) + if not board.is_online(): + raise exception.BoardNotConnected(board=board.uuid) + en_webservice = objects.enabledwebservice. \ EnabledWebservice.get_by_board_uuid(ctx, wbsrv.board_uuid) @@ -721,7 +756,6 @@ class ConductorEndpoint(object): except exception: return exception - board = objects.Board.get_by_uuid(ctx, wbsrv.board_uuid) if board.agent == None: raise exception.BoardInvalidStatus(uuid=board.uuid, status=board.status) @@ -739,10 +773,16 @@ class ConductorEndpoint(object): def enable_webservice(self, ctx, dns, zone, email, board_uuid): board = objects.Board.get_by_uuid(ctx, board_uuid) + if not board.is_online(): + raise exception.BoardNotConnected(board=board.uuid) + if board.agent == None: raise exception.BoardInvalidStatus(uuid=board.uuid, status=board.status) + mreq = new_req(ctx, board, objects.request.BOARD, + "enable_webservice", pending_requests=3) + try: create_record_dns(ctx, board, dns, zone) except exception: @@ -784,7 +824,7 @@ class ConductorEndpoint(object): service = objects.Service.get_by_name(ctx, 'webservice') res = self.execute_on_board(ctx, board.uuid, "ServiceEnable", - (service, http_port)) + (service, http_port,), main_req=mreq.uuid) result = manage_result(res, "ServiceEnable", board.uuid) exp_data = { @@ -800,7 +840,7 @@ class ConductorEndpoint(object): service = objects.Service.get_by_name(ctx, 'webservice_ssl') res = self.execute_on_board(ctx, board.uuid, "ServiceEnable", - (service, https_port)) + (service, https_port,), main_req=mreq.uuid) result = manage_result(res, "ServiceEnable", board.uuid) exp_data = { @@ -822,11 +862,11 @@ class ConductorEndpoint(object): board.uuid, dns, email) try: - # result = self.execute_on_board(ctx, board.uuid, 'EnableWebService', - (dns + "." + zone, email,)) + (dns + "." + zone, email,), + main_req=mreq.uuid) except exception: return exception @@ -840,11 +880,16 @@ class ConductorEndpoint(object): board_uuid) board = objects.Board.get_by_uuid(ctx, board_uuid) + if not board.is_online(): + raise exception.BoardNotConnected(board=board.uuid) if board.agent == None: raise exception.BoardInvalidStatus(uuid=board.uuid, status=board.status) + mreq = new_req(ctx, board, objects.request.BOARD, + "disable_webservice", pending_requests=3) + en_webservice = objects.enabledwebservice. \ EnabledWebservice.get_by_board_uuid(ctx, board.uuid) @@ -862,7 +907,7 @@ class ConductorEndpoint(object): service.uuid) res = self.execute_on_board(ctx, board.uuid, "ServiceDisable", - (service,)) + (service,), main_req=mreq.uuid) LOG.debug(res.message) exposed.destroy() @@ -872,10 +917,19 @@ class ConductorEndpoint(object): board_uuid, service.uuid) res = self.execute_on_board(ctx, board.uuid, "ServiceDisable", - (service,)) + (service,), main_req=mreq.uuid) LOG.debug(res.message) exposed.destroy() + try: + self.execute_on_board(ctx, + board.uuid, + 'DisableWebService', + (), + main_req=mreq.uuid) + except exception: + return exception + webservice = objects.EnabledWebservice.get_by_board_uuid( ctx, board_uuid) diff --git a/iotronic/db/api.py b/iotronic/db/api.py index c60f7f9..85635cf 100644 --- a/iotronic/db/api.py +++ b/iotronic/db/api.py @@ -737,6 +737,28 @@ class Connection(object): :returns: A result. """ + @abc.abstractmethod + def get_result_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + """Return a list of boards. + + :param filters: Filters to apply. Defaults to None. + + :associated: True | False + :reserved: True | False + :maintenance: True | False + :provision_state: provision state of board + :provisioned_before: + boards with provision_updated_at field before this + interval in seconds + :param limit: Maximum number of boards to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + """ + @abc.abstractmethod def update_result(self, result_id, values): """Update properties of a result. @@ -755,10 +777,10 @@ class Connection(object): :returns: A request. """ - @abc.abstractmethod - def get_results(self, request_uuid): - """get results of a request. - - :param request_uuid: the request_uuid. - :returns: a list of results - """ + # @abc.abstractmethod + # def get_results(self, request_uuid): + # """get results of a request. + # + # :param request_uuid: the request_uuid. + # :returns: a list of results + # """ diff --git a/iotronic/db/sqlalchemy/alembic/versions/76c628d60004_add_results_requests.py b/iotronic/db/sqlalchemy/alembic/versions/76c628d60004_add_results_requests.py index ea483b2..87b52b6 100644 --- a/iotronic/db/sqlalchemy/alembic/versions/76c628d60004_add_results_requests.py +++ b/iotronic/db/sqlalchemy/alembic/versions/76c628d60004_add_results_requests.py @@ -26,11 +26,18 @@ def upgrade(): sa.Column('uuid', sa.String(length=36), nullable=False), sa.Column('destination_uuid', sa.String(length=36), nullable=False), + sa.Column('main_request_uuid', sa.String(length=36), + nullable=True), + sa.Column('pending_requests', sa.Integer(), default=0, + nullable=False), + sa.Column('project', sa.String(length=36), nullable=True), sa.Column('status', sa.String(length=10), nullable=False), sa.Column('type', sa.Integer(), nullable=False), sa.Column('action', sa.String(length=20), nullable=False), sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('uuid', name='uniq_requests0uuid') + sa.UniqueConstraint('uuid', name='uniq_requests0uuid'), + sa.ForeignKeyConstraint(['main_request_uuid'], + ['requests.uuid'], ) ) op.create_table('results', diff --git a/iotronic/db/sqlalchemy/api.py b/iotronic/db/sqlalchemy/api.py index fd0b496..f208dc1 100644 --- a/iotronic/db/sqlalchemy/api.py +++ b/iotronic/db/sqlalchemy/api.py @@ -196,8 +196,18 @@ class Connection(api.Connection): if filters is None: filters = [] - if 'project' in filters: - query = query.filter(models.Fleet.project == filters['project']) + if 'project_id' in filters: + query = query.filter(models.Fleet.project == + filters['project_id']) + return query + + def _add_requests_filters(self, query, filters): + if filters is None: + filters = [] + + if 'project_id' in filters: + query = query.filter(models.Request.project == + filters['project_id']) return query def _add_wampagents_filters(self, query, filters): @@ -227,12 +237,25 @@ class Connection(api.Connection): filter(models.Port.board_uuid == filters['board_uuid']) def _add_result_filters(self, query, filters): + if filters is None: filters = [] if 'result' in filters: query = query.filter(models.Result.result == filters['result']) + if 'request_uuid' in filters: + query = query.filter( + or_( + models.Request.main_request_uuid == + filters['request_uuid'], + models.Request.uuid == + filters['request_uuid'] + ) + ) + query = query.filter(models.Request.uuid == + models.Result.request_uuid) + return query def _do_update_board(self, board_id, values): @@ -1249,6 +1272,13 @@ class Connection(api.Connection): raise exception.InvalidParameterValue(err=msg) return self._do_update_request(request_id, values) + def get_request_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Request) + query = self._add_requests_filters(query, filters) + return _paginate_query(models.Request, limit, marker, + sort_key, sort_dir, query) + # RESULT def _do_update_result(self, update_id, values): @@ -1281,11 +1311,18 @@ class Connection(api.Connection): def update_result(self, result_id, values): return self._do_update_result(result_id, values) - def get_results(self, request_uuid, filters=None): - query = model_query(models.Result).filter_by( - request_uuid=request_uuid) + def get_result_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Result) query = self._add_result_filters(query, filters) - try: - return query.all() - except NoResultFound: - raise exception.ResultNotFound() + return _paginate_query(models.Result, limit, marker, + sort_key, sort_dir, query) + + # def get_results(self, request_uuid, filters=None): + # query = model_query(models.Result).filter_by( + # request_uuid=request_uuid) + # query = self._add_result_filters(query, filters) + # try: + # return query.all() + # except NoResultFound: + # raise exception.ResultNotFound() diff --git a/iotronic/db/sqlalchemy/models.py b/iotronic/db/sqlalchemy/models.py index f5488c2..11de53f 100644 --- a/iotronic/db/sqlalchemy/models.py +++ b/iotronic/db/sqlalchemy/models.py @@ -14,20 +14,20 @@ """ SQLAlchemy models for iot data. """ - +from iotronic.common import paths import json from oslo_config import cfg from oslo_db.sqlalchemy import models -import six.moves.urllib.parse as urlparse from sqlalchemy import Boolean from sqlalchemy import Column -from sqlalchemy import ForeignKey, Integer from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ForeignKey, Integer from sqlalchemy import schema from sqlalchemy import String from sqlalchemy.types import TypeDecorator, TEXT -from iotronic.common import paths +import six.moves.urllib.parse as urlparse + sql_opts = [ cfg.StrOpt('mysql_engine', @@ -326,8 +326,12 @@ class Request(Base): table_args()) id = Column(Integer, primary_key=True) uuid = Column(String(36)) - + main_request_uuid = Column(String(36), + ForeignKey('requests.uuid'), + nullable=True) destination_uuid = Column(String(36)) + pending_requests = Column(Integer, default=0) + project = Column(String(36)) status = Column(String(10)) type = Column(Integer) action = Column(String(20)) diff --git a/iotronic/objects/request.py b/iotronic/objects/request.py index c754705..5ccddf5 100644 --- a/iotronic/objects/request.py +++ b/iotronic/objects/request.py @@ -19,7 +19,6 @@ from oslo_utils import uuidutils from iotronic.common import exception from iotronic.db import api as db_api from iotronic.objects import base -from iotronic.objects.result import Result from iotronic.objects import utils as obj_utils BOARD = 0 @@ -39,7 +38,10 @@ class Request(base.IotronicObject): 'id': int, 'uuid': obj_utils.str_or_none, 'destination_uuid': obj_utils.str_or_none, + 'main_request_uuid': obj_utils.str_or_none, + 'pending_requests': int, 'status': obj_utils.str_or_none, + 'project': obj_utils.str_or_none, 'type': int, 'action': obj_utils.str_or_none, } @@ -88,15 +90,15 @@ class Request(base.IotronicObject): request = Request._from_db_object(cls(context), db_request) return request - @base.remotable_classmethod - def get_results(cls, context, request_uuid, filters=None): - """Find a request based on uuid and return a Board object. - - :param uuid: the uuid of a request. - :returns: a :class:`Board` object. - """ - return Result.get_results_list(context, - request_uuid, filters) + # @base.remotable_classmethod + # def get_results(cls, context, filters=None): + # """Find a request based on uuid and return a Board object. + # + # :param uuid: the uuid of a request. + # :returns: a :class:`Board` object. + # """ + # return Result.get_results_list(context, + # filters) # @base.remotable_classmethod # def get_results_request(cls,context,request_uuid): @@ -115,28 +117,29 @@ class Request(base.IotronicObject): # request = Request._from_db_object(cls(context), db_request) # return request - # @base.remotable_classmethod - # def list(cls, context, limit=None, marker=None, sort_key=None, - # sort_dir=None, filters=None): - # """Return a list of Request objects. - # - # :param context: Security context. - # :param limit: maximum number of resources to return in a - # single result. - # :param marker: pagination marker for large data sets. - # :param sort_key: column to sort results by. - # :param sort_dir: direction to sort. "asc" or "desc". - # :param filters: Filters to apply. - # :returns: a list of :class:`Request` object. - # - # """ - # db_requests = cls.dbapi.get_request_list(filters=filters, - # limit=limit, - # marker=marker, - # sort_key=sort_key, - # sort_dir=sort_dir) - # return [Request._from_db_object(cls(context), obj) - # for obj in db_requests] + @base.remotable_classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Request objects. + + :param context: Security context. + :param limit: maximum number of resources to return in a + single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Request` object. + + """ + db_requests = cls.dbapi.get_request_list(filters=filters, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + + return [Request._from_db_object(cls(context), obj) + for obj in db_requests] @base.remotable def create(self, context=None): diff --git a/iotronic/objects/result.py b/iotronic/objects/result.py index e23df74..99cf487 100644 --- a/iotronic/objects/result.py +++ b/iotronic/objects/result.py @@ -62,40 +62,41 @@ class Result(base.IotronicObject): return result @base.remotable_classmethod - def get_results_list(cls, context, request_uuid, filters=None): + def get_results_list(cls, context, filters=None): """Find a result based on name and return a Board object. :param board_uuid: the board uuid result. :param request_uuid: the request_uuid. :returns: a :class:`result` object. """ - db_requests = cls.dbapi.get_results(request_uuid, - filters=filters) + db_requests = cls.dbapi.get_result_list( + filters=filters) return [Result._from_db_object(cls(context), obj) for obj in db_requests] - # @base.remotable_classmethod - # def list(cls, context, limit=None, marker=None, sort_key=None, - # sort_dir=None, filters=None): - # """Return a list of Result objects. - # - # :param context: Security context. - # :param limit: maximum number of resources to return in a - # single result. - # :param marker: pagination marker for large data sets. - # :param sort_key: column to sort results by. - # :param sort_dir: direction to sort. "asc" or "desc". - # :param filters: Filters to apply. - # :returns: a list of :class:`Result` object. - # - # """ - # db_results = cls.dbapi.get_result_list(filters=filters, - # limit=limit, - # marker=marker, - # sort_key=sort_key, - # sort_dir=sort_dir) - # return [Result._from_db_object(cls(context), obj) - # for obj in db_results] + @base.remotable_classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Result objects. + + :param context: Security context. + :param limit: maximum number of resources to return in a + single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Result` object. + + """ + + db_results = cls.dbapi.get_result_list(filters=filters, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return [Result._from_db_object(cls(context), obj) + for obj in db_results] @base.remotable def create(self, context=None): diff --git a/iotronic/wamp/agent.py b/iotronic/wamp/agent.py index 00f5ffa..67abd6e 100644 --- a/iotronic/wamp/agent.py +++ b/iotronic/wamp/agent.py @@ -96,12 +96,12 @@ connected = False async def wamp_request(kwarg): # for previous LR version (to be removed asap) - if 'req_uuid' in kwarg: + if 'req' in kwarg: LOG.debug("calling: " + kwarg['wamp_rpc_call'] + - " with request id: " + kwarg['req_uuid']) + " with request id: " + kwarg['req']['uuid']) d = await wamp_session_caller.call(kwarg['wamp_rpc_call'], - kwarg['req_uuid'], + kwarg['req'], *kwarg['data']) else: LOG.debug("calling: " + kwarg['wamp_rpc_call']) diff --git a/iotronic/wamp/functions.py b/iotronic/wamp/functions.py index 536cf27..ec3bdab 100644 --- a/iotronic/wamp/functions.py +++ b/iotronic/wamp/functions.py @@ -187,14 +187,20 @@ def notify_result(board_uuid, wampmessage): res.message = wmsg.message res.save() - filter = {"result": objects.result.RUNNING} + filter = {"result": objects.result.RUNNING, + "request_uuid": wmsg.req_id} - list_result = objects.Request.get_results(ctxt, - wmsg.req_id, - filter) + list_result = objects.Result.get_results_list(ctxt, + filter) if len(list_result) == 0: req = objects.Request.get_by_uuid(ctxt, wmsg.req_id) req.status = objects.request.COMPLETED req.save() + if req.main_request_uuid: + mreq = objects.Request.get_by_uuid(ctxt, req.main_request_uuid) + mreq.pending_requests = mreq.pending_requests - 1 + if mreq.pending_requests == 0: + mreq.status = objects.request.COMPLETED + mreq.save() return wm.WampSuccess('notification_received').serialize()