diff --git a/.zuul.yaml b/.zuul.yaml index fd4f031ce..0380edc48 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -16,6 +16,9 @@ - openstack/python-aodhclient - openstack/python-congressclient - openstack/python-muranoclient + - openstack/mistral + - openstack/python-mistralclient + - openstack/mistral-tempest-plugin run: playbooks/legacy/congress-devstack-api-base/run.yaml post-run: playbooks/legacy/congress-devstack-api-base/post.yaml timeout: 6000 @@ -40,6 +43,9 @@ - openstack/murano - openstack/murano-dashboard - openstack/python-muranoclient + - openstack/mistral + - openstack/python-mistralclient + - openstack/mistral-tempest-plugin run: playbooks/legacy/congress-pe-replicated-base/run.yaml post-run: playbooks/legacy/congress-pe-replicated-base/post.yaml timeout: 6000 @@ -75,6 +81,9 @@ - openstack/python-aodhclient - openstack/python-congressclient - openstack/python-muranoclient + - openstack/mistral + - openstack/python-mistralclient + - openstack/mistral-tempest-plugin run: playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml post-run: playbooks/legacy/congress-devstack-py35-api-mysql/post.yaml timeout: 6000 diff --git a/congress/datasources/mistral_driver.py b/congress/datasources/mistral_driver.py new file mode 100644 index 000000000..8e87ec2e6 --- /dev/null +++ b/congress/datasources/mistral_driver.py @@ -0,0 +1,204 @@ +# Copyright (c) 2018 VMware, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Mistral Driver for Congress + +This driver allows the creation of Congress datasources that interfaces with +Mistral workflows service. The Congress datasource reflects as Congress tables +the Mistral data on workflows, workflow executions, actions, and action +executions. The datasource also supports the triggering of Mistral APIs such as +initiation of a workflows or actions. The triggering of workflows or actions is +especially useful for creating Congress policies that take remedial action. + +Datasource creation CLI example: +$ openstack congress datasource create mistral mistral_datasource \ + --config username=admin \ + --config tenant_name=admin \ + --config auth_url=http://127.0.0.1/identity \ + --config password=password +""" + +from mistralclient.api.v2 import client as mistral_client + +from congress.datasources import constants +from congress.datasources import datasource_driver +from congress.datasources import datasource_utils as ds_utils + + +class MistralDriver(datasource_driver.PollingDataSourceDriver, + datasource_driver.ExecutionDriver): + WORKFLOWS = 'workflows' + ACTIONS = 'actions' + + WORKFLOW_EXECUTIONS = 'workflow_executions' + ACTION_EXECUTIONS = 'action_executions' + + value_trans = {'translation-type': 'VALUE'} + + workflows_translator = { + 'translation-type': 'HDICT', + 'table-name': WORKFLOWS, + 'selector-type': 'DOT_SELECTOR', + 'field-translators': + ({'fieldname': 'name', 'translator': value_trans}, + {'fieldname': 'id', 'translator': value_trans}, + {'fieldname': 'scope', 'translator': value_trans}, + {'fieldname': 'input', 'translator': value_trans}, + {'fieldname': 'namespace', 'translator': value_trans}, + {'fieldname': 'project_id', 'translator': value_trans}, + {'fieldname': 'created_at', 'translator': value_trans}, + {'fieldname': 'updated_at', 'translator': value_trans}, + {'fieldname': 'definition', 'translator': value_trans}, + {'fieldname': 'description', 'translator': value_trans}, + # TODO(ekcs): maybe enable tags in the future + )} + + actions_translator = { + 'translation-type': 'HDICT', + 'table-name': ACTIONS, + 'selector-type': 'DOT_SELECTOR', + 'field-translators': + ({'fieldname': 'id', 'translator': value_trans}, + {'fieldname': 'name', 'translator': value_trans}, + {'fieldname': 'input', 'translator': value_trans}, + {'fieldname': 'created_at', 'translator': value_trans}, + {'fieldname': 'updated_at', 'translator': value_trans}, + {'fieldname': 'is_system', 'translator': value_trans}, + {'fieldname': 'definition', 'translator': value_trans}, + {'fieldname': 'description', 'translator': value_trans}, + {'fieldname': 'scope', 'translator': value_trans}, + # TODO(ekcs): maybe enable tags in the future + )} + + workflow_executions_translator = { + 'translation-type': 'HDICT', + 'table-name': WORKFLOW_EXECUTIONS, + 'selector-type': 'DOT_SELECTOR', + 'field-translators': + ({'fieldname': 'id', 'translator': value_trans}, + {'fieldname': 'workflow_name', 'translator': value_trans}, + {'fieldname': 'input', 'translator': value_trans}, + {'fieldname': 'created_at', 'translator': value_trans}, + {'fieldname': 'updated_at', 'translator': value_trans}, + {'fieldname': 'state', 'translator': value_trans}, + {'fieldname': 'state_info', 'translator': value_trans}, + {'fieldname': 'description', 'translator': value_trans}, + {'fieldname': 'workflow_id', 'translator': value_trans}, + {'fieldname': 'workflow_namespace', 'translator': value_trans}, + {'fieldname': 'params', 'translator': value_trans}, + # TODO(ekcs): maybe add task_execution_ids table + )} + + action_executions_translator = { + 'translation-type': 'HDICT', + 'table-name': ACTION_EXECUTIONS, + 'selector-type': 'DOT_SELECTOR', + 'field-translators': + ({'fieldname': 'id', 'translator': value_trans}, + {'fieldname': 'name', 'translator': value_trans}, + {'fieldname': 'state_info', 'translator': value_trans}, + {'fieldname': 'workflow_name', 'translator': value_trans}, + {'fieldname': 'task_execution_id', 'translator': value_trans}, + {'fieldname': 'task_name', 'translator': value_trans}, + {'fieldname': 'description', 'translator': value_trans}, + {'fieldname': 'input', 'translator': value_trans}, + {'fieldname': 'created_at', 'translator': value_trans}, + {'fieldname': 'updated_at', 'translator': value_trans}, + {'fieldname': 'accepted', 'translator': value_trans}, + {'fieldname': 'state', 'translator': value_trans}, + {'fieldname': 'workflow_namespace', 'translator': value_trans}, + # TODO(ekcs): maybe add action execution tags + )} + + TRANSLATORS = [ + workflows_translator, actions_translator, + workflow_executions_translator, action_executions_translator] + + def __init__(self, name='', args=None): + super(MistralDriver, self).__init__(name, args=args) + datasource_driver.ExecutionDriver.__init__(self) + session = ds_utils.get_keystone_session(args) + self.mistral_client = mistral_client.Client(session=session) + + self.add_executable_client_methods( + self.mistral_client, 'mistralclient.api.v2.') + self.initialize_update_method() + self._init_end_start_poll() + + @staticmethod + def get_datasource_info(): + result = {} + result['id'] = 'mistral' + result['description'] = ('Datasource driver that interfaces with ' + 'Mistral.') + result['config'] = ds_utils.get_openstack_required_config() + result['config']['lazy_tables'] = constants.OPTIONAL + result['secret'] = ['password'] + return result + + def initialize_update_method(self): + workflows_method = lambda: self._translate_workflows( + self.mistral_client.workflows.list()) + self.add_update_method(workflows_method, self.workflows_translator) + + workflow_executions_method = ( + lambda: self._translate_workflow_executions( + self.mistral_client.executions.list())) + self.add_update_method(workflow_executions_method, + self.workflow_executions_translator) + + actions_method = lambda: self._translate_actions( + self.mistral_client.actions.list()) + self.add_update_method(actions_method, self.actions_translator) + + action_executions_method = lambda: self._translate_action_executions( + self.mistral_client.action_executions.list()) + self.add_update_method(action_executions_method, + self.action_executions_translator) + + @ds_utils.update_state_on_changed(WORKFLOWS) + def _translate_workflows(self, obj): + """Translate the workflows represented by OBJ into tables.""" + row_data = MistralDriver.convert_objs(obj, self.workflows_translator) + return row_data + + @ds_utils.update_state_on_changed(ACTIONS) + def _translate_actions(self, obj): + """Translate the workflows represented by OBJ into tables.""" + row_data = MistralDriver.convert_objs(obj, self.actions_translator) + return row_data + + @ds_utils.update_state_on_changed(WORKFLOW_EXECUTIONS) + def _translate_workflow_executions(self, obj): + """Translate the workflow_executions represented by OBJ into tables.""" + row_data = MistralDriver.convert_objs( + obj, self.workflow_executions_translator) + return row_data + + @ds_utils.update_state_on_changed(ACTION_EXECUTIONS) + def _translate_action_executions(self, obj): + """Translate the action_executions represented by OBJ into tables.""" + row_data = MistralDriver.convert_objs( + obj, self.action_executions_translator) + return row_data + + def execute(self, action, action_args): + """Overwrite ExecutionDriver.execute().""" + # action can be written as a method or an API call. + func = getattr(self, action, None) + if func and self.is_executable(func): + func(action_args) + else: + self._execute_api(self.mistral_client, action, action_args) diff --git a/congress/tests/datasources/test_mistral_driver.py b/congress/tests/datasources/test_mistral_driver.py new file mode 100644 index 000000000..32a5e2186 --- /dev/null +++ b/congress/tests/datasources/test_mistral_driver.py @@ -0,0 +1,237 @@ +# Copyright (c) 2018 VMware, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import mock +import sys + +sys.modules['mistralclient.api.v2.client'] = mock.Mock() +sys.modules['mistralclient.api.v2'] = mock.Mock() + +from congress.datasources import mistral_driver +from congress.tests import base +from congress.tests.datasources import util +from congress.tests import helper + +ResponseObj = util.ResponseObj + + +class TestMistralDriver(base.TestCase): + + def setUp(self): + super(TestMistralDriver, self).setUp() + args = helper.datasource_openstack_args() + args['poll_time'] = 0 + args['client'] = mock.MagicMock() + self.driver = mistral_driver.MistralDriver( + name='testmistral', args=args) + + def test_list_workflows(self): + raw_data = [ + ResponseObj({u'created_at': u'2017-10-12 20:06:58', + u'definition': + u'---\nversion: \'2.0\'\n\nstd.create_instance:\n' + u'...', + u'id': u'31c429eb-c439-43ec-a633-45c4e8749261', + u'input': u'name, image_id, flavor_id, ' + u'ssh_username=None, ssh_password=None, ' + u'key_name=None, security_groups=None, ' + u'nics=None', + u'name': u'std.create_instance', + u'namespace': u'', + u'project_id': u'', + u'scope': u'public', + u'tags': ['tag1', 'tag2'], + u'updated_at': None}), + ResponseObj({u'created_at': u'2017-10-12 20:06:58', + u'definition': + u'---\nversion: "2.0"\n\nstd.delete_instance:\n' + u'...', + u'id': u'55f43e39-89aa-43e6-9eec-526b5aa932b9', + u'input': u'instance_id', + u'name': u'std.delete_instance', + u'namespace': u'', + u'project_id': u'', + u'scope': u'public', + u'tags': [], + u'updated_at': None})] + + translated_data = self.driver._translate_workflows(raw_data) + self.assertIsNotNone(translated_data) + self.assertEqual(2, len(translated_data)) + + self.assertEqual({ + (u'std.create_instance', + u'31c429eb-c439-43ec-a633-45c4e8749261', + u'public', + u'name, image_id, flavor_id, ssh_username=None, ssh_password=' + u'None, key_name=None, security_groups=None, nics=None', + u'', + u'', + u'2017-10-12 20:06:58', + u'None', + u"---\nversion: '2.0'\n\nstd.create_instance:\n...", + u'None'), + (u'std.delete_instance', + u'55f43e39-89aa-43e6-9eec-526b5aa932b9', + u'public', + u'instance_id', + u'', + u'', + u'2017-10-12 20:06:58', + u'None', + u'---\nversion: "2.0"\n\nstd.delete_instance:\n...', + u'None')}, + self.driver.state['workflows']) + + def test_list_actions(self): + raw_data = [ + ResponseObj({ + u'created_at': u'2017-10-12 20:06:56', + u'definition': None, + u'description': u'Updates a load balancer health monitor.', + u'id': u'f794925d-ed65-41d4-a68d-076412d6ce9d', + u'input': u'health_monitor, action_region="", body=null', + u'is_system': True, + u'name': u'neutron.update_health_monitor', + u'scope': u'public', + u'tags': None, + u'updated_at': None}), + ResponseObj({ + u'created_at': u'2017-10-13 20:06:56', + u'definition': u'action definition', + u'description': u'Updates a load balancer health monitor.', + u'id': u'a794925d-ed65-41d4-a68d-076412d6ce9d', + u'input': u'health_monitor, action_region="", body=null', + u'is_system': False, + u'name': u'neutron.custom_action', + u'scope': u'public', + u'tags': ['tag1', 'tag2'], + u'updated_at': u'2017-10-13 23:06:56'})] + + translated_data = self.driver._translate_actions(raw_data) + self.assertIsNotNone(translated_data) + self.assertEqual(2, len(translated_data)) + + self.assertEqual({(u'a794925d-ed65-41d4-a68d-076412d6ce9d', + u'neutron.custom_action', + u'health_monitor, action_region="", body=null', + u'2017-10-13 20:06:56', + u'2017-10-13 23:06:56', + u'False', + u'action definition', + u'Updates a load balancer health monitor.', + u'public'), + (u'f794925d-ed65-41d4-a68d-076412d6ce9d', + u'neutron.update_health_monitor', + u'health_monitor, action_region="", body=null', + u'2017-10-12 20:06:56', + u'None', + u'True', + u'None', + u'Updates a load balancer health monitor.', + u'public')}, + self.driver.state['actions']) + + def test_list_workflow_executions(self): + raw_data = [ + ResponseObj({u'created_at': u'2017-12-19 22:56:50', + u'description': u'', + u'id': u'46bbba4b-8a2e-4281-be61-1e92ebfdd6b6', + u'input': u'{"instance_id": 1}', + u'params': u'{"namespace": "", "task_name": ""}', + u'state': u'ERROR', + u'state_info': u"Failure caused by error ...", + u'task_execution_id': None, + u'updated_at': u'2017-12-19 22:57:00', + u'workflow_id': + u'55f43e39-89aa-43e6-9eec-526b5aa932b9', + u'workflow_name': u'std.delete_instance', + u'workflow_namespace': u''})] + + translated_data = self.driver._translate_workflow_executions(raw_data) + self.assertIsNotNone(translated_data) + self.assertEqual(1, len(translated_data)) + + self.assertEqual({(u'46bbba4b-8a2e-4281-be61-1e92ebfdd6b6', + u'std.delete_instance', + u'{"instance_id": 1}', + u'2017-12-19 22:56:50', + u'2017-12-19 22:57:00', + u'ERROR', + u'Failure caused by error ...', + u'', + u'55f43e39-89aa-43e6-9eec-526b5aa932b9', + u'', + u'{"namespace": "", "task_name": ""}')}, + self.driver.state['workflow_executions']) + + def test_list_action_executions(self): + raw_data = [ + ResponseObj({u'accepted': True, + u'created_at': u'2017-12-19 22:56:50', + u'description': u'', + u'id': u'5c377055-5590-479a-beec-3d4a47a2dfb6', + u'input': u'{"server": 1}', + u'name': u'nova.servers_delete', + u'state': u'ERROR', + u'state_info': None, + u'tags': None, + u'task_execution_id': + u'f40a0a20-958d-4948-b0c0-e1961649f2e2', + u'task_name': u'delete_vm', + u'updated_at': u'2017-12-19 22:56:50', + u'workflow_name': u'std.delete_instance', + u'workflow_namespace': u''})] + + translated_data = self.driver._translate_action_executions(raw_data) + self.assertIsNotNone(translated_data) + self.assertEqual(1, len(translated_data)) + + self.assertEqual({(u'5c377055-5590-479a-beec-3d4a47a2dfb6', + u'nova.servers_delete', + u'None', + u'std.delete_instance', + u'f40a0a20-958d-4948-b0c0-e1961649f2e2', + u'delete_vm', + u'', + u'{"server": 1}', + u'2017-12-19 22:56:50', + u'2017-12-19 22:56:50', + u'True', + u'ERROR', + u'')}, + self.driver.state['action_executions']) + + def test_execute(self): + class MockClient(object): + def __init__(self): + self.testkey = None + + def mock_action(self, arg1): + self.testkey = 'arg1=%s' % arg1 + + mock_client = MockClient() + self.driver.mistral_client = mock_client + api_args = { + 'positional': ['1'] + } + expected_ans = 'arg1=1' + + self.driver.execute('mock_action', api_args) + + self.assertEqual(expected_ans, mock_client.testkey) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 3afb95b76..61acd49dc 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -75,8 +75,9 @@ function configure_congress { CONGRESS_DRIVERS+="congress.datasources.heatv1_driver.HeatV1Driver," CONGRESS_DRIVERS+="congress.datasources.doctor_driver.DoctorDriver," CONGRESS_DRIVERS+="congress.datasources.aodh_driver.AodhDriver," - CONGRESS_DRIVERS+="congress.tests.fake_datasource.FakeDataSource," - CONGRESS_DRIVERS+="congress.datasources.cfgvalidator_driver.ValidatorDriver" + CONGRESS_DRIVERS+="congress.datasources.cfgvalidator_driver.ValidatorDriver," + CONGRESS_DRIVERS+="congress.datasources.mistral_driver.MistralDriver," + CONGRESS_DRIVERS+="congress.tests.fake_datasource.FakeDataSource" iniset $CONGRESS_CONF DEFAULT drivers $CONGRESS_DRIVERS @@ -98,9 +99,9 @@ function configure_congress_datasources { _configure_service ironic ironic _configure_service heat heat _configure_service aodh aodh + _configure_service mistral mistral # FIXME(ekcs): congress-agent temporarily disabled while gate issue being resolved # _configure_service congress-agent config - } function _configure_service { diff --git a/playbooks/legacy/congress-devstack-api-base/run.yaml b/playbooks/legacy/congress-devstack-api-base/run.yaml index 6ed9618ab..9abe5317b 100644 --- a/playbooks/legacy/congress-devstack-api-base/run.yaml +++ b/playbooks/legacy/congress-devstack-api-base/run.yaml @@ -33,6 +33,7 @@ enable_plugin congress git://git.openstack.org/openstack/congress enable_plugin murano git://git.openstack.org/openstack/murano enable_plugin aodh git://git.openstack.org/openstack/aodh + enable_plugin mistral git://git.openstack.org/openstack/mistral enable_plugin neutron https://git.openstack.org/openstack/neutron # To deploy congress as multi-process (api, pe, datasources) CONGRESS_MULTIPROCESS_DEPLOYMENT=True @@ -61,6 +62,9 @@ #export DEVSTACK_PROJECT_FROM_GIT=python-congressclient export PROJECTS="openstack/murano $PROJECTS" export PROJECTS="openstack/aodh $PROJECTS" + export PROJECTS="openstack/mistral $PROJECTS" + export PROJECTS="openstack/python-mistralclient $PROJECTS" + export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS" export PROJECTS="openstack/murano-dashboard $PROJECTS" export PROJECTS="openstack/python-muranoclient $PROJECTS" export PROJECTS="openstack/python-aodhclient $PROJECTS" diff --git a/playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml b/playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml index 3505d6dc8..c388264c4 100644 --- a/playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml +++ b/playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml @@ -41,6 +41,7 @@ enable_plugin heat git://git.openstack.org/openstack/heat enable_plugin congress git://git.openstack.org/openstack/congress enable_plugin murano git://git.openstack.org/openstack/murano + enable_plugin mistral git://git.openstack.org/openstack/mistral enable_plugin neutron https://git.openstack.org/openstack/neutron # To deploy congress as multi-process (api, pe, datasources) CONGRESS_MULTIPROCESS_DEPLOYMENT=True @@ -70,6 +71,9 @@ export PROJECTS="openstack/python-congressclient $PROJECTS" export PROJECTS="openstack/murano $PROJECTS" export PROJECTS="openstack/aodh $PROJECTS" + export PROJECTS="openstack/mistral $PROJECTS" + export PROJECTS="openstack/python-mistralclient $PROJECTS" + export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS" export PROJECTS="openstack/murano-dashboard $PROJECTS" export PROJECTS="openstack/python-muranoclient $PROJECTS" export PROJECTS="openstack/python-aodhclient $PROJECTS" diff --git a/playbooks/legacy/congress-pe-replicated-base/run.yaml b/playbooks/legacy/congress-pe-replicated-base/run.yaml index 6656c6729..18a7352f5 100644 --- a/playbooks/legacy/congress-pe-replicated-base/run.yaml +++ b/playbooks/legacy/congress-pe-replicated-base/run.yaml @@ -33,6 +33,7 @@ enable_plugin congress git://git.openstack.org/openstack/congress enable_plugin murano git://git.openstack.org/openstack/murano enable_plugin aodh git://git.openstack.org/openstack/aodh + enable_plugin mistral git://git.openstack.org/openstack/mistral enable_plugin neutron https://git.openstack.org/openstack/neutron CONGRESS_REPLICATED=True # To deploy congress as multi-process (api, pe, datasources) @@ -61,6 +62,9 @@ export PROJECTS="openstack/python-congressclient $PROJECTS" export PROJECTS="openstack/murano $PROJECTS" export PROJECTS="openstack/aodh $PROJECTS" + export PROJECTS="openstack/mistral $PROJECTS" + export PROJECTS="openstack/python-mistralclient $PROJECTS" + export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS" export PROJECTS="openstack/murano-dashboard $PROJECTS" export PROJECTS="openstack/python-muranoclient $PROJECTS" export PROJECTS="openstack/python-aodhclient $PROJECTS" diff --git a/releasenotes/notes/mistral-driver-457e325bdae1a3bd.yaml b/releasenotes/notes/mistral-driver-457e325bdae1a3bd.yaml new file mode 100644 index 000000000..9d2d16e29 --- /dev/null +++ b/releasenotes/notes/mistral-driver-457e325bdae1a3bd.yaml @@ -0,0 +1,9 @@ +--- +prelude: > +features: + - Mistral datasource driver for Congress added, enabling Congress policies to + trigger and monitor Mistral workflows and actions. +upgrade: + - To enable Mistral datasource driver, add the following class path to the + drivers option in the DEFAULT section of congress config file. + congress.datasources.mistral_driver.MistralDriver diff --git a/requirements.txt b/requirements.txt index f0d96257e..9902a9c88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ python-neutronclient>=6.3.0 # Apache-2.0 python-cinderclient>=3.3.0 # Apache-2.0 python-swiftclient>=3.2.0 # Apache-2.0 python-ironicclient>=1.14.0 # Apache-2.0 +python-mistralclient>=3.1.0 # Apache-2.0 alembic>=0.8.10 # MIT cryptography!=2.0,>=1.9 # BSD/Apache-2.0 python-dateutil>=2.4.2 # BSD