Mistral driver

Mistral driver and unit tests. Allows congress to
create workflows and trigger execution in Mistral.

Also configures mistral for gate integration tests.
Tempest tests in separate patch.

Implements blueprint add-mistral-driver

Change-Id: I79d3b9c2f659302f43164d5eef3b23ddf2f2d056
This commit is contained in:
Eric Kao 2017-12-20 15:25:46 -08:00 committed by Eric K
parent 0bc5ab038b
commit d9f6b258f7
9 changed files with 476 additions and 3 deletions

View File

@ -16,6 +16,9 @@
- openstack/python-aodhclient
- openstack/python-congressclient
- openstack/python-muranoclient
- openstack/mistral
- openstack/python-mistralclient
- openstack/mistral-tempest-plugin
run: playbooks/legacy/congress-devstack-api-base/run.yaml
post-run: playbooks/legacy/congress-devstack-api-base/post.yaml
timeout: 6000
@ -40,6 +43,9 @@
- openstack/murano
- openstack/murano-dashboard
- openstack/python-muranoclient
- openstack/mistral
- openstack/python-mistralclient
- openstack/mistral-tempest-plugin
run: playbooks/legacy/congress-pe-replicated-base/run.yaml
post-run: playbooks/legacy/congress-pe-replicated-base/post.yaml
timeout: 6000
@ -75,6 +81,9 @@
- openstack/python-aodhclient
- openstack/python-congressclient
- openstack/python-muranoclient
- openstack/mistral
- openstack/python-mistralclient
- openstack/mistral-tempest-plugin
run: playbooks/legacy/congress-devstack-py35-api-mysql/run.yaml
post-run: playbooks/legacy/congress-devstack-py35-api-mysql/post.yaml
timeout: 6000

View File

@ -0,0 +1,204 @@
# Copyright (c) 2018 VMware, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Mistral Driver for Congress
This driver allows the creation of Congress datasources that interfaces with
Mistral workflows service. The Congress datasource reflects as Congress tables
the Mistral data on workflows, workflow executions, actions, and action
executions. The datasource also supports the triggering of Mistral APIs such as
initiation of a workflows or actions. The triggering of workflows or actions is
especially useful for creating Congress policies that take remedial action.
Datasource creation CLI example:
$ openstack congress datasource create mistral mistral_datasource \
--config username=admin \
--config tenant_name=admin \
--config auth_url=http://127.0.0.1/identity \
--config password=password
"""
from mistralclient.api.v2 import client as mistral_client
from congress.datasources import constants
from congress.datasources import datasource_driver
from congress.datasources import datasource_utils as ds_utils
class MistralDriver(datasource_driver.PollingDataSourceDriver,
datasource_driver.ExecutionDriver):
WORKFLOWS = 'workflows'
ACTIONS = 'actions'
WORKFLOW_EXECUTIONS = 'workflow_executions'
ACTION_EXECUTIONS = 'action_executions'
value_trans = {'translation-type': 'VALUE'}
workflows_translator = {
'translation-type': 'HDICT',
'table-name': WORKFLOWS,
'selector-type': 'DOT_SELECTOR',
'field-translators':
({'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'id', 'translator': value_trans},
{'fieldname': 'scope', 'translator': value_trans},
{'fieldname': 'input', 'translator': value_trans},
{'fieldname': 'namespace', 'translator': value_trans},
{'fieldname': 'project_id', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans},
{'fieldname': 'definition', 'translator': value_trans},
{'fieldname': 'description', 'translator': value_trans},
# TODO(ekcs): maybe enable tags in the future
)}
actions_translator = {
'translation-type': 'HDICT',
'table-name': ACTIONS,
'selector-type': 'DOT_SELECTOR',
'field-translators':
({'fieldname': 'id', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'input', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans},
{'fieldname': 'is_system', 'translator': value_trans},
{'fieldname': 'definition', 'translator': value_trans},
{'fieldname': 'description', 'translator': value_trans},
{'fieldname': 'scope', 'translator': value_trans},
# TODO(ekcs): maybe enable tags in the future
)}
workflow_executions_translator = {
'translation-type': 'HDICT',
'table-name': WORKFLOW_EXECUTIONS,
'selector-type': 'DOT_SELECTOR',
'field-translators':
({'fieldname': 'id', 'translator': value_trans},
{'fieldname': 'workflow_name', 'translator': value_trans},
{'fieldname': 'input', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans},
{'fieldname': 'state', 'translator': value_trans},
{'fieldname': 'state_info', 'translator': value_trans},
{'fieldname': 'description', 'translator': value_trans},
{'fieldname': 'workflow_id', 'translator': value_trans},
{'fieldname': 'workflow_namespace', 'translator': value_trans},
{'fieldname': 'params', 'translator': value_trans},
# TODO(ekcs): maybe add task_execution_ids table
)}
action_executions_translator = {
'translation-type': 'HDICT',
'table-name': ACTION_EXECUTIONS,
'selector-type': 'DOT_SELECTOR',
'field-translators':
({'fieldname': 'id', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'state_info', 'translator': value_trans},
{'fieldname': 'workflow_name', 'translator': value_trans},
{'fieldname': 'task_execution_id', 'translator': value_trans},
{'fieldname': 'task_name', 'translator': value_trans},
{'fieldname': 'description', 'translator': value_trans},
{'fieldname': 'input', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans},
{'fieldname': 'accepted', 'translator': value_trans},
{'fieldname': 'state', 'translator': value_trans},
{'fieldname': 'workflow_namespace', 'translator': value_trans},
# TODO(ekcs): maybe add action execution tags
)}
TRANSLATORS = [
workflows_translator, actions_translator,
workflow_executions_translator, action_executions_translator]
def __init__(self, name='', args=None):
super(MistralDriver, self).__init__(name, args=args)
datasource_driver.ExecutionDriver.__init__(self)
session = ds_utils.get_keystone_session(args)
self.mistral_client = mistral_client.Client(session=session)
self.add_executable_client_methods(
self.mistral_client, 'mistralclient.api.v2.')
self.initialize_update_method()
self._init_end_start_poll()
@staticmethod
def get_datasource_info():
result = {}
result['id'] = 'mistral'
result['description'] = ('Datasource driver that interfaces with '
'Mistral.')
result['config'] = ds_utils.get_openstack_required_config()
result['config']['lazy_tables'] = constants.OPTIONAL
result['secret'] = ['password']
return result
def initialize_update_method(self):
workflows_method = lambda: self._translate_workflows(
self.mistral_client.workflows.list())
self.add_update_method(workflows_method, self.workflows_translator)
workflow_executions_method = (
lambda: self._translate_workflow_executions(
self.mistral_client.executions.list()))
self.add_update_method(workflow_executions_method,
self.workflow_executions_translator)
actions_method = lambda: self._translate_actions(
self.mistral_client.actions.list())
self.add_update_method(actions_method, self.actions_translator)
action_executions_method = lambda: self._translate_action_executions(
self.mistral_client.action_executions.list())
self.add_update_method(action_executions_method,
self.action_executions_translator)
@ds_utils.update_state_on_changed(WORKFLOWS)
def _translate_workflows(self, obj):
"""Translate the workflows represented by OBJ into tables."""
row_data = MistralDriver.convert_objs(obj, self.workflows_translator)
return row_data
@ds_utils.update_state_on_changed(ACTIONS)
def _translate_actions(self, obj):
"""Translate the workflows represented by OBJ into tables."""
row_data = MistralDriver.convert_objs(obj, self.actions_translator)
return row_data
@ds_utils.update_state_on_changed(WORKFLOW_EXECUTIONS)
def _translate_workflow_executions(self, obj):
"""Translate the workflow_executions represented by OBJ into tables."""
row_data = MistralDriver.convert_objs(
obj, self.workflow_executions_translator)
return row_data
@ds_utils.update_state_on_changed(ACTION_EXECUTIONS)
def _translate_action_executions(self, obj):
"""Translate the action_executions represented by OBJ into tables."""
row_data = MistralDriver.convert_objs(
obj, self.action_executions_translator)
return row_data
def execute(self, action, action_args):
"""Overwrite ExecutionDriver.execute()."""
# action can be written as a method or an API call.
func = getattr(self, action, None)
if func and self.is_executable(func):
func(action_args)
else:
self._execute_api(self.mistral_client, action, action_args)

View File

@ -0,0 +1,237 @@
# Copyright (c) 2018 VMware, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import mock
import sys
sys.modules['mistralclient.api.v2.client'] = mock.Mock()
sys.modules['mistralclient.api.v2'] = mock.Mock()
from congress.datasources import mistral_driver
from congress.tests import base
from congress.tests.datasources import util
from congress.tests import helper
ResponseObj = util.ResponseObj
class TestMistralDriver(base.TestCase):
def setUp(self):
super(TestMistralDriver, self).setUp()
args = helper.datasource_openstack_args()
args['poll_time'] = 0
args['client'] = mock.MagicMock()
self.driver = mistral_driver.MistralDriver(
name='testmistral', args=args)
def test_list_workflows(self):
raw_data = [
ResponseObj({u'created_at': u'2017-10-12 20:06:58',
u'definition':
u'---\nversion: \'2.0\'\n\nstd.create_instance:\n'
u'...',
u'id': u'31c429eb-c439-43ec-a633-45c4e8749261',
u'input': u'name, image_id, flavor_id, '
u'ssh_username=None, ssh_password=None, '
u'key_name=None, security_groups=None, '
u'nics=None',
u'name': u'std.create_instance',
u'namespace': u'',
u'project_id': u'<default-project>',
u'scope': u'public',
u'tags': ['tag1', 'tag2'],
u'updated_at': None}),
ResponseObj({u'created_at': u'2017-10-12 20:06:58',
u'definition':
u'---\nversion: "2.0"\n\nstd.delete_instance:\n'
u'...',
u'id': u'55f43e39-89aa-43e6-9eec-526b5aa932b9',
u'input': u'instance_id',
u'name': u'std.delete_instance',
u'namespace': u'',
u'project_id': u'<default-project>',
u'scope': u'public',
u'tags': [],
u'updated_at': None})]
translated_data = self.driver._translate_workflows(raw_data)
self.assertIsNotNone(translated_data)
self.assertEqual(2, len(translated_data))
self.assertEqual({
(u'std.create_instance',
u'31c429eb-c439-43ec-a633-45c4e8749261',
u'public',
u'name, image_id, flavor_id, ssh_username=None, ssh_password='
u'None, key_name=None, security_groups=None, nics=None',
u'',
u'<default-project>',
u'2017-10-12 20:06:58',
u'None',
u"---\nversion: '2.0'\n\nstd.create_instance:\n...",
u'None'),
(u'std.delete_instance',
u'55f43e39-89aa-43e6-9eec-526b5aa932b9',
u'public',
u'instance_id',
u'',
u'<default-project>',
u'2017-10-12 20:06:58',
u'None',
u'---\nversion: "2.0"\n\nstd.delete_instance:\n...',
u'None')},
self.driver.state['workflows'])
def test_list_actions(self):
raw_data = [
ResponseObj({
u'created_at': u'2017-10-12 20:06:56',
u'definition': None,
u'description': u'Updates a load balancer health monitor.',
u'id': u'f794925d-ed65-41d4-a68d-076412d6ce9d',
u'input': u'health_monitor, action_region="", body=null',
u'is_system': True,
u'name': u'neutron.update_health_monitor',
u'scope': u'public',
u'tags': None,
u'updated_at': None}),
ResponseObj({
u'created_at': u'2017-10-13 20:06:56',
u'definition': u'action definition',
u'description': u'Updates a load balancer health monitor.',
u'id': u'a794925d-ed65-41d4-a68d-076412d6ce9d',
u'input': u'health_monitor, action_region="", body=null',
u'is_system': False,
u'name': u'neutron.custom_action',
u'scope': u'public',
u'tags': ['tag1', 'tag2'],
u'updated_at': u'2017-10-13 23:06:56'})]
translated_data = self.driver._translate_actions(raw_data)
self.assertIsNotNone(translated_data)
self.assertEqual(2, len(translated_data))
self.assertEqual({(u'a794925d-ed65-41d4-a68d-076412d6ce9d',
u'neutron.custom_action',
u'health_monitor, action_region="", body=null',
u'2017-10-13 20:06:56',
u'2017-10-13 23:06:56',
u'False',
u'action definition',
u'Updates a load balancer health monitor.',
u'public'),
(u'f794925d-ed65-41d4-a68d-076412d6ce9d',
u'neutron.update_health_monitor',
u'health_monitor, action_region="", body=null',
u'2017-10-12 20:06:56',
u'None',
u'True',
u'None',
u'Updates a load balancer health monitor.',
u'public')},
self.driver.state['actions'])
def test_list_workflow_executions(self):
raw_data = [
ResponseObj({u'created_at': u'2017-12-19 22:56:50',
u'description': u'',
u'id': u'46bbba4b-8a2e-4281-be61-1e92ebfdd6b6',
u'input': u'{"instance_id": 1}',
u'params': u'{"namespace": "", "task_name": ""}',
u'state': u'ERROR',
u'state_info': u"Failure caused by error ...",
u'task_execution_id': None,
u'updated_at': u'2017-12-19 22:57:00',
u'workflow_id':
u'55f43e39-89aa-43e6-9eec-526b5aa932b9',
u'workflow_name': u'std.delete_instance',
u'workflow_namespace': u''})]
translated_data = self.driver._translate_workflow_executions(raw_data)
self.assertIsNotNone(translated_data)
self.assertEqual(1, len(translated_data))
self.assertEqual({(u'46bbba4b-8a2e-4281-be61-1e92ebfdd6b6',
u'std.delete_instance',
u'{"instance_id": 1}',
u'2017-12-19 22:56:50',
u'2017-12-19 22:57:00',
u'ERROR',
u'Failure caused by error ...',
u'',
u'55f43e39-89aa-43e6-9eec-526b5aa932b9',
u'',
u'{"namespace": "", "task_name": ""}')},
self.driver.state['workflow_executions'])
def test_list_action_executions(self):
raw_data = [
ResponseObj({u'accepted': True,
u'created_at': u'2017-12-19 22:56:50',
u'description': u'',
u'id': u'5c377055-5590-479a-beec-3d4a47a2dfb6',
u'input': u'{"server": 1}',
u'name': u'nova.servers_delete',
u'state': u'ERROR',
u'state_info': None,
u'tags': None,
u'task_execution_id':
u'f40a0a20-958d-4948-b0c0-e1961649f2e2',
u'task_name': u'delete_vm',
u'updated_at': u'2017-12-19 22:56:50',
u'workflow_name': u'std.delete_instance',
u'workflow_namespace': u''})]
translated_data = self.driver._translate_action_executions(raw_data)
self.assertIsNotNone(translated_data)
self.assertEqual(1, len(translated_data))
self.assertEqual({(u'5c377055-5590-479a-beec-3d4a47a2dfb6',
u'nova.servers_delete',
u'None',
u'std.delete_instance',
u'f40a0a20-958d-4948-b0c0-e1961649f2e2',
u'delete_vm',
u'',
u'{"server": 1}',
u'2017-12-19 22:56:50',
u'2017-12-19 22:56:50',
u'True',
u'ERROR',
u'')},
self.driver.state['action_executions'])
def test_execute(self):
class MockClient(object):
def __init__(self):
self.testkey = None
def mock_action(self, arg1):
self.testkey = 'arg1=%s' % arg1
mock_client = MockClient()
self.driver.mistral_client = mock_client
api_args = {
'positional': ['1']
}
expected_ans = 'arg1=1'
self.driver.execute('mock_action', api_args)
self.assertEqual(expected_ans, mock_client.testkey)

View File

@ -75,8 +75,9 @@ function configure_congress {
CONGRESS_DRIVERS+="congress.datasources.heatv1_driver.HeatV1Driver,"
CONGRESS_DRIVERS+="congress.datasources.doctor_driver.DoctorDriver,"
CONGRESS_DRIVERS+="congress.datasources.aodh_driver.AodhDriver,"
CONGRESS_DRIVERS+="congress.tests.fake_datasource.FakeDataSource,"
CONGRESS_DRIVERS+="congress.datasources.cfgvalidator_driver.ValidatorDriver"
CONGRESS_DRIVERS+="congress.datasources.cfgvalidator_driver.ValidatorDriver,"
CONGRESS_DRIVERS+="congress.datasources.mistral_driver.MistralDriver,"
CONGRESS_DRIVERS+="congress.tests.fake_datasource.FakeDataSource"
iniset $CONGRESS_CONF DEFAULT drivers $CONGRESS_DRIVERS
@ -98,9 +99,9 @@ function configure_congress_datasources {
_configure_service ironic ironic
_configure_service heat heat
_configure_service aodh aodh
_configure_service mistral mistral
# FIXME(ekcs): congress-agent temporarily disabled while gate issue being resolved
# _configure_service congress-agent config
}
function _configure_service {

View File

@ -33,6 +33,7 @@
enable_plugin congress git://git.openstack.org/openstack/congress
enable_plugin murano git://git.openstack.org/openstack/murano
enable_plugin aodh git://git.openstack.org/openstack/aodh
enable_plugin mistral git://git.openstack.org/openstack/mistral
enable_plugin neutron https://git.openstack.org/openstack/neutron
# To deploy congress as multi-process (api, pe, datasources)
CONGRESS_MULTIPROCESS_DEPLOYMENT=True
@ -61,6 +62,9 @@
#export DEVSTACK_PROJECT_FROM_GIT=python-congressclient
export PROJECTS="openstack/murano $PROJECTS"
export PROJECTS="openstack/aodh $PROJECTS"
export PROJECTS="openstack/mistral $PROJECTS"
export PROJECTS="openstack/python-mistralclient $PROJECTS"
export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS"
export PROJECTS="openstack/murano-dashboard $PROJECTS"
export PROJECTS="openstack/python-muranoclient $PROJECTS"
export PROJECTS="openstack/python-aodhclient $PROJECTS"

View File

@ -41,6 +41,7 @@
enable_plugin heat git://git.openstack.org/openstack/heat
enable_plugin congress git://git.openstack.org/openstack/congress
enable_plugin murano git://git.openstack.org/openstack/murano
enable_plugin mistral git://git.openstack.org/openstack/mistral
enable_plugin neutron https://git.openstack.org/openstack/neutron
# To deploy congress as multi-process (api, pe, datasources)
CONGRESS_MULTIPROCESS_DEPLOYMENT=True
@ -70,6 +71,9 @@
export PROJECTS="openstack/python-congressclient $PROJECTS"
export PROJECTS="openstack/murano $PROJECTS"
export PROJECTS="openstack/aodh $PROJECTS"
export PROJECTS="openstack/mistral $PROJECTS"
export PROJECTS="openstack/python-mistralclient $PROJECTS"
export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS"
export PROJECTS="openstack/murano-dashboard $PROJECTS"
export PROJECTS="openstack/python-muranoclient $PROJECTS"
export PROJECTS="openstack/python-aodhclient $PROJECTS"

View File

@ -33,6 +33,7 @@
enable_plugin congress git://git.openstack.org/openstack/congress
enable_plugin murano git://git.openstack.org/openstack/murano
enable_plugin aodh git://git.openstack.org/openstack/aodh
enable_plugin mistral git://git.openstack.org/openstack/mistral
enable_plugin neutron https://git.openstack.org/openstack/neutron
CONGRESS_REPLICATED=True
# To deploy congress as multi-process (api, pe, datasources)
@ -61,6 +62,9 @@
export PROJECTS="openstack/python-congressclient $PROJECTS"
export PROJECTS="openstack/murano $PROJECTS"
export PROJECTS="openstack/aodh $PROJECTS"
export PROJECTS="openstack/mistral $PROJECTS"
export PROJECTS="openstack/python-mistralclient $PROJECTS"
export PROJECTS="openstack/mistral-tempest-plugin $PROJECTS"
export PROJECTS="openstack/murano-dashboard $PROJECTS"
export PROJECTS="openstack/python-muranoclient $PROJECTS"
export PROJECTS="openstack/python-aodhclient $PROJECTS"

View File

@ -0,0 +1,9 @@
---
prelude: >
features:
- Mistral datasource driver for Congress added, enabling Congress policies to
trigger and monitor Mistral workflows and actions.
upgrade:
- To enable Mistral datasource driver, add the following class path to the
drivers option in the DEFAULT section of congress config file.
congress.datasources.mistral_driver.MistralDriver

View File

@ -21,6 +21,7 @@ python-neutronclient>=6.3.0 # Apache-2.0
python-cinderclient>=3.3.0 # Apache-2.0
python-swiftclient>=3.2.0 # Apache-2.0
python-ironicclient>=1.14.0 # Apache-2.0
python-mistralclient>=3.1.0 # Apache-2.0
alembic>=0.8.10 # MIT
cryptography!=2.0,>=1.9 # BSD/Apache-2.0
python-dateutil>=2.4.2 # BSD