Allow to install plugin to operational environment

- Added attribute 'is_runtime' to plugin. this attribute means
  that plugin can be installed to operational env.
- Make attribute 'is_locked' public and remove logic to calculate
  the cluster is locked or not from UI.

Change-Id: I372bfb2c502bcd5927533b03aea1557cf49d9afb
Closes-Bug: #1519050
This commit is contained in:
Bulat Gaifullin 2015-11-23 21:28:35 +03:00
parent 2e90f7a577
commit 34e4f4f0dc
16 changed files with 429 additions and 90 deletions

View File

@ -15,6 +15,8 @@
import copy
from distutils.version import StrictVersion
import six
import sqlalchemy as sa
from nailgun.api.v1.validators.base import BaseDefferedTaskValidator
@ -218,6 +220,10 @@ class AttributesValidator(BasicValidator):
u"Provisioning method is not set. Unable to continue",
log_message=True)
cls.validate_plugin_attributes(
cluster, attrs.get('editable', {})
)
cls.validate_editable_attributes(attrs)
return d
@ -299,6 +305,56 @@ class AttributesValidator(BasicValidator):
raise errors.InvalidData(
'[{0}] {1}'.format(attr_name, regex_err))
@classmethod
def validate_plugin_attributes(cls, cluster, attributes):
"""Validates Cluster-Plugins relations attributes
:param cluster: A cluster instance
:type cluster: nailgun.objects.cluster.Cluster
:param attributes: The editable attributes of the Cluster
:type attributes: dict
:raises: errors.NotAllowed
"""
# TODO(need to enable restrictions check for cluster attributes[1])
# [1] https://bugs.launchpad.net/fuel/+bug/1519904
# Validates only that plugin can be installed on deployed env.
if not cluster.is_locked:
return
enabled_plugins = set(
p.id for p in objects.ClusterPlugins.get_enabled(cluster.id)
)
for attrs in six.itervalues(attributes):
if not isinstance(attrs, dict):
continue
plugin_versions = attrs.get('plugin_versions', None)
if plugin_versions is None:
continue
if not attrs.get('metadata', {}).get('enabled'):
continue
for version in plugin_versions['values']:
plugin_id = version.get('data')
plugin = objects.Plugin.get_by_uid(plugin_id)
if not plugin:
continue
if plugin_id != plugin_versions['value']:
continue
if plugin.is_hotpluggable or plugin.id in enabled_plugins:
break
raise errors.NotAllowed(
"This plugin version can be enabled only "
"before environment is deployed.",
log_message=True
)
class ClusterChangesValidator(BaseDefferedTaskValidator):

View File

@ -50,7 +50,8 @@ PLUGIN_SCHEMA = {
'homepage': {'type': 'string'},
'releases': {
'type': 'array',
'items': PLUGIN_RELEASE_SCHEMA}
'items': PLUGIN_RELEASE_SCHEMA},
'is_hotpluggable': {"type": "boolean"},
},
'required': [
'name',

View File

@ -120,9 +120,11 @@ def upgrade():
upgrade_all_network_data_from_string_to_appropriate_data_type()
create_openstack_configs_table()
upgrade_master_node_ui_settings()
upgrade_plugins_parameters()
def downgrade():
downgrade_plugins_parameters()
downgrade_master_node_ui_settings()
downgrade_openstack_configs()
downgrade_all_network_data_to_string()
@ -713,3 +715,13 @@ def downgrade_master_node_ui_settings():
connection.execute(q_update_master_node_settings,
master_node_uid=master_node_uid,
settings=jsonutils.dumps(master_node_settings))
def upgrade_plugins_parameters():
op.add_column(
'plugins', sa.Column('is_hotpluggable', sa.Boolean(), nullable=True)
)
def downgrade_plugins_parameters():
op.drop_column('plugins', 'is_hotpluggable')

View File

@ -71,6 +71,7 @@ class Plugin(Base):
licenses = Column(JSON, server_default='[]', nullable=False)
homepage = Column(Text, nullable=True)
package_version = Column(String(32), nullable=False)
is_hotpluggable = Column(Boolean, default=False)
attributes_metadata = Column(JSON, server_default='{}', nullable=False)
volumes_metadata = Column(JSON, server_default='{}', nullable=False)
roles_metadata = Column(JSON, server_default='{}', nullable=False)

View File

@ -104,6 +104,7 @@
"licenses": ["Apache License Version 2.0"],
"homepage": "https://github.com/openstack/fuel-plugin-external-zabbix",
"groups": ["monitoring"],
"is_hotpluggable": true,
"releases": [
{
"os": "ubuntu",
@ -152,6 +153,7 @@
"licenses": ["Apache License Version 2.0"],
"homepage": "https://github.com/openstack/fuel-plugin-external-zabbix",
"groups": ["monitoring"],
"is_hotpluggable": true,
"releases": [
{
"os": "ubuntu",

View File

@ -137,16 +137,22 @@ class ClusterPlugins(NailgunObject):
:return: True if compatible, False if not
:rtype: bool
"""
cluster_os = cluster.release.operating_system.lower()
for release in plugin.releases:
os_compat = cluster.release.operating_system.lower()\
== release['os'].lower()
if cluster_os != release['os'].lower():
continue
# plugin writer should be able to specify ha in release['mode']
# and know nothing about ha_compact
mode_compat = any(mode in cluster.mode for mode in release['mode'])
release_version_compat = cls.is_release_version_compatible(
cluster.release.version, release['version'])
if all((os_compat, mode_compat, release_version_compat)):
return True
if not any(
cluster.mode.startswith(mode) for mode in release['mode']
):
continue
if not cls.is_release_version_compatible(
cluster.release.version, release['version']
):
continue
return True
return False
@staticmethod
@ -259,8 +265,9 @@ class ClusterPlugins(NailgunObject):
models.Plugin.name,
models.Plugin.title,
models.Plugin.version,
cls.model.enabled,
models.Plugin.is_hotpluggable,
models.Plugin.attributes_metadata,
cls.model.enabled,
cls.model.attributes
).join(cls.model)\
.filter(cls.model.cluster_id == cluster_id)\
@ -316,4 +323,4 @@ class ClusterPlugins(NailgunObject):
.join(cls.model)\
.filter(cls.model.cluster_id == cluster_id)\
.filter(cls.model.enabled.is_(True))\
.all()
.order_by(models.Plugin.id)

View File

@ -31,5 +31,6 @@ class PluginSerializer(BasicSerializer):
"authors",
"licenses",
"homepage",
"fuel_version"
"fuel_version",
"is_hotpluggable"
)

View File

@ -74,7 +74,7 @@ class PluginManager(object):
continue
enabled = plugin_enabled and\
str(plugin.id) == plugin_versions['value']
pid == plugin_versions['value']
ClusterPlugins.set_attributes(
cluster.id, plugin.id, enabled=enabled,
@ -97,62 +97,80 @@ class PluginManager(object):
:rtype: dict
"""
versions = {
u'type': u'radio',
u'values': [],
u'weight': 10,
u'value': None,
u'label': 'Choose a plugin version'
'type': 'radio',
'values': [],
'weight': 10,
'value': None,
'label': 'Choose a plugin version'
}
def _convert_attr(pid, name, title, attr):
restrictions = attr.setdefault('restrictions', [])
restrictions.append({
'action': 'hide',
'condition': "settings:{0}.plugin_versions.value != '{1}'"
.format(name, pid)
})
return "#{0}_{1}".format(pid, title), attr
plugins_attributes = {}
for pid, name, title, version, enabled, default_attrs, cluster_attrs\
in ClusterPlugins.get_connected_plugins_data(cluster.id):
if all_versions:
enabled = enabled and not default
data = plugins_attributes.get(name, {})
metadata = data.setdefault('metadata', {
u'toggleable': True,
u'weight': 70
for plugin in ClusterPlugins.get_connected_plugins_data(cluster.id):
plugin_id = str(plugin.id)
enabled = plugin.enabled and not (all_versions and default)
plugin_attributes = plugins_attributes.setdefault(plugin.name, {})
metadata = plugin_attributes.setdefault('metadata', {
'toggleable': True,
'weight': 70
})
metadata['enabled'] = enabled or metadata.get('enabled', False)
metadata['label'] = title
metadata['label'] = plugin.title
if plugin.is_hotpluggable:
metadata["always_editable"] = True
if all_versions:
metadata['default'] = default
attrs = default_attrs if default else cluster_attrs
data.update(_convert_attr(pid, name, key, attrs[key])
for key in attrs)
plugin_attributes.update(
cls.convert_plugin_attributes(
plugin,
plugin.attributes_metadata
if default else plugin.attributes
)
)
if 'plugin_versions' in data:
plugin_versions = data['plugin_versions']
plugin_version = {
'data': plugin_id,
'description': '',
'label': plugin.version,
}
if not plugin.is_hotpluggable:
plugin_version['restrictions'] = [{
'action': 'disable',
'condition': 'cluster:is_locked'
}]
plugin_versions = plugin_attributes.get('plugin_versions')
if plugin_versions is not None:
if enabled:
plugin_versions['value'] = plugin_id
else:
plugin_versions = copy.deepcopy(versions)
plugin_versions['values'].append({
u'data': str(pid),
u'description': '',
u'label': version
})
if not plugin_versions['value'] or enabled:
plugin_versions['value'] = str(pid)
plugin_versions['value'] = plugin_id
plugin_attributes['plugin_versions'] = plugin_versions
data['plugin_versions'] = plugin_versions
else:
data.update(cluster_attrs if enabled else {})
plugins_attributes[name] = data
plugin_versions['values'].append(plugin_version)
elif enabled:
plugin_attributes.update(plugin.attributes)
return plugins_attributes
@classmethod
def convert_plugin_attributes(cls, plugin, attributes):
def converter(plugin_id, plugin_name, title, attr):
restrictions = attr.setdefault('restrictions', [])
restrictions.append({
'action': 'hide',
'condition': "settings:{0}.plugin_versions.value != '{1}'"
.format(plugin_name, plugin_id)
})
return "#{0}_{1}".format(plugin_id, title), attr
return (
converter(plugin.id, plugin.name, k, v)
for k, v in six.iteritems(attributes)
)
@classmethod
def get_cluster_plugins_with_tasks(cls, cluster):
cluster_plugins = []
@ -238,17 +256,14 @@ class PluginManager(object):
# and afterwards show them in error message;
# thus role names for which following checks
# fails are accumulated in err_info variable
err_roles = set()
if set(plugin_roles) & core_roles:
err_roles |= set(plugin_roles) & core_roles
if set(plugin_roles) & set(result):
err_roles |= set(plugin_roles) & set(result)
err_roles = set(
r for r in plugin_roles if r in core_roles or r in result
)
if err_roles:
raise errors.AlreadyExists(
"Plugin (ID={0}) is unable to register the following "
"node roles: {1}".format(plugin_db.id,
", ".join(err_roles))
", ".join(sorted(err_roles)))
)
# update info on processed roles in case of

View File

@ -494,7 +494,7 @@ class EnvironmentManager(object):
resp = self.neutron_networks_put(cluster_id, netconfig)
return resp
def create_plugin(self, api=False, cluster=None, **kwargs):
def create_plugin(self, api=False, cluster=None, enabled=True, **kwargs):
plugin_data = self.get_default_plugin_metadata(**kwargs)
if api:
@ -513,7 +513,9 @@ class EnvironmentManager(object):
# Enable plugin for specific cluster
if cluster:
cluster.plugins.append(plugin)
ClusterPlugins.set_attributes(cluster.id, plugin.id, enabled=True)
ClusterPlugins.set_attributes(
cluster.id, plugin.id, enabled=enabled
)
return plugin
def create_cluster_plugin_link(self, **kwargs):
@ -768,6 +770,7 @@ class EnvironmentManager(object):
'licenses': ['License 1'],
'authors': ['Author1'],
'homepage': 'http://some-plugin-url.com/',
'is_hotpluggable': False,
'releases': [
{'repository_path': 'repositories/ubuntu',
'version': '2014.2-6.0', 'os': 'ubuntu',

View File

@ -713,7 +713,12 @@ class TestAttributesWithPlugins(BaseIntegrationTest):
'mode': consts.CLUSTER_MODES.ha_compact,
'net_provider': consts.CLUSTER_NET_PROVIDERS.neutron,
'net_segment_type': consts.NEUTRON_SEGMENT_TYPES.vlan,
})
},
nodes_kwargs=[
{'roles': ['controller'], 'pending_addition': True},
{'roles': ['compute'], 'pending_addition': True},
]
)
self.cluster = self.env.clusters[0]
@ -751,7 +756,7 @@ class TestAttributesWithPlugins(BaseIntegrationTest):
kwargs={'cluster_id': self.cluster['id']}),
params=jsonutils.dumps({
'editable': {
'testing_plugin': {
plugin.name: {
'metadata': {
'label': 'Test plugin',
'toggleable': True,
@ -786,13 +791,86 @@ class TestAttributesWithPlugins(BaseIntegrationTest):
resp = _modify_plugin(enabled=True)
self.assertEqual(200, resp.status_code)
editable = objects.Cluster.get_editable_attributes(self.cluster)
self.assertIn('testing_plugin', editable)
self.assertTrue(editable['testing_plugin']['metadata']['enabled'])
self.assertEqual('1', editable['testing_plugin']['attr']['value'])
self.assertIn(plugin.name, editable)
self.assertTrue(editable[plugin.name]['metadata']['enabled'])
self.assertEqual('1', editable[plugin.name]['attr']['value'])
resp = _modify_plugin(enabled=False)
self.assertEqual(200, resp.status_code)
editable = objects.Cluster.get_editable_attributes(self.cluster)
self.assertIn('testing_plugin', editable)
self.assertFalse(editable['testing_plugin']['metadata']['enabled'])
self.assertNotIn(attr, editable['testing_plugin'])
self.assertIn(plugin.name, editable)
self.assertFalse(editable[plugin.name]['metadata']['enabled'])
self.assertNotIn(attr, editable[plugin.name])
def _modify_plugin(self, plugin, enabled, **kwargs):
return self.app.put(
reverse(
'ClusterAttributesHandler',
kwargs={'cluster_id': self.cluster.id}
),
params=jsonutils.dumps({
'editable': {
plugin.name: dict(
metadata={'enabled': enabled},
plugin_versions={
'type': 'radio',
'values': [{
'data': str(plugin.id),
'label': plugin.version
}],
'value': str(plugin.id),
},
**kwargs
)
}
}),
headers=self.default_headers,
expect_errors=True
)
def test_install_plugins_after_deployment(self):
self.cluster.status = consts.CLUSTER_STATUSES.operational
self.assertTrue(self.cluster.is_locked)
runtime_plugin = self.env.create_plugin(
cluster=self.cluster,
is_hotpluggable=True,
version='1.0.1',
enabled=False,
**self.plugin_data
)
plugin = self.env.create_plugin(
cluster=self.cluster,
name=runtime_plugin.name,
version='1.0.3',
is_hotpluggable=False,
enabled=False,
**self.plugin_data
)
resp = self._modify_plugin(runtime_plugin, True)
self.assertEqual(200, resp.status_code, resp.body)
editable = objects.Cluster.get_editable_attributes(self.cluster)
self.assertIn(runtime_plugin.name, editable)
self.assertTrue(editable[runtime_plugin.name]['metadata']['enabled'])
resp = self._modify_plugin(plugin, True)
self.assertEqual(403, resp.status_code)
def test_enable_plugin_is_idempotent(self):
plugin = self.env.create_plugin(
cluster=self.cluster,
version='1.0.1',
is_hotpluggable=False,
enabled=True,
**self.plugin_data
)
self.env.create_plugin(
cluster=self.cluster,
is_hotpluggable=True,
version='1.0.2',
enabled=False,
**self.plugin_data
)
self.cluster.status = consts.CLUSTER_STATUSES.operational
self.assertTrue(self.cluster.is_locked)
resp = self._modify_plugin(plugin, True)
self.assertEqual(200, resp.status_code, resp.body)

View File

@ -269,6 +269,158 @@ class TestPluginManager(base.BaseIntegrationTest):
enabled_plugins = ClusterPlugins.get_enabled(cluster.id)
self.assertItemsEqual([plugin], enabled_plugins)
def test_get_plugins_attributes_when_cluster_is_locked(self):
self.env.create(api=False)
cluster = self.env.clusters[-1]
plugin_a1 = self.env.create_plugin(
name='plugin_a', version='1.0.1',
cluster=cluster, enabled=False
)
plugin_a2 = self.env.create_plugin(
name='plugin_a', version='1.0.2', is_hotpluggable=True,
cluster=cluster, enabled=False
)
plugin_b = self.env.create_plugin(
name='plugin_b', title='plugin_a_title', cluster=cluster
)
cluster.status = consts.CLUSTER_STATUSES.operational
self.db.flush()
self.assertTrue(cluster.is_locked)
attributes = PluginManager.get_plugins_attributes(
cluster, True, True
)
self.assertItemsEqual(
['plugin_a', 'plugin_b'], attributes
)
self.assertTrue(
attributes['plugin_a']['metadata']['always_editable']
)
self.assertItemsEqual(
[
{
'data': str(plugin_a1.id),
'description': '',
'label': plugin_a1.version,
'restrictions': [
{
'action': 'disable',
'condition': 'cluster:is_locked'
}
],
},
{
'data': str(plugin_a2.id),
'description': '',
'label': plugin_a2.version
}
],
attributes['plugin_a']['plugin_versions']['values']
)
self.assertEqual(
str(plugin_a1.id),
attributes['plugin_a']['plugin_versions']['value']
)
self.assertNotIn(
'always_editable', attributes['plugin_b']['metadata']
)
self.assertItemsEqual(
[
{
'restrictions': [
{
'action': 'disable',
'condition': 'cluster:is_locked'
}
],
'data': str(plugin_b.id),
'description': '',
'label': plugin_b.version,
},
],
attributes['plugin_b']['plugin_versions']['values']
)
self.assertEqual(
str(plugin_b.id),
attributes['plugin_b']['plugin_versions']['value']
)
def test_get_plugins_attributes_when_cluster_is_not_locked(self):
self.env.create(api=False)
cluster = self.env.clusters[-1]
plugin_a1 = self.env.create_plugin(
name='plugin_a', version='1.0.1',
cluster=cluster, enabled=False
)
plugin_a2 = self.env.create_plugin(
name='plugin_a', version='1.0.2', is_hotpluggable=True,
cluster=cluster, enabled=True
)
plugin_b = self.env.create_plugin(
name='plugin_b', title='plugin_a_title', cluster=cluster
)
self.assertFalse(plugin_a1.is_hotpluggable)
self.assertTrue(plugin_a2.is_hotpluggable)
self.assertFalse(plugin_b.is_hotpluggable)
self.assertFalse(cluster.is_locked)
attributes = PluginManager.get_plugins_attributes(
cluster, True, True
)
self.assertItemsEqual(
['plugin_a', 'plugin_b'], attributes
)
self.assertTrue(
attributes['plugin_a']['metadata']['always_editable']
)
self.assertItemsEqual(
[
{
'data': str(plugin_a1.id),
'description': '',
'label': plugin_a1.version,
'restrictions': [
{
'action': 'disable',
'condition': 'cluster:is_locked'
}
],
},
{
'data': str(plugin_a2.id),
'description': '',
'label': plugin_a2.version
}
],
attributes['plugin_a']['plugin_versions']['values']
)
self.assertEqual(
str(plugin_a1.id),
attributes['plugin_a']['plugin_versions']['value']
)
self.assertNotIn(
'always_editable', attributes['plugin_b']['metadata']
)
self.assertItemsEqual(
[
{
'restrictions': [
{
'action': 'disable',
'condition': 'cluster:is_locked'
}
],
'data': str(plugin_b.id),
'description': '',
'label': plugin_b.version,
},
],
attributes['plugin_b']['plugin_versions']['values']
)
self.assertEqual(
str(plugin_b.id),
attributes['plugin_b']['plugin_versions']['value']
)
class TestClusterPluginIntegration(base.BaseTestCase):

View File

@ -182,15 +182,24 @@ class TestPluginsApi(BasePluginTest):
resp = self.create_plugin()
plugin = objects.Plugin.get_by_uid(resp.json['id'])
cluster = self.create_cluster()
self.assertEqual(objects.ClusterPlugins.get_enabled(cluster.id), [])
self.assertItemsEqual(
[],
objects.ClusterPlugins.get_enabled(cluster.id)
)
resp = self.enable_plugin(cluster, plugin.name, plugin.id)
self.assertEqual(resp.status_code, 200)
self.assertIn(plugin, objects.ClusterPlugins.get_enabled(cluster.id))
self.assertItemsEqual(
[plugin],
objects.ClusterPlugins.get_enabled(cluster.id)
)
resp = self.disable_plugin(cluster, plugin.name)
self.assertEqual(resp.status_code, 200)
self.assertEqual(objects.ClusterPlugins.get_enabled(cluster.id), [])
self.assertItemsEqual(
[],
objects.ClusterPlugins.get_enabled(cluster.id)
)
def test_delete_plugin(self):
resp = self.create_plugin()
@ -258,10 +267,10 @@ class TestPluginsApi(BasePluginTest):
return response.json_body['id']
def get_num_enabled(cluster_id):
return len(objects.ClusterPlugins.get_enabled(cluster_id))
return objects.ClusterPlugins.get_enabled(cluster_id).count()
def get_enabled_version(cluster_id):
plugin = objects.ClusterPlugins.get_enabled(cluster_id)[0]
plugin = objects.ClusterPlugins.get_enabled(cluster_id).first()
return plugin.version
plugin_ids = []

View File

@ -299,7 +299,9 @@ class TestAttributesValidator(BaseTestCase):
attrs = {'editable': {'provision': {'method':
{'value': 'image', 'type': 'text'}}}}
mock_cluster_attrs.return_value = attrs
cluster_mock = Mock(release=Mock(environment_version='7.0'))
cluster_mock = Mock(
is_locked=False, release=Mock(environment_version='7.0')
)
self.assertNotRaises(errors.InvalidData,
AttributesValidator.validate,
json.dumps(attrs), cluster_mock)
@ -309,7 +311,9 @@ class TestAttributesValidator(BaseTestCase):
attrs = {'editable': {'provision': {'method':
{'value': 'image', 'type': 'text'}}}}
mock_cluster_attrs.return_value = attrs
cluster_mock = Mock(release=Mock(environment_version='6.0'))
cluster_mock = Mock(
is_locked=False, release=Mock(environment_version='6.0')
)
self.assertNotRaises(errors.InvalidData,
AttributesValidator.validate,
json.dumps(attrs), cluster_mock)

View File

@ -473,6 +473,12 @@ class TestComponentsMigration(base.BaseAlembicMigrationTest):
for idx, db_value in enumerate(db_values):
self.assertEqual(jsonutils.loads(db_value), column_values[idx][1])
def test_hotplug_field_exists(self):
result = db.execute(
sa.select([self.meta.tables['plugins'].c.is_hotpluggable])
)
self.assertTrue(all(x[0] is None for x in result))
class TestMasterNodeSettingsMigration(base.BaseAlembicMigrationTest):

View File

@ -114,7 +114,7 @@ class TestClusterPlugins(ExtraFunctions):
self._create_test_plugins()
cluster = self._create_test_cluster()
plugin = ClusterPlugins.get_connected_plugins(cluster)[0]
plugin = ClusterPlugins.get_connected_plugins(cluster).first()
ClusterPlugins.set_attributes(cluster.id, plugin.id, enabled=True)
columns = meta.tables['cluster_plugins'].c
@ -159,8 +159,8 @@ class TestClusterPlugins(ExtraFunctions):
self._create_test_plugins()
cluster = self._create_test_cluster()
plugin = ClusterPlugins.get_connected_plugins(cluster)[0]
plugin = ClusterPlugins.get_connected_plugins(cluster).first()
ClusterPlugins.set_attributes(cluster.id, plugin.id, enabled=True)
enabled_plugin = ClusterPlugins.get_enabled(cluster.id)[0].id
self.assertEqual(enabled_plugin, plugin.id)
enabled_plugin = ClusterPlugins.get_enabled(cluster.id).first()
self.assertEqual(plugin.id, enabled_plugin.id)

View File

@ -21,7 +21,6 @@ import mock
from itertools import cycle
from itertools import ifilter
import re
import uuid
from sqlalchemy import inspect as sqlalchemy_inspect
@ -1399,19 +1398,12 @@ class TestClusterObjectGetRoles(BaseTestCase):
message_pattern = (
'^Plugin \(ID={0}\) is unable to register the following node '
'roles: (.*)'
'roles: role_a, role_x'
.format(plugin_in_conflict.id))
with self.assertRaisesRegexp(
errors.AlreadyExists, message_pattern) as cm:
with self.assertRaisesRegexp(errors.AlreadyExists, message_pattern):
objects.Cluster.get_roles(self.cluster)
# 0 - the whole message, 1 - is first match of (.*) pattern
roles = re.match(message_pattern, str(cm.exception)).group(1)
roles = [role.lstrip().rstrip() for role in roles.split(',')]
self.assertItemsEqual(roles, ['role_x', 'role_a'])
class TestClusterObjectGetNetworkManager(BaseTestCase):
def setUp(self):