openstack
/
fuel-web
Code Issues Proposed changes

[Nailgun] Mix plugin volume metadata with core 

Implement functionality for merging plugin volumes metadata
with release volumes metadata and using it in Volumes Manager.

Implements: blueprint role-as-a-plugin
Change-Id: Ie8e1236e4aa38b8220d937febaa50d146c9caa2a
This commit is contained in:
Andriy Popovych 2015-07-08 12:04:25 +03:00 committed by Igor Kalnitsky
parent 82f2523001
commit 1cfedb1f53
11 changed files with 407 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nailgun/nailgun/errors/__init__.py
View File

@ -92,11 +92,6 @@ default_messages = {
# Plugin errors
"PackageVersionIsNotCompatible": "Package version is not compatible",
"PluginsTasksOverlapping":
"There is task with same id supplied by another plugin",
"PluginRolesConflict":
("Plugin is unable to register its node role due to conflict with "
"core roles"),
# Extensions
"CannotFindExtension": "Cannot find extension",

11
nailgun/nailgun/extensions/volume_manager/manager.py
View File

@ -111,16 +111,17 @@ def get_node_spaces(node):
Sets key `_allocate_size` which used only for internal calculation
and not used in partitioning system.
"""
# FIXME(apopovych): ugly hack to avoid circular dependency
from nailgun import objects
node_spaces = []
role_mapping = node.cluster.release.volumes_metadata[
'volumes_roles_mapping']
volumes_metadata = objects.Cluster.get_volumes_metadata(node.cluster)
role_mapping = volumes_metadata['volumes_roles_mapping']
all_spaces = volumes_metadata['volumes']
# TODO(dshulyak)
# This logic should go to openstack.yaml (or other template)
# when it will be extended with flexible template engine
modify_volumes_hook(role_mapping, node)
all_spaces = node.cluster.release.volumes_metadata['volumes']
for role in node.all_roles:
if not role_mapping.get(role):

19
nailgun/nailgun/objects/cluster.py
View File

@ -817,6 +817,25 @@ class Cluster(NailgunObject):
PluginManager.get_plugins_deployment_tasks(instance)
return release_deployment_tasks + plugin_deployment_tasks
@classmethod
def get_volumes_metadata(cls, instance):
"""Return proper volumes metadata for cluster and consists
with general volumes metadata from release and volumes
metadata from plugins which releated to this cluster
:param instance: Cluster DB instance
:returns: dict -- object with merged volumes metadata
"""
volumes_metadata = copy.deepcopy(instance.release.volumes_metadata)
plugin_volumes = PluginManager.get_volumes_metadata(instance)
volumes_metadata['volumes_roles_mapping'].update(
plugin_volumes['volumes_roles_mapping'])
volumes_metadata['volumes'].extend(plugin_volumes['volumes'])
return volumes_metadata
@classmethod
def create_vmware_attributes(cls, instance):
"""Store VmwareAttributes instance into DB.

15
nailgun/nailgun/objects/serializers/role.py
View File

@ -32,15 +32,12 @@ class RoleSerializer(BasicSerializer):
@classmethod
def serialize_from_cluster(cls, cluster, role_name):
meta = objects.Cluster.get_roles(cluster)[role_name]
# TODO(ikalnitsky): Use volumes mapping from both release and plugins.
# Currently, we try to retrieve them only from release and fallback
# to empty list if nothing is found.
volumes = cluster.release.volumes_metadata['volumes_roles_mapping']
volumes = volumes.get(role_name, [])
role_metadata = objects.Cluster.get_roles(cluster)[role_name]
volumes_metadata = objects.Cluster.get_volumes_metadata(cluster)
role_mapping = volumes_metadata.get('volumes_roles_mapping').get(
role_name, [])
return {
'name': role_name,
'meta': meta,
'volumes_roles_mapping': volumes}
'meta': role_metadata,
'volumes_roles_mapping': role_mapping}

8
nailgun/nailgun/plugins/adapters.py
View File

@ -201,6 +201,10 @@ class PluginAdapterBase(object):
deployment_tasks.append(task)
return deployment_tasks
@property
def volumes_metadata(self):
return self.plugin.volumes_metadata
def get_release_info(self, release):
"""Returns plugin release information which corresponds to
a provided release.
@ -289,7 +293,7 @@ class PluginAdapterV2(PluginAdapterBase):
class PluginAdapterV3(PluginAdapterV2):
"""Plugin wrapper class for package version >= 3.0.0
"""Plugin wrapper class for package version 3.0.0
"""
node_roles_config_name = 'node_roles.yaml'
@ -298,6 +302,8 @@ class PluginAdapterV3(PluginAdapterV2):
network_roles_config_name = 'network_roles.yaml'
def sync_metadata_to_db(self):
"""Sync metadata from all config yaml files to DB
"""
super(PluginAdapterV3, self).sync_metadata_to_db()
data_to_update = {}

75
nailgun/nailgun/plugins/manager.py
View File

@ -106,20 +106,6 @@ class PluginManager(object):
return plugin_roles
@classmethod
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given ids. If there is not
ids all newest plugins will be synced
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()
@classmethod
def get_plugins_deployment_tasks(cls, cluster):
deployment_tasks = []
@ -131,7 +117,7 @@ class PluginManager(object):
for t in depl_tasks:
t_id = t['id']
if t_id in processed_tasks:
raise errors.PluginsTasksOverlapping(
raise errors.AlreadyExists(
'Plugin {0} is overlapping with plugin {1} '
'by introducing the same deployment task with '
'id {2}'
@ -166,7 +152,7 @@ class PluginManager(object):
err_roles |= set(plugin_roles) & set(result)
if err_roles:
raise errors.PluginRolesConflict(
raise errors.AlreadyExists(
"Plugin (ID={0}) is unable to register the following "
"node roles: {1}".format(plugin_db.id,
", ".join(err_roles))
@ -177,3 +163,60 @@ class PluginManager(object):
result.update(plugin_roles)
return result
@classmethod
def get_volumes_metadata(cls, cluster):
"""Get volumes metadata for specific cluster from all
plugins which enabled for it.
:param cluster: Cluster DB model
:returns: dict -- object with merged volumes data from plugins
"""
volumes_metadata = {
'volumes': [],
'volumes_roles_mapping': {}
}
release_volumes = cluster.release.volumes_metadata.get('volumes', [])
release_volumes_ids = [v['id'] for v in release_volumes]
processed_volumes = {}
for plugin_adapter in map(wrap_plugin, cluster.plugins):
metadata = plugin_adapter.volumes_metadata
for volume in metadata.get('volumes', []):
volume_id = volume['id']
if volume_id in release_volumes_ids:
raise errors.AlreadyExists(
'Plugin {0} is overlapping with release '
'by introducing the same volume with id "{1}"'
.format(plugin_adapter.full_name, volume_id))
elif volume_id in processed_volumes:
raise errors.AlreadyExists(
'Plugin {0} is overlapping with plugin {1} '
'by introducing the same volume with id "{2}"'
.format(plugin_adapter.full_name,
processed_volumes[volume_id],
volume_id))
processed_volumes[volume_id] = plugin_adapter.full_name
volumes_metadata.get('volumes_roles_mapping', {}).update(
metadata.get('volumes_roles_mapping', {}))
volumes_metadata.get('volumes', []).extend(
metadata.get('volumes', []))
return volumes_metadata
@classmethod
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given ids. If there is not
ids all newest plugins will be synced
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()

45
nailgun/nailgun/test/base.py
View File

@ -62,6 +62,7 @@ from nailgun.objects import Cluster
from nailgun.objects import MasterNodeSettings
from nailgun.objects import Node
from nailgun.objects import NodeGroup
from nailgun.objects import Plugin
from nailgun.objects import Release
from nailgun.app import build_app
@ -107,6 +108,7 @@ class EnvironmentManager(object):
self.releases = []
self.clusters = []
self.nodes = []
self.plugins = []
self.network_manager = NetworkManager
def create(self, **kwargs):
@ -390,6 +392,30 @@ class EnvironmentManager(object):
return ng
def create_plugin(self, api=False, cluster=None, **kwargs):
plugin_data = self.get_default_plugin_metadata()
plugin_data.update(kwargs)
if api:
resp = self.app.post(
reverse('PluginCollectionHandler'),
jsonutils.dumps(plugin_data),
headers=self.default_headers,
expect_errors=False
)
plugin = Plugin.get_by_uid(resp.json_body['id'])
self.plugins.append(plugin)
else:
plugin = Plugin.create(plugin_data)
self.plugins.append(plugin)
# Enable plugin for specific cluster
if cluster:
cluster.plugins.append(plugin)
return plugin
def default_metadata(self):
item = self.find_item_by_pk_model(
self.read_fixtures(("sample_environment",)),
@ -532,11 +558,9 @@ class EnvironmentManager(object):
def get_default_plugin_node_roles_config(self, **kwargs):
node_roles = {
'test_node_role': {
'metadata': {
'name': 'Some plugin role',
'description': 'Some description'
}
'testing_plugin': {
'name': 'Some plugin role',
'description': 'Some description'
}
}
@ -545,11 +569,14 @@ class EnvironmentManager(object):
def get_default_plugin_volumes_config(self, **kwargs):
volumes = {
'volumes_roles_mapping': {
'testing_plugin': [
{'allocate_size': 'min', 'id': 'os'},
{'allocate_size': 'all', 'id': 'test_volume'}
]
},
'volumes': [
{
'id': 'test_node_volume',
'type': 'vg'
}
{'id': 'test_volume', 'type': 'vg'}
]
}

19
nailgun/nailgun/test/integration/test_cluster_roles_handler.py
View File

@ -21,8 +21,8 @@ from nailgun.test import base
class TestClusterRolesHandler(base.BaseTestCase):
ROLES = yaml.load("""
# TODO(apopovych): use test data from base test file
ROLES = yaml.safe_load("""
test_role:
name: "Some plugin role"
description: "Some description"
@ -35,6 +35,14 @@ class TestClusterRolesHandler(base.BaseTestCase):
message: "Some message for restriction warning"
""")
VOLUMES = yaml.safe_load("""
volumes_roles_mapping:
test_role:
- {allocate_size: "min", id: "os"}
- {allocate_size: "all", id: "image"}
""")
def setUp(self):
super(TestClusterRolesHandler, self).setUp()
@ -150,6 +158,7 @@ class TestClusterRolesHandler(base.BaseTestCase):
def test_get_particular_role_for_cluster_w_plugin(self):
plugin_data = self.env.get_default_plugin_metadata()
plugin_data['roles_metadata'] = self.ROLES
plugin_data['volumes_metadata'] = self.VOLUMES
plugin = objects.Plugin.create(plugin_data)
self.cluster.plugins.append(plugin)
self.db.flush()
@ -166,6 +175,6 @@ class TestClusterRolesHandler(base.BaseTestCase):
role['meta'],
self.ROLES['test_role']
)
# TODO(ikalnitsky): Add check for volumes when volumes for plugins
# are implemented.
self.assertItemsEqual(
role['volumes_roles_mapping'],
self.VOLUMES['volumes_roles_mapping']['test_role'])

159
nailgun/nailgun/test/integration/test_plugin_manager.py Normal file
View File

@ -0,0 +1,159 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from nailgun.errors import errors
from nailgun.plugins.adapters import PluginAdapterV3
from nailgun.plugins.manager import PluginManager
from nailgun.test import base
class TestPluginManager(base.BaseIntegrationTest):
def setUp(self):
super(TestPluginManager, self).setUp()
self.env.create()
self.cluster = self.env.clusters[0]
# Create two plugins with package verion 3.0.0
for name in ['test_plugin_1', 'test_plugin_2']:
volumes_metadata = {
'volumes_roles_mapping': {
name: [{'allocate_size': 'min', 'id': name}]
},
'volumes': [{'id': name, 'type': 'vg'}]
}
self.env.create_plugin(
api=True,
cluster=self.cluster,
name=name,
package_version='3.0.0',
fuel_version=['7.0'],
volumes_metadata=volumes_metadata
)
def test_get_plugin_volumes_metadata_for_cluster(self):
volumes_metadata = PluginManager.get_volumes_metadata(
self.cluster)
expected_volumes_metadata = {
'volumes_roles_mapping': {
'test_plugin_1': [
{'allocate_size': 'min', 'id': 'test_plugin_1'}
],
'test_plugin_2': [
{'allocate_size': 'min', 'id': 'test_plugin_2'}
],
},
'volumes': [
{'id': 'test_plugin_1', 'type': 'vg'},
{'id': 'test_plugin_2', 'type': 'vg'}
]
}
self.assertEqual(
volumes_metadata['volumes_roles_mapping'],
expected_volumes_metadata['volumes_roles_mapping'])
self.assertItemsEqual(
volumes_metadata['volumes'],
expected_volumes_metadata['volumes'])
def test_get_empty_plugin_volumes_metadata_for_cluster(self):
cluster = self.env.create_cluster(api=False)
self.env.create_plugin(
api=True,
cluster=cluster,
package_version='3.0.0',
fuel_version=['7.0']
)
volumes_metadata = PluginManager.get_volumes_metadata(cluster)
expected_volumes_metadata = {
'volumes_roles_mapping': {}, 'volumes': []}
self.assertEqual(
volumes_metadata, expected_volumes_metadata)
def test_raise_exception_when_plugin_overlap_release_volumes(self):
cluster = self.env.create_cluster(api=False)
plugin_name = 'test_plugin_3'
volumes_metadata = {
'volumes_roles_mapping': {
plugin_name: [
{'allocate_size': 'min', 'id': plugin_name}
]
},
'volumes': [
{'id': 'os', 'type': 'vg'},
{'id': plugin_name, 'type': 'vg'}
]
}
self.env.create_plugin(
api=True,
cluster=cluster,
name=plugin_name,
package_version='3.0.0',
fuel_version=['7.0'],
volumes_metadata=volumes_metadata
)
expected_message = (
'Plugin test_plugin_3-0.1.0 is overlapping with release '
'by introducing the same volume with id "os"')
with self.assertRaisesRegexp(errors.AlreadyExists,
expected_message):
PluginManager.get_volumes_metadata(cluster)
def test_raise_exception_when_plugin_overlap_another_plugin_volumes(self):
plugin_name = 'test_plugin_4'
volumes_metadata = {
'volumes_roles_mapping': {
plugin_name: [
{'allocate_size': 'min', 'id': plugin_name}
]
},
'volumes': [
{'id': 'test_plugin_2', 'type': 'vg'},
{'id': plugin_name, 'type': 'vg'}
]
}
self.env.create_plugin(
api=True,
cluster=self.cluster,
name=plugin_name,
package_version='3.0.0',
fuel_version=['7.0'],
volumes_metadata=volumes_metadata
)
expected_message = (
'Plugin test_plugin_4-0.1.0 is overlapping with plugin '
'test_plugin_2-0.1.0 by introducing the same volume '
'with id "test_plugin_2"')
with self.assertRaisesRegexp(errors.AlreadyExists,
expected_message):
PluginManager.get_volumes_metadata(self.cluster)
@mock.patch.object(PluginAdapterV3, 'sync_metadata_to_db')
def test_sync_metadata_for_all_plugins(self, sync_mock):
PluginManager.sync_plugins_metadata()
self.assertEqual(sync_mock.call_count, 2)
@mock.patch.object(PluginAdapterV3, 'sync_metadata_to_db')
def test_sync_metadata_for_specific_plugin(self, sync_mock):
PluginManager.sync_plugins_metadata([self.env.plugins[0].id])
self.assertEqual(sync_mock.call_count, 1)

63
nailgun/nailgun/test/unit/test_node_disks.py
View File

@ -24,6 +24,7 @@ from nailgun.errors import errors
from nailgun.extensions.volume_manager.extension import VolumeManagerExtension
from nailgun.extensions.volume_manager.manager import Disk
from nailgun.extensions.volume_manager.manager import DisksFormatConvertor
from nailgun.extensions.volume_manager.manager import get_node_spaces
from nailgun.extensions.volume_manager.manager import only_disks
from nailgun.extensions.volume_manager.manager import only_vg
from nailgun.test.base import BaseIntegrationTest
@ -836,6 +837,68 @@ class TestVolumeManager(BaseIntegrationTest):
self.update_ram_and_assert_swap_size(node, 81920, 4096)
def test_get_node_spaces(self):
cluster = self._prepare_env()
node = self.env.create_node(
api=False,
cluster_id=cluster.id,
roles=['controller'],
pending_addition=True
)
expected_spaces = [
{'id': 'os', 'type': 'vg', '_allocate_size': 'min'},
{'id': 'image', 'type': 'vg', '_allocate_size': 'all'}
]
self.assertItemsEqual(get_node_spaces(node), expected_spaces)
def test_get_node_spaces_with_plugins(self):
cluster = self._prepare_env()
node = self.env.create_node(
api=False,
cluster_id=cluster.id,
roles=['controller', 'testing_plugin'],
pending_addition=True
)
self.env.create_plugin(
cluster=cluster,
package_version='3.0.0',
fuel_version=['7.0'],
volumes_metadata=self.env.get_default_plugin_volumes_config()
)
expected_spaces = [
{'id': 'os', 'type': 'vg', '_allocate_size': 'min'},
{'id': 'image', 'type': 'vg', '_allocate_size': 'all'},
{'id': 'test_volume', 'type': 'vg', '_allocate_size': 'all'}
]
self.assertItemsEqual(get_node_spaces(node), expected_spaces)
def _prepare_env(self):
volumes_metadata = {
'volumes_roles_mapping': {
'controller': [
{'allocate_size': 'min', 'id': 'os'},
{'allocate_size': 'all', 'id': 'image'}
]
},
'volumes': [
{'id': 'os', 'type': 'vg'},
{'id': 'image', 'type': 'vg'},
{'id': 'vm', 'type': 'vg'}
]
}
release = self.env.create_release(
volumes_metadata=volumes_metadata)
cluster = self.env.create_cluster(
api=False, release_id=release.id
)
return cluster
class TestDisks(BaseIntegrationTest):

42
nailgun/nailgun/test/unit/test_objects.py
View File

@ -17,6 +17,8 @@
import copy
import datetime
import hashlib
import mock
from itertools import cycle
from itertools import ifilter
import re
@ -44,6 +46,7 @@ from nailgun.network.neutron import NeutronManager
from nailgun.network.neutron import NeutronManager70
from nailgun import objects
from nailgun.plugins.manager import PluginManager
class TestObjects(BaseIntegrationTest):
@ -857,7 +860,7 @@ class TestClusterObject(BaseTestCase):
'test_plugin_first-0.1.0 by introducing the same '
'deployment task with id role-name'
)
with self.assertRaisesRegexp(errors.PluginsTasksOverlapping,
with self.assertRaisesRegexp(errors.AlreadyExists,
expected_message):
objects.Cluster.get_deployment_tasks(cluster)
@ -884,6 +887,37 @@ class TestClusterObject(BaseTestCase):
errors.NetworkRoleConflict,
objects.Cluster.get_network_roles, cluster)
def test_get_volumes_metadata_when_plugins_are_enabled(self):
plugin_volumes_metadata = {
'volumes_roles_mapping': {
'test_plugin_1': [
{'allocate_size': 'min', 'id': 'test_plugin_1'}
],
'test_plugin_2': [
{'allocate_size': 'min', 'id': 'test_plugin_2'}
],
},
'volumes': [
{'id': 'test_plugin_1', 'type': 'vg'},
{'id': 'test_plugin_2', 'type': 'vg'}
]
}
with mock.patch.object(
PluginManager, 'get_volumes_metadata') as plugin_volumes:
plugin_volumes.return_value = plugin_volumes_metadata
expected_volumes_metadata = copy.deepcopy(
self.env.releases[0].volumes_metadata)
expected_volumes_metadata['volumes_roles_mapping'].update(
plugin_volumes_metadata['volumes_roles_mapping'])
expected_volumes_metadata['volumes'].extend(
plugin_volumes_metadata['volumes'])
volumes_metadata = objects.Cluster.get_volumes_metadata(
self.env.clusters[0])
self.assertDictEqual(
volumes_metadata, expected_volumes_metadata)
class TestClusterObjectGetRoles(BaseTestCase):
@ -947,7 +981,7 @@ class TestClusterObjectGetRoles(BaseTestCase):
"the following node roles: role_a"
.format(plugin.id)
)
with self.assertRaisesRegexp(errors.PluginRolesConflict,
with self.assertRaisesRegexp(errors.AlreadyExists,
expected_message):
objects.Cluster.get_roles(self.cluster)
@ -965,7 +999,7 @@ class TestClusterObjectGetRoles(BaseTestCase):
"the following node roles: role_x"
.format(plugin_in_conflict.id)
)
with self.assertRaisesRegexp(errors.PluginRolesConflict,
with self.assertRaisesRegexp(errors.AlreadyExists,
expected_message):
objects.Cluster.get_roles(self.cluster)
@ -987,7 +1021,7 @@ class TestClusterObjectGetRoles(BaseTestCase):
.format(plugin_in_conflict.id))
with self.assertRaisesRegexp(
errors.PluginRolesConflict, message_pattern) as cm:
errors.AlreadyExists, message_pattern) as cm:
objects.Cluster.get_roles(self.cluster)
# 0 - the whole message, 1 - is first match of (.*) pattern