Add baremetal data model

Change-Id: I57b7bb53b3bc84ad383ae485069274f5c5362c50
Implements: blueprint build-baremetal-data-model-in-watcher
This commit is contained in:
Yumeng_Bao 2017-07-10 18:50:58 +08:00
parent 97799521f9
commit 54da2a75fb
16 changed files with 587 additions and 1 deletions

View File

@ -0,0 +1,4 @@
---
features:
- |
Adds baremetal data model in Watcher

View File

@ -97,6 +97,7 @@ watcher_planners =
watcher_cluster_data_model_collectors =
compute = watcher.decision_engine.model.collector.nova:NovaClusterDataModelCollector
storage = watcher.decision_engine.model.collector.cinder:CinderClusterDataModelCollector
baremetal = watcher.decision_engine.model.collector.ironic:BaremetalClusterDataModelCollector
[pbr]

View File

@ -473,6 +473,14 @@ class VolumeNotFound(StorageResourceNotFound):
msg_fmt = _("The volume '%(name)s' could not be found")
class BaremetalResourceNotFound(WatcherException):
msg_fmt = _("The baremetal resource '%(name)s' could not be found")
class IronicNodeNotFound(BaremetalResourceNotFound):
msg_fmt = _("The ironic node %(uuid)s could not be found")
class LoadingError(WatcherException):
msg_fmt = _("Error loading plugin '%(name)s'")

View File

@ -0,0 +1,49 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 ZTE Corporation
#
# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_log import log
from watcher.common import clients
from watcher.common import exception
from watcher.common import utils
LOG = log.getLogger(__name__)
class IronicHelper(object):
def __init__(self, osc=None):
""":param osc: an OpenStackClients instance"""
self.osc = osc if osc else clients.OpenStackClients()
self.ironic = self.osc.ironic()
def get_ironic_node_list(self):
return self.ironic.node.list()
def get_ironic_node_by_uuid(self, node_uuid):
"""Get ironic node by node UUID"""
try:
node = self.ironic.node.get(utils.Struct(uuid=node_uuid))
if not node:
raise exception.IronicNodeNotFound(uuid=node_uuid)
except Exception as exc:
LOG.exception(exc)
raise exception.IronicNodeNotFound(uuid=node_uuid)
# We need to pass an object with an 'uuid' attribute to make it work
return node

View File

@ -0,0 +1,97 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 ZTE Corporation
#
# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log
from watcher.common import ironic_helper
from watcher.decision_engine.model.collector import base
from watcher.decision_engine.model import element
from watcher.decision_engine.model import model_root
LOG = log.getLogger(__name__)
class BaremetalClusterDataModelCollector(base.BaseClusterDataModelCollector):
"""Baremetal cluster data model collector
The Baremetal cluster data model collector creates an in-memory
representation of the resources exposed by the baremetal service.
"""
def __init__(self, config, osc=None):
super(BaremetalClusterDataModelCollector, self).__init__(config, osc)
@property
def notification_endpoints(self):
"""Associated notification endpoints
:return: Associated notification endpoints
:rtype: List of :py:class:`~.EventsNotificationEndpoint` instances
"""
return None
def get_audit_scope_handler(self, audit_scope):
return None
def execute(self):
"""Build the baremetal cluster data model"""
LOG.debug("Building latest Baremetal cluster data model")
builder = ModelBuilder(self.osc)
return builder.execute()
class ModelBuilder(object):
"""Build the graph-based model
This model builder adds the following data"
- Baremetal-related knowledge (Ironic)
"""
def __init__(self, osc):
self.osc = osc
self.model = model_root.BaremetalModelRoot()
self.ironic_helper = ironic_helper.IronicHelper(osc=self.osc)
def add_ironic_node(self, node):
# Build and add base node.
ironic_node = self.build_ironic_node(node)
self.model.add_node(ironic_node)
def build_ironic_node(self, node):
"""Build a Baremetal node from a Ironic node
:param node: A ironic node
:type node: :py:class:`~ironicclient.v1.node.Node`
"""
# build up the ironic node.
node_attributes = {
"uuid": node.uuid,
"power_state": node.power_state,
"maintenance": node.maintenance,
"maintenance_reason": node.maintenance_reason,
"extra": {"compute_node_id": node.extra.compute_node_id}
}
ironic_node = element.IronicNode(**node_attributes)
return ironic_node
def execute(self):
for node in self.ironic_helper.get_ironic_node_list():
self.add_ironic_node(node)
return self.model

View File

@ -23,6 +23,7 @@ from watcher.decision_engine.model.element import volume
ServiceState = node.ServiceState
ComputeNode = node.ComputeNode
StorageNode = node.StorageNode
IronicNode = node.IronicNode
Pool = node.Pool
InstanceState = instance.InstanceState
@ -37,4 +38,5 @@ __all__ = ['ServiceState',
'StorageNode',
'Pool',
'VolumeState',
'Volume']
'Volume',
'IronicNode']

View File

@ -0,0 +1,33 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 ZTE Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
from watcher.decision_engine.model.element import base
from watcher.objects import fields as wfields
@six.add_metaclass(abc.ABCMeta)
class BaremetalResource(base.Element):
VERSION = '1.0'
fields = {
"uuid": wfields.StringField(),
"human_id": wfields.StringField(default=""),
}

View File

@ -16,6 +16,7 @@
import enum
from watcher.decision_engine.model.element import baremetal_resource
from watcher.decision_engine.model.element import compute_resource
from watcher.decision_engine.model.element import storage_resource
from watcher.objects import base
@ -78,3 +79,17 @@ class Pool(storage_resource.StorageResource):
def accept(self, visitor):
raise NotImplementedError()
@base.WatcherObjectRegistry.register_if(False)
class IronicNode(baremetal_resource.BaremetalResource):
fields = {
"power_state": wfields.StringField(),
"maintenance": wfields.BooleanField(),
"maintenance_reason": wfields.StringField(),
"extra": wfields.DictField()
}
def accept(self, visitor):
raise NotImplementedError()

View File

@ -545,3 +545,85 @@ class StorageModelRoot(nx.DiGraph, base.Model):
def is_isomorphic(cls, G1, G2):
return nx.algorithms.isomorphism.isomorph.is_isomorphic(
G1, G2)
class BaremetalModelRoot(nx.DiGraph, base.Model):
"""Cluster graph for an Openstack cluster: Baremetal Cluster."""
def __init__(self, stale=False):
super(BaremetalModelRoot, self).__init__()
self.stale = stale
def __nonzero__(self):
return not self.stale
__bool__ = __nonzero__
@staticmethod
def assert_node(obj):
if not isinstance(obj, element.IronicNode):
raise exception.IllegalArgumentException(
message=_("'obj' argument type is not valid: %s") % type(obj))
@lockutils.synchronized("baremetal_model")
def add_node(self, node):
self.assert_node(node)
super(BaremetalModelRoot, self).add_node(node.uuid, node)
@lockutils.synchronized("baremetal_model")
def remove_node(self, node):
self.assert_node(node)
try:
super(BaremetalModelRoot, self).remove_node(node.uuid)
except nx.NetworkXError as exc:
LOG.exception(exc)
raise exception.IronicNodeNotFound(name=node.uuid)
@lockutils.synchronized("baremetal_model")
def get_all_ironic_nodes(self):
return {uuid: cn for uuid, cn in self.nodes(data=True)
if isinstance(cn, element.IronicNode)}
@lockutils.synchronized("baremetal_model")
def get_node_by_uuid(self, uuid):
try:
return self._get_by_uuid(uuid)
except exception.BaremetalResourceNotFound:
raise exception.IronicNodeNotFound(name=uuid)
def _get_by_uuid(self, uuid):
try:
return self.node[uuid]
except Exception as exc:
LOG.exception(exc)
raise exception.BaremetalResourceNotFound(name=uuid)
def to_string(self):
return self.to_xml()
def to_xml(self):
root = etree.Element("ModelRoot")
# Build Ironic node tree
for cn in sorted(self.get_all_ironic_nodes().values(),
key=lambda cn: cn.uuid):
ironic_node_el = cn.as_xml_element()
root.append(ironic_node_el)
return etree.tostring(root, pretty_print=True).decode('utf-8')
@classmethod
def from_xml(cls, data):
model = cls()
root = etree.fromstring(data)
for cn in root.findall('.//IronicNode'):
node = element.IronicNode(**cn.attrib)
model.add_node(node)
return model
@classmethod
def is_isomorphic(cls, G1, G2):
return nx.algorithms.isomorphism.isomorph.is_isomorphic(
G1, G2)

View File

@ -0,0 +1,46 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2018 ZTE Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log
from watcher.decision_engine.scope import base
LOG = log.getLogger(__name__)
class BaremetalScope(base.BaseScope):
"""Baremetal Audit Scope Handler"""
def __init__(self, scope, config, osc=None):
super(BaremetalScope, self).__init__(scope, config)
self._osc = osc
def get_scoped_model(self, cluster_model):
"""Leave only nodes and instances proposed in the audit scope"""
if not cluster_model:
return None
for scope in self.scope:
baremetal_scope = scope.get('baremetal')
if not baremetal_scope:
return cluster_model
# TODO(yumeng-bao): currently self.scope is always []
# Audit scoper for baremetal data model will be implemented:
# https://blueprints.launchpad.net/watcher/+spec/audit-scoper-for-baremetal-data-model
return cluster_model

View File

@ -85,6 +85,7 @@ class BaseStrategy(loadable.Loadable):
self._collector_manager = None
self._compute_model = None
self._storage_model = None
self._baremetal_model = None
self._input_parameters = utils.Struct()
self._audit_scope = None
self._datasource_backend = None
@ -219,6 +220,29 @@ class BaseStrategy(loadable.Loadable):
return self._storage_model
@property
def baremetal_model(self):
"""Cluster data model
:returns: Cluster data model the strategy is executed on
:rtype model: :py:class:`~.ModelRoot` instance
"""
if self._baremetal_model is None:
collector = self.collector_manager.get_cluster_model_collector(
'baremetal', osc=self.osc)
audit_scope_handler = collector.get_audit_scope_handler(
audit_scope=self.audit_scope)
self._baremetal_model = audit_scope_handler.get_scoped_model(
collector.get_latest_cluster_data_model())
if not self._baremetal_model:
raise exception.ClusterStateNotDefined()
if self._baremetal_model.stale:
raise exception.ClusterStateStale()
return self._baremetal_model
@classmethod
def get_schema(cls):
"""Defines a Schema that the input parameters shall comply to

View File

@ -0,0 +1,63 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 ZTE Corporation
#
# Authors:Yumeng Bao <bao.yumeng@zte.com.cn>
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import mock
from watcher.common import clients
from watcher.common import exception
from watcher.common import ironic_helper
from watcher.common import utils as w_utils
from watcher.tests import base
class TestIronicHelper(base.TestCase):
def setUp(self):
super(TestIronicHelper, self).setUp()
osc = clients.OpenStackClients()
p_ironic = mock.patch.object(osc, 'ironic')
p_ironic.start()
self.addCleanup(p_ironic.stop)
self.ironic_util = ironic_helper.IronicHelper(osc=osc)
@staticmethod
def fake_ironic_node():
node = mock.MagicMock()
node.uuid = w_utils.generate_uuid()
return node
def test_get_ironic_node_list(self):
node1 = self.fake_ironic_node()
self.ironic_util.ironic.node.list.return_value = [node1]
rt_nodes = self.ironic_util.get_ironic_node_list()
self.assertEqual(rt_nodes, [node1])
def test_get_ironic_node_by_uuid_success(self):
node1 = self.fake_ironic_node()
self.ironic_util.ironic.node.get.return_value = node1
node = self.ironic_util.get_ironic_node_by_uuid(node1.uuid)
self.assertEqual(node, node1)
def test_get_ironic_node_by_uuid_failure(self):
self.ironic_util.ironic.node.get.return_value = None
self.assertRaisesRegex(
exception.IronicNodeNotFound,
"The ironic node node1 could not be found",
self.ironic_util.get_ironic_node_by_uuid, 'node1')

View File

@ -0,0 +1,12 @@
<ModelRoot>
<IronicNode uuid="c5941348-5a87-4016-94d4-4f9e0ce2b87a" power_state="power on" maintenance="false" maintenance_reason="null">
<extra>
<compute_node_id> 1</compute_node_id>
</extra>
</IronicNode>
<IronicNode uuid="c5941348-5a87-4016-94d4-4f9e0ce2b87c" power_state="power on" maintenance="false" maintenance_reason="null">
<extra>
<compute_node_id> 2</compute_node_id>
</extra>
</IronicNode>
</ModelRoot>

View File

@ -20,6 +20,7 @@ import os
import mock
from watcher.common import utils
from watcher.decision_engine.model.collector import base
from watcher.decision_engine.model import element
from watcher.decision_engine.model import model_root as modelroot
@ -261,3 +262,55 @@ class FakerStorageModelCollector(base.BaseClusterDataModelCollector):
def generate_scenario_1(self):
return self.load_model('storage_scenario_1.xml')
class FakerBaremetalModelCollector(base.BaseClusterDataModelCollector):
def __init__(self, config=None, osc=None):
if config is None:
config = mock.Mock(period=777)
super(FakerBaremetalModelCollector, self).__init__(config)
@property
def notification_endpoints(self):
return []
def get_audit_scope_handler(self, audit_scope):
return None
def load_data(self, filename):
cwd = os.path.abspath(os.path.dirname(__file__))
data_folder = os.path.join(cwd, "data")
with open(os.path.join(data_folder, filename), 'rb') as xml_file:
xml_data = xml_file.read()
return xml_data
def load_model(self, filename):
return modelroot.BaremetalModelRoot.from_xml(self.load_data(filename))
def execute(self):
return self._cluster_data_model or self.build_scenario_1()
def build_scenario_1(self):
model = modelroot.BaremetalModelRoot()
# number of nodes
node_count = 2
for i in range(0, node_count):
uuid = utils.generate_uuid()
node_attributes = {
"uuid": uuid,
"power_state": "power on",
"maintenance": "false",
"maintenance_reason": "null",
"extra": {"compute_node_id": i}
}
node = element.IronicNode(**node_attributes)
model.add_node(node)
return model
def generate_scenario_1(self):
return self.load_model('ironic_scenario_1.xml')

View File

@ -152,3 +152,30 @@ class TestStorageElement(base.TestCase):
def test_as_xml_element(self):
el = self.cls(**self.data)
el.as_xml_element()
class TestIronicElement(base.TestCase):
scenarios = [
("IronicNode_with_all_fields", dict(
cls=element.IronicNode,
data={
"uuid": 'FAKE_UUID',
"power_state": 'up',
"maintenance": "false",
"maintenance_reason": "null",
"extra": {"compute_node_id": 1}
})),
("IronicNode_with_some_fields", dict(
cls=element.IronicNode,
data={
"uuid": 'FAKE_UUID',
"power_state": 'up',
"maintenance": "false",
"extra": {"compute_node_id": 1}
})),
]
def test_as_xml_element(self):
el = self.cls(**self.data)
el.as_xml_element()

View File

@ -367,3 +367,73 @@ class TestStorageModel(base.TestCase):
self.assertEqual(volume, model.get_volume_by_uuid(uuid_))
model.map_volume(volume, pool)
self.assertEqual([volume], model.get_pool_volumes(pool))
class TestBaremetalModel(base.TestCase):
def load_data(self, filename):
cwd = os.path.abspath(os.path.dirname(__file__))
data_folder = os.path.join(cwd, "data")
with open(os.path.join(data_folder, filename), 'rb') as xml_file:
xml_data = xml_file.read()
return xml_data
def load_model(self, filename):
return model_root.StorageModelRoot.from_xml(self.load_data(filename))
def test_model_structure(self):
fake_cluster = faker_cluster_state.FakerBaremetalModelCollector()
model1 = fake_cluster.build_scenario_1()
self.assertEqual(2, len(model1.get_all_ironic_nodes()))
expected_struct_str = self.load_data('ironic_scenario_1.xml')
model2 = model_root.BaremetalModelRoot.from_xml(expected_struct_str)
self.assertTrue(
model_root.BaremetalModelRoot.is_isomorphic(model2, model1))
def test_build_model_from_xml(self):
fake_cluster = faker_cluster_state.FakerBaremetalModelCollector()
expected_model = fake_cluster.generate_scenario_1()
struct_str = self.load_data('ironic_scenario_1.xml')
model = model_root.BaremetalModelRoot.from_xml(struct_str)
self.assertEqual(expected_model.to_string(), model.to_string())
def test_assert_node_raise(self):
model = model_root.BaremetalModelRoot()
node_uuid = uuidutils.generate_uuid()
node = element.IronicNode(uuid=node_uuid)
model.add_node(node)
self.assertRaises(exception.IllegalArgumentException,
model.assert_node, "obj")
def test_add_node(self):
model = model_root.BaremetalModelRoot()
node_uuid = uuidutils.generate_uuid()
node = element.IronicNode(uuid=node_uuid)
model.add_node(node)
self.assertEqual(node, model.get_node_by_uuid(node_uuid))
def test_remove_node(self):
model = model_root.BaremetalModelRoot()
node_uuid = uuidutils.generate_uuid()
node = element.IronicNode(uuid=node_uuid)
model.add_node(node)
self.assertEqual(node, model.get_node_by_uuid(node_uuid))
model.remove_node(node)
self.assertRaises(exception.IronicNodeNotFound,
model.get_node_by_uuid, node_uuid)
def test_get_all_ironic_nodes(self):
model = model_root.BaremetalModelRoot()
for i in range(10):
node_uuid = uuidutils.generate_uuid()
node = element.IronicNode(uuid=node_uuid)
model.add_node(node)
all_nodes = model.get_all_ironic_nodes()
for node_uuid in all_nodes:
node = model.get_node_by_uuid(node_uuid)
model.assert_node(node)