Add implement of monitoring host
This patch adds implementation of monitoring host. Change-Id: I066a0b632389e261dcd8e5ca6e7067d47be9eab1 Implements: bp pythonize-host-and-process-monitor
This commit is contained in:
Kengo Takahara
takahara.kengo
parent
3caf57eaba
commit
29d82c3ce2
|
|
@ -17,6 +17,9 @@ monitor_host_opts = [
|
|||
cfg.StrOpt('monitoring_driver',
|
||||
default='default',
|
||||
help='Driver that hostmonitor uses for monitoring hosts.'),
|
||||
cfg.IntOpt('monitoring_interval',
|
||||
default=60,
|
||||
help='Monitoring interval(in seconds) of node status.'),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,20 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import socket
|
||||
|
||||
import eventlet
|
||||
from oslo_log import log as oslo_logging
|
||||
|
||||
import masakarimonitors.conf
|
||||
import masakarimonitors.hostmonitor.host_handler.driver as driver
|
||||
from masakarimonitors.hostmonitor.host_handler import parse_cib_xml
|
||||
from masakarimonitors.i18n import _LE
|
||||
from masakarimonitors.i18n import _LW
|
||||
from masakarimonitors import utils
|
||||
|
||||
LOG = oslo_logging.getLogger(__name__)
|
||||
CONF = masakarimonitors.conf.CONF
|
||||
|
||||
|
||||
class HandleHost(driver.DriverBase):
|
||||
|
|
@ -23,6 +36,100 @@ class HandleHost(driver.DriverBase):
|
|||
|
||||
def __init__(self):
|
||||
super(HandleHost, self).__init__()
|
||||
self.my_hostname = socket.gethostname()
|
||||
self.xml_parser = parse_cib_xml.ParseCibXml()
|
||||
|
||||
def _check_host_status_by_crmadmin(self):
|
||||
try:
|
||||
# Execute crmadmin command.
|
||||
out, err = utils.execute('crmadmin', '-S', self.my_hostname,
|
||||
run_as_root=True)
|
||||
|
||||
if err:
|
||||
msg = ("crmadmin command output stderr: %s") % err
|
||||
raise Exception(msg)
|
||||
|
||||
# If own host is stable status, crmadmin outputs
|
||||
# 'S_IDLE' or 'S_NOT_DC'
|
||||
if 'S_IDLE' in out or 'S_NOT_DC' in out:
|
||||
return 0
|
||||
else:
|
||||
raise Exception(
|
||||
"crmadmin command output unexpected host status.")
|
||||
|
||||
except Exception as e:
|
||||
LOG.warning(_LW("Exception caught: %s"), e)
|
||||
LOG.warning(_LW("'%s' is unstable state on cluster."),
|
||||
self.my_hostname)
|
||||
return 1
|
||||
|
||||
def _get_cib_xml(self):
|
||||
try:
|
||||
# Execute cibadmin command.
|
||||
out, err = utils.execute('cibadmin', '--query', run_as_root=True)
|
||||
|
||||
if err:
|
||||
msg = ("cibadmin command output stderr: %s") % err
|
||||
raise Exception(msg)
|
||||
|
||||
except Exception as e:
|
||||
LOG.warning(_LW("Exception caught: %s"), e)
|
||||
return
|
||||
|
||||
return out
|
||||
|
||||
def _check_host_status_by_cibadmin(self):
|
||||
# Get xml of cib info.
|
||||
cib_xml = self._get_cib_xml()
|
||||
if cib_xml is None:
|
||||
# cibadmin command failure.
|
||||
return 1
|
||||
|
||||
# Set to the ParseCibXml object.
|
||||
self.xml_parser.set_cib_xml(cib_xml)
|
||||
|
||||
# Check if pacemaker cluster have quorum.
|
||||
if self.xml_parser.have_quorum() == 0:
|
||||
msg = "Pacemaker cluster doesn't have quorum."
|
||||
LOG.warning(_LW("%s"), msg)
|
||||
|
||||
# Get node_state tag list.
|
||||
node_state_tag_list = self.xml_parser.get_node_state_tag_list()
|
||||
if len(node_state_tag_list) == 0:
|
||||
# If cib xml doesn't have node_state tag,
|
||||
# it is an unexpected result.
|
||||
raise Exception(
|
||||
"Failed to get node_state tag from cib xml.")
|
||||
|
||||
return 0
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
|
||||
def monitor_hosts(self):
|
||||
pass
|
||||
"""Host monitoring main method.
|
||||
|
||||
This method monitors hosts.
|
||||
"""
|
||||
try:
|
||||
self.running = True
|
||||
while self.running:
|
||||
|
||||
# Check the host status is stable or unstable by crmadmin.
|
||||
if self._check_host_status_by_crmadmin() != 0:
|
||||
LOG.warning(_LW("hostmonitor skips monitoring hosts."))
|
||||
eventlet.greenthread.sleep(CONF.host.monitoring_interval)
|
||||
continue
|
||||
|
||||
# Check the host status is online or offline by cibadmin.
|
||||
if self._check_host_status_by_cibadmin() != 0:
|
||||
LOG.warning(_LW("hostmonitor skips monitoring hosts."))
|
||||
eventlet.greenthread.sleep(CONF.host.monitoring_interval)
|
||||
continue
|
||||
|
||||
eventlet.greenthread.sleep(CONF.host.monitoring_interval)
|
||||
|
||||
except Exception as e:
|
||||
LOG.exception(_LE("Exception caught: %s"), e)
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from oslo_log import log as oslo_logging
|
||||
|
||||
from masakarimonitors.i18n import _LE
|
||||
|
||||
LOG = oslo_logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ParseCibXml(object):
|
||||
"""ParseCibXml class
|
||||
|
||||
This class parses the cib xml.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.cib_tag = None
|
||||
|
||||
def set_cib_xml(self, cib_xml):
|
||||
"""Set xml.etree.ElementTree.Element object.
|
||||
|
||||
This method recieves string of cib xml, and convert it
|
||||
to xml.etree.ElementTree.Element object.
|
||||
|
||||
:params cib_xml: String of cib xml
|
||||
"""
|
||||
# Convert xml.etree.ElementTree.Element object.
|
||||
self.cib_tag = ElementTree.fromstring(cib_xml)
|
||||
|
||||
def have_quorum(self):
|
||||
"""Returns if cluster has quorum or not.
|
||||
|
||||
:returns: 0 on no-quorum, 1 if cluster has quorum.
|
||||
"""
|
||||
return int(self.cib_tag.get('have-quorum'))
|
||||
|
||||
def _get_status_tag(self):
|
||||
# status tag exists in the cib tag.
|
||||
child_list = self.cib_tag.getchildren()
|
||||
for child in child_list:
|
||||
if child.tag == 'status':
|
||||
return child
|
||||
return None
|
||||
|
||||
def _get_node_states(self, status_tag):
|
||||
node_state_tag_list = []
|
||||
|
||||
# node_state tag exists in the status tag.
|
||||
child_list = status_tag.getchildren()
|
||||
for child in child_list:
|
||||
if child.tag == 'node_state':
|
||||
node_state_tag_list.append(child)
|
||||
|
||||
return node_state_tag_list
|
||||
|
||||
def get_node_state_tag_list(self):
|
||||
"""Get node_state tag list.
|
||||
|
||||
This method gets node_state tag list from cib xml.
|
||||
|
||||
:returns: node_state tag list
|
||||
"""
|
||||
# Get status tag.
|
||||
status_tag = self._get_status_tag()
|
||||
if status_tag is None:
|
||||
LOG.error(_LE("Cib xml doesn't have status tag."))
|
||||
return []
|
||||
|
||||
# Get node_state tag list.
|
||||
node_state_tag_list = self._get_node_states(status_tag)
|
||||
if len(node_state_tag_list) == 0:
|
||||
LOG.error(_LE("Cib xml doesn't have node_state tag."))
|
||||
|
||||
return node_state_tag_list
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import testtools
|
||||
|
||||
import eventlet
|
||||
|
||||
from masakarimonitors.hostmonitor.host_handler import handle_host
|
||||
from masakarimonitors.hostmonitor.host_handler import parse_cib_xml
|
||||
from masakarimonitors import utils
|
||||
|
||||
eventlet.monkey_patch(os=False)
|
||||
|
||||
EXECUTE_RETURN = 'Status of crmd@masakari-node: S_NOT_DC (ok)'
|
||||
|
||||
|
||||
class TestHandleHost(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestHandleHost, self).setUp()
|
||||
|
||||
@mock.patch.object(parse_cib_xml.ParseCibXml, 'get_node_state_tag_list')
|
||||
@mock.patch.object(parse_cib_xml.ParseCibXml, 'have_quorum')
|
||||
@mock.patch.object(parse_cib_xml.ParseCibXml, 'set_cib_xml')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_monitor_hosts(self,
|
||||
mock_execute,
|
||||
mock_set_cib_xml,
|
||||
mock_have_quorum,
|
||||
mock_get_node_state_tag_list):
|
||||
|
||||
obj = handle_host.HandleHost()
|
||||
|
||||
mock_execute.return_value = (EXECUTE_RETURN, '')
|
||||
mock_set_cib_xml.return_value = None
|
||||
mock_have_quorum.return_value = 0
|
||||
mock_get_node_state_tag_list.return_value = []
|
||||
|
||||
ret = obj.monitor_hosts()
|
||||
self.assertEqual(None, ret)
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# Copyright(c) 2016 Nippon Telegraph and Telephone Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import testtools
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import eventlet
|
||||
|
||||
from masakarimonitors.hostmonitor.host_handler import parse_cib_xml
|
||||
|
||||
eventlet.monkey_patch(os=False)
|
||||
|
||||
CIB_XML = '<cib have-quorum="1">' \
|
||||
' <status>' \
|
||||
' <node_state uname="masakari-node" crmd="online">' \
|
||||
' <test hoge="hoge"/>' \
|
||||
' </node_state>' \
|
||||
' <node_state crmd="online" uname="compute-node">' \
|
||||
' <test hoge="hoge"/>' \
|
||||
' </node_state>' \
|
||||
' </status>' \
|
||||
'</cib>'
|
||||
CIB_TAG = ElementTree.fromstring(CIB_XML)
|
||||
|
||||
|
||||
class TestParseCibXml(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestParseCibXml, self).setUp()
|
||||
|
||||
@mock.patch.object(ElementTree, 'fromstring')
|
||||
def test_set_cib_xml(self,
|
||||
mock_fromstring):
|
||||
|
||||
obj = parse_cib_xml.ParseCibXml()
|
||||
mock_fromstring.return_value = CIB_TAG
|
||||
obj.set_cib_xml(CIB_XML)
|
||||
|
||||
def test_have_quorum(self):
|
||||
|
||||
obj = parse_cib_xml.ParseCibXml()
|
||||
obj.set_cib_xml(CIB_XML)
|
||||
self.assertEqual(1, obj.have_quorum())
|
||||
|
||||
def test_get_node_state_tag_list(self):
|
||||
|
||||
obj = parse_cib_xml.ParseCibXml()
|
||||
obj.set_cib_xml(CIB_XML)
|
||||
|
||||
node_state_tag_list = obj.get_node_state_tag_list()
|
||||
|
||||
for node_state_tag in node_state_tag_list:
|
||||
self.assertEqual('online', node_state_tag.get('crmd'))
|
||||
Loading…
Reference in New Issue