diff --git a/freezer_dr/fencers/common/driver.py b/freezer_dr/fencers/common/driver.py index 568bb74..4b8f33f 100644 --- a/freezer_dr/fencers/common/driver.py +++ b/freezer_dr/fencers/common/driver.py @@ -15,7 +15,6 @@ """Abstract fencer""" import abc - import six @@ -28,38 +27,29 @@ class FencerBaseDriver(object): needed. """ - def __init__(self, node, **kwargs): + def __init__(self, nodes, fencer_conf): """Initialize the driver. Any fencer driver requires the following parameters to do the api calls. All these parameters can be passed from the configuration file in /etc/freezer/dr.conf (default). - :param node: dict with all node details. (/etc/freezer/servers.yml) ? - :param kwargs: any additional parameters can be passed using this - config option. + :param nodes: A list of failed nodes to be fenced! + :param fencer_conf: dict contains configuration options loaded + from the config file. """ - self.node = node - self.kwargs = kwargs + self.nodes = nodes + self.fencer_conf = fencer_conf + + def update_nodes(self, nodes): + """Allows changing the nodes during the evacuation...""" + self.nodes = nodes @abc.abstractmethod - def graceful_shutdown(self): - """Gracefully shutdown the compute node to evacuate it.""" - - @abc.abstractmethod - def force_shutdown(self): - """Force shutdown the compute node to evacuate it. - - May be you can try force shutdown if the graceful shutdown failed. - """ - - @abc.abstractmethod - def status(self): - """Get compute node status. - - Should return 1 if on and 0 if off or -1 if error or unknown power - status. - """ + def fence(self): + """This function to be implemented by each driver. Each driver will + implement its own fencing logic and the manager will just load it and + call the fence function""" @abc.abstractmethod def get_info(self): diff --git a/freezer_dr/fencers/common/manager.py b/freezer_dr/fencers/common/manager.py index d04b87c..518d99a 100644 --- a/freezer_dr/fencers/common/manager.py +++ b/freezer_dr/fencers/common/manager.py @@ -14,8 +14,7 @@ from oslo_config import cfg from oslo_log import log from oslo_utils import importutils -from freezer_dr.common.yaml_parser import YamlParser -from time import sleep + CONF = cfg.CONF LOG = log.getLogger(__name__) @@ -24,51 +23,20 @@ LOG = log.getLogger(__name__) class FencerManager(object): def __init__(self, nodes): - self.fencer = CONF.get('fencer') + self.fencer_conf = CONF.get('fencer') self.nodes = nodes - self.parser = YamlParser(self.fencer.get('credentials_file')) def fence(self): """ Try to shutdown nodes and wait for configurable amount of times :return: list of nodes and either they are shutdown or failed """ - processed_nodes = [] - for node in self.nodes: - node_details = self.parser.find_server_by_ip(node.get('ip')) or\ - self.parser.find_server_by_hostname(node.get('host')) - driver = importutils.import_object( - self.fencer.get('driver'), - node=node_details, - **self.fencer.get('options') - ) - node['status'] = self.do_shutdown_procedure(driver) - processed_nodes.append(node) - return processed_nodes - - def do_shutdown_procedure(self, driver): - for retry in range(0, self.fencer.get('retries', 1)): - if driver.status(): - try: - driver.graceful_shutdown() - except Exception as e: - LOG.error(e) - else: - return True - # try to wait a pre-configured amount of time before redoing - # the fence call again :) - sleep(self.fencer.get('hold_period', 10)) - LOG.info('wait for %d seconds before retrying to gracefully ' - 'shutdown' % self.fencer.get('hold_period', 10)) - LOG.info('Retrying to gracefully shutdown the node.') - - try: - driver.force_shutdown() - except Exception as e: - LOG.error(e) - - if not driver.status(): - return True - - return False + driver_name = self.fencer_conf['driver'] + driver = importutils.import_object( + driver_name, + self.fencer_conf + ) + LOG.debug('Loaded fencing driver {0} with config: ' + '{1}'.format(driver.get_info(), self.fencer_conf)) + return driver.fence() diff --git a/freezer_dr/fencers/drivers/ipmi/driver.py b/freezer_dr/fencers/drivers/ipmi/driver.py index 43bb9b9..303b16e 100644 --- a/freezer_dr/fencers/drivers/ipmi/driver.py +++ b/freezer_dr/fencers/drivers/ipmi/driver.py @@ -11,18 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from freezer_dr.common.yaml_parser import YamlParser from freezer_dr.fencers.common.driver import FencerBaseDriver from freezer_dr.fencers.drivers.ipmi.ipmitool import IpmiInterface from oslo_log import log from oslo_config import cfg +import time CONF = cfg.CONF LOG = log.getLogger(__name__) class IpmiDriver(FencerBaseDriver): - def __init__(self, node, **kwargs): - super(IpmiDriver, self).__init__(node, **kwargs) + + def __init__(self, nodes, fencer_conf): + + super(IpmiDriver, self).__init__(nodes, fencer_conf) + self.parser = YamlParser(self.fencer_conf['credentials_file']) + + def prepare_node(self, node): + """Prepares the subprocess to call ``ipmitool`` with the node details! + :param node: dict contains node fencing information + """ self.interface = IpmiInterface( node.get('fencer-ip'), node.get('fencer-user'), @@ -48,11 +59,57 @@ class IpmiDriver(FencerBaseDriver): def power_on(self): self.interface.power_on() + def get_node_details(self, node): + """Loads the node's fencing information from ``credentials_file`` + :param node: a dict contains node ip or hostname + :return: a dict contains node fencing information + """ + node_details = self.parser.find_server_by_ip(node.get('ip')) or \ + self.parser.find_server_by_hostname(node.get('host')) + + return node_details + + def fence(self): + """Implements the fencing procedure for server fencing using ipmi + :return: a list of nodes and weather they're fenced or not! + """ + fenced_nodes = [] + for node in self.nodes: + LOG.debug("fencing node {0}".format(node)) + # load node details + node_details = self.get_node_details(node) + # loop on the node number of n times trying to fence it gently, + # if not force it! + self.prepare_node(node_details) + for retry in range(0, self.fencer_conf['retries']): + if self.status(): + try: + self.graceful_shutdown() + except Exception as e: + LOG.debug(e) + else: + node['status'] = True + break + time.sleep(self.fencer_conf['hold_period']) + LOG.info('wait for %d seconds before retrying to gracefully ' + 'shutdown' % self.fencer_conf['hold_period']) + + try: + self.force_shutdown() + except Exception as e: + LOG.error(e) + + if not self.status(): + node['status'] = True + else: + node['status'] = False + fenced_nodes.append(node) + + return fenced_nodes + def get_info(self): return { 'name': 'IPMI Interface driver', - 'version': 1.0, - 'author': 'Hewlett-Packard Development Company, L.P' + 'version': 1.1, + 'author': 'Hewlett-Packard Enterprise Company, L.P' } - - diff --git a/freezer_dr/fencers/drivers/libvirt/driver.py b/freezer_dr/fencers/drivers/libvirt/driver.py index d3e3cb5..f370993 100644 --- a/freezer_dr/fencers/drivers/libvirt/driver.py +++ b/freezer_dr/fencers/drivers/libvirt/driver.py @@ -11,38 +11,90 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from freezer_dr.common.yaml_parser import YamlParser from freezer_dr.fencers.common.driver import FencerBaseDriver +import libvirt from oslo_log import log from oslo_config import cfg -import libvirt +import time CONF = cfg.CONF LOG = log.getLogger(__name__) class LibvirtDriver(FencerBaseDriver): - def __init__(self, node, **kwargs): - super(LibvirtDriver, self).__init__(node, **kwargs) - conn_name = kwargs.get('name', None) + + def __init__(self, nodes, fencer_conf): + super(LibvirtDriver, self).__init__(nodes, fencer_conf) + self.parser = YamlParser(self.fencer_conf['credentials_file']) + # initiate libvirt connection + conn_name = self.fencer_conf.get('name', None) self.connection = libvirt.open(name=conn_name) - def force_shutdown(self): - target = self.connection.lookupByName(name=self.node.get('domain-name')) + def force_shutdown(self, node): + target = self.connection.lookupByName(name=node.get('domain-name')) return target.destroy() - def graceful_shutdown(self): - target = self.connection.lookupByName(name=self.node.get('domain-name')) + def graceful_shutdown(self, node): + target = self.connection.lookupByName(name=node.get('domain-name')) return target.shutdown() - def status(self): - target = self.connection.lookupByName(name=self.node.get('domain-name')) + def status(self, node): + target = self.connection.lookupByName(name=node.get('domain-name')) return target.isActive() + def get_node_details(self, node): + """Loads the node's fencing information from ``credentials_file`` + :param node: a dict contains node ip or hostname + :return: a dict contains node fencing information + """ + node_details = self.parser.find_server_by_ip(node.get('ip')) or \ + self.parser.find_server_by_hostname(node.get('host')) + + return node_details + + def fence(self): + """Implements the fencing procedure for server fencing using ipmi + :return: a list of nodes and weather they're fenced or not! + """ + fenced_nodes = [] + for node in self.nodes: + LOG.debug("fencing node {0}".format(node)) + # load node details + node_details = self.get_node_details(node) + # loop on the node number of n times trying to fence it gently, + # if not force it! + for retry in range(0, self.fencer_conf['retries']): + if self.status(node=node_details): + try: + self.graceful_shutdown(node=node_details) + except Exception as e: + LOG.debug(e) + else: + node['status'] = True + break + time.sleep(self.fencer_conf['hold_period']) + LOG.info('wait for %d seconds before retrying to gracefully ' + 'shutdown' % self.fencer_conf['hold_period']) + + try: + self.force_shutdown(node=node_details) + except Exception as e: + LOG.error(e) + + if not self.status(node=node_details): + node['status'] = True + else: + node['status'] = False + fenced_nodes.append(node) + + return fenced_nodes + def get_info(self): return { 'name': 'Libvirt Interface driver', - 'version': 1.0, - 'author': 'Hewlett-Packard Development Company, L.P' + 'version': 1.1, + 'author': 'Hewlett-Packard Enterprise Company, L.P' }