performa
/
os-faults
Code Issues Proposed changes

Adding threads to IPMI driver 

This patch adds threads to poweroff/poweron/reset methods to increase
the velocity of IPMI nodes management. The common code with the libvirt
driver was moved to a separate module utils.py.

Change-Id: Ic6e65079636cef6f32b13876fe3fb56ee302c6e9
This commit is contained in:
Yaroslav Lobankov 2016-08-31 18:01:26 +03:00
parent 096744ac7d
commit 0b9824726a
5 changed files with 91 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/power_off_on_vm_node.py
View File

@ -45,7 +45,7 @@ def main():
nodes = destructor.get_nodes()
logging.info('All cluster nodes: %s', nodes)
logging.info('Pick and power off one of cluster nodes')
logging.info('Pick and power off/on one of cluster nodes')
one = nodes.pick()
one.poweroff()
one.poweron()

2
os_failures/api/error.py
View File

@ -20,5 +20,5 @@ class OSFError(OSFException):
"""Base Error class"""
class PowerManagmentError(OSFError):
class PowerManagementError(OSFError):
"""Base Error class for Power Management API"""

48
os_failures/drivers/ipmi.py
View File

@ -18,6 +18,7 @@ from pyghmi.ipmi import command as ipmi_command
from os_failures.api import error
from os_failures.api import power_management
from os_failures import utils
class IPMIDriver(power_management.PowerManagement):
@ -26,7 +27,7 @@ class IPMIDriver(power_management.PowerManagement):
def _find_bmc_by_mac_address(self, mac_address):
if mac_address not in self.mac_to_bmc:
raise error.PowerManagmentError(
raise error.PowerManagementError(
'BMC for Node(%s) not found!' % mac_address)
return self.mac_to_bmc[mac_address]
@ -38,12 +39,11 @@ class IPMIDriver(power_management.PowerManagement):
userid=bmc['username'],
password=bmc['password'])
ret = ipmicmd.set_power(cmd, wait=True)
except pyghmi_exception.IpmiException as e:
except pyghmi_exception.IpmiException:
msg = 'IPMI cmd {!r} failed on bmc {!r}, Node({})'.format(
cmd, bmc['address'], mac_address)
logging.error(msg)
logging.exception(e)
raise error.PowerManagmentError(msg)
raise
logging.debug('IPMI response: {}'.format(ret))
if ret.get('powerstate') != expected_state or 'error' in ret:
@ -51,26 +51,32 @@ class IPMIDriver(power_management.PowerManagement):
'Node({})'.format(expected_state,
bmc['address'],
mac_address))
raise error.PowerManagmentError(msg)
raise error.PowerManagementError(msg)
def _poweroff(self, mac_address):
logging.info('Power off Node with MAC address: %s', mac_address)
self._run_set_power_cmd(
mac_address, cmd='off', expected_state='off')
logging.info('Node(%s) was powered off' % mac_address)
def _poweron(self, mac_address):
logging.info('Power on Node with MAC address: %s', mac_address)
self._run_set_power_cmd(
mac_address, cmd='on', expected_state='on')
logging.info('Node(%s) was powered on' % mac_address)
def _reset(self, mac_address):
logging.info('Reset Node with MAC address: %s', mac_address)
# boot -- If system is off, then 'on', else 'reset'
self._run_set_power_cmd(mac_address, cmd='boot')
# NOTE(astudenov): This command does not wait for node to boot
logging.info('Node(%s) was reset' % mac_address)
def poweroff(self, mac_addresses_list):
for mac_address in mac_addresses_list:
logging.info('Power off Node with MAC address: %s', mac_address)
self._run_set_power_cmd(
mac_address, cmd='off', expected_state='off')
logging.info('Node(%s) was powered off' % mac_address)
utils.run(self._poweroff, mac_addresses_list)
def poweron(self, mac_addresses_list):
for mac_address in mac_addresses_list:
logging.info('Power on Node with MAC address: %s', mac_address)
self._run_set_power_cmd(
mac_address, cmd='on', expected_state='on')
logging.info('Node(%s) was powered on' % mac_address)
utils.run(self._poweron, mac_addresses_list)
def reset(self, mac_addresses_list):
for mac_address in mac_addresses_list:
logging.info('Reset Node with MAC address: %s', mac_address)
# boot -- If system is off, then 'on', else 'reset'
self._run_set_power_cmd(mac_address, cmd='boot')
# NOTE(astudenov): this command does not wait for node to boot
logging.info('Node(%s) was reset' % mac_address)
utils.run(self._reset, mac_addresses_list)

54
os_failures/drivers/libvirt_driver.py
View File

@ -12,36 +12,12 @@
# limitations under the License.
import logging
import threading
import traceback
import libvirt
from os_failures.api import error
from os_failures.api import power_management
class ThreadsWrapper(object):
def __init__(self, target):
self.target = target
self.threads = []
self.errors = []
def _target(self, **kwargs):
try:
self.target(**kwargs)
except Exception as exc:
logging.error(traceback.format_exc())
self.errors.append(exc)
def start_thread(self, **kwargs):
thread = threading.Thread(target=self._target, kwargs=kwargs)
thread.start()
self.threads.append(thread)
def join_threads(self):
for thread in self.threads:
thread.join()
from os_failures import utils
class LibvirtDriver(power_management.PowerManagement):
@ -64,44 +40,32 @@ class LibvirtDriver(power_management.PowerManagement):
if mac_address in domain.XMLDesc():
return domain
raise error.PowerManagmentError(
'Node with MAC address %s not found!' % mac_address)
raise error.PowerManagementError(
'Domain with MAC address %s not found!' % mac_address)
def _poweroff(self, mac_address):
logging.info('Power off domain with MAC address: %s', mac_address)
domain = self._find_domain_by_mac_address(mac_address)
domain.destroy()
logging.info('Domain (%s) was powered off' % mac_address)
logging.info('Domain(%s) was powered off' % mac_address)
def _poweron(self, mac_address):
logging.info('Power on domain with MAC address: %s', mac_address)
domain = self._find_domain_by_mac_address(mac_address)
domain.create()
logging.info('Domain (%s) was powered on' % mac_address)
logging.info('Domain(%s) was powered on' % mac_address)
def _reset(self, mac_address):
logging.info('Reset domain with MAC address: %s', mac_address)
domain = self._find_domain_by_mac_address(mac_address)
domain.reset()
logging.info('Domain (%s) was reset' % mac_address)
@staticmethod
def _run(target, mac_addresses_list):
tw = ThreadsWrapper(target)
for mac_address in mac_addresses_list:
tw.start_thread(mac_address=mac_address)
tw.join_threads()
if tw.errors:
raise error.PowerManagmentError(
'There are some errors when working the libvirt driver. '
'Please, check logs for more details.')
logging.info('Domain(%s) was reset' % mac_address)
def poweroff(self, mac_addresses_list):
self._run(self._poweroff, mac_addresses_list)
utils.run(self._poweroff, mac_addresses_list)
def poweron(self, mac_addresses_list):
self._run(self._poweron, mac_addresses_list)
utils.run(self._poweron, mac_addresses_list)
def reset(self, mac_addresses_list):
self._run(self._reset, mac_addresses_list)
utils.run(self._reset, mac_addresses_list)

53
os_failures/utils.py Normal file
View File

@ -0,0 +1,53 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import threading
import traceback
from os_failures.api import error
def run(target, mac_addresses_list):
tw = ThreadsWrapper(target)
for mac_address in mac_addresses_list:
tw.start_thread(mac_address=mac_address)
tw.join_threads()
if tw.errors:
raise error.PowerManagementError(
'There are some errors when working the driver. '
'Please, check logs for more details.')
class ThreadsWrapper(object):
def __init__(self, target):
self.target = target
self.threads = []
self.errors = []
def _target(self, **kwargs):
try:
self.target(**kwargs)
except Exception as exc:
logging.error(traceback.format_exc())
self.errors.append(exc)
def start_thread(self, **kwargs):
thread = threading.Thread(target=self._target, kwargs=kwargs)
thread.start()
self.threads.append(thread)
def join_threads(self):
for thread in self.threads:
thread.join()