performa
/
os-faults
Code Issues Proposed changes

Add node_list driver 

* Added node_list driver that allows specifying list
  of nodes.
* Added default discovering to existing cloud drivers.
* Moved get_nodes to CloudManagment class
* Hid execute_on_master_node on tcpcloud and fuel drivers.

Change-Id: I5cd567b3afba12ad98a36474c739fa3b01ab2a8f
This commit is contained in:
Anton Studenov 2017-01-23 17:20:19 +03:00
parent 6c31641cfc
commit 8bbbc75bdb
14 changed files with 261 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
os_faults/__init__.py
View File

@ -54,6 +54,15 @@ CONFIG_SCHEMA = {
'type': 'object',
'$schema': 'http://json-schema.org/draft-04/schema#',
'properties': {
'node_discover': {
'type': 'object',
'properties': {
'driver': {'type': 'string'},
'args': {},
},
'required': ['driver', 'args'],
'additionalProperties': False,
},
'cloud_management': {
'type': 'object',
'properties': {
@ -112,6 +121,11 @@ def connect(cloud_config=None, config_filename=None):
cloud_management_conf = cloud_config['cloud_management']
cloud_management = _init_driver(cloud_management_conf)
node_discover_conf = cloud_config.get('node_discover')
if node_discover_conf:
node_discover = _init_driver(node_discover_conf)
cloud_management.set_node_discover(node_discover)
power_management_conf = cloud_config.get('power_management')
if power_management_conf:
power_management = _init_driver(power_management_conf)

25
os_faults/api/cloud_management.py
View File

@ -12,11 +12,15 @@
# limitations under the License.
import abc
import logging
import six
from os_faults.api import base_driver
from os_faults.api import error
from os_faults.api import node_collection
LOG = logging.getLogger(__name__)
@six.add_metaclass(abc.ABCMeta)
@ -24,20 +28,24 @@ class CloudManagement(base_driver.BaseDriver):
SERVICE_NAME_TO_CLASS = {}
SUPPORTED_SERVICES = []
SUPPORTED_NETWORKS = []
NODE_CLS = node_collection.NodeCollection
def __init__(self):
self.power_management = None
self.node_discover = None
def set_power_management(self, power_management):
self.power_management = power_management
def set_node_discover(self, node_discover):
self.node_discover = node_discover
@abc.abstractmethod
def verify(self):
"""Verify connection to the cloud.
"""
@abc.abstractmethod
def get_nodes(self, fqdns=None):
"""Get nodes in the cloud
@ -47,6 +55,21 @@ class CloudManagement(base_driver.BaseDriver):
:return: NodesCollection
"""
if self.node_discover is None:
raise error.OSFError(
'node_discover is not specified and "{}" '
'driver does not support discovering'.format(self.NAME))
hosts = self.node_discover.discover_hosts()
nodes = self.NODE_CLS(cloud_management=self,
power_management=self.power_management,
hosts=hosts)
if fqdns:
LOG.debug('Trying to find nodes with FQDNs: %s', fqdns)
nodes = nodes.filter(lambda node: node.fqdn in fqdns)
LOG.debug('The following nodes were found: %s', nodes.hosts)
return nodes
def get_service(self, name):
"""Get service with specified name

30
os_faults/api/node_discover.py Normal file
View File

@ -0,0 +1,30 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
from os_faults.api import base_driver
@six.add_metaclass(abc.ABCMeta)
class NodeDiscover(base_driver.BaseDriver):
"""Node discover base driver."""
@abc.abstractmethod
def discover_hosts(self):
"""Discover hosts
:returns: list of Host instances
"""

14
os_faults/drivers/devstack.py
View File

@ -16,6 +16,7 @@ import logging
from os_faults.ansible import executor
from os_faults.api import cloud_management
from os_faults.api import node_collection
from os_faults.api import node_discover
from os_faults.common import service
from os_faults import utils
@ -117,7 +118,8 @@ class IronicConductorService(ServiceInScreen):
WINDOW_NAME = 'ir-cond'
class DevStackManagement(cloud_management.CloudManagement):
class DevStackManagement(cloud_management.CloudManagement,
node_discover.NodeDiscover):
NAME = 'devstack'
DESCRIPTION = 'Single node DevStack management driver'
NODE_CLS = DevStackNode
@ -153,6 +155,7 @@ class DevStackManagement(cloud_management.CloudManagement):
def __init__(self, cloud_management_params):
super(DevStackManagement, self).__init__()
self.node_discover = self # supports discovering
self.address = cloud_management_params['address']
self.username = cloud_management_params['username']
@ -171,8 +174,9 @@ class DevStackManagement(cloud_management.CloudManagement):
def verify(self):
"""Verify connection to the cloud."""
nodes = self.get_nodes()
task = {'shell': 'screen -ls | grep -P "\\d+\\.stack"'}
results = self.execute_on_cloud(self.hosts, task)
results = self.execute_on_cloud(nodes.get_ips(), task)
hostnames = [result.host for result in results]
LOG.debug('DevStack hostnames: %s', hostnames)
LOG.info('Connected to cloud successfully')
@ -190,7 +194,7 @@ class DevStackManagement(cloud_management.CloudManagement):
else:
return self.cloud_executor.execute(hosts, task, [])
def get_nodes(self, fqdns=None):
def discover_hosts(self):
if self.nodes is None:
get_mac_cmd = 'cat /sys/class/net/{}/address'.format(self.iface)
task = {'command': get_mac_cmd}
@ -202,6 +206,4 @@ class DevStackManagement(cloud_management.CloudManagement):
fqdn='')
for r in results]
return self.NODE_CLS(cloud_management=self,
power_management=self.power_management,
hosts=self.nodes)
return self.nodes

36
os_faults/drivers/fuel.py
View File

@ -17,6 +17,7 @@ import logging
from os_faults.ansible import executor
from os_faults.api import cloud_management
from os_faults.api import node_collection
from os_faults.api import node_discover
from os_faults.common import service
from os_faults import utils
@ -339,7 +340,8 @@ class SwiftProxyService(service.LinuxService):
LINUX_SERVICE = 'swift-proxy'
class FuelManagement(cloud_management.CloudManagement):
class FuelManagement(cloud_management.CloudManagement,
node_discover.NodeDiscover):
NAME = 'fuel'
DESCRIPTION = 'Fuel 9.x cloud management driver'
NODE_CLS = FuelNodeCollection
@ -404,6 +406,7 @@ class FuelManagement(cloud_management.CloudManagement):
def __init__(self, cloud_management_params):
super(FuelManagement, self).__init__()
self.node_discover = self # supports discovering
self.master_node_address = cloud_management_params['address']
self.username = cloud_management_params['username']
@ -420,21 +423,20 @@ class FuelManagement(cloud_management.CloudManagement):
def verify(self):
"""Verify connection to the cloud."""
hosts = self._get_cloud_hosts()
LOG.debug('Cloud nodes: %s', hosts)
nodes = self.get_nodes()
LOG.debug('Cloud nodes: %s', nodes)
task = {'command': 'hostname'}
host_addrs = [host.ip for host in hosts]
task_result = self.execute_on_cloud(host_addrs, task)
task_result = self.execute_on_cloud(nodes.get_ips(), task)
LOG.debug('Hostnames of cloud nodes: %s',
[r.payload['stdout'] for r in task_result])
LOG.info('Connected to cloud successfully!')
def _get_cloud_hosts(self):
def discover_hosts(self):
if not self.cached_cloud_hosts:
task = {'command': 'fuel node --json'}
result = self.execute_on_master_node(task)
result = self._execute_on_master_node(task)
for r in json.loads(result[0].payload['stdout']):
host = node_collection.Host(ip=r['ip'], mac=r['mac'],
fqdn=r['fqdn'])
@ -442,7 +444,7 @@ class FuelManagement(cloud_management.CloudManagement):
return self.cached_cloud_hosts
def execute_on_master_node(self, task):
def _execute_on_master_node(self, task):
"""Execute task on Fuel master node.
:param task: Ansible task
@ -463,21 +465,3 @@ class FuelManagement(cloud_management.CloudManagement):
return self.cloud_executor.execute(hosts, task)
else:
return self.cloud_executor.execute(hosts, task, [])
def get_nodes(self, fqdns=None):
"""Get nodes in the cloud
This function returns NodesCollection representing all nodes in the
cloud or only those that were specified by FQDNs.
:param fqdns: list of FQDNs or None to retrieve all nodes
:return: NodesCollection
"""
nodes = self.NODE_CLS(cloud_management=self,
power_management=self.power_management,
hosts=self._get_cloud_hosts())
if fqdns:
LOG.debug('Trying to find nodes with FQDNs: %s', fqdns)
nodes = nodes.filter(lambda node: node.fqdn in fqdns)
LOG.debug('The following nodes were found: %s', nodes.hosts)
return nodes

2
os_faults/drivers/ipmi.py
View File

@ -33,7 +33,7 @@ class IPMIDriver(power_management.PowerManagement):
'mac_to_bmc': {
'type': 'object',
'patternProperties': {
'^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$': {
utils.MACADDR_REGEXP: {
'type': 'object',
'properties': {
'address': {'type': 'string'},

51
os_faults/drivers/node_list.py Normal file
View File

@ -0,0 +1,51 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from os_faults.api import node_collection
from os_faults.api import node_discover
from os_faults import utils
class NodeListDiscover(node_discover.NodeDiscover):
NAME = 'node_list'
DESCRIPTION = 'Reads hosts from configuration file'
CONFIG_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'array',
'items': {
'type': 'object',
'properties': {
'ip': {'type': 'string'},
'mac': {
'type': 'string',
'pattern': utils.MACADDR_REGEXP,
},
'fqdn': {'type': 'string'},
},
'required': ['ip', 'mac', 'fqdn'],
'additionalProperties': False,
},
'minItems': 1,
}
def __init__(self, conf):
self.hosts = [node_collection.Host(ip=host['ip'], mac=host['mac'],
fqdn=host['fqdn']) for host in conf]
def discover_hosts(self):
"""Discover hosts
:returns: list of Host instances
"""
return self.hosts

36
os_faults/drivers/tcpcloud.py
View File

@ -18,6 +18,7 @@ import yaml
from os_faults.ansible import executor
from os_faults.api import cloud_management
from os_faults.api import node_collection
from os_faults.api import node_discover
from os_faults.common import service
from os_faults import utils
@ -201,7 +202,8 @@ class CinderBackupService(SaltService):
SALT_SERVICE = 'cinder-backup'
class TCPCloudManagement(cloud_management.CloudManagement):
class TCPCloudManagement(cloud_management.CloudManagement,
node_discover.NodeDiscover):
NAME = 'tcpcloud'
DESCRIPTION = 'TCPCloud management driver'
NODE_CLS = TCPCloudNodeCollection
@ -253,6 +255,7 @@ class TCPCloudManagement(cloud_management.CloudManagement):
def __init__(self, cloud_management_params):
super(TCPCloudManagement, self).__init__()
self.node_discover = self # supports discovering
self.master_node_address = cloud_management_params['address']
self.username = cloud_management_params['username']
@ -282,22 +285,21 @@ class TCPCloudManagement(cloud_management.CloudManagement):
def verify(self):
"""Verify connection to the cloud."""
hosts = self._get_cloud_hosts()
LOG.debug('Cloud nodes: %s', hosts)
nodes = self.get_nodes()
LOG.debug('Cloud nodes: %s', nodes)
task = {'command': 'hostname'}
host_addrs = [host.ip for host in hosts]
task_result = self.execute_on_cloud(host_addrs, task)
task_result = self.execute_on_cloud(nodes.get_ips(), task)
LOG.debug('Hostnames of cloud nodes: %s',
[r.payload['stdout'] for r in task_result])
LOG.info('Connected to cloud successfully!')
def _get_cloud_hosts(self):
def discover_hosts(self):
if not self.cached_cloud_hosts:
cmd = "salt -E '{}' network.interfaces --out=yaml".format(
self.slave_name_regexp)
result = self.execute_on_master_node({'command': cmd})
result = self._execute_on_master_node({'command': cmd})
stdout = result[0].payload['stdout']
for fqdn, net_data in yaml.load(stdout).items():
host = node_collection.Host(
@ -309,7 +311,7 @@ class TCPCloudManagement(cloud_management.CloudManagement):
return self.cached_cloud_hosts
def execute_on_master_node(self, task):
def _execute_on_master_node(self, task):
"""Execute task on salt master node.
:param task: Ansible task
@ -330,21 +332,3 @@ class TCPCloudManagement(cloud_management.CloudManagement):
return self.cloud_executor.execute(hosts, task)
else:
return self.cloud_executor.execute(hosts, task, [])
def get_nodes(self, fqdns=None):
"""Get nodes in the cloud
This function returns NodesCollection representing all nodes in the
cloud or only those that were specified by FQDNs.
:param fqdns: list of FQDNs or None to retrieve all nodes
:return: NodesCollection
"""
nodes = self.NODE_CLS(cloud_management=self,
power_management=self.power_management,
hosts=self._get_cloud_hosts())
if fqdns:
LOG.debug('Trying to find nodes with FQDNs: %s', fqdns)
nodes = nodes.filter(lambda node: node.fqdn in fqdns)
LOG.debug('The following nodes were found: %s', nodes.hosts)
return nodes

25
os_faults/tests/unit/drivers/test_devstack.py
View File

@ -57,20 +57,32 @@ class DevStackManagementTestCase(test.TestCase):
def test_verify(self, mock_ansible_runner):
ansible_runner_inst = mock_ansible_runner.return_value
ansible_runner_inst.execute.side_effect = [
[fakes.FakeAnsibleResult(payload={'stdout': 'mac'},
host='10.0.0.2')],
[fakes.FakeAnsibleResult(payload={'stdout': ''},
host='10.0.0.2')],
]
devstack_management = devstack.DevStackManagement(self.conf)
devstack_management.verify()
ansible_runner_inst.execute.assert_called_once_with(
['10.0.0.2'], {'shell': 'screen -ls | grep -P "\\d+\\.stack"'})
ansible_runner_inst.execute.assert_has_calls([
mock.call(['10.0.0.2'],
{'command': 'cat /sys/class/net/eth0/address'}),
mock.call(['10.0.0.2'],
{'shell': 'screen -ls | grep -P "\\d+\\.stack"'})
])
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_verify_slaves(self, mock_ansible_runner):
self.conf['slaves'] = ['10.0.0.3', '10.0.0.4']
ansible_runner_inst = mock_ansible_runner.return_value
ansible_runner_inst.execute.side_effect = [
[fakes.FakeAnsibleResult(payload={'stdout': 'mac1'},
host='10.0.0.2'),
fakes.FakeAnsibleResult(payload={'stdout': 'mac2'},
host='10.0.0.3'),
fakes.FakeAnsibleResult(payload={'stdout': 'mac3'},
host='10.0.0.4')],
[fakes.FakeAnsibleResult(payload={'stdout': ''},
host='10.0.0.2'),
fakes.FakeAnsibleResult(payload={'stdout': ''},
@ -81,9 +93,12 @@ class DevStackManagementTestCase(test.TestCase):
devstack_management = devstack.DevStackManagement(self.conf)
devstack_management.verify()
ansible_runner_inst.execute.assert_called_once_with(
['10.0.0.2', '10.0.0.3', '10.0.0.4'],
{'shell': 'screen -ls | grep -P "\\d+\\.stack"'})
ansible_runner_inst.execute.assert_has_calls([
mock.call(['10.0.0.2', '10.0.0.3', '10.0.0.4'],
{'command': 'cat /sys/class/net/eth0/address'}),
mock.call(['10.0.0.2', '10.0.0.3', '10.0.0.4'],
{'shell': 'screen -ls | grep -P "\\d+\\.stack"'})
])
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_execute_on_cloud(self, mock_ansible_runner):

21
os_faults/tests/unit/drivers/test_fuel_management.py
View File

@ -73,6 +73,27 @@ class FuelManagementTestCase(test.TestCase):
]
self.assertEqual(nodes.hosts, hosts)
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_get_nodes_from_discover_driver(self, mock_ansible_runner):
ansible_runner_inst = mock_ansible_runner.return_value
hosts = [
node_collection.Host(ip='10.0.2.2', mac='09:7b:74:90:63:c2',
fqdn='mynode1.local'),
node_collection.Host(ip='10.0.2.3', mac='09:7b:74:90:63:c3',
fqdn='mynode2.local'),
]
node_discover_driver = mock.Mock()
node_discover_driver.discover_hosts.return_value = hosts
fuel_managment = fuel.FuelManagement({
'address': 'fuel.local',
'username': 'root',
})
fuel_managment.set_node_discover(node_discover_driver)
nodes = fuel_managment.get_nodes()
self.assertFalse(ansible_runner_inst.execute.called)
self.assertEqual(hosts, nodes.hosts)
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_execute_on_cloud(self, mock_ansible_runner):
ansible_runner_inst = mock_ansible_runner.return_value

33
os_faults/tests/unit/drivers/test_node_list.py Normal file
View File

@ -0,0 +1,33 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from os_faults.api import node_collection
from os_faults.drivers import node_list
from os_faults.tests.unit import test
class NodeListDiscoverTestCase(test.TestCase):
def test_discover_hosts(self):
conf = [
{'ip': '10.0.0.11', 'mac': '01', 'fqdn': 'node-1'},
{'ip': '10.0.0.12', 'mac': '02', 'fqdn': 'node-2'},
]
expected_hosts = [
node_collection.Host(ip='10.0.0.11', mac='01', fqdn='node-1'),
node_collection.Host(ip='10.0.0.12', mac='02', fqdn='node-2'),
]
node_list_discover = node_list.NodeListDiscover(conf)
hosts = node_list_discover.discover_hosts()
self.assertEqual(expected_hosts, hosts)

18
os_faults/tests/unit/drivers/test_tcpcloud.py
View File

@ -117,6 +117,24 @@ class TCPCloudManagementTestCase(test.TestCase):
]
self.assertEqual(nodes.hosts, hosts)
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_get_nodes_from_discover_driver(self, mock_ansible_runner):
ansible_runner_inst = mock_ansible_runner.return_value
hosts = [
node_collection.Host(ip='10.0.2.2', mac='09:7b:74:90:63:c2',
fqdn='mynode1.local'),
node_collection.Host(ip='10.0.2.3', mac='09:7b:74:90:63:c3',
fqdn='mynode2.local'),
]
node_discover_driver = mock.Mock()
node_discover_driver.discover_hosts.return_value = hosts
tcp_managment = tcpcloud.TCPCloudManagement(self.tcp_conf)
tcp_managment.set_node_discover(node_discover_driver)
nodes = tcp_managment.get_nodes()
self.assertFalse(ansible_runner_inst.execute.called)
self.assertEqual(hosts, nodes.hosts)
@mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True)
def test_execute_on_cloud(self, mock_ansible_runner):
ansible_runner_inst = mock_ansible_runner.return_value

21
os_faults/tests/unit/test_os_faults.py
View File

@ -23,6 +23,7 @@ from os_faults.drivers import devstack
from os_faults.drivers import fuel
from os_faults.drivers import ipmi
from os_faults.drivers import libvirt_driver
from os_faults.drivers import node_list
from os_faults.tests.unit import test
@ -64,17 +65,31 @@ class OSFaultsTestCase(test.TestCase):
def test_connect_fuel_with_libvirt(self):
destructor = os_faults.connect(self.cloud_config)
self.assertIsInstance(destructor, fuel.FuelManagement)
self.assertIsInstance(destructor.node_discover, fuel.FuelManagement)
self.assertIsInstance(destructor.power_management,
libvirt_driver.LibvirtDriver)
def test_connect_fuel_with_ipmi(self):
def test_connect_fuel_with_ipmi_and_node_list(self):
cloud_config = {
'node_discover': {
'driver': 'node_list',
'args': [
{
'ip': '10.0.0.11',
'mac': '01:ab:cd:01:ab:cd',
'fqdn': 'node-1'
}, {
'ip': '10.0.0.12',
'mac': '02:ab:cd:02:ab:cd',
'fqdn': 'node-2'},
]
},
'cloud_management': {
'driver': 'fuel',
'args': {
'address': '10.30.00.5',
'username': 'root',
}
},
},
'power_management': {
'driver': 'ipmi',
@ -91,6 +106,8 @@ class OSFaultsTestCase(test.TestCase):
}
destructor = os_faults.connect(cloud_config)
self.assertIsInstance(destructor, fuel.FuelManagement)
self.assertIsInstance(destructor.node_discover,
node_list.NodeListDiscover)
self.assertIsInstance(destructor.power_management, ipmi.IPMIDriver)
def test_connect_driver_not_found(self):

2
os_faults/utils.py
View File

@ -19,6 +19,8 @@ from os_faults.api import error
LOG = logging.getLogger(__name__)
MACADDR_REGEXP = '^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$'
def run(target, kwargs_list):
tw = ThreadsWrapper(target)