openstack
/
nova
Code Issues Proposed changes

Fibre channel block storage support (nova changes) 

implements blueprint libvirt-fibre-channel

These changes constitute the required libvirt changes to
support attaching Fibre Channel volumes to a VM.

This requires a few new packages to be available on the system
sysfsutils -- This is needed to discover the FC HBAs
sg3-utils -- this package is needed for scsi device discovery
multipath -- This package is needed for multipath support.

Typical Fibre Channel arrays support exporting volumes via
multiple ports, so multipath support is highly desirable
for fault tolerance.  If multipath is not installed,
the new FibreChannel libvirt volume driver will still work.

If multipath is enabled, the new Fibre Channel volume driver
detects each of the attached devices for the volume, and
properly removes every one of them on detach.

In order to use this, the cinder volume driver's
initialize_connection will simply return a dictionary
with a new driver_volume_type called 'fibrechan'.

The target_wwn can be a single entry or a list of wwns
that correspond to the list of remote wwn(s) that will
export the volume.

return {'driver_volume_type': 'fibre_channel',
        'data': {'target_lun', 1,
                 'target_wwn': '1234567890123'}}

or

return {'driver_volume_type': 'fibre_channel',
        'data': {'target_lun', 1,
                 'target_wwn': ['1234567890123',
                                '0987654321321']}}

Change-Id: Ifccc56f960ef434f7cb56a9367e4cad288358440
This commit is contained in:
Walter A. Boring IV 2013-01-17 16:36:22 -08:00
parent 7106ba954e
commit 052859fbdb
9 changed files with 565 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
etc/nova/rootwrap.d/compute.filters
View File

@ -186,4 +186,11 @@ read_passwd: RegExpFilter, cat, root, cat, (/var|/usr)?/tmp/openstack-vfs-localf
read_shadow: RegExpFilter, cat, root, cat, (/var|/usr)?/tmp/openstack-vfs-localfs[^/]+/etc/shadow
# nova/virt/libvirt/volume.py: 'multipath' '-R'
multipath: CommandFilter, /sbin/multipath, root
multipath: CommandFilter, /sbin/multipath, root
# nova/virt/libvirt/utils.py:
systool: CommandFilter, /usr/bin/systool, root
# nova/virt/libvirt/volume.py:
sginfo: CommandFilter, /usr/bin/sginfo, root
sg_scan: CommandFilter, /usr/bin/sg_scan, root

15
nova/storage/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2013 Hewlett-Packard, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

139
nova/storage/linuxscsi.py Normal file
View File

@ -0,0 +1,139 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Generic linux scsi subsystem utilities."""
from nova import exception
from nova.openstack.common import log as logging
from nova import utils
LOG = logging.getLogger(__name__)
def echo_scsi_command(path, content):
"""Used to echo strings to scsi subsystem."""
args = ["-a", path]
kwargs = dict(process_input=content, run_as_root=True)
utils.execute('tee', *args, **kwargs)
def rescan_hosts(hbas):
for hba in hbas:
echo_scsi_command("/sys/class/scsi_host/%s/scan"
% hba['host_device'], "- - -")
def get_device_list():
(out, err) = utils.execute('sginfo', '-r', run_as_root=True)
devices = []
if out:
line = out.strip()
devices = line.split(" ")
return devices
def get_device_info(device):
(out, err) = utils.execute('sg_scan', device, run_as_root=True)
dev_info = {'device': device, 'host': None,
'channel': None, 'id': None, 'lun': None}
if out:
line = out.strip()
line = line.replace(device + ": ", "")
info = line.split(" ")
for item in info:
if '=' in item:
pair = item.split('=')
dev_info[pair[0]] = pair[1]
elif 'scsi' in item:
dev_info['host'] = item.replace('scsi', '')
return dev_info
def _wait_for_remove(device, tries):
tries = tries + 1
LOG.debug(_("Trying (%(tries)s) to remove device %(device)s")
% {'tries': tries, 'device': device["device"]})
path = "/sys/bus/scsi/drivers/sd/%s:%s:%s:%s/delete"
echo_scsi_command(path % (device["host"], device["channel"],
device["id"], device["lun"]),
"1")
devices = get_device_list()
if device["device"] not in devices:
raise utils.LoopingCallDone()
def remove_device(device):
tries = 0
timer = utils.FixedIntervalLoopingCall(_wait_for_remove, device, tries)
timer.start(interval=2).wait()
timer.stop()
def find_multipath_device(device):
"""Try and discover the multipath device for a volume."""
mdev = None
devices = []
out = None
try:
(out, err) = utils.execute('multipath', '-l', device,
run_as_root=True)
except exception.ProcessExecutionError as exc:
LOG.warn(_("Multipath call failed exit (%(code)s)")
% {'code': exc.exit_code})
return None
if out:
lines = out.strip()
lines = lines.split("\n")
if lines:
line = lines[0]
info = line.split(" ")
# device line output is different depending
# on /etc/multipath.conf settings.
if info[1][:2] == "dm":
mdev = "/dev/%s" % info[1]
elif info[2][:2] == "dm":
mdev = "/dev/%s" % info[2]
if mdev is None:
LOG.warn(_("Couldn't find multipath device %(line)s")
% locals())
return None
LOG.debug(_("Found multipath device = %(mdev)s") % locals())
device_lines = lines[3:]
for dev_line in device_lines:
dev_line = dev_line.strip()
dev_line = dev_line[3:]
dev_info = dev_line.split(" ")
if dev_line.find("policy") != -1:
address = dev_info[0].split(":")
dev = {'device': '/dev/%s' % dev_info[1],
'host': address[0], 'channel': address[1],
'id': address[2], 'lun': address[3]
}
devices.append(dev)
if mdev is not None:
info = {"device": mdev,
"devices": devices}
return info
return None

54
nova/tests/fake_libvirt_utils.py
View File

@ -34,6 +34,60 @@ def get_iscsi_initiator():
return "fake.initiator.iqn"
def get_fc_hbas():
return [{'ClassDevice': 'host1',
'ClassDevicePath': '/sys/devices/pci0000:00/0000:00:03.0'
'/0000:05:00.2/host1/fc_host/host1',
'dev_loss_tmo': '30',
'fabric_name': '0x1000000533f55566',
'issue_lip': '<store method only>',
'max_npiv_vports': '255',
'maxframe_size': '2048 bytes',
'node_name': '0x200010604b019419',
'npiv_vports_inuse': '0',
'port_id': '0x680409',
'port_name': '0x100010604b019419',
'port_state': 'Online',
'port_type': 'NPort (fabric via point-to-point)',
'speed': '10 Gbit',
'supported_classes': 'Class 3',
'supported_speeds': '10 Gbit',
'symbolic_name': 'Emulex 554M FV4.0.493.0 DV8.3.27',
'tgtid_bind_type': 'wwpn (World Wide Port Name)',
'uevent': None,
'vport_create': '<store method only>',
'vport_delete': '<store method only>'}]
def get_fc_hbas_info():
hbas = get_fc_hbas()
info = [{'port_name': hbas[0]['port_name'].replace('0x', ''),
'node_name': hbas[0]['node_name'].replace('0x', ''),
'host_device': hbas[0]['ClassDevice'],
'device_path': hbas[0]['ClassDevicePath']}]
return info
def get_fc_wwpns():
hbas = get_fc_hbas()
wwpns = []
for hba in hbas:
wwpn = hba['port_name'].replace('0x', '')
wwpns.append(wwpn)
return wwpns
def get_fc_wwnns():
hbas = get_fc_hbas()
wwnns = []
for hba in hbas:
wwnn = hba['node_name'].replace('0x', '')
wwnns.append(wwnn)
return wwnns
def create_image(disk_format, path, size):
pass

6
nova/tests/test_libvirt.py
View File

@ -338,6 +338,8 @@ class LibvirtConnTestCase(test.TestCase):
initiator = 'fake.initiator.iqn'
ip = 'fakeip'
host = 'fakehost'
wwpns = ['100010604b019419']
wwnns = ['200010604b019419']
self.flags(my_ip=ip)
self.flags(host=host)
@ -345,7 +347,9 @@ class LibvirtConnTestCase(test.TestCase):
expected = {
'ip': ip,
'initiator': initiator,
'host': host
'host': host,
'wwpns': wwpns,
'wwnns': wwnns
}
volume = {
'id': 'fake'

79
nova/tests/test_libvirt_volume.py
View File

@ -17,10 +17,14 @@
import os
from nova import exception
from nova.openstack.common import cfg
from nova.storage import linuxscsi
from nova import test
from nova.tests import fake_libvirt_utils
from nova import utils
from nova.virt import fake
from nova.virt.libvirt import utils as libvirt_utils
from nova.virt.libvirt import volume
CONF = cfg.CONF
@ -467,3 +471,78 @@ class LibvirtVolumeTestCase(test.TestCase):
('stat', export_mnt_base),
('mount', '-t', 'glusterfs', export_string, export_mnt_base)]
self.assertEqual(self.executes, expected_commands)
def fibrechan_connection(self, volume, location, wwn):
return {
'driver_volume_type': 'fibrechan',
'data': {
'volume_id': volume['id'],
'target_portal': location,
'target_wwn': wwn,
'target_lun': 1,
}
}
def test_libvirt_fibrechan_driver(self):
self.stubs.Set(libvirt_utils, 'get_fc_hbas',
fake_libvirt_utils.get_fc_hbas)
self.stubs.Set(libvirt_utils, 'get_fc_hbas_info',
fake_libvirt_utils.get_fc_hbas_info)
# NOTE(vish) exists is to make driver assume connecting worked
self.stubs.Set(os.path, 'exists', lambda x: True)
self.stubs.Set(os.path, 'realpath', lambda x: '/dev/sdb')
libvirt_driver = volume.LibvirtFibreChannelVolumeDriver(self.fake_conn)
multipath_devname = '/dev/md-1'
devices = {"device": multipath_devname,
"devices": [{'device': '/dev/sdb',
'address': '1:0:0:1',
'host': 1, 'channel': 0,
'id': 0, 'lun': 1}]}
self.stubs.Set(linuxscsi, 'find_multipath_device', lambda x: devices)
self.stubs.Set(linuxscsi, 'remove_device', lambda x: None)
location = '10.0.2.15:3260'
name = 'volume-00000001'
wwn = '1234567890123456'
vol = {'id': 1, 'name': name}
connection_info = self.fibrechan_connection(vol, location, wwn)
mount_device = "vde"
disk_info = {
"bus": "virtio",
"dev": mount_device,
"type": "disk"
}
conf = libvirt_driver.connect_volume(connection_info, disk_info)
tree = conf.format_dom()
dev_str = '/dev/disk/by-path/pci-0000:05:00.2-fc-0x%s-lun-1' % wwn
self.assertEqual(tree.get('type'), 'block')
self.assertEqual(tree.find('./source').get('dev'), multipath_devname)
connection_info["data"]["devices"] = devices["devices"]
libvirt_driver.disconnect_volume(connection_info, mount_device)
expected_commands = []
self.assertEqual(self.executes, expected_commands)
self.stubs.Set(libvirt_utils, 'get_fc_hbas',
lambda: [])
self.stubs.Set(libvirt_utils, 'get_fc_hbas_info',
lambda: [])
self.assertRaises(exception.NovaException,
libvirt_driver.connect_volume,
connection_info, disk_info)
self.stubs.Set(libvirt_utils, 'get_fc_hbas', lambda: [])
self.stubs.Set(libvirt_utils, 'get_fc_hbas_info', lambda: [])
self.assertRaises(exception.NovaException,
libvirt_driver.connect_volume,
connection_info, disk_info)
def test_libvirt_fibrechan_getpci_num(self):
libvirt_driver = volume.LibvirtFibreChannelVolumeDriver(self.fake_conn)
hba = {'device_path': "/sys/devices/pci0000:00/0000:00:03.0"
"/0000:05:00.3/host2/fc_host/host2"}
pci_num = libvirt_driver._get_pci_num(hba)
self.assertEqual("0000:05:00.3", pci_num)
hba = {'device_path': "/sys/devices/pci0000:00/0000:00:03.0"
"/0000:05:00.3/0000:06:00.6/host2/fc_host/host2"}
pci_num = libvirt_driver._get_pci_num(hba)
self.assertEqual("0000:06:00.6", pci_num)

45
nova/virt/libvirt/driver.py
View File

@ -6,6 +6,7 @@
# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright (c) 2011 Piston Cloud Computing, Inc
# Copyright (c) 2012 University Of Minho
# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -154,7 +155,9 @@ libvirt_opts = [
'nfs=nova.virt.libvirt.volume.LibvirtNFSVolumeDriver',
'aoe=nova.virt.libvirt.volume.LibvirtAOEVolumeDriver',
'glusterfs='
'nova.virt.libvirt.volume.LibvirtGlusterfsVolumeDriver'
'nova.virt.libvirt.volume.LibvirtGlusterfsVolumeDriver',
'fibre_channel=nova.virt.libvirt.volume.'
'LibvirtFibreChannelVolumeDriver'
],
help='Libvirt handlers for remote volumes.'),
cfg.StrOpt('libvirt_disk_prefix',
@ -281,6 +284,8 @@ class LibvirtDriver(driver.ComputeDriver):
self._host_state = None
self._initiator = None
self._fc_wwnns = None
self._fc_wwpns = None
self._wrapped_conn = None
self._caps = None
self.read_only = read_only
@ -644,13 +649,37 @@ class LibvirtDriver(driver.ComputeDriver):
if not self._initiator:
self._initiator = libvirt_utils.get_iscsi_initiator()
if not self._initiator:
LOG.warn(_('Could not determine iscsi initiator name'),
instance=instance)
return {
'ip': CONF.my_ip,
'initiator': self._initiator,
'host': CONF.host
}
LOG.debug(_('Could not determine iscsi initiator name'),
instance=instance)
if not self._fc_wwnns:
self._fc_wwnns = libvirt_utils.get_fc_wwnns()
if not self._fc_wwnns or len(self._fc_wwnns) == 0:
LOG.debug(_('Could not determine fibre channel '
'world wide node names'),
instance=instance)
if not self._fc_wwpns:
self._fc_wwpns = libvirt_utils.get_fc_wwpns()
if not self._fc_wwpns or len(self._fc_wwpns) == 0:
LOG.debug(_('Could not determine fibre channel '
'world wide port names'),
instance=instance)
if not self._initiator and not self._fc_wwnns and not self._fc_wwpns:
msg = _("No Volume Connector found.")
LOG.error(msg)
raise exception.NovaException(msg)
connector = {'ip': CONF.my_ip,
'initiator': self._initiator,
'host': CONF.host}
if self._fc_wwnns and self._fc_wwpns:
connector["wwnns"] = self._fc_wwnns
connector["wwpns"] = self._fc_wwpns
return connector
def _cleanup_resize(self, instance, network_info):
target = libvirt_utils.get_instance_path(instance) + "_resize"

88
nova/virt/libvirt/utils.py
View File

@ -6,6 +6,7 @@
# Copyright (c) 2010 Citrix Systems, Inc.
# Copyright (c) 2011 Piston Cloud Computing, Inc
# Copyright (c) 2011 OpenStack LLC
# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -56,6 +57,93 @@ def get_iscsi_initiator():
return l[l.index('=') + 1:].strip()
def get_fc_hbas():
"""Get the Fibre Channel HBA information."""
try:
out, err = execute('systool', '-c', 'fc_host', '-v',
run_as_root=True)
except exception.ProcessExecutionError as exc:
if exc.exit_code == 96:
LOG.warn(_("systool is not installed"))
return []
if out is None:
raise RuntimeError(_("Cannot find any Fibre Channel HBAs"))
lines = out.split('\n')
# ignore the first 2 lines
lines = lines[2:]
hbas = []
hba = {}
lastline = None
for line in lines:
line = line.strip()
# 2 newlines denotes a new hba port
if line == '' and lastline == '':
if len(hba) > 0:
hbas.append(hba)
hba = {}
else:
val = line.split('=')
if len(val) == 2:
key = val[0].strip().replace(" ", "")
value = val[1].strip()
hba[key] = value.replace('"', '')
lastline = line
return hbas
def get_fc_hbas_info():
"""Get Fibre Channel WWNs and device paths from the system, if any."""
# Note modern linux kernels contain the FC HBA's in /sys
# and are obtainable via the systool app
hbas = get_fc_hbas()
hbas_info = []
for hba in hbas:
wwpn = hba['port_name'].replace('0x', '')
wwnn = hba['node_name'].replace('0x', '')
device_path = hba['ClassDevicepath']
device = hba['ClassDevice']
hbas_info.append({'port_name': wwpn,
'node_name': wwnn,
'host_device': device,
'device_path': device_path})
return hbas_info
def get_fc_wwpns():
"""Get Fibre Channel WWPNs from the system, if any."""
# Note modern linux kernels contain the FC HBA's in /sys
# and are obtainable via the systool app
hbas = get_fc_hbas()
wwpns = []
if hbas:
for hba in hbas:
if hba['port_state'] == 'Online':
wwpn = hba['port_name'].replace('0x', '')
wwpns.append(wwpn)
return wwpns
def get_fc_wwnns():
"""Get Fibre Channel WWNNs from the system, if any."""
# Note modern linux kernels contain the FC HBA's in /sys
# and are obtainable via the systool app
hbas = get_fc_hbas()
wwnns = []
if hbas:
for hba in hbas:
if hba['port_state'] == 'Online':
wwnn = hba['node_name'].replace('0x', '')
wwnns.append(wwnn)
return wwnns
def create_image(disk_format, path, size):
"""Create a disk image

140
nova/virt/libvirt/volume.py
View File

@ -1,6 +1,7 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# (c) Copyright 2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -26,6 +27,7 @@ from nova.openstack.common import cfg
from nova.openstack.common import lockutils
from nova.openstack.common import log as logging
from nova import paths
from nova.storage import linuxscsi
from nova import utils
from nova.virt.libvirt import config as vconfig
from nova.virt.libvirt import utils as virtutils
@ -608,3 +610,141 @@ class LibvirtGlusterfsVolumeDriver(LibvirtBaseVolumeDriver):
return utils.execute('stat', path, run_as_root=True)
except exception.ProcessExecutionError:
return False
class LibvirtFibreChannelVolumeDriver(LibvirtBaseVolumeDriver):
"""Driver to attach Fibre Channel Network volumes to libvirt."""
def __init__(self, connection):
super(LibvirtFibreChannelVolumeDriver,
self).__init__(connection, is_block_dev=False)
def _get_pci_num(self, hba):
# NOTE(walter-boring)
# device path is in format of
# /sys/devices/pci0000:00/0000:00:03.0/0000:05:00.3/host2/fc_host/host2
# sometimes an extra entry exists before the host2 value
# we always want the value prior to the host2 value
pci_num = None
if hba is not None:
if "device_path" in hba:
index = 0
device_path = hba['device_path'].split('/')
for value in device_path:
if value.startswith('host'):
break
index = index + 1
if index > 0:
pci_num = device_path[index - 1]
return pci_num
@lockutils.synchronized('connect_volume', 'nova-')
def connect_volume(self, connection_info, disk_info):
"""Attach the volume to instance_name."""
fc_properties = connection_info['data']
mount_device = disk_info["dev"]
ports = fc_properties['target_wwn']
wwns = []
# we support a list of wwns or a single wwn
if isinstance(ports, list):
for wwn in ports:
wwns.append(wwn)
elif isinstance(ports, str):
wwns.append(ports)
# We need to look for wwns on every hba
# because we don't know ahead of time
# where they will show up.
hbas = virtutils.get_fc_hbas_info()
host_devices = []
for hba in hbas:
pci_num = self._get_pci_num(hba)
if pci_num is not None:
for wwn in wwns:
target_wwn = "0x%s" % wwn.lower()
host_device = ("/dev/disk/by-path/pci-%s-fc-%s-lun-%s" %
(pci_num,
target_wwn,
fc_properties.get('target_lun', 0)))
host_devices.append(host_device)
if len(host_devices) == 0:
# this is empty because we don't have any FC HBAs
msg = _("We are unable to locate any Fibre Channel devices")
raise exception.NovaException(msg)
# The /dev/disk/by-path/... node is not always present immediately
# We only need to find the first device. Once we see the first device
# multipath will have any others.
def _wait_for_device_discovery(host_devices, mount_device):
tries = self.tries
for device in host_devices:
LOG.debug(_("Looking for Fibre Channel dev %(device)s")
% locals())
if os.path.exists(device):
self.host_device = device
# get the /dev/sdX device. This is used
# to find the multipath device.
self.device_name = os.path.realpath(device)
raise utils.LoopingCallDone()
if self.tries >= CONF.num_iscsi_scan_tries:
msg = _("Fibre Channel device not found.")
raise exception.NovaException(msg)
LOG.warn(_("Fibre volume not yet found at: %(mount_device)s. "
"Will rescan & retry. Try number: %(tries)s") %
locals())
linuxscsi.rescan_hosts(hbas)
self.tries = self.tries + 1
self.host_device = None
self.device_name = None
self.tries = 0
timer = utils.FixedIntervalLoopingCall(_wait_for_device_discovery,
host_devices, mount_device)
timer.start(interval=2).wait()
tries = self.tries
if self.host_device is not None and self.device_name is not None:
LOG.debug(_("Found Fibre Channel volume %(mount_device)s "
"(after %(tries)s rescans)") % locals())
# see if the new drive is part of a multipath
# device. If so, we'll use the multipath device.
mdev_info = linuxscsi.find_multipath_device(self.device_name)
if mdev_info is not None:
LOG.debug(_("Multipath device discovered %(device)s")
% {'device': mdev_info['device']})
device_path = mdev_info['device']
connection_info['data']['devices'] = mdev_info['devices']
else:
# we didn't find a multipath device.
# so we assume the kernel only sees 1 device
device_path = self.host_device
device_info = linuxscsi.get_device_info(self.device_name)
connection_info['data']['devices'] = [device_info]
conf = super(LibvirtFibreChannelVolumeDriver,
self).connect_volume(connection_info, disk_info)
conf.source_type = "block"
conf.source_path = device_path
return conf
@lockutils.synchronized('connect_volume', 'nova-')
def disconnect_volume(self, connection_info, mount_device):
"""Detach the volume from instance_name."""
super(LibvirtFibreChannelVolumeDriver,
self).disconnect_volume(connection_info, mount_device)
devices = connection_info['data']['devices']
# There may have been more than 1 device mounted
# by the kernel for this volume. We have to remove
# all of them
for device in devices:
linuxscsi.remove_device(device)