openstack
/
os-win
Code Issues Proposed changes

FC: add support for retrieving FC LUN UIDs 

This change allows retrieving SCSI Unique IDs for FibreChannel LUNs.

We're sending a SCSI INQUIRY request on the specified LUN, parsing
the retrieved VPD page. We're also selecting the identifiers based
on Windows support and order of precedence.

This will allow discovering FibreChannel disks in a more efficient
way.

Related-Bug: #1694671
Change-Id: I86d953d000e6d0244e8e8f2aaf2cbd76305cb63b
This commit is contained in:
Lucian Petrut 2017-06-29 17:47:28 +03:00
parent 7618208a7f
commit 6a6fc77b96
8 changed files with 504 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
os_win/constants.py
View File

@ -212,3 +212,19 @@ VM_SNAPSHOT_TYPE_PROD_ENFORCED = 4
VM_SNAPSHOT_TYPE_STANDARD = 5
DEFAULT_WMI_EVENT_TIMEOUT_MS = 2000
SCSI_UID_SCSI_NAME_STRING = 8
SCSI_UID_FCPH_NAME = 3
SCSI_UID_EUI64 = 2
SCSI_UID_VENDOR_ID = 1
SCSI_UID_VENDOR_SPECIFIC = 0
# The following SCSI Unique ID formats are accepted by Windows, in this
# specific order of precedence.
SUPPORTED_SCSI_UID_FORMATS = [
SCSI_UID_SCSI_NAME_STRING,
SCSI_UID_FCPH_NAME,
SCSI_UID_EUI64,
SCSI_UID_VENDOR_ID,
SCSI_UID_VENDOR_SPECIFIC
]

10
os_win/exceptions.py
View File

@ -251,3 +251,13 @@ class ClusterPropertyListEntryNotFound(ClusterPropertyRetrieveFailed):
class ClusterPropertyListParsingError(ClusterPropertyRetrieveFailed):
msg_fmt = _("Parsing a cluster property list failed.")
class SCSIPageParsingError(Invalid):
msg_fmt = _("Parsing SCSI Page %(page)s failed. "
"Reason: %(reason)s.")
class SCSIIdDescriptorParsingError(Invalid):
msg_fmt = _("Parsing SCSI identification descriptor failed. "
"Reason: %(reason)s.")

122
os_win/tests/unit/utils/storage/initiator/test_fc_utils.py
View File

@ -36,6 +36,10 @@ class FCUtilsTestCase(base.BaseTestCase):
self._setup_lib_mocks()
self._fc_utils = fc_utils.FCUtils()
self._fc_utils._diskutils = mock.Mock()
self._diskutils = self._fc_utils._diskutils
self._run_mocker = mock.patch.object(self._fc_utils,
'_run_and_check_output')
self._run_mocker.start()
@ -330,3 +334,121 @@ class FCUtilsTestCase(base.BaseTestCase):
expected_func = fc_utils.hbaapi.HBA_RefreshAdapterConfiguration
expected_func.assert_called_once_with()
def test_send_scsi_inquiry_v2(self):
self._ctypes_mocker.stop()
fake_port_wwn = fc_struct.HBA_WWN()
fake_remote_port_wwn = fc_struct.HBA_WWN()
fake_fcp_lun = 11
fake_cdb_byte_1 = 1
fake_cdb_byte_2 = 0x80
fake_resp = bytearray(range(200))
fake_sense_data = bytearray(range(200)[::-1])
fake_scsi_status = 5
def mock_run(func, hba_handle, port_wwn_struct,
remote_port_wwn_struct, fcp_lun, cdb_byte1,
cdb_byte2, p_resp_buff, p_resp_buff_sz,
p_scsi_status, p_sense_buff, p_sense_buff_sz):
self.assertEqual(fc_utils.hbaapi.HBA_ScsiInquiryV2, func)
self.assertEqual(mock.sentinel.hba_handle, hba_handle)
self.assertEqual(fake_port_wwn, port_wwn_struct)
self.assertEqual(fake_remote_port_wwn, remote_port_wwn_struct)
self.assertEqual(fake_fcp_lun, fcp_lun.value)
self.assertEqual(fake_cdb_byte_1, cdb_byte1.value)
self.assertEqual(fake_cdb_byte_2, cdb_byte2.value)
resp_buff_sz = ctypes.cast(
p_resp_buff_sz,
ctypes.POINTER(ctypes.c_uint32)).contents
sense_buff_sz = ctypes.cast(
p_sense_buff_sz,
ctypes.POINTER(ctypes.c_uint32)).contents
scsi_status = ctypes.cast(
p_scsi_status,
ctypes.POINTER(ctypes.c_ubyte)).contents
self.assertEqual(fc_utils.SCSI_INQ_BUFF_SZ, resp_buff_sz.value)
self.assertEqual(fc_utils.SENSE_BUFF_SZ, sense_buff_sz.value)
resp_buff_type = (ctypes.c_ubyte * resp_buff_sz.value)
sense_buff_type = (ctypes.c_ubyte * sense_buff_sz.value)
resp_buff = ctypes.cast(p_resp_buff,
ctypes.POINTER(resp_buff_type)).contents
sense_buff = ctypes.cast(p_sense_buff,
ctypes.POINTER(sense_buff_type)).contents
resp_buff[:len(fake_resp)] = fake_resp
sense_buff[:len(fake_sense_data)] = fake_sense_data
resp_buff_sz.value = len(fake_resp)
sense_buff_sz.value = len(fake_sense_data)
scsi_status.value = fake_scsi_status
self._mock_run.side_effect = mock_run
resp_buff = self._fc_utils._send_scsi_inquiry_v2(
mock.sentinel.hba_handle,
fake_port_wwn,
fake_remote_port_wwn,
fake_fcp_lun,
fake_cdb_byte_1,
fake_cdb_byte_2)
self.assertEqual(fake_resp, bytearray(resp_buff[:len(fake_resp)]))
@mock.patch.object(fc_utils.FCUtils, '_send_scsi_inquiry_v2')
def test_get_scsi_device_id_vpd(self, mock_send_scsi_inq):
self._fc_utils._get_scsi_device_id_vpd(
mock.sentinel.hba_handle, mock.sentinel.port_wwn,
mock.sentinel.remote_port_wwn, mock.sentinel.fcp_lun)
mock_send_scsi_inq.assert_called_once_with(
mock.sentinel.hba_handle, mock.sentinel.port_wwn,
mock.sentinel.remote_port_wwn, mock.sentinel.fcp_lun,
1, 0x83)
@mock.patch.object(fc_utils.FCUtils, '_wwn_struct_from_hex_str')
@mock.patch.object(fc_utils.FCUtils, '_open_adapter_by_wwn')
@mock.patch.object(fc_utils.FCUtils, '_close_adapter')
@mock.patch.object(fc_utils.FCUtils, '_get_scsi_device_id_vpd')
def test_get_scsi_device_identifiers(self, mock_get_scsi_dev_id_vpd,
mock_close_adapter, mock_open_adapter,
mock_wwn_struct_from_hex_str):
mock_wwn_struct_from_hex_str.side_effect = (
mock.sentinel.local_wwnn_struct, mock.sentinel.local_wwpn_struct,
mock.sentinel.remote_wwpn_struct)
self._diskutils._parse_scsi_page_83.return_value = (
mock.sentinel.identifiers)
identifiers = self._fc_utils.get_scsi_device_identifiers(
mock.sentinel.local_wwnn, mock.sentinel.local_wwpn,
mock.sentinel.remote_wwpn, mock.sentinel.fcp_lun,
mock.sentinel.select_supp_ids)
self.assertEqual(mock.sentinel.identifiers, identifiers)
mock_wwn_struct_from_hex_str.assert_has_calls(
[mock.call(wwn)
for wwn in (mock.sentinel.local_wwnn, mock.sentinel.local_wwpn,
mock.sentinel.remote_wwpn)])
mock_get_scsi_dev_id_vpd.assert_called_once_with(
mock_open_adapter.return_value,
mock.sentinel.local_wwpn_struct,
mock.sentinel.remote_wwpn_struct,
mock.sentinel.fcp_lun)
self._diskutils._parse_scsi_page_83.assert_called_once_with(
mock_get_scsi_dev_id_vpd.return_value,
select_supported_identifiers=mock.sentinel.select_supp_ids)
mock_open_adapter.assert_called_once_with(
mock.sentinel.local_wwnn_struct)
mock_close_adapter.assert_called_once_with(
mock_open_adapter.return_value)

116
os_win/tests/unit/utils/storage/test_diskutils.py
View File

@ -16,6 +16,8 @@
import ddt
import mock
from os_win import _utils
from os_win import constants
from os_win import exceptions
from os_win.tests.unit import test_base
from os_win.utils.storage import diskutils
@ -185,3 +187,117 @@ class DiskUtilsTestCase(test_base.OsWinBaseTestCase):
def test_get_disk_capacity_raised_exc(self):
self._test_get_disk_capacity(
raised_exc=exceptions.Win32Exception)
def test_parse_scsi_id_desc(self):
vpd_str = ('008300240103001060002AC00000000000000EA0'
'0000869902140004746573740115000400000001')
buff = _utils.hex_str_to_byte_array(vpd_str)
identifiers = self._diskutils._parse_scsi_page_83(buff)
exp_scsi_id_0 = '60002AC00000000000000EA000008699'
exp_scsi_id_1 = '74657374'
exp_scsi_id_2 = '00000001'
exp_identifiers = [
{'protocol': None,
'raw_id_desc_size': 20,
'raw_id': _utils.hex_str_to_byte_array(exp_scsi_id_0),
'code_set': 1,
'type': 3,
'id': exp_scsi_id_0,
'association': 0},
{'protocol': None,
'raw_id_desc_size': 8,
'raw_id': _utils.hex_str_to_byte_array(exp_scsi_id_1),
'code_set': 2,
'type': 4,
'id': 'test',
'association': 1},
{'protocol': None,
'raw_id_desc_size': 8,
'raw_id': _utils.hex_str_to_byte_array(exp_scsi_id_2),
'code_set': 1,
'type': 5,
'id': exp_scsi_id_2,
'association': 1}]
self.assertEqual(exp_identifiers, identifiers)
def test_parse_supported_scsi_id_desc(self):
vpd_str = ('008300240103001060002AC00000000000000EA0'
'0000869901140004000003F40115000400000001')
buff = _utils.hex_str_to_byte_array(vpd_str)
identifiers = self._diskutils._parse_scsi_page_83(
buff, select_supported_identifiers=True)
exp_scsi_id = '60002AC00000000000000EA000008699'
exp_identifiers = [
{'protocol': None,
'raw_id_desc_size': 20,
'raw_id': _utils.hex_str_to_byte_array(exp_scsi_id),
'code_set': 1,
'type': 3,
'id': exp_scsi_id,
'association': 0}]
self.assertEqual(exp_identifiers, identifiers)
def test_parse_scsi_page_83_no_desc(self):
# We've set the page length field to 0, so we're expecting an
# empty list to be returned.
vpd_str = ('008300000103001060002AC00000000000000EA0'
'0000869901140004000003F40115000400000001')
buff = _utils.hex_str_to_byte_array(vpd_str)
identifiers = self._diskutils._parse_scsi_page_83(buff)
self.assertEqual([], identifiers)
def test_parse_scsi_id_desc_exc(self):
vpd_str = '0083'
# Invalid VPD page data (buffer too small)
self.assertRaises(exceptions.SCSIPageParsingError,
self._diskutils._parse_scsi_page_83,
_utils.hex_str_to_byte_array(vpd_str))
vpd_str = ('00FF00240103001060002AC00000000000000EA0'
'0000869901140004000003F40115000400000001')
# Unexpected page code
self.assertRaises(exceptions.SCSIPageParsingError,
self._diskutils._parse_scsi_page_83,
_utils.hex_str_to_byte_array(vpd_str))
vpd_str = ('008300F40103001060002AC00000000000000EA0'
'0000869901140004000003F40115000400000001')
# VPD page overflow
self.assertRaises(exceptions.SCSIPageParsingError,
self._diskutils._parse_scsi_page_83,
_utils.hex_str_to_byte_array(vpd_str))
vpd_str = ('00830024010300FF60002AC00000000000000EA0'
'0000869901140004000003F40115000400000001')
# Identifier overflow
self.assertRaises(exceptions.SCSIIdDescriptorParsingError,
self._diskutils._parse_scsi_page_83,
_utils.hex_str_to_byte_array(vpd_str))
vpd_str = ('0083001F0103001060002AC00000000000000EA0'
'0000869901140004000003F4011500')
# Invalid identifier structure (too small)
self.assertRaises(exceptions.SCSIIdDescriptorParsingError,
self._diskutils._parse_scsi_page_83,
_utils.hex_str_to_byte_array(vpd_str))
def test_select_supported_scsi_identifiers(self):
identifiers = [
{'type': id_type}
for id_type in constants.SUPPORTED_SCSI_UID_FORMATS[::-1]]
identifiers.append({'type': mock.sentinel.scsi_id_format})
expected_identifiers = [
{'type': id_type}
for id_type in constants.SUPPORTED_SCSI_UID_FORMATS]
result = self._diskutils._select_supported_scsi_identifiers(
identifiers)
self.assertEqual(expected_identifiers, result)

138
os_win/utils/storage/diskutils.py
View File

@ -22,6 +22,7 @@ from oslo_log import log as logging
from os_win._i18n import _
from os_win import _utils
from os_win import constants
from os_win import exceptions
from os_win.utils import baseutils
from os_win.utils import win32utils
@ -32,6 +33,36 @@ kernel32 = w_lib.get_shared_lib_handle(w_lib.KERNEL32)
LOG = logging.getLogger(__name__)
class DEVICE_ID_VPD_PAGE(ctypes.BigEndianStructure):
_fields_ = [
('DeviceType', ctypes.c_ubyte, 5),
('Qualifier', ctypes.c_ubyte, 3),
('PageCode', ctypes.c_ubyte),
('PageLength', ctypes.c_uint16)
]
class IDENTIFICATION_DESCRIPTOR(ctypes.Structure):
_fields_ = [
('CodeSet', ctypes.c_ubyte, 4),
('ProtocolIdentifier', ctypes.c_ubyte, 4),
('IdentifierType', ctypes.c_ubyte, 4),
('Association', ctypes.c_ubyte, 2),
('_reserved', ctypes.c_ubyte, 1),
('Piv', ctypes.c_ubyte, 1),
('_reserved', ctypes.c_ubyte),
('IdentifierLength', ctypes.c_ubyte)
]
PDEVICE_ID_VPD_PAGE = ctypes.POINTER(DEVICE_ID_VPD_PAGE)
PIDENTIFICATION_DESCRIPTOR = ctypes.POINTER(IDENTIFICATION_DESCRIPTOR)
SCSI_ID_ASSOC_TYPE_DEVICE = 0
SCSI_ID_CODE_SET_BINARY = 1
SCSI_ID_CODE_SET_ASCII = 2
class DiskUtils(baseutils.BaseUtils):
_wmi_namespace = 'root/microsoft/windows/storage'
@ -106,3 +137,110 @@ class DiskUtils(baseutils.BaseUtils):
return 0, 0
else:
raise exc
def _parse_scsi_page_83(self, buff,
select_supported_identifiers=False):
"""Parse SCSI Device Identification VPD (page 0x83 data).
:param buff: a byte array containing the SCSI page 0x83 data.
:param select_supported_identifiers: select identifiers supported
by Windows, in the order of precedence.
:returns: a list of identifiers represented as dicts, containing
SCSI Unique IDs.
"""
identifiers = []
buff_sz = len(buff)
buff = (ctypes.c_ubyte * buff_sz)(*bytearray(buff))
vpd_pg_struct_sz = ctypes.sizeof(DEVICE_ID_VPD_PAGE)
if buff_sz < vpd_pg_struct_sz:
reason = _('Invalid VPD page data.')
raise exceptions.SCSIPageParsingError(page='0x83',
reason=reason)
vpd_page = ctypes.cast(buff, PDEVICE_ID_VPD_PAGE).contents
vpd_page_addr = ctypes.addressof(vpd_page)
total_page_sz = vpd_page.PageLength + vpd_pg_struct_sz
if vpd_page.PageCode != 0x83:
reason = _('Unexpected page code: %s') % vpd_page.PageCode
raise exceptions.SCSIPageParsingError(page='0x83',
reason=reason)
if total_page_sz > buff_sz:
reason = _('VPD page overflow.')
raise exceptions.SCSIPageParsingError(page='0x83',
reason=reason)
if not vpd_page.PageLength:
LOG.info('Page 0x83 data does not contain any '
'identification descriptors.')
return identifiers
id_desc_offset = vpd_pg_struct_sz
while id_desc_offset < total_page_sz:
id_desc_addr = vpd_page_addr + id_desc_offset
# Remaining buffer size
id_desc_buff_sz = buff_sz - id_desc_offset
identifier = self._parse_scsi_id_desc(id_desc_addr,
id_desc_buff_sz)
identifiers.append(identifier)
id_desc_offset += identifier['raw_id_desc_size']
if select_supported_identifiers:
identifiers = self._select_supported_scsi_identifiers(identifiers)
return identifiers
def _parse_scsi_id_desc(self, id_desc_addr, buff_sz):
"""Parse SCSI VPD identification descriptor."""
id_desc_struct_sz = ctypes.sizeof(IDENTIFICATION_DESCRIPTOR)
if buff_sz < id_desc_struct_sz:
reason = _('Identifier descriptor overflow.')
raise exceptions.SCSIIdDescriptorParsingError(reason=reason)
id_desc = IDENTIFICATION_DESCRIPTOR.from_address(id_desc_addr)
id_desc_sz = id_desc_struct_sz + id_desc.IdentifierLength
identifier_addr = id_desc_addr + id_desc_struct_sz
if id_desc_sz > buff_sz:
reason = _('Identifier overflow.')
raise exceptions.SCSIIdDescriptorParsingError(reason=reason)
identifier = (ctypes.c_ubyte *
id_desc.IdentifierLength).from_address(
identifier_addr)
raw_id = bytearray(identifier)
if id_desc.CodeSet == SCSI_ID_CODE_SET_ASCII:
parsed_id = bytes(
bytearray(identifier)).decode('ascii').strip('\x00')
else:
parsed_id = _utils.byte_array_to_hex_str(raw_id)
id_dict = {
'code_set': id_desc.CodeSet,
'protocol': (id_desc.ProtocolIdentifier
if id_desc.Piv else None),
'type': id_desc.IdentifierType,
'association': id_desc.Association,
'raw_id': raw_id,
'id': parsed_id,
'raw_id_desc_size': id_desc_sz,
}
return id_dict
def _select_supported_scsi_identifiers(self, identifiers):
# This method will filter out unsupported SCSI identifiers,
# also sorting them based on the order of precedence.
selected_identifiers = []
for id_type in constants.SUPPORTED_SCSI_UID_FORMATS:
for identifier in identifiers:
if identifier['type'] == id_type:
selected_identifiers.append(identifier)
return selected_identifiers

82
os_win/utils/storage/initiator/fc_utils.py
View File

@ -23,6 +23,7 @@ from os_win._i18n import _
from os_win import _utils
import os_win.conf
from os_win import exceptions
from os_win.utils.storage import diskutils
from os_win.utils import win32utils
from os_win.utils.winapi import constants as w_const
from os_win.utils.winapi import libs as w_lib
@ -37,10 +38,14 @@ LOG = logging.getLogger(__name__)
HBA_STATUS_OK = 0
HBA_STATUS_ERROR_MORE_DATA = 7
SCSI_INQ_BUFF_SZ = 256
SENSE_BUFF_SZ = 256
class FCUtils(object):
def __init__(self):
self._win32_utils = win32utils.Win32Utils()
self._diskutils = diskutils.DiskUtils()
def _run_and_check_output(self, *args, **kwargs):
kwargs['failure_exc'] = exceptions.FCWin32Exception
@ -206,3 +211,80 @@ class FCUtils(object):
@_utils.avoid_blocking_call_decorator
def refresh_hba_configuration(self):
hbaapi.HBA_RefreshAdapterConfiguration()
def _send_scsi_inquiry_v2(self, hba_handle, port_wwn_struct,
remote_port_wwn_struct,
fcp_lun, cdb_byte1, cdb_byte2):
port_wwn = _utils.byte_array_to_hex_str(port_wwn_struct.wwn)
remote_port_wwn = _utils.byte_array_to_hex_str(
remote_port_wwn_struct.wwn)
LOG.debug("Sending SCSI INQUIRY to WWPN %(remote_port_wwn)s, "
"FCP LUN %(fcp_lun)s from WWPN %(port_wwn)s. "
"CDB byte 1 %(cdb_byte1)s, CDB byte 2: %(cdb_byte2)s.",
dict(port_wwn=port_wwn,
remote_port_wwn=remote_port_wwn,
fcp_lun=fcp_lun,
cdb_byte1=hex(cdb_byte1),
cdb_byte2=hex(cdb_byte2)))
resp_buffer_sz = ctypes.c_uint32(SCSI_INQ_BUFF_SZ)
resp_buffer = (ctypes.c_ubyte * resp_buffer_sz.value)()
sense_buffer_sz = ctypes.c_uint32(SENSE_BUFF_SZ)
sense_buffer = (ctypes.c_ubyte * sense_buffer_sz.value)()
scsi_status = ctypes.c_ubyte()
try:
self._run_and_check_output(
hbaapi.HBA_ScsiInquiryV2,
hba_handle,
port_wwn_struct,
remote_port_wwn_struct,
ctypes.c_uint64(fcp_lun),
ctypes.c_uint8(cdb_byte1),
ctypes.c_uint8(cdb_byte2),
ctypes.byref(resp_buffer),
ctypes.byref(resp_buffer_sz),
ctypes.byref(scsi_status),
ctypes.byref(sense_buffer),
ctypes.byref(sense_buffer_sz))
finally:
sense_data = _utils.byte_array_to_hex_str(
sense_buffer[:sense_buffer_sz.value])
LOG.debug("SCSI inquiry returned sense data: %(sense_data)s. "
"SCSI status: %(scsi_status)s.",
dict(sense_data=sense_data,
scsi_status=scsi_status.value))
return resp_buffer
def _get_scsi_device_id_vpd(self, hba_handle, port_wwn_struct,
remote_port_wwn_struct, fcp_lun):
# The following bytes will be included in the CDB passed to the
# lun, requesting the 0x83 VPD page.
cdb_byte1 = 1
cdb_byte2 = 0x83
return self._send_scsi_inquiry_v2(hba_handle, port_wwn_struct,
remote_port_wwn_struct, fcp_lun,
cdb_byte1, cdb_byte2)
def get_scsi_device_identifiers(self, node_wwn, port_wwn,
remote_port_wwn, fcp_lun,
select_supported_identifiers=True):
node_wwn_struct = self._wwn_struct_from_hex_str(node_wwn)
port_wwn_struct = self._wwn_struct_from_hex_str(port_wwn)
remote_port_wwn_struct = self._wwn_struct_from_hex_str(
remote_port_wwn)
with self._get_hba_handle(
adapter_wwn_struct=node_wwn_struct) as hba_handle:
vpd_data = self._get_scsi_device_id_vpd(hba_handle,
port_wwn_struct,
remote_port_wwn_struct,
fcp_lun)
identifiers = self._diskutils._parse_scsi_page_83(
vpd_data,
select_supported_identifiers=select_supported_identifiers)
return identifiers

15
os_win/utils/winapi/libs/hbaapi.py
View File

@ -142,5 +142,20 @@ def register():
HBA_WWN]
lib_handle.HBA_OpenAdapterByWWN.restype = HBA_STATUS
lib_handle.HBA_ScsiInquiryV2.argtypes = [
HBA_HANDLE,
HBA_WWN,
HBA_WWN,
ctypes.c_uint64,
ctypes.c_uint8,
ctypes.c_uint8,
wintypes.PVOID,
ctypes.POINTER(ctypes.c_uint32),
ctypes.POINTER(ctypes.c_uint8),
wintypes.PVOID,
ctypes.POINTER(ctypes.c_uint32)
]
lib_handle.HBA_ScsiInquiryV2.restype = HBA_STATUS
lib_handle.HBA_RefreshAdapterConfiguration.argtypes = []
lib_handle.HBA_RefreshAdapterConfiguration.restype = None

5
releasenotes/notes/fc_scsi_id-c29d2e17c4d83453.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
os-win now supports retrieving SCSI unique ids for FibreChannel disks. This
allows discovering FibreChannel disks in a more efficient way.