hyperv: Retry after WMI query fails to find dev
During long stress runs the WMI query that is looking for
the iSCSI device number can incorrectly return no data.
If the query is retried the appropriate data can then be
obtained.
This commit adds a retry loop, calling
basevolumeutils.get_device_number_for_target to avoid this situation.
It also handles the case where the devices list returned in
get_device_number_for_target is empty. The retry loop is
implemented with new mounted_disk_query_retry_count and
mounted_disk_query_retry_interval configuration options.
Unit tests have been added to check the good and bad paths for
get_mounted_disk_from_lun.
DocImpact
Closes-bug: 1247901
Change-Id: I082c4b1694efcd20cce65293cd330b7a0cf7d470
This commit is contained in:
parent
80efcae3a3
commit
d143540ad1
|
@ -2592,6 +2592,14 @@
|
|||
# (integer value)
|
||||
#volume_attach_retry_interval=5
|
||||
|
||||
# The number of times to retry checking for a disk mounted via
|
||||
# iSCSI. (integer value)
|
||||
#mounted_disk_query_retry_count=10
|
||||
|
||||
# Interval between checks for a mounted iSCSI disk, in
|
||||
# seconds. (integer value)
|
||||
#mounted_disk_query_retry_interval=5
|
||||
|
||||
|
||||
[image_file_url]
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
Test suite for the Hyper-V driver and related APIs.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import platform
|
||||
|
@ -60,6 +61,7 @@ from nova.virt.hyperv import vhdutils
|
|||
from nova.virt.hyperv import vhdutilsv2
|
||||
from nova.virt.hyperv import vmutils
|
||||
from nova.virt.hyperv import vmutilsv2
|
||||
from nova.virt.hyperv import volumeops
|
||||
from nova.virt.hyperv import volumeutils
|
||||
from nova.virt.hyperv import volumeutilsv2
|
||||
from nova.virt import images
|
||||
|
@ -1636,3 +1638,52 @@ class HyperVAPITestCase(test.NoDBTestCase):
|
|||
self._test_spawn_instance, [], None)
|
||||
mock_destroy.assert_called_once_with(self._context,
|
||||
self._test_spawn_instance, [], None)
|
||||
|
||||
|
||||
class VolumeOpsTestCase(HyperVAPITestCase):
|
||||
"""Unit tests for VolumeOps class."""
|
||||
|
||||
def setUp(self):
|
||||
super(VolumeOpsTestCase, self).setUp()
|
||||
self.volumeops = volumeops.VolumeOps()
|
||||
|
||||
def test_get_mounted_disk_from_lun(self):
|
||||
with contextlib.nested(
|
||||
mock.patch.object(self.volumeops._volutils,
|
||||
'get_device_number_for_target'),
|
||||
mock.patch.object(self.volumeops._vmutils,
|
||||
'get_mounted_disk_by_drive_number')
|
||||
) as (mock_get_device_number_for_target,
|
||||
mock_get_mounted_disk_by_drive_number):
|
||||
|
||||
mock_get_device_number_for_target.return_value = 0
|
||||
mock_get_mounted_disk_by_drive_number.return_value = 'disk_path'
|
||||
|
||||
block_device_info = db_fakes.get_fake_block_device_info(
|
||||
self._volume_target_portal, self._volume_id)
|
||||
|
||||
mapping = driver.block_device_info_get_mapping(block_device_info)
|
||||
data = mapping[0]['connection_info']['data']
|
||||
target_lun = data['target_lun']
|
||||
target_iqn = data['target_iqn']
|
||||
|
||||
disk = self.volumeops._get_mounted_disk_from_lun(target_iqn,
|
||||
target_lun)
|
||||
self.assertEqual(disk, 'disk_path')
|
||||
|
||||
def test_get_mounted_disk_from_lun_failure(self):
|
||||
with mock.patch.object(self.volumeops._volutils,
|
||||
'get_device_number_for_target') as m_device_num:
|
||||
m_device_num.return_value = None
|
||||
|
||||
block_device_info = db_fakes.get_fake_block_device_info(
|
||||
self._volume_target_portal, self._volume_id)
|
||||
|
||||
mapping = driver.block_device_info_get_mapping(block_device_info)
|
||||
data = mapping[0]['connection_info']['data']
|
||||
target_lun = data['target_lun']
|
||||
target_iqn = data['target_iqn']
|
||||
|
||||
self.assertRaises(exception.NotFound,
|
||||
self.volumeops._get_mounted_disk_from_lun,
|
||||
target_iqn, target_lun)
|
||||
|
|
|
@ -127,6 +127,10 @@ class BaseVolumeUtils(object):
|
|||
return None
|
||||
|
||||
devices = initiator_sessions[0].Devices
|
||||
|
||||
if not devices:
|
||||
return None
|
||||
|
||||
for device in devices:
|
||||
if device.ScsiLun == target_lun:
|
||||
return device.DeviceNumber
|
||||
|
|
|
@ -39,6 +39,14 @@ hyper_volumeops_opts = [
|
|||
cfg.IntOpt('volume_attach_retry_interval',
|
||||
default=5,
|
||||
help='Interval between volume attachment attempts, in seconds'),
|
||||
cfg.IntOpt('mounted_disk_query_retry_count',
|
||||
default=10,
|
||||
help='The number of times to retry checking for a disk mounted '
|
||||
'via iSCSI.'),
|
||||
cfg.IntOpt('mounted_disk_query_retry_interval',
|
||||
default=5,
|
||||
help='Interval between checks for a mounted iSCSI '
|
||||
'disk, in seconds.'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
@ -192,8 +200,21 @@ class VolumeOps(object):
|
|||
|
||||
def _get_mounted_disk_from_lun(self, target_iqn, target_lun,
|
||||
wait_for_device=False):
|
||||
device_number = self._volutils.get_device_number_for_target(target_iqn,
|
||||
target_lun)
|
||||
# The WMI query in get_device_number_for_target can incorrectly
|
||||
# return no data when the system is under load. This issue can
|
||||
# be avoided by adding a retry.
|
||||
for i in xrange(CONF.hyperv.mounted_disk_query_retry_count):
|
||||
device_number = self._volutils.get_device_number_for_target(
|
||||
target_iqn, target_lun)
|
||||
if device_number is None:
|
||||
attempt = i + 1
|
||||
LOG.debug(_('Attempt %d to get device_number '
|
||||
'from get_device_number_for_target failed. '
|
||||
'Retrying...') % attempt)
|
||||
time.sleep(CONF.hyperv.mounted_disk_query_retry_interval)
|
||||
else:
|
||||
break
|
||||
|
||||
if device_number is None:
|
||||
raise exception.NotFound(_('Unable to find a mounted disk for '
|
||||
'target_iqn: %s') % target_iqn)
|
||||
|
|
Loading…
Reference in New Issue