hyperv: Retry after WMI query fails to find dev
During long stress runs the WMI query that is looking for the iSCSI device number can incorrectly return no data. If the query is retried the appropriate data can then be obtained. This commit adds a retry loop, calling basevolumeutils.get_device_number_for_target to avoid this situation. It also handles the case where the devices list returned in get_device_number_for_target is empty. The retry loop is implemented with new mounted_disk_query_retry_count and mounted_disk_query_retry_interval configuration options. Unit tests have been added to check the good and bad paths for get_mounted_disk_from_lun. DocImpact Closes-bug:1247901
Change-Id: I082c4b1694efcd20cce65293cd330b7a0cf7d470 (cherry picked from commitd143540ad1
)
This commit is contained in:
parent
bcdc813194
commit
1c7ff2af93
|
@ -2612,6 +2612,14 @@
|
|||
# (integer value)
|
||||
#volume_attach_retry_interval=5
|
||||
|
||||
# The number of times to retry checking for a disk mounted via
|
||||
# iSCSI. (integer value)
|
||||
#mounted_disk_query_retry_count=10
|
||||
|
||||
# Interval between checks for a mounted iSCSI disk, in
|
||||
# seconds. (integer value)
|
||||
#mounted_disk_query_retry_interval=5
|
||||
|
||||
|
||||
[zookeeper]
|
||||
|
||||
|
|
|
@ -18,7 +18,9 @@
|
|||
Test suite for the Hyper-V driver and related APIs.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import mock
|
||||
import mox
|
||||
import os
|
||||
import platform
|
||||
|
@ -58,6 +60,7 @@ from nova.virt.hyperv import vhdutils
|
|||
from nova.virt.hyperv import vhdutilsv2
|
||||
from nova.virt.hyperv import vmutils
|
||||
from nova.virt.hyperv import vmutilsv2
|
||||
from nova.virt.hyperv import volumeops
|
||||
from nova.virt.hyperv import volumeutils
|
||||
from nova.virt.hyperv import volumeutilsv2
|
||||
from nova.virt import images
|
||||
|
@ -1612,3 +1615,52 @@ class HyperVAPITestCase(test.NoDBTestCase):
|
|||
|
||||
def test_finish_revert_migration_with_ephemeral_storage(self):
|
||||
self._test_finish_revert_migration(False, ephemeral_storage=True)
|
||||
|
||||
|
||||
class VolumeOpsTestCase(HyperVAPITestCase):
|
||||
"""Unit tests for VolumeOps class."""
|
||||
|
||||
def setUp(self):
|
||||
super(VolumeOpsTestCase, self).setUp()
|
||||
self.volumeops = volumeops.VolumeOps()
|
||||
|
||||
def test_get_mounted_disk_from_lun(self):
|
||||
with contextlib.nested(
|
||||
mock.patch.object(self.volumeops._volutils,
|
||||
'get_device_number_for_target'),
|
||||
mock.patch.object(self.volumeops._vmutils,
|
||||
'get_mounted_disk_by_drive_number')
|
||||
) as (mock_get_device_number_for_target,
|
||||
mock_get_mounted_disk_by_drive_number):
|
||||
|
||||
mock_get_device_number_for_target.return_value = 0
|
||||
mock_get_mounted_disk_by_drive_number.return_value = 'disk_path'
|
||||
|
||||
block_device_info = db_fakes.get_fake_block_device_info(
|
||||
self._volume_target_portal, self._volume_id)
|
||||
|
||||
mapping = driver.block_device_info_get_mapping(block_device_info)
|
||||
data = mapping[0]['connection_info']['data']
|
||||
target_lun = data['target_lun']
|
||||
target_iqn = data['target_iqn']
|
||||
|
||||
disk = self.volumeops._get_mounted_disk_from_lun(target_iqn,
|
||||
target_lun)
|
||||
self.assertEqual(disk, 'disk_path')
|
||||
|
||||
def test_get_mounted_disk_from_lun_failure(self):
|
||||
with mock.patch.object(self.volumeops._volutils,
|
||||
'get_device_number_for_target') as m_device_num:
|
||||
m_device_num.return_value = None
|
||||
|
||||
block_device_info = db_fakes.get_fake_block_device_info(
|
||||
self._volume_target_portal, self._volume_id)
|
||||
|
||||
mapping = driver.block_device_info_get_mapping(block_device_info)
|
||||
data = mapping[0]['connection_info']['data']
|
||||
target_lun = data['target_lun']
|
||||
target_iqn = data['target_iqn']
|
||||
|
||||
self.assertRaises(exception.NotFound,
|
||||
self.volumeops._get_mounted_disk_from_lun,
|
||||
target_iqn, target_lun)
|
||||
|
|
|
@ -127,6 +127,10 @@ class BaseVolumeUtils(object):
|
|||
return None
|
||||
|
||||
devices = initiator_sessions[0].Devices
|
||||
|
||||
if not devices:
|
||||
return None
|
||||
|
||||
for device in devices:
|
||||
if device.ScsiLun == target_lun:
|
||||
return device.DeviceNumber
|
||||
|
|
|
@ -39,6 +39,14 @@ hyper_volumeops_opts = [
|
|||
cfg.IntOpt('volume_attach_retry_interval',
|
||||
default=5,
|
||||
help='Interval between volume attachment attempts, in seconds'),
|
||||
cfg.IntOpt('mounted_disk_query_retry_count',
|
||||
default=10,
|
||||
help='The number of times to retry checking for a disk mounted '
|
||||
'via iSCSI.'),
|
||||
cfg.IntOpt('mounted_disk_query_retry_interval',
|
||||
default=5,
|
||||
help='Interval between checks for a mounted iSCSI '
|
||||
'disk, in seconds.'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
@ -192,8 +200,21 @@ class VolumeOps(object):
|
|||
|
||||
def _get_mounted_disk_from_lun(self, target_iqn, target_lun,
|
||||
wait_for_device=False):
|
||||
device_number = self._volutils.get_device_number_for_target(target_iqn,
|
||||
target_lun)
|
||||
# The WMI query in get_device_number_for_target can incorrectly
|
||||
# return no data when the system is under load. This issue can
|
||||
# be avoided by adding a retry.
|
||||
for i in xrange(CONF.hyperv.mounted_disk_query_retry_count):
|
||||
device_number = self._volutils.get_device_number_for_target(
|
||||
target_iqn, target_lun)
|
||||
if device_number is None:
|
||||
attempt = i + 1
|
||||
LOG.debug(_('Attempt %d to get device_number '
|
||||
'from get_device_number_for_target failed. '
|
||||
'Retrying...') % attempt)
|
||||
time.sleep(CONF.hyperv.mounted_disk_query_retry_interval)
|
||||
else:
|
||||
break
|
||||
|
||||
if device_number is None:
|
||||
raise exception.NotFound(_('Unable to find a mounted disk for '
|
||||
'target_iqn: %s') % target_iqn)
|
||||
|
|
Loading…
Reference in New Issue