hyperv: Retry after WMI query fails to find dev

During long stress runs the WMI query that is looking for
the iSCSI device number can incorrectly return no data.
If the query is retried the appropriate data can then be
obtained.

This commit adds a retry loop, calling
basevolumeutils.get_device_number_for_target to avoid this situation.
It also handles the case where the devices list returned in
get_device_number_for_target is empty.  The retry loop is
implemented with new mounted_disk_query_retry_count and
mounted_disk_query_retry_interval configuration options.

Unit tests have been added to check the good and bad paths for
get_mounted_disk_from_lun.

DocImpact
Closes-bug: 1247901
Change-Id: I082c4b1694efcd20cce65293cd330b7a0cf7d470
(cherry picked from commit d143540ad1)
This commit is contained in:
Jay S. Bryant 2013-11-06 10:49:00 -06:00
parent bcdc813194
commit 1c7ff2af93
4 changed files with 87 additions and 2 deletions

View File

@ -2612,6 +2612,14 @@
# (integer value)
#volume_attach_retry_interval=5
# The number of times to retry checking for a disk mounted via
# iSCSI. (integer value)
#mounted_disk_query_retry_count=10
# Interval between checks for a mounted iSCSI disk, in
# seconds. (integer value)
#mounted_disk_query_retry_interval=5
[zookeeper]

View File

@ -18,7 +18,9 @@
Test suite for the Hyper-V driver and related APIs.
"""
import contextlib
import io
import mock
import mox
import os
import platform
@ -58,6 +60,7 @@ from nova.virt.hyperv import vhdutils
from nova.virt.hyperv import vhdutilsv2
from nova.virt.hyperv import vmutils
from nova.virt.hyperv import vmutilsv2
from nova.virt.hyperv import volumeops
from nova.virt.hyperv import volumeutils
from nova.virt.hyperv import volumeutilsv2
from nova.virt import images
@ -1612,3 +1615,52 @@ class HyperVAPITestCase(test.NoDBTestCase):
def test_finish_revert_migration_with_ephemeral_storage(self):
self._test_finish_revert_migration(False, ephemeral_storage=True)
class VolumeOpsTestCase(HyperVAPITestCase):
"""Unit tests for VolumeOps class."""
def setUp(self):
super(VolumeOpsTestCase, self).setUp()
self.volumeops = volumeops.VolumeOps()
def test_get_mounted_disk_from_lun(self):
with contextlib.nested(
mock.patch.object(self.volumeops._volutils,
'get_device_number_for_target'),
mock.patch.object(self.volumeops._vmutils,
'get_mounted_disk_by_drive_number')
) as (mock_get_device_number_for_target,
mock_get_mounted_disk_by_drive_number):
mock_get_device_number_for_target.return_value = 0
mock_get_mounted_disk_by_drive_number.return_value = 'disk_path'
block_device_info = db_fakes.get_fake_block_device_info(
self._volume_target_portal, self._volume_id)
mapping = driver.block_device_info_get_mapping(block_device_info)
data = mapping[0]['connection_info']['data']
target_lun = data['target_lun']
target_iqn = data['target_iqn']
disk = self.volumeops._get_mounted_disk_from_lun(target_iqn,
target_lun)
self.assertEqual(disk, 'disk_path')
def test_get_mounted_disk_from_lun_failure(self):
with mock.patch.object(self.volumeops._volutils,
'get_device_number_for_target') as m_device_num:
m_device_num.return_value = None
block_device_info = db_fakes.get_fake_block_device_info(
self._volume_target_portal, self._volume_id)
mapping = driver.block_device_info_get_mapping(block_device_info)
data = mapping[0]['connection_info']['data']
target_lun = data['target_lun']
target_iqn = data['target_iqn']
self.assertRaises(exception.NotFound,
self.volumeops._get_mounted_disk_from_lun,
target_iqn, target_lun)

View File

@ -127,6 +127,10 @@ class BaseVolumeUtils(object):
return None
devices = initiator_sessions[0].Devices
if not devices:
return None
for device in devices:
if device.ScsiLun == target_lun:
return device.DeviceNumber

View File

@ -39,6 +39,14 @@ hyper_volumeops_opts = [
cfg.IntOpt('volume_attach_retry_interval',
default=5,
help='Interval between volume attachment attempts, in seconds'),
cfg.IntOpt('mounted_disk_query_retry_count',
default=10,
help='The number of times to retry checking for a disk mounted '
'via iSCSI.'),
cfg.IntOpt('mounted_disk_query_retry_interval',
default=5,
help='Interval between checks for a mounted iSCSI '
'disk, in seconds.'),
]
CONF = cfg.CONF
@ -192,8 +200,21 @@ class VolumeOps(object):
def _get_mounted_disk_from_lun(self, target_iqn, target_lun,
wait_for_device=False):
device_number = self._volutils.get_device_number_for_target(target_iqn,
target_lun)
# The WMI query in get_device_number_for_target can incorrectly
# return no data when the system is under load. This issue can
# be avoided by adding a retry.
for i in xrange(CONF.hyperv.mounted_disk_query_retry_count):
device_number = self._volutils.get_device_number_for_target(
target_iqn, target_lun)
if device_number is None:
attempt = i + 1
LOG.debug(_('Attempt %d to get device_number '
'from get_device_number_for_target failed. '
'Retrying...') % attempt)
time.sleep(CONF.hyperv.mounted_disk_query_retry_interval)
else:
break
if device_number is None:
raise exception.NotFound(_('Unable to find a mounted disk for '
'target_iqn: %s') % target_iqn)