[Unity] Retype volume support

To support change volume type after its creation in Unity driver.

Change-Id: I0885c83a2bdfffd3bbba23310665e5dff74fe5dd
This commit is contained in:
dingd 2019-04-29 14:58:09 +08:00 committed by Ryan Liang
parent a039f5e7a5
commit 007e45ba59
10 changed files with 324 additions and 22 deletions

View File

@ -118,6 +118,10 @@ class MockClient(object):
lun_id += '_low'
return test_client.MockResource(_id=lun_id, name=name)
@staticmethod
def lun_has_snapshot(lun):
return lun.name == 'volume_has_snapshot'
@staticmethod
def get_lun(name=None, lun_id=None):
if lun_id is None:
@ -214,7 +218,9 @@ class MockClient(object):
@staticmethod
def get_io_limit_policy(specs):
return None
mock_io_policy = (test_client.MockResource(name=specs.get('id'))
if specs else None)
return mock_io_policy
@staticmethod
def extend_lun(lun_id, size_gib):
@ -257,7 +263,7 @@ class MockClient(object):
'PoolC': 'pool_3'}
return pools.get(name, None)
def migrate_lun(self, lun_id, dest_pool_id):
def migrate_lun(self, lun_id, dest_pool_id, provision=None):
if dest_pool_id == 'pool_2':
return True
if dest_pool_id == 'pool_3':
@ -409,6 +415,20 @@ def get_connection_info(adapter, hlu, host, connector):
return {}
def get_volume_type_qos_specs(qos_id):
if qos_id == 'qos':
return {'qos_specs': {'id': u'qos_type_id_1',
'consumer': u'back-end',
u'maxBWS': u'102400',
u'maxIOPS': u'500'}}
if qos_id == 'qos_2':
return {'qos_specs': {'id': u'qos_type_id_2',
'consumer': u'back-end',
u'maxBWS': u'102402',
u'maxIOPS': u'502'}}
return {'qos_specs': {}}
def get_volume_type_extra_specs(type_id):
if type_id == 'thick':
return {'provisioning:type': 'thick',
@ -419,6 +439,9 @@ def get_volume_type_extra_specs(type_id):
if type_id == 'tier_lowest':
return {'storagetype:tiering': 'LowestAvailable',
'fast_support': '<is> True'}
if type_id == 'compressed':
return {'provisioning:type': 'compressed',
'compression_support': '<is> True'}
return {}
@ -439,6 +462,8 @@ def patch_for_unity_adapter(func):
new=get_volume_type_extra_specs)
@mock.patch('cinder.volume.group_types.get_group_type_specs',
new=get_group_type_specs)
@mock.patch('cinder.volume.volume_types.get_volume_type_qos_specs',
new=get_volume_type_qos_specs)
@mock.patch('cinder.volume.drivers.dell_emc.unity.utils.'
'get_backend_qos_specs',
new=get_backend_qos_specs)
@ -601,6 +626,65 @@ class CommonAdapterTest(test.TestCase):
volume = MockOSResource(provider_location='id^lun_4')
self.adapter.delete_volume(volume)
@patch_for_unity_adapter
def test_retype_volume_has_snapshot(self):
volume = MockOSResource(name='volume_has_snapshot', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'type01', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertFalse(result)
@patch_for_unity_adapter
def test_retype_volume_thick_to_compressed(self):
volume = MockOSResource(name='thick_volume', size=5,
host='HostA@BackendB#PoolA',
provider_location='id^lun_33')
ctxt = None
diff = None
new_type = {'name': u'compressed_type', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertEqual((True, {}), result)
@patch_for_unity_adapter
def test_retype_volume_to_compressed(self):
volume = MockOSResource(name='thin_volume', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'compressed_type', 'id': 'compressed'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type, diff, host)
self.assertTrue(result)
@patch_for_unity_adapter
def test_retype_volume_to_qos(self):
volume = MockOSResource(name='thin_volume', size=5,
host='HostA@BackendB#PoolB')
ctxt = None
diff = None
new_type = {'name': u'qos_type', 'id': 'qos'}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type,
diff, host)
self.assertTrue(result)
@patch_for_unity_adapter
def test_retype_volume_revert_qos(self):
volume = MockOSResource(name='qos_volume', size=5,
host='HostA@BackendB#PoolB',
volume_type_id='qos_2')
ctxt = None
diff = None
new_type = {'name': u'no_qos_type', 'id': ''}
host = {'host': 'HostA@BackendB#PoolB'}
result = self.adapter.retype(ctxt, volume, new_type,
diff, host)
self.assertTrue(result)
def test_get_pool_stats(self):
stats_list = self.adapter.get_pools_stats()
self.assertEqual(1, len(stats_list))

View File

@ -59,6 +59,7 @@ class MockResource(object):
self.lun = None
self.tiering_policy = None
self.pool_fast_vp = None
self.snap = True
@property
def id(self):
@ -199,9 +200,12 @@ class MockResource(object):
def storage_resource(self, value):
self._storage_resource = value
def modify(self, name=None):
def modify(self, name=None, is_compression=None, io_limit_policy=None):
self.name = name
def remove_from_storage(self, lun):
pass
def thin_clone(self, name, io_limit_policy=None, description=None):
if name == 'thin_clone_name_in_use':
raise ex.UnityLunNameInUseError
@ -213,8 +217,8 @@ class MockResource(object):
def restore(self, delete_backup):
return MockResource(_id='snap_1', name="internal_snap")
def migrate(self, dest_pool):
if dest_pool.id == 'pool_2':
def migrate(self, dest_pool, **kwargs):
if dest_pool.id == 'fail_migration_pool':
return False
return True
@ -685,9 +689,17 @@ class ClientTest(unittest.TestCase):
self.assertTrue(ret)
def test_migrate_lun_failed(self):
ret = self.client.migrate_lun('lun_0', 'pool_2')
ret = self.client.migrate_lun('lun_0', 'fail_migration_pool')
self.assertFalse(ret)
def test_migrate_lun_thick(self):
ret = self.client.migrate_lun('lun_thick', 'pool_2', 'thick')
self.assertTrue(ret)
def test_migrate_lun_compressed(self):
ret = self.client.migrate_lun('lun_compressed', 'pool_2', 'compressed')
self.assertTrue(ret)
def test_get_pool_id_by_name(self):
self.assertEqual('pool_3', self.client.get_pool_id_by_name('Pool 3'))

View File

@ -175,6 +175,9 @@ class MockAdapter(object):
return group_update, volumes_update
return group_update, None
def retype(self, ctxt, volume, new_type, diff, host):
return True
class MockReplicationManager(object):
def __init__(self):
@ -318,6 +321,17 @@ class UnityDriverTest(unittest.TestCase):
'HostA@BackendB#PoolC')
self.assertEqual((True, {}), ret)
def test_retype_volume(self):
volume = self.get_volume()
new_type = {'name': u'type01', 'qos_specs_id': 'test_qos_id',
'extra_specs': {},
'id': u'd67c4480-a61b-44c0-a58b-24c0357cadeb'}
diff = None
ret = self.driver.retype(self.get_context(),
volume, new_type, diff,
'HostA@BackendB#PoolC')
self.assertTrue(ret)
def test_create_snapshot(self):
snapshot = self.get_snapshot()
self.driver.create_snapshot(snapshot)

View File

@ -329,3 +329,52 @@ class UnityUtilsTest(unittest.TestCase):
cg = test_driver.UnityDriverTest.get_cg()
result = utils.get_group_specs(cg, 'test_key')
self.assertIsNone(result)
@patch_volume_types
def test_retype_no_need_migration_when_same_host(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
ret = utils.retype_need_migration(volume, None, None, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_need_migration_when_diff_host(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'new_host'}
ret = utils.retype_need_migration(volume, None, None, new_host)
self.assertTrue(ret)
@patch_volume_types
def test_retype_no_need_migration_thin_to_compressed(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = ''
new_provision = 'compressed'
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_no_need_migration_compressed_to_thin(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = 'compressed'
new_provision = ''
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertFalse(ret)
@patch_volume_types
def test_retype_need_migration_thin_to_thick(self):
volume = test_adapter.MockOSResource(volume_type_id='host_1',
host='host_1')
new_host = {'name': 'new_name', 'host': 'host_1'}
old_provision = ''
new_provision = 'thick'
ret = utils.retype_need_migration(volume, old_provision,
new_provision, new_host)
self.assertTrue(ret)

View File

@ -30,6 +30,7 @@ from cinder.objects import fields
from cinder import utils as cinder_utils
from cinder.volume.drivers.dell_emc.unity import client
from cinder.volume.drivers.dell_emc.unity import utils
from cinder.volume import volume_types
from cinder.volume import volume_utils
storops = importutils.try_import('storops')
@ -120,7 +121,8 @@ class VolumeParams(object):
@property
def is_thick(self):
if self._is_thick is None:
provision = utils.get_extra_spec(self._volume, 'provisioning:type')
provision = utils.get_extra_spec(self._volume,
utils.PROVISIONING_TYPE)
support = utils.get_extra_spec(self._volume,
'thick_provisioning_support')
self._is_thick = (provision == 'thick' and support == '<is> True')
@ -129,10 +131,12 @@ class VolumeParams(object):
@property
def is_compressed(self):
if self._is_compressed is None:
provision = utils.get_extra_spec(self._volume, 'provisioning:type')
provision = utils.get_extra_spec(self._volume,
utils.PROVISIONING_TYPE)
compression = utils.get_extra_spec(self._volume,
'compression_support')
if provision == 'compressed' and compression == '<is> True':
if (provision == utils.PROVISIONING_COMPRESSED and
compression == '<is> True'):
self._is_compressed = True
return self._is_compressed
@ -443,6 +447,60 @@ class CommonAdapter(object):
else:
self.client.delete_lun(lun_id)
def retype(self, ctxt, volume, new_type, diff, host):
"""Changes volume from one type to another."""
old_qos_specs = {utils.QOS_SPECS: None}
old_provision = None
new_specs = volume_types.get_volume_type_extra_specs(
new_type.get(utils.QOS_ID))
new_qos_specs = volume_types.get_volume_type_qos_specs(
new_type.get(utils.QOS_ID))
lun = self.client.get_lun(name=volume.name)
volume_type_id = volume.volume_type_id
if volume_type_id:
old_provision = utils.get_extra_spec(volume,
utils.PROVISIONING_TYPE)
old_qos_specs = volume_types.get_volume_type_qos_specs(
volume_type_id)
need_migration = utils.retype_need_migration(
volume, old_provision,
new_specs.get(utils.PROVISIONING_TYPE), host)
need_change_compress = utils.retype_need_change_compression(
old_provision, new_specs.get(utils.PROVISIONING_TYPE))
need_change_qos = utils.retype_need_change_qos(
old_qos_specs, new_qos_specs)
if need_migration or need_change_compress[0] or need_change_qos:
if self.client.lun_has_snapshot(lun):
LOG.warning('Driver is not able to do retype because '
'the volume %s has snapshot(s).',
volume.id)
return False
new_qos_dict = new_qos_specs.get(utils.QOS_SPECS)
if need_change_qos:
new_io_policy = (self.client.get_io_limit_policy(new_qos_dict)
if need_change_qos else None)
# Modify lun to change qos settings
if new_io_policy:
lun.modify(io_limit_policy=new_io_policy)
else:
# remove current qos settings
old_qos_dict = old_qos_specs.get(utils.QOS_SPECS)
old_io_policy = self.client.get_io_limit_policy(old_qos_dict)
old_io_policy.remove_from_storage(lun)
if need_migration:
LOG.debug('Driver needs to use storage-assisted migration '
'to retype the volume.')
return self.migrate_volume(volume, host, new_specs)
if need_change_compress[0]:
# Modify lun to change compression
lun.modify(is_compression=need_change_compress[1])
return True
def _create_host_and_attach(self, host_name, lun_or_snap):
@utils.lock_if(self.to_lock_host, '{lock_name}')
def _lock_helper(lock_name):
@ -908,18 +966,22 @@ class CommonAdapter(object):
def restore_snapshot(self, volume, snapshot):
return self.client.restore_snapshot(snapshot.name)
def migrate_volume(self, volume, host):
def migrate_volume(self, volume, host, extra_specs=None):
"""Leverage the Unity move session functionality.
This method is invoked at the source backend.
:param extra_specs: Instance of ExtraSpecs. The new volume will be
changed to align with the new extra specs.
"""
log_params = {
'name': volume.name,
'src_host': volume.host,
'dest_host': host['host']
'dest_host': host['host'],
'extra_specs': extra_specs,
}
LOG.info('Migrate Volume: %(name)s, host: %(src_host)s, destination: '
'%(dest_host)s', log_params)
'%(dest_host)s, extra_specs: %(extra_specs)s', log_params)
src_backend = utils.get_backend_name_from_volume(volume)
dest_backend = utils.get_backend_name_from_host(host)
@ -930,10 +992,12 @@ class CommonAdapter(object):
return False, None
lun_id = self.get_lun_id(volume)
provision = None
if extra_specs:
provision = extra_specs.get(utils.PROVISIONING_TYPE)
dest_pool_name = utils.get_pool_name_from_host(host)
dest_pool_id = self.get_pool_id_by_name(dest_pool_name)
if self.client.migrate_lun(lun_id, dest_pool_id):
if self.client.migrate_lun(lun_id, dest_pool_id, provision):
LOG.debug('Volume migrated successfully.')
model_update = {}
return True, model_update

View File

@ -177,10 +177,23 @@ class UnityClient(object):
lun_id)
return lun
def migrate_lun(self, lun_id, dest_pool_id):
def migrate_lun(self, lun_id, dest_pool_id, dest_provision=None):
# dest_provision possible value ('thin', 'thick', 'compressed')
lun = self.system.get_lun(lun_id)
dest_pool = self.system.get_pool(dest_pool_id)
return lun.migrate(dest_pool)
is_thin = True if dest_provision == 'thin' else None
if dest_provision == 'compressed':
# compressed needs work with thin
is_compressed = True
is_thin = True
else:
is_compressed = False
if dest_provision == 'thick':
# thick needs work with uncompressed
is_thin = False
is_compressed = False
return lun.migrate(dest_pool, is_compressed=is_compressed,
is_thin=is_thin)
def get_pools(self):
"""Gets all storage pools on the Unity system.
@ -236,6 +249,10 @@ class UnityClient(object):
{'name': name, 'err': err})
return None
def lun_has_snapshot(self, lun):
snaps = lun.snapshots
return len(snaps) != 0
@coordination.synchronized('{self.host}-{name}')
def create_host(self, name):
return self.create_host_wo_lock(name)

View File

@ -83,9 +83,10 @@ class UnityDriver(driver.ManageableVD,
6.1.0 - Support volume replication
7.0.0 - Support tiering policy
7.1.0 - Support consistency group replication
7.2.0 - Support retype volume
"""
VERSION = '07.01.00'
VERSION = '07.02.00'
VENDOR = 'Dell EMC'
# ThirdPartySystems wiki page
CI_WIKI_NAME = "EMC_UNITY_CI"
@ -142,6 +143,10 @@ class UnityDriver(driver.ManageableVD,
"""Migrates a volume."""
return self.adapter.migrate_volume(volume, host)
def retype(self, ctxt, volume, new_type, diff, host):
"""Convert the volume to be of the new type."""
return self.adapter.retype(ctxt, volume, new_type, diff, host)
def create_snapshot(self, snapshot):
"""Creates a snapshot."""
self.adapter.create_snapshot(snapshot)

View File

@ -38,6 +38,11 @@ LOG = logging.getLogger(__name__)
BACKEND_QOS_CONSUMERS = frozenset(['back-end', 'both'])
QOS_MAX_IOPS = 'maxIOPS'
QOS_MAX_BWS = 'maxBWS'
PROVISIONING_TYPE = 'provisioning:type'
PROVISIONING_COMPRESSED = 'compressed'
QOS_SPECS = 'qos_specs'
SPECS_OF_QOS = 'specs'
QOS_ID = 'id'
def dump_provider_location(location_dict):
@ -113,6 +118,36 @@ def validate_pool_names(conf_pools, array_pools):
return existed
def retype_need_migration(volume, old_provision, new_provision, host):
if volume['host'] != host['host']:
return True
if old_provision != new_provision:
if retype_need_change_compression(old_provision, new_provision)[0]:
return False
else:
return True
return False
def retype_need_change_compression(old_provision, new_provision):
""":return: whether need change compression and the new value"""
if ((not old_provision or old_provision == 'thin') and
new_provision == PROVISIONING_COMPRESSED):
return True, True
elif (old_provision == PROVISIONING_COMPRESSED and
(not new_provision or old_provision == 'thin')):
return True, False
# no need change compression
return False, None
def retype_need_change_qos(old_qos=None, new_qos=None):
old = old_qos.get(QOS_SPECS).get(QOS_ID) if old_qos.get(QOS_SPECS) else ''
new = new_qos.get(QOS_SPECS).get(QOS_ID) if new_qos.get(QOS_SPECS) else ''
return old != new
def extract_iscsi_uids(connector):
if 'initiator' not in connector:
msg = _("Host %s doesn't have iSCSI initiator.") % connector['host']
@ -281,7 +316,7 @@ def get_backend_qos_specs(volume):
if qos_specs is None:
return None
qos_specs = qos_specs['qos_specs']
qos_specs = qos_specs[QOS_SPECS]
if qos_specs is None:
return None
@ -290,8 +325,8 @@ def get_backend_qos_specs(volume):
if consumer not in BACKEND_QOS_CONSUMERS:
return None
max_iops = qos_specs['specs'].get(QOS_MAX_IOPS)
max_bws = qos_specs['specs'].get(QOS_MAX_BWS)
max_iops = qos_specs[SPECS_OF_QOS].get(QOS_MAX_IOPS)
max_bws = qos_specs[SPECS_OF_QOS].get(QOS_MAX_BWS)
if max_iops is None and max_bws is None:
return None

View File

@ -296,12 +296,29 @@ triggered. Instead, host-assisted volume migration will be triggered:
the storage-assisted volume migration of vol_2 will not be triggered.
Retype volume support
~~~~~~~~~~~~~~~~~~~~~
Unity driver supports to change a volume's type after its creation.
.. code-block:: console
$ cinder retype [--migration-policy <never|on-demand>] <volume> <volume-type>
The --migration-policy is not enabled by default.
Some retype operations will require migration based on back-end support.
In these cases, the storage-assisted migration will be triggered regardless
the --migration-policy. For examples: retype between 'thin' and 'thick', retype
between 'thick' and 'compressed', retype to type(s) current host doesn't
support.
QoS support
~~~~~~~~~~~
Unity driver supports ``maxBWS`` and ``maxIOPS`` specs for the back-end
consumer type. ``maxBWS`` represents the ``Maximum IO/S`` absolute limit,
``maxIOPS`` represents the ``Maximum Bandwidth (KBPS)`` absolute limit on the
consumer type. ``maxBWS`` represents the ``Maximum Bandwidth (KBPS)`` absolute
limit, ``maxIOPS`` represents the ``Maximum IO/S`` absolute limit on the
Unity respectively.
@ -655,3 +672,4 @@ to track specific Block Storage command logs.
# grep "req-3a459e0e-871a-49f9-9796-b63cc48b5015" cinder-volume.log

View File

@ -0,0 +1,4 @@
---
features:
- |
Dell EMC Unity driver: Add efficient retype support when new type uses the same Unity device.