openstack
/
cinder
Code Issues Proposed changes

Merge "3PAR: Enable HPE-3PAR Compression Feature"

This commit is contained in:
Jenkins 2017-04-18 17:11:06 +00:00 committed by Gerrit Code Review
parent 18be8fd750 da181434d0
commit ff2cba6a4f
3 changed files with 242 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py
View File

@ -100,6 +100,7 @@ class HPE3PARBaseDriver(object):
CLONE_ID = 'd03338a9-9115-48a3-8dfc-000000000000'
VOLUME_TYPE_ID_REPLICATED = 'be9181f1-4040-46f2-8298-e7532f2bf9db'
VOLUME_TYPE_ID_DEDUP = 'd03338a9-9115-48a3-8dfc-11111111111'
VOL_TYPE_ID_DEDUP_COMPRESS = 'd03338a9-9115-48a3-8dfc-33333333333'
VOLUME_TYPE_ID_FLASH_CACHE = 'd03338a9-9115-48a3-8dfc-22222222222'
VOLUME_NAME = 'volume-' + VOLUME_ID
SRC_CG_VOLUME_NAME = 'volume-' + SRC_CG_VOLUME_ID
@ -200,6 +201,14 @@ class HPE3PARBaseDriver(object):
'volume_type_id': None,
'encryption_key_id': 'fake_key'}
volume_dedup_compression = {'name': VOLUME_NAME,
'id': VOLUME_ID,
'display_name': 'Foo Volume',
'size': 16,
'host': FAKE_CINDER_HOST,
'volume_type': 'dedup_compression',
'volume_type_id': VOL_TYPE_ID_DEDUP_COMPRESS}
volume_dedup = {'name': VOLUME_NAME,
'id': VOLUME_ID,
'display_name': 'Foo Volume',
@ -284,6 +293,15 @@ class HPE3PARBaseDriver(object):
'deleted_at': None,
'id': VOLUME_TYPE_ID_REPLICATED}
volume_type_dedup_compression = {'name': 'dedup',
'deleted': False,
'updated_at': None,
'extra_specs': {'cpg': HPE3PAR_CPG2,
'provisioning': 'dedup',
'compression': 'true'},
'deleted_at': None,
'id': VOL_TYPE_ID_DEDUP_COMPRESS}
volume_type_dedup = {'name': 'dedup',
'deleted': False,
'updated_at': None,
@ -543,6 +561,11 @@ class HPE3PARBaseDriver(object):
'minor': 3,
'revision': 1}
wsapi_version_for_compression = {'major': 1,
'build': 30301215,
'minor': 6,
'revision': 0}
wsapi_version_for_dedup = {'major': 1,
'build': 30201120,
'minor': 4,
@ -559,7 +582,7 @@ class HPE3PARBaseDriver(object):
'revision': 0}
# Use this to point to latest version of wsapi
wsapi_version_latest = wsapi_version_for_remote_copy
wsapi_version_latest = wsapi_version_for_compression
standard_login = [
mock.call.login(HPE3PAR_USER_NAME, HPE3PAR_USER_PASS),
@ -1185,6 +1208,69 @@ class HPE3PARBaseDriver(object):
'provider_location': self.CLIENT_ID},
return_model)
@mock.patch.object(volume_types, 'get_volume_type')
def test_create_volume_dedup_compression(self, _mock_volume_types):
# setup_mock_client drive with default configuration
# and return the mock HTTP 3PAR client
mock_client = self.setup_driver()
_mock_volume_types.return_value = {
'name': 'dedup_compression',
'extra_specs': {
'cpg': HPE3PAR_CPG_QOS,
'snap_cpg': HPE3PAR_CPG_SNAP,
'vvs_name': self.VVS_NAME,
'qos': self.QOS,
'hpe3par:provisioning': 'dedup',
'hpe3par:compression': 'True',
'volume_type': self.volume_type_dedup_compression}}
mock_client.getStorageSystemInfo.return_value = {
'id': self.CLIENT_ID,
'serialNumber': '1234',
'licenseInfo': {
'licenses': [{'name': 'Compression'},
{'name': 'Thin Provisioning (102400G)'},
{'name': 'System Reporter'}]
}
}
with mock.patch.object(hpecommon.HPE3PARCommon,
'_create_client') as mock_create_client:
mock_create_client.return_value = mock_client
return_model = self.driver.create_volume(
self.volume_dedup_compression)
comment = Comment({
"volume_type_name": "dedup_compression",
"display_name": "Foo Volume",
"name": "volume-d03338a9-9115-48a3-8dfc-35cdfcdc15a7",
"volume_type_id": "d03338a9-9115-48a3-8dfc-33333333333",
"volume_id": "d03338a9-9115-48a3-8dfc-35cdfcdc15a7",
"qos": {},
"type": "OpenStack"})
expectedcall = [
mock.call.getStorageSystemInfo()]
expected = [
mock.call.getCPG(HPE3PAR_CPG),
mock.call.getStorageSystemInfo(),
mock.call.createVolume(
self.VOLUME_3PAR_NAME,
HPE3PAR_CPG,
16384, {
'comment': comment,
'tpvv': False,
'tdvv': True,
'compression': True,
'snapCPG': HPE3PAR_CPG_SNAP})]
mock_client.assert_has_calls(
self.standard_login +
expectedcall +
self.standard_logout +
self.standard_login +
expected +
self.standard_logout)
self.assertIsNone(return_model)
@mock.patch.object(volume_types, 'get_volume_type')
def test_create_volume_dedup(self, _mock_volume_types):
# setup_mock_client drive with default configuration
@ -1594,8 +1680,9 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'OpenStackCPG',
'conversionOperation': 1,
'compression': False,
'tuneOperation': 1}),
mock.call.getTask(1)
mock.call.getTask(1),
]
mock_client.assert_has_calls(expected + self.standard_logout)
@ -1619,7 +1706,7 @@ class HPE3PARBaseDriver(object):
True, False, False, True, None, None,
self.QOS_SPECS, self.RETYPE_QOS_SPECS,
None, None,
"{}")
"{}", None)
expected = [
mock.call.createVolumeSet('vvs-0DM4qZEVSKON-DXN-NwVpw', None),
@ -1654,16 +1741,19 @@ class HPE3PARBaseDriver(object):
True, False, False, True, None, None,
self.QOS_SPECS, self.RETYPE_QOS_SPECS,
None, None,
"{}")
"{}", None)
expected = [
mock.call.addVolumeToVolumeSet(u'vvs-0DM4qZEVSKON-DXN-NwVpw',
'osv-0DM4qZEVSKON-DXN-NwVpw'),
mock.call.modifyVolume('osv-0DM4qZEVSKON-DXN-NwVpw',
{'action': 6,
'userCPG': 'any_cpg',
'conversionOperation': 3,
'compression': False,
'tuneOperation': 1}),
mock.call.getTask(1)]
mock_client.assert_has_calls(expected)
mock_client.assert_has_calls(expected)
def test_delete_volume(self):
# setup_mock_client drive with default configuration
@ -1826,6 +1916,10 @@ class HPE3PARBaseDriver(object):
mock_client = self.setup_driver()
mock_client.getVolume.return_value = {'name': mock.ANY}
mock_client.copyVolume.return_value = {'taskid': 1}
mock_client.getStorageSystemInfo.return_value = {
'id': self.CLIENT_ID,
'serialNumber': 'XXXXXXX'}
with mock.patch.object(hpecommon.HPE3PARCommon,
'_create_client') as mock_create_client:
mock_create_client.return_value = mock_client
@ -1842,6 +1936,8 @@ class HPE3PARBaseDriver(object):
'size': 2, 'status': 'available'}
model_update = self.driver.create_cloned_volume(volume, src_vref)
self.assertIsNone(model_update)
expectedcall = [
mock.call.getStorageSystemInfo()]
expected = [
mock.call.copyVolume(
@ -1852,6 +1948,9 @@ class HPE3PARBaseDriver(object):
'tdvv': False, 'online': True})]
mock_client.assert_has_calls(
self.standard_login +
expectedcall +
self.standard_logout +
self.standard_login +
expected +
self.standard_logout)
@ -2120,7 +2219,10 @@ class HPE3PARBaseDriver(object):
osv_matcher = 'osv-' + volume_name_3par
comment = Comment({"qos": {}, "display_name": "Foo Volume"})
comment = Comment({
"display_name": "Foo Volume",
"qos": {},
})
expected = [
mock.call.modifyVolume(
@ -2131,6 +2233,7 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'CPG-FC1',
'conversionOperation': 1,
'compression': False,
'tuneOperation': 1}),
mock.call.getTask(mock.ANY)
]
@ -2206,7 +2309,8 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'CPG-FC1',
'conversionOperation': 1,
'tuneOperation': 1}),
'tuneOperation': 1,
'compression': False}),
mock.call.getTask(mock.ANY)
]
@ -2303,7 +2407,8 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'CPG-FC1',
'conversionOperation': 1,
'tuneOperation': 1}),
'tuneOperation': 1,
'compression': False}),
mock.call.getTask(mock.ANY),
]
@ -2356,7 +2461,8 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'OpenStackCPG',
'conversionOperation': 1,
'tuneOperation': 1}),
'tuneOperation': 1,
'compression': False}),
mock.call.getTask(1),
mock.call.logout()
]
@ -3323,7 +3429,8 @@ class HPE3PARBaseDriver(object):
osv_matcher,
{'action': 6,
'userCPG': HPE3PAR_CPG,
'conversionOperation': 1, 'tuneOperation': 1}),
'conversionOperation': 1, 'tuneOperation': 1,
'compression': False}),
mock.call.getTask(1)
]
@ -3445,7 +3552,8 @@ class HPE3PARBaseDriver(object):
{'action': 6,
'userCPG': 'CPGNOTUSED',
'conversionOperation': 1,
'tuneOperation': 1}),
'tuneOperation': 1,
'compression': False}),
mock.call.getTask(1)
]
@ -4814,7 +4922,6 @@ class TestHPE3PARFCDriver(HPE3PARBaseDriver, test.TestCase):
}}}
def setup_driver(self, config=None, mock_conf=None, wsapi_version=None):
self.ctxt = context.get_admin_context()
mock_client = self.setup_mock_client(
conf=config,

129
cinder/volume/drivers/hpe/hpe_3par_common.py
View File

@ -74,6 +74,7 @@ LOG = logging.getLogger(__name__)
MIN_CLIENT_VERSION = '4.2.0'
DEDUP_API_VERSION = 30201120
FLASH_CACHE_API_VERSION = 30201200
COMPRESSION_API_VERSION = 30301215
SRSTATLD_API_VERSION = 30201200
REMOTE_COPY_API_VERSION = 30202290
@ -252,10 +253,11 @@ class HPE3PARCommon(object):
Bug #1661541
3.0.29 - Fix convert snapshot volume to base volume type. bug #1656186
3.0.30 - Handle manage and unmanage hosts present. bug #1648067
3.0.31 - Enable HPE-3PAR Compression Feature.
"""
VERSION = "3.0.30"
VERSION = "3.0.31"
stats = {}
@ -289,6 +291,7 @@ class HPE3PARCommon(object):
THIN_PROV_LIC = "Thin Provisioning"
REMOTE_COPY_LIC = "Remote Copy"
SYSTEM_REPORTER_LIC = "System Reporter"
COMPRESSION_LIC = "Compression"
# Valid values for volume type extra specs
# The first value in the list is the default value
@ -308,7 +311,7 @@ class HPE3PARCommon(object):
'priority']
qos_priority_level = {'low': 1, 'normal': 2, 'high': 3}
hpe3par_valid_keys = ['cpg', 'snap_cpg', 'provisioning', 'persona', 'vvs',
'flash_cache']
'flash_cache', 'compression']
def __init__(self, config, active_backend_id=None):
self.config = config
@ -571,8 +574,16 @@ class HPE3PARCommon(object):
cpg = type_info['cpg']
tpvv = type_info.get('tpvv', False)
tdvv = type_info.get('tdvv', False)
compression = self.get_compression_policy(
type_info['hpe3par_keys'])
optional = {'online': True, 'snapCPG': cpg,
'tpvv': tpvv, 'tdvv': tdvv}
if compression is not None:
optional['compression'] = compression
self.client.copyVolume(snap_name, volume_name, cpg, optional)
self.client.addVolumeToVolumeSet(vvs_name, volume_name)
@ -1259,6 +1270,7 @@ class HPE3PARCommon(object):
thin_support = True
remotecopy_support = True
sr_support = True
compression_support = False
if 'licenseInfo' in info:
if 'licenses' in info['licenseInfo']:
valid_licenses = info['licenseInfo']['licenses']
@ -1274,6 +1286,9 @@ class HPE3PARCommon(object):
sr_support = self._check_license_enabled(
valid_licenses, self.SYSTEM_REPORTER_LIC,
"System_reporter_support")
compression_support = self._check_license_enabled(
valid_licenses, self.COMPRESSION_LIC,
"Compression")
for cpg_name in self._client_conf['hpe3par_cpg']:
try:
@ -1364,6 +1379,7 @@ class HPE3PARCommon(object):
'goodness_function': goodness_function,
'multiattach': False,
'consistencygroup_support': True,
'compression': compression_support,
}
if remotecopy_support:
@ -1610,6 +1626,40 @@ class HPE3PARCommon(object):
return None
def get_compression_policy(self, hpe3par_keys):
if hpe3par_keys is not None:
# here it should return true/false/None
val = self._get_key_value(hpe3par_keys, 'compression', None)
compression_support = False
if val is not None:
info = self.client.getStorageSystemInfo()
if 'licenseInfo' in info:
if 'licenses' in info['licenseInfo']:
valid_licenses = info['licenseInfo']['licenses']
compression_support = self._check_license_enabled(
valid_licenses, self.COMPRESSION_LIC,
"Compression")
# here check the wsapi version
if self.API_VERSION < COMPRESSION_API_VERSION:
err = (_("Compression Policy requires "
"WSAPI version '%(compression_version)s' "
"version '%(version)s' is installed.") %
{'compression_version': COMPRESSION_API_VERSION,
'version': self.API_VERSION})
LOG.error(err)
raise exception.InvalidInput(reason=err)
else:
if val.lower() == 'true':
if not compression_support:
msg = _('Compression is not supported on '
'underlying hardware')
LOG.error(msg)
raise exception.InvalidInput(reason=msg)
return True
else:
return False
return None
def _set_flash_cache_policy_in_vvs(self, flash_cache, vvs_name):
# Update virtual volume set
if flash_cache:
@ -1855,6 +1905,8 @@ class HPE3PARCommon(object):
tdvv = type_info['tdvv']
flash_cache = self.get_flash_cache_policy(
type_info['hpe3par_keys'])
compression = self.get_compression_policy(
type_info['hpe3par_keys'])
cg_id = volume.get('consistencygroup_id', None)
if cg_id:
@ -1879,6 +1931,10 @@ class HPE3PARCommon(object):
capacity = self._capacity_from_size(volume['size'])
volume_name = self._get_3par_vol_name(volume['id'])
if compression is not None:
extras['compression'] = compression
self.client.createVolume(volume_name, cpg, capacity, extras)
if qos or vvs_name or flash_cache is not None:
try:
@ -1919,7 +1975,7 @@ class HPE3PARCommon(object):
provider_location=self.client.id)
def _copy_volume(self, src_name, dest_name, cpg, snap_cpg=None,
tpvv=True, tdvv=False):
tpvv=True, tdvv=False, compression=None):
# Virtual volume sets are not supported with the -online option
LOG.debug('Creating clone of a volume %(src)s to %(dest)s.',
{'src': src_name, 'dest': dest_name})
@ -1931,6 +1987,10 @@ class HPE3PARCommon(object):
if self.API_VERSION >= DEDUP_API_VERSION:
optional['tdvv'] = tdvv
if (compression is not None and
self.API_VERSION >= COMPRESSION_API_VERSION):
optional['compression'] = compression
body = self.client.copyVolume(src_name, dest_name, cpg, optional)
return body['taskid']
@ -2042,12 +2102,15 @@ class HPE3PARCommon(object):
type_info = self.get_volume_settings_from_type(volume)
cpg = type_info['cpg']
compression_val = self.get_compression_policy(
type_info['hpe3par_keys'])
# make the 3PAR copy the contents.
# can't delete the original until the copy is done.
self._copy_volume(src_vol_name, vol_name, cpg=cpg,
snap_cpg=type_info['snap_cpg'],
tpvv=type_info['tpvv'],
tdvv=type_info['tdvv'])
tdvv=type_info['tdvv'],
compression=compression_val)
# v2 replication check
replication_flag = False
@ -2362,7 +2425,6 @@ class HPE3PARCommon(object):
'status': volume['status']}
LOG.debug('enter: migrate_volume: id=%(id)s, host=%(host)s, '
'status=%(status)s.', dbg)
ret = False, None
if volume['status'] in ['available', 'in-use']:
@ -2453,10 +2515,13 @@ class HPE3PARCommon(object):
volume_name = self._get_3par_vol_name(volume['id'])
temp_vol_name = volume_name.replace("osv-", "omv-")
compression = self.get_compression_policy(
type_info['hpe3par_keys'])
# Create a physical copy of the volume
task_id = self._copy_volume(volume_name, temp_vol_name,
cpg, cpg, type_info['tpvv'],
type_info['tdvv'])
type_info['tdvv'],
compression)
LOG.debug('Copy volume scheduled: convert_to_base_volume: '
'id=%s.', volume['id'])
@ -2632,7 +2697,7 @@ class HPE3PARCommon(object):
return portPos
def tune_vv(self, old_tpvv, new_tpvv, old_tdvv, new_tdvv,
old_cpg, new_cpg, volume_name):
old_cpg, new_cpg, volume_name, new_compression):
"""Tune the volume to change the userCPG and/or provisioningType.
The volume will be modified/tuned/converted to the new userCPG and
@ -2643,6 +2708,10 @@ class HPE3PARCommon(object):
either be done or it is in a state that we need to treat as an error.
"""
compression = False
if new_compression is not None:
compression = new_compression
if old_tpvv == new_tpvv and old_tdvv == new_tdvv:
if new_cpg != old_cpg:
LOG.info("Modifying %(volume_name)s userCPG "
@ -2681,12 +2750,21 @@ class HPE3PARCommon(object):
{'volume_name': volume_name, 'new_cpg': new_cpg})
try:
response, body = self.client.modifyVolume(
volume_name,
{'action': 6,
'tuneOperation': 1,
'userCPG': new_cpg,
'conversionOperation': cop})
if self.API_VERSION < COMPRESSION_API_VERSION:
response, body = self.client.modifyVolume(
volume_name,
{'action': 6,
'tuneOperation': 1,
'userCPG': new_cpg,
'conversionOperation': cop})
else:
response, body = self.client.modifyVolume(
volume_name,
{'action': 6,
'tuneOperation': 1,
'userCPG': new_cpg,
'compression': compression,
'conversionOperation': cop})
except hpeexceptions.HTTPBadRequest as ex:
if ex.get_code() == 40 and "keepVV" in six.text_type(ex):
# Cannot retype with snapshots because we don't want to
@ -2753,7 +2831,7 @@ class HPE3PARCommon(object):
old_tpvv, new_tpvv, old_tdvv, new_tdvv,
old_vvs, new_vvs, old_qos, new_qos,
old_flash_cache, new_flash_cache,
old_comment):
old_comment, new_compression):
action = "volume:retype"
@ -2784,7 +2862,8 @@ class HPE3PARCommon(object):
'old_flash_cache': old_flash_cache,
'new_flash_cache': new_flash_cache,
'new_type_name': new_type_name, 'new_type_id': new_type_id,
'old_comment': old_comment
'old_comment': old_comment,
'new_compression': new_compression
})
def _retype_from_old_to_new(self, volume, new_type, old_volume_settings,
@ -2828,6 +2907,9 @@ class HPE3PARCommon(object):
new_persona = new_hpe3par_keys['persona']
new_flash_cache = self.get_flash_cache_policy(new_hpe3par_keys)
# it will return None / True /False$
new_compression = self.get_compression_policy(new_hpe3par_keys)
old_qos = old_volume_settings['qos']
old_vvs = old_volume_settings['vvs_name']
old_hpe3par_keys = old_volume_settings['hpe3par_keys']
@ -2854,7 +2936,7 @@ class HPE3PARCommon(object):
old_snap_cpg, new_snap_cpg, old_tpvv, new_tpvv,
old_tdvv, new_tdvv, old_vvs, new_vvs,
old_qos, new_qos, old_flash_cache, new_flash_cache,
old_comment)
old_comment, new_compression)
if host:
return True, self._get_model_update(host['host'], new_cpg)
@ -3564,6 +3646,7 @@ class ModifyVolumeTask(flow_utils.CinderTask):
def _get_new_comment(self, old_comment, new_vvs, new_qos,
new_type_name, new_type_id):
# Modify the comment during ModifyVolume
comment_dict = dict(ast.literal_eval(old_comment))
if 'vvs' in comment_dict:
@ -3634,19 +3717,21 @@ class TuneVolumeTask(flow_utils.CinderTask):
"""Task to change a volume's CPG and/or provisioning type.
This is a task for changing the CPG and/or provisioning type. It is
intended for use during retype(). This task has no revert. The current
design is to do this task last and do revert-able tasks first. Un-doing a
tunevv can be expensive and should be avoided.
This is a task for changing the CPG and/or provisioning type.
It is intended for use during retype().
This task has no revert. The current design is to do this task last
and do revert-able tasks first. Un-doing a tunevv can be expensive
and should be avoided.
"""
def __init__(self, action, **kwargs):
super(TuneVolumeTask, self).__init__(addons=[action])
def execute(self, common, old_tpvv, new_tpvv, old_tdvv, new_tdvv,
old_cpg, new_cpg, volume_name):
old_cpg, new_cpg, volume_name, new_compression):
common.tune_vv(old_tpvv, new_tpvv, old_tdvv, new_tdvv,
old_cpg, new_cpg, volume_name)
old_cpg, new_cpg, volume_name, new_compression)
class ModifySpecsTask(flow_utils.CinderTask):

16
releasenotes/notes/Enable-HPE-3PAR-Compression-Feature-90e4de4b64a74a46.yaml Normal file
View File

@ -0,0 +1,16 @@
---
prelude: >
Enabling Compression Feature On 3PAR
features:
- |
* This adds following functionalities
* Creating thin/dedup compresssed volume.
* Retype for tpvv/tdvv volumes to be compressed.
* Migration of compressed volumes.
* Create compressed volume from compressed volume/snapshot source
* Compression support to create cg from source