From da181434d01dadc4b1adb17ee6ceb979c733d01e Mon Sep 17 00:00:00 2001 From: Swapnil Nilangekar Date: Fri, 9 Dec 2016 05:28:22 -0700 Subject: [PATCH] 3PAR: Enable HPE-3PAR Compression Feature 3PAR provides tpvv/tdvv provisioned volume types to be compressed. This adds following functionalities * Creating thin/dedup compresssed volume. * Retype for already exsting tpvv/tdvv volumes to be compressed. * Migration of compressed volumes. * Create compressed volume from compressed volume/snapshot source * Compression support to create cg from source Change-Id: I6f9261ec893f714766f1500382ad830e3aa38c92 Implements: blueprint 3par-compression-enable-support --- .../unit/volume/drivers/hpe/test_hpe3par.py | 131 ++++++++++++++++-- cinder/volume/drivers/hpe/hpe_3par_common.py | 129 ++++++++++++++--- ...-Compression-Feature-90e4de4b64a74a46.yaml | 16 +++ 3 files changed, 242 insertions(+), 34 deletions(-) create mode 100644 releasenotes/notes/Enable-HPE-3PAR-Compression-Feature-90e4de4b64a74a46.yaml diff --git a/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py b/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py index 0081e199e29..b3078a64999 100644 --- a/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py +++ b/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py @@ -100,6 +100,7 @@ class HPE3PARBaseDriver(object): CLONE_ID = 'd03338a9-9115-48a3-8dfc-000000000000' VOLUME_TYPE_ID_REPLICATED = 'be9181f1-4040-46f2-8298-e7532f2bf9db' VOLUME_TYPE_ID_DEDUP = 'd03338a9-9115-48a3-8dfc-11111111111' + VOL_TYPE_ID_DEDUP_COMPRESS = 'd03338a9-9115-48a3-8dfc-33333333333' VOLUME_TYPE_ID_FLASH_CACHE = 'd03338a9-9115-48a3-8dfc-22222222222' VOLUME_NAME = 'volume-' + VOLUME_ID SRC_CG_VOLUME_NAME = 'volume-' + SRC_CG_VOLUME_ID @@ -200,6 +201,14 @@ class HPE3PARBaseDriver(object): 'volume_type_id': None, 'encryption_key_id': 'fake_key'} + volume_dedup_compression = {'name': VOLUME_NAME, + 'id': VOLUME_ID, + 'display_name': 'Foo Volume', + 'size': 16, + 'host': FAKE_CINDER_HOST, + 'volume_type': 'dedup_compression', + 'volume_type_id': VOL_TYPE_ID_DEDUP_COMPRESS} + volume_dedup = {'name': VOLUME_NAME, 'id': VOLUME_ID, 'display_name': 'Foo Volume', @@ -284,6 +293,15 @@ class HPE3PARBaseDriver(object): 'deleted_at': None, 'id': VOLUME_TYPE_ID_REPLICATED} + volume_type_dedup_compression = {'name': 'dedup', + 'deleted': False, + 'updated_at': None, + 'extra_specs': {'cpg': HPE3PAR_CPG2, + 'provisioning': 'dedup', + 'compression': 'true'}, + 'deleted_at': None, + 'id': VOL_TYPE_ID_DEDUP_COMPRESS} + volume_type_dedup = {'name': 'dedup', 'deleted': False, 'updated_at': None, @@ -543,6 +561,11 @@ class HPE3PARBaseDriver(object): 'minor': 3, 'revision': 1} + wsapi_version_for_compression = {'major': 1, + 'build': 30301215, + 'minor': 6, + 'revision': 0} + wsapi_version_for_dedup = {'major': 1, 'build': 30201120, 'minor': 4, @@ -559,7 +582,7 @@ class HPE3PARBaseDriver(object): 'revision': 0} # Use this to point to latest version of wsapi - wsapi_version_latest = wsapi_version_for_remote_copy + wsapi_version_latest = wsapi_version_for_compression standard_login = [ mock.call.login(HPE3PAR_USER_NAME, HPE3PAR_USER_PASS), @@ -1185,6 +1208,69 @@ class HPE3PARBaseDriver(object): 'provider_location': self.CLIENT_ID}, return_model) + @mock.patch.object(volume_types, 'get_volume_type') + def test_create_volume_dedup_compression(self, _mock_volume_types): + # setup_mock_client drive with default configuration + # and return the mock HTTP 3PAR client + + mock_client = self.setup_driver() + + _mock_volume_types.return_value = { + 'name': 'dedup_compression', + 'extra_specs': { + 'cpg': HPE3PAR_CPG_QOS, + 'snap_cpg': HPE3PAR_CPG_SNAP, + 'vvs_name': self.VVS_NAME, + 'qos': self.QOS, + 'hpe3par:provisioning': 'dedup', + 'hpe3par:compression': 'True', + 'volume_type': self.volume_type_dedup_compression}} + mock_client.getStorageSystemInfo.return_value = { + 'id': self.CLIENT_ID, + 'serialNumber': '1234', + 'licenseInfo': { + 'licenses': [{'name': 'Compression'}, + {'name': 'Thin Provisioning (102400G)'}, + {'name': 'System Reporter'}] + } + } + with mock.patch.object(hpecommon.HPE3PARCommon, + '_create_client') as mock_create_client: + mock_create_client.return_value = mock_client + + return_model = self.driver.create_volume( + self.volume_dedup_compression) + comment = Comment({ + "volume_type_name": "dedup_compression", + "display_name": "Foo Volume", + "name": "volume-d03338a9-9115-48a3-8dfc-35cdfcdc15a7", + "volume_type_id": "d03338a9-9115-48a3-8dfc-33333333333", + "volume_id": "d03338a9-9115-48a3-8dfc-35cdfcdc15a7", + "qos": {}, + "type": "OpenStack"}) + expectedcall = [ + mock.call.getStorageSystemInfo()] + expected = [ + mock.call.getCPG(HPE3PAR_CPG), + mock.call.getStorageSystemInfo(), + mock.call.createVolume( + self.VOLUME_3PAR_NAME, + HPE3PAR_CPG, + 16384, { + 'comment': comment, + 'tpvv': False, + 'tdvv': True, + 'compression': True, + 'snapCPG': HPE3PAR_CPG_SNAP})] + mock_client.assert_has_calls( + self.standard_login + + expectedcall + + self.standard_logout + + self.standard_login + + expected + + self.standard_logout) + self.assertIsNone(return_model) + @mock.patch.object(volume_types, 'get_volume_type') def test_create_volume_dedup(self, _mock_volume_types): # setup_mock_client drive with default configuration @@ -1594,8 +1680,9 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'OpenStackCPG', 'conversionOperation': 1, + 'compression': False, 'tuneOperation': 1}), - mock.call.getTask(1) + mock.call.getTask(1), ] mock_client.assert_has_calls(expected + self.standard_logout) @@ -1619,7 +1706,7 @@ class HPE3PARBaseDriver(object): True, False, False, True, None, None, self.QOS_SPECS, self.RETYPE_QOS_SPECS, None, None, - "{}") + "{}", None) expected = [ mock.call.createVolumeSet('vvs-0DM4qZEVSKON-DXN-NwVpw', None), @@ -1654,16 +1741,19 @@ class HPE3PARBaseDriver(object): True, False, False, True, None, None, self.QOS_SPECS, self.RETYPE_QOS_SPECS, None, None, - "{}") + "{}", None) expected = [ + mock.call.addVolumeToVolumeSet(u'vvs-0DM4qZEVSKON-DXN-NwVpw', + 'osv-0DM4qZEVSKON-DXN-NwVpw'), mock.call.modifyVolume('osv-0DM4qZEVSKON-DXN-NwVpw', {'action': 6, 'userCPG': 'any_cpg', 'conversionOperation': 3, + 'compression': False, 'tuneOperation': 1}), mock.call.getTask(1)] - mock_client.assert_has_calls(expected) + mock_client.assert_has_calls(expected) def test_delete_volume(self): # setup_mock_client drive with default configuration @@ -1826,6 +1916,10 @@ class HPE3PARBaseDriver(object): mock_client = self.setup_driver() mock_client.getVolume.return_value = {'name': mock.ANY} mock_client.copyVolume.return_value = {'taskid': 1} + mock_client.getStorageSystemInfo.return_value = { + 'id': self.CLIENT_ID, + 'serialNumber': 'XXXXXXX'} + with mock.patch.object(hpecommon.HPE3PARCommon, '_create_client') as mock_create_client: mock_create_client.return_value = mock_client @@ -1842,6 +1936,8 @@ class HPE3PARBaseDriver(object): 'size': 2, 'status': 'available'} model_update = self.driver.create_cloned_volume(volume, src_vref) self.assertIsNone(model_update) + expectedcall = [ + mock.call.getStorageSystemInfo()] expected = [ mock.call.copyVolume( @@ -1852,6 +1948,9 @@ class HPE3PARBaseDriver(object): 'tdvv': False, 'online': True})] mock_client.assert_has_calls( + self.standard_login + + expectedcall + + self.standard_logout + self.standard_login + expected + self.standard_logout) @@ -2120,7 +2219,10 @@ class HPE3PARBaseDriver(object): osv_matcher = 'osv-' + volume_name_3par - comment = Comment({"qos": {}, "display_name": "Foo Volume"}) + comment = Comment({ + "display_name": "Foo Volume", + "qos": {}, + }) expected = [ mock.call.modifyVolume( @@ -2131,6 +2233,7 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'CPG-FC1', 'conversionOperation': 1, + 'compression': False, 'tuneOperation': 1}), mock.call.getTask(mock.ANY) ] @@ -2206,7 +2309,8 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'CPG-FC1', 'conversionOperation': 1, - 'tuneOperation': 1}), + 'tuneOperation': 1, + 'compression': False}), mock.call.getTask(mock.ANY) ] @@ -2303,7 +2407,8 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'CPG-FC1', 'conversionOperation': 1, - 'tuneOperation': 1}), + 'tuneOperation': 1, + 'compression': False}), mock.call.getTask(mock.ANY), ] @@ -2356,7 +2461,8 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'OpenStackCPG', 'conversionOperation': 1, - 'tuneOperation': 1}), + 'tuneOperation': 1, + 'compression': False}), mock.call.getTask(1), mock.call.logout() ] @@ -3323,7 +3429,8 @@ class HPE3PARBaseDriver(object): osv_matcher, {'action': 6, 'userCPG': HPE3PAR_CPG, - 'conversionOperation': 1, 'tuneOperation': 1}), + 'conversionOperation': 1, 'tuneOperation': 1, + 'compression': False}), mock.call.getTask(1) ] @@ -3445,7 +3552,8 @@ class HPE3PARBaseDriver(object): {'action': 6, 'userCPG': 'CPGNOTUSED', 'conversionOperation': 1, - 'tuneOperation': 1}), + 'tuneOperation': 1, + 'compression': False}), mock.call.getTask(1) ] @@ -4814,7 +4922,6 @@ class TestHPE3PARFCDriver(HPE3PARBaseDriver, test.TestCase): }}} def setup_driver(self, config=None, mock_conf=None, wsapi_version=None): - self.ctxt = context.get_admin_context() mock_client = self.setup_mock_client( conf=config, diff --git a/cinder/volume/drivers/hpe/hpe_3par_common.py b/cinder/volume/drivers/hpe/hpe_3par_common.py index 2c363e07e3e..699d4d1753f 100644 --- a/cinder/volume/drivers/hpe/hpe_3par_common.py +++ b/cinder/volume/drivers/hpe/hpe_3par_common.py @@ -74,6 +74,7 @@ LOG = logging.getLogger(__name__) MIN_CLIENT_VERSION = '4.2.0' DEDUP_API_VERSION = 30201120 FLASH_CACHE_API_VERSION = 30201200 +COMPRESSION_API_VERSION = 30301215 SRSTATLD_API_VERSION = 30201200 REMOTE_COPY_API_VERSION = 30202290 @@ -252,10 +253,11 @@ class HPE3PARCommon(object): Bug #1661541 3.0.29 - Fix convert snapshot volume to base volume type. bug #1656186 3.0.30 - Handle manage and unmanage hosts present. bug #1648067 + 3.0.31 - Enable HPE-3PAR Compression Feature. """ - VERSION = "3.0.30" + VERSION = "3.0.31" stats = {} @@ -289,6 +291,7 @@ class HPE3PARCommon(object): THIN_PROV_LIC = "Thin Provisioning" REMOTE_COPY_LIC = "Remote Copy" SYSTEM_REPORTER_LIC = "System Reporter" + COMPRESSION_LIC = "Compression" # Valid values for volume type extra specs # The first value in the list is the default value @@ -308,7 +311,7 @@ class HPE3PARCommon(object): 'priority'] qos_priority_level = {'low': 1, 'normal': 2, 'high': 3} hpe3par_valid_keys = ['cpg', 'snap_cpg', 'provisioning', 'persona', 'vvs', - 'flash_cache'] + 'flash_cache', 'compression'] def __init__(self, config, active_backend_id=None): self.config = config @@ -571,8 +574,16 @@ class HPE3PARCommon(object): cpg = type_info['cpg'] tpvv = type_info.get('tpvv', False) tdvv = type_info.get('tdvv', False) + + compression = self.get_compression_policy( + type_info['hpe3par_keys']) + optional = {'online': True, 'snapCPG': cpg, 'tpvv': tpvv, 'tdvv': tdvv} + + if compression is not None: + optional['compression'] = compression + self.client.copyVolume(snap_name, volume_name, cpg, optional) self.client.addVolumeToVolumeSet(vvs_name, volume_name) @@ -1259,6 +1270,7 @@ class HPE3PARCommon(object): thin_support = True remotecopy_support = True sr_support = True + compression_support = False if 'licenseInfo' in info: if 'licenses' in info['licenseInfo']: valid_licenses = info['licenseInfo']['licenses'] @@ -1274,6 +1286,9 @@ class HPE3PARCommon(object): sr_support = self._check_license_enabled( valid_licenses, self.SYSTEM_REPORTER_LIC, "System_reporter_support") + compression_support = self._check_license_enabled( + valid_licenses, self.COMPRESSION_LIC, + "Compression") for cpg_name in self._client_conf['hpe3par_cpg']: try: @@ -1364,6 +1379,7 @@ class HPE3PARCommon(object): 'goodness_function': goodness_function, 'multiattach': False, 'consistencygroup_support': True, + 'compression': compression_support, } if remotecopy_support: @@ -1610,6 +1626,40 @@ class HPE3PARCommon(object): return None + def get_compression_policy(self, hpe3par_keys): + if hpe3par_keys is not None: + # here it should return true/false/None + val = self._get_key_value(hpe3par_keys, 'compression', None) + compression_support = False + if val is not None: + info = self.client.getStorageSystemInfo() + if 'licenseInfo' in info: + if 'licenses' in info['licenseInfo']: + valid_licenses = info['licenseInfo']['licenses'] + compression_support = self._check_license_enabled( + valid_licenses, self.COMPRESSION_LIC, + "Compression") + # here check the wsapi version + if self.API_VERSION < COMPRESSION_API_VERSION: + err = (_("Compression Policy requires " + "WSAPI version '%(compression_version)s' " + "version '%(version)s' is installed.") % + {'compression_version': COMPRESSION_API_VERSION, + 'version': self.API_VERSION}) + LOG.error(err) + raise exception.InvalidInput(reason=err) + else: + if val.lower() == 'true': + if not compression_support: + msg = _('Compression is not supported on ' + 'underlying hardware') + LOG.error(msg) + raise exception.InvalidInput(reason=msg) + return True + else: + return False + return None + def _set_flash_cache_policy_in_vvs(self, flash_cache, vvs_name): # Update virtual volume set if flash_cache: @@ -1855,6 +1905,8 @@ class HPE3PARCommon(object): tdvv = type_info['tdvv'] flash_cache = self.get_flash_cache_policy( type_info['hpe3par_keys']) + compression = self.get_compression_policy( + type_info['hpe3par_keys']) cg_id = volume.get('consistencygroup_id', None) if cg_id: @@ -1879,6 +1931,10 @@ class HPE3PARCommon(object): capacity = self._capacity_from_size(volume['size']) volume_name = self._get_3par_vol_name(volume['id']) + + if compression is not None: + extras['compression'] = compression + self.client.createVolume(volume_name, cpg, capacity, extras) if qos or vvs_name or flash_cache is not None: try: @@ -1919,7 +1975,7 @@ class HPE3PARCommon(object): provider_location=self.client.id) def _copy_volume(self, src_name, dest_name, cpg, snap_cpg=None, - tpvv=True, tdvv=False): + tpvv=True, tdvv=False, compression=None): # Virtual volume sets are not supported with the -online option LOG.debug('Creating clone of a volume %(src)s to %(dest)s.', {'src': src_name, 'dest': dest_name}) @@ -1931,6 +1987,10 @@ class HPE3PARCommon(object): if self.API_VERSION >= DEDUP_API_VERSION: optional['tdvv'] = tdvv + if (compression is not None and + self.API_VERSION >= COMPRESSION_API_VERSION): + optional['compression'] = compression + body = self.client.copyVolume(src_name, dest_name, cpg, optional) return body['taskid'] @@ -2042,12 +2102,15 @@ class HPE3PARCommon(object): type_info = self.get_volume_settings_from_type(volume) cpg = type_info['cpg'] + compression_val = self.get_compression_policy( + type_info['hpe3par_keys']) # make the 3PAR copy the contents. # can't delete the original until the copy is done. self._copy_volume(src_vol_name, vol_name, cpg=cpg, snap_cpg=type_info['snap_cpg'], tpvv=type_info['tpvv'], - tdvv=type_info['tdvv']) + tdvv=type_info['tdvv'], + compression=compression_val) # v2 replication check replication_flag = False @@ -2362,7 +2425,6 @@ class HPE3PARCommon(object): 'status': volume['status']} LOG.debug('enter: migrate_volume: id=%(id)s, host=%(host)s, ' 'status=%(status)s.', dbg) - ret = False, None if volume['status'] in ['available', 'in-use']: @@ -2453,10 +2515,13 @@ class HPE3PARCommon(object): volume_name = self._get_3par_vol_name(volume['id']) temp_vol_name = volume_name.replace("osv-", "omv-") + compression = self.get_compression_policy( + type_info['hpe3par_keys']) # Create a physical copy of the volume task_id = self._copy_volume(volume_name, temp_vol_name, cpg, cpg, type_info['tpvv'], - type_info['tdvv']) + type_info['tdvv'], + compression) LOG.debug('Copy volume scheduled: convert_to_base_volume: ' 'id=%s.', volume['id']) @@ -2632,7 +2697,7 @@ class HPE3PARCommon(object): return portPos def tune_vv(self, old_tpvv, new_tpvv, old_tdvv, new_tdvv, - old_cpg, new_cpg, volume_name): + old_cpg, new_cpg, volume_name, new_compression): """Tune the volume to change the userCPG and/or provisioningType. The volume will be modified/tuned/converted to the new userCPG and @@ -2643,6 +2708,10 @@ class HPE3PARCommon(object): either be done or it is in a state that we need to treat as an error. """ + compression = False + if new_compression is not None: + compression = new_compression + if old_tpvv == new_tpvv and old_tdvv == new_tdvv: if new_cpg != old_cpg: LOG.info("Modifying %(volume_name)s userCPG " @@ -2681,12 +2750,21 @@ class HPE3PARCommon(object): {'volume_name': volume_name, 'new_cpg': new_cpg}) try: - response, body = self.client.modifyVolume( - volume_name, - {'action': 6, - 'tuneOperation': 1, - 'userCPG': new_cpg, - 'conversionOperation': cop}) + if self.API_VERSION < COMPRESSION_API_VERSION: + response, body = self.client.modifyVolume( + volume_name, + {'action': 6, + 'tuneOperation': 1, + 'userCPG': new_cpg, + 'conversionOperation': cop}) + else: + response, body = self.client.modifyVolume( + volume_name, + {'action': 6, + 'tuneOperation': 1, + 'userCPG': new_cpg, + 'compression': compression, + 'conversionOperation': cop}) except hpeexceptions.HTTPBadRequest as ex: if ex.get_code() == 40 and "keepVV" in six.text_type(ex): # Cannot retype with snapshots because we don't want to @@ -2753,7 +2831,7 @@ class HPE3PARCommon(object): old_tpvv, new_tpvv, old_tdvv, new_tdvv, old_vvs, new_vvs, old_qos, new_qos, old_flash_cache, new_flash_cache, - old_comment): + old_comment, new_compression): action = "volume:retype" @@ -2784,7 +2862,8 @@ class HPE3PARCommon(object): 'old_flash_cache': old_flash_cache, 'new_flash_cache': new_flash_cache, 'new_type_name': new_type_name, 'new_type_id': new_type_id, - 'old_comment': old_comment + 'old_comment': old_comment, + 'new_compression': new_compression }) def _retype_from_old_to_new(self, volume, new_type, old_volume_settings, @@ -2828,6 +2907,9 @@ class HPE3PARCommon(object): new_persona = new_hpe3par_keys['persona'] new_flash_cache = self.get_flash_cache_policy(new_hpe3par_keys) + # it will return None / True /False$ + new_compression = self.get_compression_policy(new_hpe3par_keys) + old_qos = old_volume_settings['qos'] old_vvs = old_volume_settings['vvs_name'] old_hpe3par_keys = old_volume_settings['hpe3par_keys'] @@ -2854,7 +2936,7 @@ class HPE3PARCommon(object): old_snap_cpg, new_snap_cpg, old_tpvv, new_tpvv, old_tdvv, new_tdvv, old_vvs, new_vvs, old_qos, new_qos, old_flash_cache, new_flash_cache, - old_comment) + old_comment, new_compression) if host: return True, self._get_model_update(host['host'], new_cpg) @@ -3564,6 +3646,7 @@ class ModifyVolumeTask(flow_utils.CinderTask): def _get_new_comment(self, old_comment, new_vvs, new_qos, new_type_name, new_type_id): + # Modify the comment during ModifyVolume comment_dict = dict(ast.literal_eval(old_comment)) if 'vvs' in comment_dict: @@ -3634,19 +3717,21 @@ class TuneVolumeTask(flow_utils.CinderTask): """Task to change a volume's CPG and/or provisioning type. - This is a task for changing the CPG and/or provisioning type. It is - intended for use during retype(). This task has no revert. The current - design is to do this task last and do revert-able tasks first. Un-doing a - tunevv can be expensive and should be avoided. + This is a task for changing the CPG and/or provisioning type. + It is intended for use during retype(). + + This task has no revert. The current design is to do this task last + and do revert-able tasks first. Un-doing a tunevv can be expensive + and should be avoided. """ def __init__(self, action, **kwargs): super(TuneVolumeTask, self).__init__(addons=[action]) def execute(self, common, old_tpvv, new_tpvv, old_tdvv, new_tdvv, - old_cpg, new_cpg, volume_name): + old_cpg, new_cpg, volume_name, new_compression): common.tune_vv(old_tpvv, new_tpvv, old_tdvv, new_tdvv, - old_cpg, new_cpg, volume_name) + old_cpg, new_cpg, volume_name, new_compression) class ModifySpecsTask(flow_utils.CinderTask): diff --git a/releasenotes/notes/Enable-HPE-3PAR-Compression-Feature-90e4de4b64a74a46.yaml b/releasenotes/notes/Enable-HPE-3PAR-Compression-Feature-90e4de4b64a74a46.yaml new file mode 100644 index 00000000000..06ea6b5c998 --- /dev/null +++ b/releasenotes/notes/Enable-HPE-3PAR-Compression-Feature-90e4de4b64a74a46.yaml @@ -0,0 +1,16 @@ +--- +prelude: > + Enabling Compression Feature On 3PAR +features: + - | + * This adds following functionalities + + * Creating thin/dedup compresssed volume. + + * Retype for tpvv/tdvv volumes to be compressed. + + * Migration of compressed volumes. + + * Create compressed volume from compressed volume/snapshot source + + * Compression support to create cg from source