From 23ddc7cbc668ed9b2e38583ce84924ea09e70994 Mon Sep 17 00:00:00 2001 From: zengyingzhe Date: Sat, 22 Apr 2017 15:33:52 +0800 Subject: [PATCH] Use requests lib for Huawei array connection Updates Huawei driver logic to use requests lib for Huawei array connection instead. This stable branch merging attempt is to fix the problem that Huawei driver cannot login Huawei array, if python env user used is SSL certificate verification default enabled. At this circumstance, Huawei driver will fail to login array, and never operate properly. So it's necessary to merge back to Ocata stable branch for Huawei driver. Change-Id: I397e4edde4d82baceb324c1fe9896b1c0f914194 (cherry picked from commit 59ddeabacabdbbd9e396d1d62ae65c1fdfa3f455) --- .../drivers/huawei/test_huawei_drivers.py | 98 +++++++++++---- cinder/volume/drivers/huawei/rest_client.py | 118 +++++++++--------- 2 files changed, 135 insertions(+), 81 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py b/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py index 47d7003ecdf..7ca9bdc1959 100644 --- a/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py +++ b/cinder/tests/unit/volume/drivers/huawei/test_huawei_drivers.py @@ -19,6 +19,7 @@ import ddt import json import mock import re +import requests import tempfile import unittest from xml.dom import minidom @@ -1202,10 +1203,10 @@ FAKE_GET_METROROUP_ID_RESPONSE = """ # mock login info map MAP_COMMAND_TO_FAKE_RESPONSE = {} -MAP_COMMAND_TO_FAKE_RESPONSE['/xx/sessions'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/xx/sessions/POST'] = ( FAKE_GET_LOGIN_STORAGE_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/sessions'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/sessions/DELETE'] = ( FAKE_LOGIN_OUT_STORAGE_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUN_MIGRATION/POST'] = ( @@ -1218,11 +1219,11 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/LUN_MIGRATION/11/DELETE'] = ( FAKE_COMMON_SUCCESS_RESPONSE) # mock storage info map -MAP_COMMAND_TO_FAKE_RESPONSE['/storagepool'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/storagepool/GET'] = ( FAKE_STORAGE_POOL_RESPONSE) # mock lun info map -MAP_COMMAND_TO_FAKE_RESPONSE['/lun'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lun/POST'] = ( FAKE_LUN_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/lun/11/GET'] = ( @@ -1278,10 +1279,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/associate?TYPE=27&ASSOCIATEOBJTYPE=256' MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup?range=[0-8191]/GET'] = ( FAKE_QUERY_LUN_GROUP_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/POST'] = ( FAKE_QUERY_LUN_GROUP_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate/POST'] = ( FAKE_QUERY_LUN_GROUP_ASSOCIAT_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUNGroup/11/DELETE'] = ( @@ -1331,14 +1332,14 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/lungroup/associate?ID=12&ASSOCIATEOBJTYPE=11' FAKE_COMMON_SUCCESS_RESPONSE) # mock snapshot info map -MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/POST'] = ( FAKE_CREATE_SNAPSHOT_INFO_RESPONSE) # mock snapshot info map MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/11/GET'] = ( FAKE_GET_SNAPSHOT_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/activate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/activate/POST'] = ( FAKE_COMMON_SUCCESS_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/snapshot/stop/PUT'] = ( @@ -1363,10 +1364,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/11/PUT'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/active/11/PUT'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass//POST'] = ( FAKE_QOS_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/ioclass/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) # mock iscsi info map @@ -1380,13 +1381,13 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/eth_port/associate?TYPE=213&ASSOCIATEOBJTYPE' '=257&ASSOCIATEOBJID=11/GET'] = ( FAKE_GET_ETH_ASSOCIATE_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/iscsidevicename'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/iscsidevicename/GET'] = ( FAKE_GET_ISCSI_DEVICE_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator?range=[0-256]/GET'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/GET'] = ( FAKE_ISCSI_INITIATOR_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/iscsi_initiator/POST'] = ( @@ -1415,13 +1416,13 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/host/1/DELETE'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/host/1/GET'] = ( FAKE_GET_HOST_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/host'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/host/POST'] = ( FAKE_CREATE_HOST_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup?range=[0-8191]/GET'] = ( FAKE_GET_ALL_HOST_GROUP_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/GET'] = ( FAKE_GET_HOST_GROUP_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/host/associate?TYPE=14&ID=0' @@ -1445,11 +1446,11 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/host/associate?TYPE=21&' FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/associate'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/hostgroup/associate/POST'] = ( FAKE_COMMON_SUCCESS_RESPONSE) # mock copy info map -MAP_COMMAND_TO_FAKE_RESPONSE['/luncopy'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/luncopy/POST'] = ( FAKE_GET_LUN_COPY_INFO_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/LUNCOPY?range=[0-1023]/GET'] = ( @@ -1465,7 +1466,7 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/LUNCOPY/0/DELETE'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview?range=[0-8191]/GET'] = ( FAKE_GET_MAPPING_VIEW_INFO_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview/POST'] = ( FAKE_GET_MAPPING_VIEW_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/mappingview/PUT'] = ( @@ -1582,7 +1583,7 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/0/GET'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/REMOVE_ASSOCIATE/PUT'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/SMARTCACHEPARTITION/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) MAP_COMMAND_TO_FAKE_RESPONSE['/cachepartition/0/GET'] = ( @@ -1621,10 +1622,10 @@ MAP_COMMAND_TO_FAKE_RESPONSE['/HyperMetroPair/synchronize_hcpair/PUT'] = ( MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror?range=[0-8191]/GET'] = ( FAKE_COMMON_SUCCESS_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/splitmirror/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) -MAP_COMMAND_TO_FAKE_RESPONSE['/smartcachepool/count'] = ( +MAP_COMMAND_TO_FAKE_RESPONSE['/smartcachepool/count/GET'] = ( FAKE_COMMON_FAIL_RESPONSE) FAKE_GET_PORTG_BY_VIEW = """ @@ -2109,7 +2110,7 @@ class FakeClient(rest_client.RestClient): def add_lun_to_cache(self, lunid, cache_id): pass - def do_call(self, url=False, data=None, method=None, calltimeout=4, + def do_call(self, url, data, method, calltimeout=4, log_filter_flag=False): url = url.replace('http://192.0.2.69:8082/deviceManager/rest', '') command = url.replace('/210235G7J20000000000/', '') @@ -5239,3 +5240,58 @@ class HuaweiConfTestCase(test.TestCase): fakefile = open(self.conf.cinder_huawei_conf_file, 'w') fakefile.write(doc.toprettyxml(indent='')) fakefile.close() + + +@ddt.ddt +class HuaweiRestClientTestCase(test.TestCase): + def setUp(self): + super(HuaweiRestClientTestCase, self).setUp() + config = mock.Mock(spec=conf.Configuration) + huawei_conf = FakeHuaweiConf(config, 'iSCSI') + huawei_conf.update_config_value() + self.client = rest_client.RestClient( + config, config.san_address, config.san_user, config.san_password) + + def test_init_http_head(self): + self.client.init_http_head() + self.assertIsNone(self.client.url) + self.assertEqual("keep-alive", + self.client.session.headers["Connection"]) + self.assertEqual("application/json", + self.client.session.headers["Content-Type"]) + self.assertEqual(False, self.client.session.verify) + + @ddt.data('POST', 'PUT', 'GET', 'DELETE') + def test_do_call_method(self, method): + self.client.init_http_head() + + if method: + mock_func = self.mock_object(self.client.session, method.lower()) + else: + mock_func = self.mock_object(self.client.session, 'post') + + self.client.do_call("http://fake-rest-url", None, method) + mock_func.assert_called_once_with("http://fake-rest-url", + timeout=constants.SOCKET_TIMEOUT) + + def test_do_call_method_invalid(self): + self.assertRaises(exception.VolumeBackendAPIException, + self.client.do_call, + "http://fake-rest-url", None, 'fake-method') + + def test_do_call_http_error(self): + self.client.init_http_head() + + fake_res = requests.Response() + fake_res.reason = 'something wrong' + fake_res.status_code = 500 + fake_res.url = "http://fake-rest-url" + + self.mock_object(self.client.session, 'post', return_value=fake_res) + res = self.client.do_call("http://fake-rest-url", None, 'POST') + + expected = {"error": {"code": 500, + "description": + '500 Server Error: something wrong for ' + 'url: http://fake-rest-url'}} + self.assertEqual(expected, res) diff --git a/cinder/volume/drivers/huawei/rest_client.py b/cinder/volume/drivers/huawei/rest_client.py index 0f1ce189e5e..77c3bade5e4 100644 --- a/cinder/volume/drivers/huawei/rest_client.py +++ b/cinder/volume/drivers/huawei/rest_client.py @@ -15,14 +15,12 @@ import json import re +import requests import six -import socket import time from oslo_log import log as logging from oslo_utils import excutils -from six.moves import http_cookiejar -from six.moves import urllib from cinder import exception from cinder.i18n import _, _LE, _LI, _LW @@ -41,7 +39,6 @@ class RestClient(object): self.san_address = san_address self.san_user = san_user self.san_password = san_password - self.init_http_head() self.storage_pools = kwargs.get('storage_pools', self.configuration.storage_pools) self.iscsi_info = kwargs.get('iscsi_info', @@ -49,17 +46,19 @@ class RestClient(object): self.iscsi_default_target_ip = kwargs.get( 'iscsi_default_target_ip', self.configuration.iscsi_default_target_ip) - - def init_http_head(self): - self.cookie = http_cookiejar.CookieJar() + self.session = None self.url = None self.device_id = None - self.headers = { - "Connection": "keep-alive", - "Content-Type": "application/json", - } - def do_call(self, url=None, data=None, method=None, + def init_http_head(self): + self.url = None + self.session = requests.Session() + self.session.headers.update({ + "Connection": "keep-alive", + "Content-Type": "application/json"}) + self.session.verify = False + + def do_call(self, url, data, method, calltimeout=constants.SOCKET_TIMEOUT, log_filter_flag=False): """Send requests to Huawei storage server. @@ -68,43 +67,42 @@ class RestClient(object): """ if self.url: url = self.url + url - handler = urllib.request.HTTPCookieProcessor(self.cookie) - opener = urllib.request.build_opener(handler) - urllib.request.install_opener(opener) - res_json = None + + kwargs = {'timeout': calltimeout} + if data: + kwargs['data'] = json.dumps(data) + + if method in ('POST', 'PUT', 'GET', 'DELETE'): + func = getattr(self.session, method.lower()) + else: + msg = _("Request method %s is invalid.") % method + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) try: - socket.setdefaulttimeout(calltimeout) - if data: - data = json.dumps(data) - req = urllib.request.Request(url, data, self.headers) - if method: - req.get_method = lambda: method - res = urllib.request.urlopen(req).read().decode("utf-8") - - if not log_filter_flag: - LOG.info(_LI('\n\n\n\nRequest URL: %(url)s\n\n' - 'Call Method: %(method)s\n\n' - 'Request Data: %(data)s\n\n' - 'Response Data:%(res)s\n\n'), - {'url': url, - 'method': method, - 'data': data, - 'res': res}) - + res = func(url, **kwargs) except Exception as err: - LOG.error(_LE('Bad response from server: %(url)s.' - ' Error: %(err)s'), {'url': url, 'err': err}) - json_msg = ('{"error":{"code": %s,"description": "Connect to ' - 'server error."}}') % constants.ERROR_CONNECT_TO_SERVER - res_json = json.loads(json_msg) - return res_json + LOG.exception(_LE('Bad response from server: %(url)s.' + ' Error: %(err)s'), {'url': url, 'err': err}) + return {"error": {"code": constants.ERROR_CONNECT_TO_SERVER, + "description": "Connect to server error."}} try: - res_json = json.loads(res) - except Exception as err: - LOG.error(_LE('JSON transfer error: %s.'), err) - raise + res.raise_for_status() + except requests.HTTPError as exc: + return {"error": {"code": exc.response.status_code, + "description": six.text_type(exc)}} + + res_json = res.json() + if not log_filter_flag: + LOG.info(_LI('\n\n\n\nRequest URL: %(url)s\n\n' + 'Call Method: %(method)s\n\n' + 'Request Data: %(data)s\n\n' + 'Response Data:%(res)s\n\n'), + {'url': url, + 'method': method, + 'data': data, + 'res': res_json}) return res_json @@ -117,7 +115,7 @@ class RestClient(object): "password": self.san_password, "scope": "0"} self.init_http_head() - result = self.do_call(url, data, + result = self.do_call(url, data, 'POST', calltimeout=constants.LOGIN_SOCKET_TIMEOUT, log_filter_flag=True) @@ -131,7 +129,7 @@ class RestClient(object): device_id = result['data']['deviceid'] self.device_id = device_id self.url = item_url + device_id - self.headers['iBaseToken'] = result['data']['iBaseToken'] + self.session.headers['iBaseToken'] = result['data']['iBaseToken'] if (result['data']['accountstate'] in (constants.PWD_EXPIRED, constants.PWD_RESET)): self.logout() @@ -204,7 +202,7 @@ class RestClient(object): def create_lun(self, lun_params): url = "/lun" - result = self.call(url, lun_params) + result = self.call(url, lun_params, 'POST') if result['error']['code'] == constants.ERROR_VOLUME_ALREADY_EXIST: lun_id = self.get_lun_id_by_name(lun_params['NAME']) if lun_id: @@ -239,7 +237,7 @@ class RestClient(object): def get_all_pools(self): url = "/storagepool" - result = self.call(url, None, log_filter_flag=True) + result = self.call(url, None, "GET", log_filter_flag=True) msg = _('Query resource pool error.') self._assert_rest_result(result, msg) self._assert_data_in_result(result, msg) @@ -305,7 +303,7 @@ class RestClient(object): data = ({"SNAPSHOTLIST": snapshot_id} if type(snapshot_id) in (list, tuple) else {"SNAPSHOTLIST": [snapshot_id]}) - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Activate snapshot error.')) def create_snapshot(self, lun_id, snapshot_name, snapshot_description): @@ -315,7 +313,7 @@ class RestClient(object): "PARENTTYPE": "11", "DESCRIPTION": snapshot_description, "PARENTID": lun_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create snapshot error.') self._assert_rest_result(result, msg) @@ -381,7 +379,7 @@ class RestClient(object): % srclunid), "TARGETLUN": ("INVALID;%s;INVALID;INVALID;INVALID" % tgtlunid)} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create luncopy error.') self._assert_rest_result(result, msg) @@ -512,7 +510,7 @@ class RestClient(object): def _get_iscsi_tgt_port(self): url = "/iscsidevicename" - result = self.call(url, None) + result = self.call(url, None, 'GET') msg = _('Get iSCSI target port error.') self._assert_rest_result(result, msg) @@ -576,7 +574,7 @@ class RestClient(object): def _create_hostgroup(self, hostgroup_name): url = "/hostgroup" data = {"TYPE": "14", "NAME": hostgroup_name} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create hostgroup error.') self._assert_rest_result(result, msg) @@ -590,7 +588,7 @@ class RestClient(object): "APPTYPE": '0', "GROUPTYPE": '0', "NAME": lungroup_name} - result = self.call(url, data) + result = self.call(url, data, 'POST') msg = _('Create lungroup error.') self._assert_rest_result(result, msg) @@ -695,7 +693,7 @@ class RestClient(object): "NAME": hostname, "OPERATIONSYSTEM": "0", "DESCRIPTION": host_name_before_hash} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add new host error.')) if 'data' in result: @@ -737,7 +735,7 @@ class RestClient(object): "ASSOCIATEOBJTYPE": "21", "ASSOCIATEOBJID": host_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Associate host to hostgroup ' 'error.')) @@ -748,7 +746,7 @@ class RestClient(object): data = {"ID": lungroup_id, "ASSOCIATEOBJTYPE": lun_type, "ASSOCIATEOBJID": lun_id} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Associate lun to lungroup error.')) def remove_lun_from_lungroup(self, lungroup_id, lun_id, @@ -902,7 +900,7 @@ class RestClient(object): def _add_mapping_view(self, name): url = "/mappingview" data = {"NAME": name, "TYPE": "245"} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add mapping view error.')) return result['data']['ID'] @@ -1417,7 +1415,7 @@ class RestClient(object): data.update(qos) url = "/ioclass/" - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Create QoS policy error.')) return result['data']['ID'] @@ -1852,7 +1850,7 @@ class RestClient(object): url = '/fc_initiator/' data = {"TYPE": '223', "ID": ininame} - result = self.call(url, data) + result = self.call(url, data, 'POST') self._assert_rest_result(result, _('Add fc initiator to array error.')) def ensure_fc_initiator_added(self, initiator_name, host_id):