From 02251481e11d0a7fdcc128dc694a26d3372a6bdb Mon Sep 17 00:00:00 2001 From: Felipe Rodrigues Date: Wed, 6 May 2020 23:54:59 +0000 Subject: [PATCH] [NetApp] Fix HTTPS connection for python 3.7 With python3.7, the eventlet is breaking the ssl.py, so the https is not working. This patch fixes it by changing the request library (urllib by requests), the new library can be built over the pyopenssl.py instead of ssl.py. Closes-Bug: #1878993 Change-Id: I9c0b1f332ead25634f3dc3aebfdc8b51dfbc4178 (cherry picked from commit 29622725e411402b835e3e45328a9e695f75ea78) (cherry picked from commit 2dbdace6616d34bfb43c72905dda04c91bcf1dc6) --- .../drivers/netapp/dataontap/client/api.py | 51 ++++++++++--------- .../drivers/netapp/dataontap/client/fakes.py | 4 +- .../netapp/dataontap/client/test_api.py | 47 ++++++++--------- ...993-netapp-fix-https-3eddf9eb5b762f3a.yaml | 6 +++ 4 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml diff --git a/manila/share/drivers/netapp/dataontap/client/api.py b/manila/share/drivers/netapp/dataontap/client/api.py index 833a501416..4c7691d540 100644 --- a/manila/share/drivers/netapp/dataontap/client/api.py +++ b/manila/share/drivers/netapp/dataontap/client/api.py @@ -23,8 +23,9 @@ import re from lxml import etree from oslo_log import log +import requests +from requests import auth import six -from six.moves import urllib from manila import exception from manila.i18n import _ @@ -61,6 +62,7 @@ class NaServer(object): TRANSPORT_TYPE_HTTP = 'http' TRANSPORT_TYPE_HTTPS = 'https' + SSL_CERT_DEFAULT = "/etc/ssl/certs/" SERVER_TYPE_FILER = 'filer' SERVER_TYPE_DFM = 'dfm' URL_FILER = 'servlets/netapp.servlets.admin.XMLrequest_filer' @@ -232,8 +234,8 @@ class NaServer(object): """Invoke the API on the server.""" if na_element and not isinstance(na_element, NaElement): ValueError('NaElement must be supplied to invoke API') - request, request_element = self._create_request(na_element, - enable_tunneling) + request_element = self._create_request(na_element, enable_tunneling) + request_d = request_element.to_string() api_name = na_element.get_name() api_name_matches_regex = (re.match(self._api_trace_pattern, api_name) @@ -242,23 +244,26 @@ class NaServer(object): if self._trace and api_name_matches_regex: LOG.debug("Request: %s", request_element.to_string(pretty=True)) - if (not hasattr(self, '_opener') or not self._opener + if (not hasattr(self, '_session') or not self._session or self._refresh_conn): - self._build_opener() + self._build_session() try: if hasattr(self, '_timeout'): - response = self._opener.open(request, timeout=self._timeout) + response = self._session.post( + self._get_url(), data=request_d, timeout=self._timeout) else: - response = self._opener.open(request) - except urllib.error.HTTPError as e: - raise NaApiError(e.code, e.msg) - except urllib.error.URLError as e: + response = self._session.post( + self._get_url(), data=request_d) + except requests.HTTPError as e: + raise NaApiError(e.errno, e.strerror) + except requests.URLRequired as e: raise exception.StorageCommunicationException(six.text_type(e)) except Exception as e: raise NaApiError(message=e) - response_xml = response.read() - response_element = self._get_result(response_xml) + response_xml = response.text + response_element = self._get_result( + bytes(bytearray(response_xml, encoding='utf-8'))) if self._trace and api_name_matches_regex: LOG.debug("Response: %s", response_element.to_string(pretty=True)) @@ -296,11 +301,7 @@ class NaServer(object): if enable_tunneling: self._enable_tunnel_request(netapp_elem) netapp_elem.add_child_elem(na_element) - request_d = netapp_elem.to_string() - request = urllib.request.Request( - self._get_url(), data=request_d, - headers={'Content-Type': 'text/xml', 'charset': 'utf-8'}) - return request, netapp_elem + return netapp_elem def _enable_tunnel_request(self, netapp_elem): """Enables vserver or vfiler tunneling.""" @@ -341,20 +342,20 @@ class NaServer(object): host = '[%s]' % host return '%s://%s:%s/%s' % (self._protocol, host, self._port, self._url) - def _build_opener(self): + def _build_session(self): if self._auth_style == NaServer.STYLE_LOGIN_PASSWORD: auth_handler = self._create_basic_auth_handler() else: auth_handler = self._create_certificate_auth_handler() - opener = urllib.request.build_opener(auth_handler) - self._opener = opener + + self._session = requests.Session() + self._session.auth = auth_handler + self._session.verify = NaServer.SSL_CERT_DEFAULT + self._session.headers = { + 'Content-Type': 'text/xml', 'charset': 'utf-8'} def _create_basic_auth_handler(self): - password_man = urllib.request.HTTPPasswordMgrWithDefaultRealm() - password_man.add_password(None, self._get_url(), self._username, - self._password) - auth_handler = urllib.request.HTTPBasicAuthHandler(password_man) - return auth_handler + return auth.HTTPBasicAuth(self._username, self._password) def _create_certificate_auth_handler(self): raise NotImplementedError() diff --git a/manila/tests/share/drivers/netapp/dataontap/client/fakes.py b/manila/tests/share/drivers/netapp/dataontap/client/fakes.py index 1c2bc973ea..b65fb12b8d 100644 --- a/manila/tests/share/drivers/netapp/dataontap/client/fakes.py +++ b/manila/tests/share/drivers/netapp/dataontap/client/fakes.py @@ -14,7 +14,7 @@ from lxml import etree import mock -from six.moves import urllib +import requests from manila.share.drivers.netapp.dataontap.client import api @@ -2597,7 +2597,7 @@ FAKE_RESULT_API_ERRNO_VALID.add_attr('errno', '14956') FAKE_RESULT_SUCCESS = api.NaElement('result') FAKE_RESULT_SUCCESS.add_attr('status', 'passed') -FAKE_HTTP_OPENER = urllib.request.build_opener() +FAKE_HTTP_SESSION = requests.Session() FAKE_MANAGE_VOLUME = { 'aggregate': SHARE_AGGREGATE_NAME, diff --git a/manila/tests/share/drivers/netapp/dataontap/client/test_api.py b/manila/tests/share/drivers/netapp/dataontap/client/test_api.py index d3d7f527cf..9b16b54865 100644 --- a/manila/tests/share/drivers/netapp/dataontap/client/test_api.py +++ b/manila/tests/share/drivers/netapp/dataontap/client/test_api.py @@ -20,7 +20,7 @@ Tests for NetApp API layer """ import ddt import mock -from six.moves import urllib +import requests from manila import exception from manila.share.drivers.netapp.dataontap.client import api @@ -188,14 +188,12 @@ class NetAppApiServerTests(test.TestCase): """Tests handling of HTTPError""" na_element = fake.FAKE_NA_ELEMENT self.mock_object(self.root, '_create_request', mock.Mock( - return_value=('abc', fake.FAKE_NA_ELEMENT))) + return_value=fake.FAKE_NA_ELEMENT)) self.mock_object(api, 'LOG') - self.root._opener = fake.FAKE_HTTP_OPENER - self.mock_object(self.root, '_build_opener') - self.mock_object(self.root._opener, 'open', mock.Mock( - side_effect=urllib.error.HTTPError(url='', hdrs='', - fp=None, code='401', - msg='httperror'))) + self.root._session = fake.FAKE_HTTP_SESSION + self.mock_object(self.root, '_build_session') + self.mock_object(self.root._session, 'post', mock.Mock( + side_effect=requests.HTTPError())) self.assertRaises(api.NaApiError, self.root.invoke_elem, na_element) @@ -204,12 +202,12 @@ class NetAppApiServerTests(test.TestCase): """Tests handling of URLError""" na_element = fake.FAKE_NA_ELEMENT self.mock_object(self.root, '_create_request', mock.Mock( - return_value=('abc', fake.FAKE_NA_ELEMENT))) + return_value=fake.FAKE_NA_ELEMENT)) self.mock_object(api, 'LOG') - self.root._opener = fake.FAKE_HTTP_OPENER - self.mock_object(self.root, '_build_opener') - self.mock_object(self.root._opener, 'open', mock.Mock( - side_effect=urllib.error.URLError(reason='urlerror'))) + self.root._session = fake.FAKE_HTTP_SESSION + self.mock_object(self.root, '_build_session') + self.mock_object(self.root._session, 'post', mock.Mock( + side_effect=requests.URLRequired())) self.assertRaises(exception.StorageCommunicationException, self.root.invoke_elem, @@ -219,11 +217,11 @@ class NetAppApiServerTests(test.TestCase): """Tests handling of Unknown Exception""" na_element = fake.FAKE_NA_ELEMENT self.mock_object(self.root, '_create_request', mock.Mock( - return_value=('abc', fake.FAKE_NA_ELEMENT))) + return_value=fake.FAKE_NA_ELEMENT)) self.mock_object(api, 'LOG') - self.root._opener = fake.FAKE_HTTP_OPENER - self.mock_object(self.root, '_build_opener') - self.mock_object(self.root._opener, 'open', mock.Mock( + self.root._session = fake.FAKE_HTTP_SESSION + self.mock_object(self.root, '_build_session') + self.mock_object(self.root._session, 'post', mock.Mock( side_effect=Exception)) exception = self.assertRaises(api.NaApiError, self.root.invoke_elem, @@ -245,15 +243,18 @@ class NetAppApiServerTests(test.TestCase): self.root._trace = trace_enabled self.root._api_trace_pattern = trace_pattern self.mock_object(self.root, '_create_request', mock.Mock( - return_value=('abc', fake.FAKE_NA_ELEMENT))) + return_value=fake.FAKE_NA_ELEMENT)) self.mock_object(api, 'LOG') - self.root._opener = fake.FAKE_HTTP_OPENER - self.mock_object(self.root, '_build_opener') + self.root._session = fake.FAKE_HTTP_SESSION + self.mock_object(self.root, '_build_session') self.mock_object(self.root, '_get_result', mock.Mock( return_value=fake.FAKE_NA_ELEMENT)) - opener_mock = self.mock_object( - self.root._opener, 'open', mock.Mock()) - opener_mock.read.side_effect = ['resp1', 'resp2'] + + response = mock.Mock() + response.text = 'res1' + self.mock_object( + self.root._session, 'post', mock.Mock( + return_value=response)) self.root.invoke_elem(na_element) diff --git a/releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml b/releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml new file mode 100644 index 0000000000..0aa009660c --- /dev/null +++ b/releasenotes/notes/bug-1878993-netapp-fix-https-3eddf9eb5b762f3a.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixed `bug #1878993 `_ + that caused a failure on HTTPS connections within NetApp backend using + python 3.7.