From 68a689b0f3e5bcdd8939fdadef21de38d06f4dd2 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Mon, 1 Oct 2018 15:56:05 +0100 Subject: [PATCH] Clean up header encoding handling in compute API PEP 3333[1] says request and response headers (within the application) should be treated as native `str` (whatever the Python version). It's the job of the WSGI server to translate from `str` to reasonable output on the outgoing socket connection. This was already mostly correct but two issues were discovered while trying to create integration tests that use the value of the location response header when POSTing to create a server. In python2 it was working. In Python3 the header had a value of location: http://192.168.1.76/compute/v2.1/b'http:/192.168.1.76/compute/v2.1/servers/fad04042-850b-443a-9e48-773111cbc981' (note the b'...' bounding the full url on the end). This was happening for two reasons: * nova/api/openstack/compute/servers.py independently encodes the location header to utf-8, instead of using the centralized handling in nova/api/openstack/wsgi.py This meant that the value of the location header would arrive, in Python 3, at the centralized handling as a bytestring. * The centralized handling in nova/api/openstack/wsgi.py was incorrectly using the six.text_type() method. That is simply an alias to unicode in python 2 and str in python3. In python3 when given a bytestring as the only argument object.__str()__ is called on the argument. Which yields b'whatever'. At that stage, the handling in the web server which processes a location header to check for the presence of a host and prefix already at the start of the provided location will find b'...' and do a concatenation without any replace. So, because of all that, this patch includes three changes: * The server creation location header code does no encoding and relies on the centralized handling. * The centralized handling removes the use of text_type() as a function because that is redundant with the safe_encode and safe_decode changes in the same loop. * Doc strings and comments in the ResponseObject are clarified with regard to this encoding topic. Also, comments in Resource._process_stack are updated to correspond with the changes above. The code is not changed, as they are already doing the right thing: The comment was misrepresenting what was going on. There is some duplication of code between these two areas, but the code is too inscrutable for me to be willing to change a part that isn't presenting an explicit bug. Tests for the internal production of the location header are adjusted to reflect these changes. [1] https://www.python.org/dev/peps/pep-3333/#a-note-on-string-types Change-Id: I163c417375678b428ac77aac87ccafaf1f6186ab Closes-Bug: #1795425 --- nova/api/openstack/compute/servers.py | 2 +- nova/api/openstack/wsgi.py | 19 +++++++++++-------- .../openstack/compute/test_server_actions.py | 8 ++++++-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/nova/api/openstack/compute/servers.py b/nova/api/openstack/compute/servers.py index d8b874edd9a3..8347a535a56f 100644 --- a/nova/api/openstack/compute/servers.py +++ b/nova/api/openstack/compute/servers.py @@ -67,7 +67,7 @@ class ServersController(wsgi.Controller): link = [l for l in robj.obj['server']['links'] if l['rel'] == 'self'] if link: - robj['Location'] = utils.utf8(link[0]['href']) + robj['Location'] = link[0]['href'] # Convenience return return robj diff --git a/nova/api/openstack/wsgi.py b/nova/api/openstack/wsgi.py index 8f7cafbc326c..dec07d8a6c62 100644 --- a/nova/api/openstack/wsgi.py +++ b/nova/api/openstack/wsgi.py @@ -261,6 +261,9 @@ class ResponseObject(object): Utility method for serializing the wrapped object. Returns a webob.Response object. + + Header values are set to the appropriate Python type and + encoding demanded by PEP 3333: whatever the native str type is. """ serializer = self.serializer @@ -271,24 +274,24 @@ class ResponseObject(object): response = webob.Response(body=body) response.status_int = self.code for hdr, val in self._headers.items(): - if not isinstance(val, six.text_type): - val = six.text_type(val) if six.PY2: - # In Py2.X Headers must be byte strings + # In Py2.X Headers must be a UTF-8 encode str. response.headers[hdr] = encodeutils.safe_encode(val) else: - # In Py3.X Headers must be utf-8 strings + # In Py3.X Headers must be a str that was first safely + # encoded to UTF-8 (to catch any bad encodings) and then + # decoded back to a native str. response.headers[hdr] = encodeutils.safe_decode( encodeutils.safe_encode(val)) # Deal with content_type if not isinstance(content_type, six.text_type): content_type = six.text_type(content_type) if six.PY2: - # In Py2.X Headers must be byte strings + # In Py2.X Headers must be a UTF-8 encode str. response.headers['Content-Type'] = encodeutils.safe_encode( content_type) else: - # In Py3.X Headers must be utf-8 strings + # In Py3.X Headers must be a str. response.headers['Content-Type'] = encodeutils.safe_decode( encodeutils.safe_encode(content_type)) return response @@ -558,10 +561,10 @@ class Resource(wsgi.Application): if not isinstance(val, six.text_type): val = six.text_type(val) if six.PY2: - # In Py2.X Headers must be byte strings + # In Py2.X Headers must be UTF-8 encoded string response.headers[hdr] = encodeutils.safe_encode(val) else: - # In Py3.X Headers must be utf-8 strings + # In Py3.X Headers must be a string response.headers[hdr] = encodeutils.safe_decode( encodeutils.safe_encode(val)) diff --git a/nova/tests/unit/api/openstack/compute/test_server_actions.py b/nova/tests/unit/api/openstack/compute/test_server_actions.py index 53f8c3cc8924..5d612569a443 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_actions.py +++ b/nova/tests/unit/api/openstack/compute/test_server_actions.py @@ -289,7 +289,9 @@ class ServerActionsControllerTestV21(test.TestCase): self.assertEqual(len(body['server']['adminPass']), CONF.password_length) - self.assertEqual(robj['location'], self_href.encode('utf-8')) + self.assertEqual(robj['location'], self_href) + # pep3333 requires applications produces headers which are str + self.assertEqual(str, type(robj['location'])) def test_rebuild_instance_with_image_uuid(self): info = dict(image_href_in_call=None) @@ -343,7 +345,9 @@ class ServerActionsControllerTestV21(test.TestCase): self.assertEqual(body['server']['image']['id'], '2') self.assertNotIn("adminPass", body['server']) - self.assertEqual(robj['location'], self_href.encode('utf-8')) + self.assertEqual(robj['location'], self_href) + # pep3333 requires applications produces headers which are str + self.assertEqual(str, type(robj['location'])) def test_rebuild_raises_conflict_on_invalid_state(self): body = {