x
/
swift3
Code Issues Proposed changes

Stop using client headers for cross-middleware communication 

Previously, we would use client-accessible headers to pass the S3 access
key, signature, and normalized request to authentication middleware.
Specifically, we would send the following headers:

    Authorization: AWS <access key>:<signature>
    X-Auth-Token: <base64-encoded normalized request>

However, few authentication middleware would validate that the
Authorization header actually started with "AWS ", the only prefix that
Swift3 would actually handle. As a result, the authentication
middlewares had no way to validate that the normalized request came from
swift3 rather than the client itself. This leads to a security hole
wherein an attacker who has captured a single valid request through the
S3 API or who has obtained a valid pre-signed URL may impersonate the
user that issued the request or pre-signed URL indefinitely through the
Swift API.

Now, the S3 authentication information will be placed in a separate
namespace in the WSGI environment, completely inaccessible to the
client. Specifically,

    environ['swift3.auth_details'] = {
        'access_key': <access key>,
        'signature': <signature>,
        'string_to_sign': <normalized request>,
    }

(Note that the normalized request is no longer base64-encoded.)

UpgradeImpact

This is a breaking API change. No currently-deployed authentication
middlewares will work with this. This patch includes a fix for s3_token
(used to authenticate against Keystone); any deployers still using
keystonemiddleware to provide s3_token should switch to using swift3.
Similar changes are being proposed for Swauth and tempauth. Proprietary
authentication middlewares will need to be updated to use the new
environment keys as well. When upgrading Swift3, operators will need to
upgrade their Swift3-capable authentication middleware at the same time.

Closes-Bug: 1561199
Change-Id: Ia3fbb4938f0daa8845cba4137a01cc43bc1a713c
Depends-On: Ib90adcc2f059adaf203fba1c95b2154561ea7487
This commit is contained in:
Tim Burke 2016-03-21 20:40:41 -07:00
parent 74d818f035
commit cd094eea4a
6 changed files with 186 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swift3/request.py
View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from email.header import Header
from hashlib import sha256, md5
import re
@ -386,7 +385,12 @@ class Request(swob.Request):
self.bucket_in_host = self._parse_host()
self.container_name, self.object_name = self._parse_uri()
self._validate_headers()
self.token = base64.urlsafe_b64encode(self._string_to_sign())
self.environ['swift3.auth_details'] = {
'access_key': self.access_key,
'signature': signature,
'string_to_sign': self._string_to_sign(),
}
self.token = None
self.account = None
self.user_id = None
self.slo_enabled = slo_enabled
@ -1199,13 +1203,15 @@ class S3AclRequest(Request):
sw_resp.environ['HTTP_X_USER_NAME'])
self.user_id = utf8encode(self.user_id)
self.token = sw_resp.environ.get('HTTP_X_AUTH_TOKEN')
# Need to skip S3 authorization since authtoken middleware
# overwrites account in PATH_INFO
del self.headers['Authorization']
else:
# tempauth
self.user_id = self.access_key
# Need to skip S3 authorization on subsequent requests to prevent
# overwriting the account in PATH_INFO
del self.headers['Authorization']
del self.environ['swift3.auth_details']
def to_swift_req(self, method, container, obj, query=None,
body=None, headers=None):
sw_req = super(S3AclRequest, self).to_swift_req(

36
swift3/s3_token_middleware.py
View File

@ -31,6 +31,7 @@ This WSGI component:
"""
import base64
import json
import logging
@ -176,31 +177,24 @@ class S3Token(object):
return self._app(environ, start_response)
# Read request signature and access id.
if 'Authorization' not in req.headers:
msg = 'No Authorization header. skipping.'
s3_auth_details = req.environ.get('swift3.auth_details')
if not s3_auth_details:
msg = 'No authorization deatils from Swift3. skipping.'
self._logger.debug(msg)
return self._app(environ, start_response)
token = req.headers.get('X-Auth-Token',
req.headers.get('X-Storage-Token'))
if not token:
msg = 'You did not specify an auth or a storage token. skipping.'
self._logger.debug(msg)
return self._app(environ, start_response)
access = s3_auth_details['access_key']
if isinstance(access, six.binary_type):
access = access.decode('utf-8')
auth_header = req.headers['Authorization']
try:
access, signature = auth_header.split(' ')[-1].rsplit(':', 1)
except ValueError:
if self._delay_auth_decision:
self._logger.debug('Invalid Authorization header: %s - '
'deferring reject downstream', auth_header)
return self._app(environ, start_response)
else:
self._logger.debug('Invalid Authorization header: %s - '
'rejecting request', auth_header)
return self._deny_request('InvalidURI')(
environ, start_response)
signature = s3_auth_details['signature']
if isinstance(signature, six.binary_type):
signature = signature.decode('utf-8')
string_to_sign = s3_auth_details['string_to_sign']
if isinstance(string_to_sign, six.text_type):
string_to_sign = string_to_sign.encode('utf-8')
token = base64.urlsafe_b64encode(string_to_sign).encode('ascii')
# NOTE(chmou): This is to handle the special case with nova
# when we have the option s3_affix_tenant. We will force it to

32
swift3/test/unit/test_middleware.py
View File

@ -18,7 +18,6 @@ from mock import patch, MagicMock
from contextlib import nested
from datetime import datetime
import hashlib
import base64
import requests
import json
import copy
@ -102,16 +101,21 @@ class TestSwift3Middleware(Swift3TestCase):
path, query_string = path.split('?', 1)
else:
query_string = ''
env = {
'REQUEST_METHOD': 'GET',
'PATH_INFO': path,
'QUERY_STRING': query_string,
'HTTP_AUTHORIZATION': 'AWS X:Y:Z',
}
for header, value in headers.items():
header = 'HTTP_' + header.replace('-', '_').upper()
if header in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
header = header[5:]
env[header] = value
with patch('swift3.request.Request._validate_headers'):
req = S3Request({
'REQUEST_METHOD': 'GET',
'PATH_INFO': path,
'QUERY_STRING': query_string,
'HTTP_AUTHORIZATION': 'AWS X:Y:Z',
})
req.headers.update(headers)
return req._string_to_sign()
req = S3Request(env)
return req.environ['swift3.auth_details']['string_to_sign']
def verify(hash, path, headers):
s = canonical_string(path, headers)
@ -379,10 +383,12 @@ class TestSwift3Middleware(Swift3TestCase):
req.headers['Date'] = date_header
status, headers, body = self.call_swift3(req)
_, _, headers = self.swift.calls_with_headers[-1]
self.assertEqual(base64.urlsafe_b64decode(
headers['X-Auth-Token']),
'PUT\n\n\n%s\n/bucket/object?partNumber=1&uploadId=123456789abcdef'
% date_header)
self.assertEqual(req.environ['swift3.auth_details'], {
'access_key': 'test:tester',
'signature': 'hmac',
'string_to_sign': '\n'.join([
'PUT', '', '', date_header,
'/bucket/object?partNumber=1&uploadId=123456789abcdef'])})
def test_invalid_uri(self):
req = Request.blank('/bucket/invalid\xffname',

25
swift3/test/unit/test_obj.py
View File

@ -131,26 +131,51 @@ class TestSwift3Obj(Swift3TestCase):
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPForbidden, {}, None)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPNotFound, {}, None)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPPreconditionFailed, {}, None)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '412')
self.assertEqual(body, '') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPServerError, {}, None)
status, headers, body = self.call_swift3(req)
self.assertEqual(status.split()[0], '500')
self.assertEqual(body, '') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
swob.HTTPServiceUnavailable, {}, None)
status, headers, body = self.call_swift3(req)

10
swift3/test/unit/test_request.py
View File

@ -221,7 +221,7 @@ class TestRequest(Swift3TestCase):
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
def test_authenticate_delete_Authorization_from_s3req_headers(self):
def test_authenticate_delete_Authorization_from_s3req(self):
req = Request.blank('/bucket/obj',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac',
@ -232,11 +232,12 @@ class TestRequest(Swift3TestCase):
m_swift_resp.return_value = FakeSwiftResponse()
s3_req = S3AclRequest(req.environ, MagicMock())
self.assertTrue('HTTP_AUTHORIZATION' not in s3_req.environ)
self.assertTrue('Authorization' not in s3_req.headers)
self.assertNotIn('swift3.auth_details', s3_req.environ)
self.assertNotIn('HTTP_AUTHORIZATION', s3_req.environ)
self.assertNotIn('Authorization', s3_req.headers)
self.assertEqual(s3_req.token, 'token')
def test_to_swift_req_Authorization_not_exist_in_swreq_headers(self):
def test_to_swift_req_Authorization_not_exist_in_swreq(self):
container = 'bucket'
obj = 'obj'
method = 'GET'
@ -251,6 +252,7 @@ class TestRequest(Swift3TestCase):
m_swift_resp.return_value = FakeSwiftResponse()
s3_req = S3AclRequest(req.environ, MagicMock())
sw_req = s3_req.to_swift_req(method, container, obj)
self.assertNotIn('swift3.auth_details', sw_req.environ)
self.assertNotIn('HTTP_AUTHORIZATION', sw_req.environ)
self.assertNotIn('Authorization', sw_req.headers)
self.assertEqual(sw_req.headers['X-Auth-Token'], 'token')

162
swift3/test/unit/test_s3_token_middleware.py
View File

@ -13,6 +13,7 @@
# under the License.
import copy
import base64
import json
import logging
import time
@ -173,8 +174,9 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.response_status, 200)
def _assert_authorized(self, req, expect_token=True):
self.assertTrue(req.path.startswith('/v1/AUTH_TENANT_ID'))
def _assert_authorized(self, req, expect_token=True,
account_path='/v1/AUTH_TENANT_ID/'):
self.assertTrue(req.path.startswith(account_path))
expected_headers = {
'X-Identity-Status': 'Confirmed',
'X-Roles': 'swift-user,_member_',
@ -196,10 +198,20 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self.assertIsInstance(req.headers[header], str)
self.assertEqual(1, self.middleware._app.calls)
self.assertEqual(1, self.requests_mock.call_count)
request_call = self.requests_mock.request_history[0]
self.assertEqual(json.loads(request_call.body), {'credentials': {
'access': 'access',
'signature': 'signature',
'token': base64.urlsafe_b64encode(b'token').decode('ascii')}})
def test_authorized(self):
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req)
@ -211,11 +223,24 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
json=resp)
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req, expect_token=False)
def test_authorized_bytes(self):
req = Request.blank('/v1/AUTH_cfa/c/o')
req.environ['swift3.auth_details'] = {
'access_key': b'access',
'signature': b'signature',
'string_to_sign': b'token',
}
req.get_response(self.middleware)
self._assert_authorized(req)
def test_authorized_http(self):
protocol = 'http'
host = 'fakehost'
@ -229,8 +254,11 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
'auth_host': host,
'auth_port': port})(self.app))
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req)
@ -238,18 +266,23 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self.middleware = s3_token.filter_factory({
'auth_uri': self.TEST_AUTH_URI + '/'})(self.app)
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req)
def test_authorization_nova_toconnect(self):
req = Request.blank('/v1/AUTH_swiftint/c/o')
req.headers['Authorization'] = 'AWS access:FORCED_TENANT_ID:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access:FORCED_TENANT_ID',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
path = req.environ['PATH_INFO']
self.assertTrue(path.startswith('/v1/AUTH_FORCED_TENANT_ID'))
self._assert_authorized(req, account_path='/v1/AUTH_FORCED_TENANT_ID/')
@mock.patch.object(requests, 'post')
def test_insecure(self, MOCK_REQUEST):
@ -262,8 +295,11 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
'text': text_return_value})
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self.assertTrue(MOCK_REQUEST.called)
@ -331,8 +367,11 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
'text': json.dumps(GOOD_RESPONSE)})
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self.assertTrue(MOCK_REQUEST.called)
@ -369,8 +408,11 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
def test_unicode_path(self):
url = u'/v1/AUTH_cfa/c/euro\u20ac'.encode('utf8')
req = Request.blank(urllib.parse.quote(url))
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req)
@ -383,8 +425,11 @@ class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase):
"title": "Unauthorized"}}
self.requests_mock.post(self.TEST_URL, status_code=403, json=ret)
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
s3_denied_req = self.middleware._deny_request('AccessDenied')
self.assertEqual(resp.body, s3_denied_req.body)
@ -393,18 +438,20 @@ class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase):
s3_denied_req.status_int) # pylint: disable-msg=E1101
self.assertEqual(0, self.middleware._app.calls)
def test_bogus_authorization(self):
self.assertEqual(1, self.requests_mock.call_count)
request_call = self.requests_mock.request_history[0]
self.assertEqual(json.loads(request_call.body), {'credentials': {
'access': 'access',
'signature': 'signature',
'token': base64.urlsafe_b64encode(b'token').decode('ascii')}})
def test_no_s3_creds_defers_to_auth_middleware(self):
# Without an Authorization header, we should just pass through to the
# auth system to make a decision.
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS badboy'
req.headers['X-Storage-Token'] = 'token'
resp = req.get_response(self.middleware)
self.assertEqual(resp.status_int, 400) # pylint: disable-msg=E1101
s3_invalid_resp = self.middleware._deny_request('InvalidURI')
self.assertEqual(resp.body, s3_invalid_resp.body)
self.assertEqual(
resp.status_int, # pylint: disable-msg=E1101
s3_invalid_resp.status_int) # pylint: disable-msg=E1101
self.assertEqual(0, self.middleware._app.calls)
self.assertEqual(resp.status_int, 200) # pylint: disable-msg=E1101
self.assertEqual(1, self.middleware._app.calls)
def test_fail_to_connect_to_keystone(self):
with mock.patch.object(self.middleware, '_json_request') as o:
@ -412,8 +459,11 @@ class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase):
o.side_effect = s3_invalid_resp
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
self.assertEqual(resp.body, s3_invalid_resp.body)
self.assertEqual(
@ -427,8 +477,11 @@ class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase):
text=response_body)
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
s3_invalid_resp = self.middleware._deny_request('InvalidURI')
self.assertEqual(resp.body, s3_invalid_resp.body)
@ -494,8 +547,11 @@ class S3TokenMiddlewareTestDeferredAuth(S3TokenMiddlewareTestBase):
"title": "Unauthorized"}}
self.requests_mock.post(self.TEST_URL, status_code=403, json=ret)
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
self.assertEqual(
resp.status_int, # pylint: disable-msg=E1101
@ -503,24 +559,23 @@ class S3TokenMiddlewareTestDeferredAuth(S3TokenMiddlewareTestBase):
self.assertNotIn('X-Auth-Token', req.headers)
self.assertEqual(1, self.middleware._app.calls)
def test_bogus_authorization(self):
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS badboy'
req.headers['X-Storage-Token'] = 'token'
resp = req.get_response(self.middleware)
self.assertEqual(
resp.status_int, # pylint: disable-msg=E1101
200)
self.assertNotIn('X-Auth-Token', req.headers)
self.assertEqual(1, self.middleware._app.calls)
self.assertEqual(1, self.requests_mock.call_count)
request_call = self.requests_mock.request_history[0]
self.assertEqual(json.loads(request_call.body), {'credentials': {
'access': 'access',
'signature': 'signature',
'token': base64.urlsafe_b64encode(b'token').decode('ascii')}})
def test_fail_to_connect_to_keystone(self):
with mock.patch.object(self.middleware, '_json_request') as o:
o.side_effect = self.middleware._deny_request('InvalidURI')
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
self.assertEqual(
resp.status_int, # pylint: disable-msg=E1101
@ -534,8 +589,11 @@ class S3TokenMiddlewareTestDeferredAuth(S3TokenMiddlewareTestBase):
text="<badreply>")
req = Request.blank('/v1/AUTH_cfa/c/o')
req.headers['Authorization'] = 'AWS access:signature'
req.headers['X-Storage-Token'] = 'token'
req.environ['swift3.auth_details'] = {
'access_key': u'access',
'signature': u'signature',
'string_to_sign': u'token',
}
resp = req.get_response(self.middleware)
self.assertEqual(
resp.status_int, # pylint: disable-msg=E1101