Add py3 support

This also cleaned up our types enough that tests pass again on newer
Swift, so raise the version of swift that we test against.

While we're at it, move docs and pep8 tox envs to use py3.

Add tox envs to simplify testing old releases of Swift:

 - py27-min    (for 2.2.0, our documented minimum Swift)
 - py27-pike   (for 2.15.1)
 - py27-queens (for 2.17.0)
 - py27-rocky  (for 2.19.0)
 - py27-stein  (for 2.21.0)

Add py3 gate jobs.

Add gate job to test with our minimum-supported version of swift.

Only look at coverage for swauth; previously we'd see coverage stats for
swift and eventlet, too.

Change-Id: I6ce0c6278fc445932ea14b8cf5fe70217fc62764
This commit is contained in:
Tim Burke 2019-08-12 15:56:17 -07:00
parent f91a945590
commit 76a341ad11
6 changed files with 141 additions and 52 deletions

20
.zuul.yaml Normal file
View File

@ -0,0 +1,20 @@
- job:
name: swauth-tox-old-swift
parent: openstack-tox-py27
description: |
Run swauth unit tests with our minimum-supported version of swift, 2.2.0.
vars:
tox_envlist: py27-min
- project:
templates:
- openstack-python-jobs
- openstack-python3-train-jobs
- check-requirements
- publish-to-pypi
check:
jobs:
- swauth-tox-old-swift
gate:
jobs:
- swauth-tox-old-swift

View File

@ -16,6 +16,9 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
[pbr]
skip_authors = True

View File

@ -29,8 +29,10 @@ conditions:
indicates whether the match is True or False.
"""
import base64
import hashlib
import os
import six
import string
import sys
@ -117,7 +119,9 @@ class Sha1(object):
:param key: User's secret key
:returns: A string representing user credentials
"""
enc_key = '%s%s' % (salt, key)
enc_key = salt + key
if not six.PY2:
enc_key = enc_key.encode('utf8')
enc_val = hashlib.sha1(enc_key).hexdigest()
return "sha1:%s$%s" % (salt, enc_val)
@ -131,7 +135,9 @@ class Sha1(object):
:param key: User's secret key
:returns: A string representing user credentials
"""
salt = self.salt or os.urandom(32).encode('base64').rstrip()
salt = self.salt or base64.b64encode(os.urandom(32)).rstrip()
if not six.PY2 and isinstance(salt, bytes):
salt = salt.decode('ascii')
return self.encode_w_salt(salt, key)
def match(self, key, creds, salt, **kwargs):
@ -187,6 +193,8 @@ class Sha512(object):
:returns: A string representing user credentials
"""
enc_key = '%s%s' % (salt, key)
if not six.PY2:
enc_key = enc_key.encode('utf8')
enc_val = hashlib.sha512(enc_key).hexdigest()
return "sha512:%s$%s" % (salt, enc_val)
@ -200,7 +208,9 @@ class Sha512(object):
:param key: User's secret key
:returns: A string representing user credentials
"""
salt = self.salt or os.urandom(32).encode('base64').rstrip()
salt = self.salt or base64.b64encode(os.urandom(32)).rstrip()
if not six.PY2 and isinstance(salt, bytes):
salt = salt.decode('ascii')
return self.encode_w_salt(salt, key)
def match(self, key, creds, salt, **kwargs):

View File

@ -17,18 +17,18 @@ import base64
from hashlib import sha1
from hashlib import sha512
import hmac
from httplib import HTTPConnection
from httplib import HTTPSConnection
import json
import six
from six.moves.http_client import HTTPConnection
from six.moves.http_client import HTTPSConnection
from six.moves.urllib.parse import quote
from six.moves.urllib.parse import unquote
from six.moves.urllib.parse import urlparse
import swift
from time import gmtime
from time import strftime
from time import time
from traceback import format_exc
from urllib import quote
from urllib import unquote
from uuid import uuid4
from eventlet.timeout import Timeout
@ -235,12 +235,12 @@ class Swauth(object):
return self.handle(env, start_response)
s3 = env.get('swift3.auth_details')
if s3 and not self.s3_support:
msg = 'S3 support is disabled in swauth.'
msg = b'S3 support is disabled in swauth.'
return HTTPBadRequest(body=msg)(env, start_response)
token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
if token and len(token) > swauth.authtypes.MAX_TOKEN_LENGTH:
return HTTPBadRequest(body='Token exceeds maximum length.')(env,
start_response)
return HTTPBadRequest(body=b'Token exceeds maximum length.')(
env, start_response)
if s3 or (token and token.startswith(self.reseller_prefix)):
# Note: Empty reseller_prefix will match all tokens.
groups = self.get_groups(env, token)
@ -299,7 +299,9 @@ class Swauth(object):
Tokens are stored in auth account but object names are visible in Swift
logs. Object names are hashed from token.
"""
enc_key = "%s:%s:%s" % (HASH_PATH_PREFIX, token, HASH_PATH_SUFFIX)
if not isinstance(token, bytes):
token = token.encode('ascii')
enc_key = b"%s:%s:%s" % (HASH_PATH_PREFIX, token, HASH_PATH_SUFFIX)
return sha512(enc_key).hexdigest()
def get_groups(self, env, token):
@ -365,6 +367,8 @@ class Swauth(object):
detail = json.loads(resp.body)
if detail:
creds = detail.get('auth')
if six.PY2:
creds = creds.encode('utf8')
try:
auth_encoder, creds_dict = \
swauth.authtypes.validate_creds(creds)
@ -382,6 +386,8 @@ class Swauth(object):
valid_signature = base64.encodestring(hmac.new(
password, msg, sha1).digest()).strip()
if not six.PY2:
valid_signature = valid_signature.decode('ascii')
if signature_from_user != valid_signature:
return None
groups = [g['name'] for g in detail['groups']]
@ -520,7 +526,7 @@ class Swauth(object):
print("EXCEPTION IN handle: %s: %s" % (format_exc(), env))
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
return [b'Internal server error.\n']
def handle_request(self, req):
"""Entry point for auth requests (ones that match the self.auth_prefix).
@ -607,7 +613,7 @@ class Swauth(object):
if resp.status_int // 100 != 2:
raise Exception('Could not create container: %s %s' %
(path, resp.status))
for container in xrange(16):
for container in range(16):
path = quote('/v1/%s/.token_%x' % (self.auth_account, container))
resp = self.make_pre_authed_request(
req.environ, 'PUT', path).get_response(self.app)
@ -651,7 +657,7 @@ class Swauth(object):
if container['name'][0] != '.':
listing.append({'name': container['name']})
marker = sublisting[-1]['name'].encode('utf-8')
return Response(body=json.dumps({'accounts': listing}),
return Response(body=json.dumps({'accounts': listing}).encode('ascii'),
content_type=CONTENT_TYPE_JSON)
def handle_get_account(self, req):
@ -711,7 +717,7 @@ class Swauth(object):
return Response(content_type=CONTENT_TYPE_JSON,
body=json.dumps({'account_id': account_id,
'services': services,
'users': listing}))
'users': listing}).encode('ascii'))
def handle_set_services(self, req):
"""Handles the POST v2/<account>/.services call for setting services
@ -757,7 +763,8 @@ class Swauth(object):
try:
new_services = json.loads(req.body)
except ValueError as err:
return HTTPBadRequest(body=str(err))
msg = str(err) if six.PY2 else str(err).encode('utf8')
return HTTPBadRequest(body=msg)
# Get the current services information
path = quote('/v1/%s/%s/.services' % (self.auth_account, account))
resp = self.make_pre_authed_request(
@ -768,13 +775,13 @@ class Swauth(object):
raise Exception('Could not obtain services info: %s %s' %
(path, resp.status))
services = json.loads(resp.body)
for new_service, value in new_services.iteritems():
for new_service, value in new_services.items():
if new_service in services:
services[new_service].update(value)
else:
services[new_service] = value
# Save the new services information
services = json.dumps(services)
services = json.dumps(services).encode('ascii')
resp = self.make_pre_authed_request(
req.environ, 'PUT', path, services).get_response(self.app)
if resp.status_int // 100 != 2:
@ -916,7 +923,7 @@ class Swauth(object):
services = json.loads(resp.body)
# Delete the account on each cluster it is on.
deleted_any = False
for name, url in services['storage'].iteritems():
for name, url in services['storage'].items():
if name != 'default':
parsed = urlparse(url)
conn = self.get_conn(parsed)
@ -1033,10 +1040,11 @@ class Swauth(object):
break
for obj in sublisting:
if obj['name'][0] != '.':
user = (obj['name'].encode('utf8') if six.PY2
else obj['name'])
# get list of groups for each user
user_json = self.get_user_detail(req, account,
obj['name'])
user)
if user_json is None:
raise Exception('Could not retrieve user object: '
'%s:%s %s' % (account, user, 404))
@ -1044,7 +1052,8 @@ class Swauth(object):
g['name'] for g in json.loads(user_json)['groups'])
marker = sublisting[-1]['name'].encode('utf-8')
body = json.dumps(
{'groups': [{'name': g} for g in sorted(groups)]})
{'groups': [{'name': g} for g in sorted(groups)]}
).encode('ascii')
else:
# get information for specific user,
# if user doesn't exist, return HTTPNotFound
@ -1312,7 +1321,7 @@ class Swauth(object):
request=req,
content_type=CONTENT_TYPE_JSON,
body=json.dumps({'storage': {'default': 'local',
'local': url}}),
'local': url}}).encode('ascii'),
headers={'x-auth-token': token,
'x-storage-token': token,
'x-storage-url': url})
@ -1570,6 +1579,8 @@ class Swauth(object):
"""
if user_detail:
creds = user_detail.get('auth')
if six.PY2:
creds = creds.encode('utf8')
try:
auth_encoder, creds_dict = \
swauth.authtypes.validate_creds(creds)

View File

@ -18,9 +18,9 @@ from contextlib import contextmanager
import hashlib
import json
import mock
from six.moves.urllib.parse import quote
from time import time
import unittest
from urllib import quote
from swift.common.swob import Request
from swift.common.swob import Response
@ -68,7 +68,7 @@ class FakeApp(object):
self.calls = 0
self.status_headers_body_iter = status_headers_body_iter
if not self.status_headers_body_iter:
self.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.status_headers_body_iter = iter([('404 Not Found', {}, b'')])
self.acl = acl
self.sync_key = sync_key
@ -83,7 +83,9 @@ class FakeApp(object):
resp = env['swift.authorize'](self.request)
if resp:
return resp(env, start_response)
status, headers, body = self.status_headers_body_iter.next()
status, headers, body = next(self.status_headers_body_iter)
if not isinstance(body, bytes):
body = body.encode('utf8')
return Response(status=status, headers=headers,
body=body)(env, start_response)
@ -94,13 +96,15 @@ class FakeConn(object):
self.calls = 0
self.status_headers_body_iter = status_headers_body_iter
if not self.status_headers_body_iter:
self.status_headers_body_iter = iter([('404 Not Found', {}, '')])
self.status_headers_body_iter = iter([('404 Not Found', {}, b'')])
def request(self, method, path, headers):
self.calls += 1
self.request_path = path
self.status, self.headers, self.body = \
self.status_headers_body_iter.next()
self.status, self.headers, self.body = next(
self.status_headers_body_iter)
if not isinstance(self.body, bytes):
self.body = self.body.encode('utf8')
self.status, self.reason = self.status.split(' ', 1)
self.status = int(self.status)
@ -109,7 +113,7 @@ class FakeConn(object):
def read(self):
body = self.body
self.body = ''
self.body = b''
return body
@ -132,11 +136,12 @@ class TestAuth(unittest.TestCase):
'max_token_life': str(MAX_TOKEN_LIFE),
'auth_type': auth_type})(FakeApp())
self.assertEqual(test_auth.auth_encoder.salt, None)
mock_urandom = mock.Mock(return_value="abc")
mock_urandom = mock.Mock(return_value=b"abc")
with mock.patch("os.urandom", mock_urandom):
h_key = test_auth.auth_encoder().encode("key")
self.assertTrue(mock_urandom.called)
prefix = auth_type + ":" + "abc".encode('base64').rstrip() + '$'
prefix = auth_type + ":" + base64.b64encode(
b"abc").rstrip().decode('ascii') + '$'
self.assertTrue(h_key.startswith(prefix))
# Salt manually set
@ -341,7 +346,7 @@ class TestAuth(unittest.TestCase):
local_auth = \
auth.filter_factory({'super_admin_key': 'supertest',
'reseller_prefix': ''})(FakeApp())
local_authorize = lambda req: Response('test')
local_authorize = lambda req: Response(body=b'test')
resp = Request.blank('/v1/account', environ={'swift.authorize':
local_authorize}).get_response(local_auth)
self.assertEqual(resp.status_int, 200)
@ -1103,7 +1108,7 @@ class TestAuth(unittest.TestCase):
# PUT of .account_id container
('201 Created', {}, '')]
# PUT of .token* containers
for x in xrange(16):
for x in range(16):
list_to_iter.append(('201 Created', {}, ''))
self.test_auth.app = FakeApp(iter(list_to_iter))
resp = Request.blank('/auth/v2/.prep',
@ -2474,7 +2479,7 @@ class TestAuth(unittest.TestCase):
self.assertEqual(resp.body, json.dumps(
{"groups": [{"name": "act:usr"}, {"name": "act"},
{"name": ".admin"}],
"auth": "plaintext:key"}))
"auth": "plaintext:key"}).encode('ascii'))
self.assertEqual(self.test_auth.app.calls, 1)
def test_get_user_fail_no_super_admin_key(self):
@ -2524,7 +2529,8 @@ class TestAuth(unittest.TestCase):
self.assertEqual(resp.content_type, CONTENT_TYPE_JSON)
self.assertEqual(resp.body, json.dumps(
{"groups": [{"name": ".admin"}, {"name": "act"},
{"name": "act:tester"}, {"name": "act:tester3"}]}))
{"name": "act:tester"}, {"name": "act:tester3"}]}
).encode('ascii'))
self.assertEqual(self.test_auth.app.calls, 4)
def test_get_user_groups_success2(self):
@ -2563,7 +2569,8 @@ class TestAuth(unittest.TestCase):
self.assertEqual(resp.content_type, CONTENT_TYPE_JSON)
self.assertEqual(resp.body, json.dumps(
{"groups": [{"name": ".admin"}, {"name": "act"},
{"name": "act:tester"}, {"name": "act:tester3"}]}))
{"name": "act:tester"}, {"name": "act:tester3"}]}
).encode('ascii'))
self.assertEqual(self.test_auth.app.calls, 5)
def test_get_user_fail_invalid_account(self):
@ -2620,7 +2627,7 @@ class TestAuth(unittest.TestCase):
self.assertEqual(resp.content_type, CONTENT_TYPE_JSON)
self.assertEqual(resp.body, json.dumps(
{"groups": [{"name": "act:usr"}, {"name": "act"}],
"auth": "plaintext:key"}))
"auth": "plaintext:key"}).encode('ascii'))
self.assertEqual(self.test_auth.app.calls, 2)
def test_get_user_account_admin_fail_getting_account_admin(self):
@ -2697,7 +2704,7 @@ class TestAuth(unittest.TestCase):
self.assertEqual(resp.body, json.dumps(
{"groups": [{"name": "act:usr"}, {"name": "act"},
{"name": ".reseller_admin"}],
"auth": "plaintext:key"}))
"auth": "plaintext:key"}).encode('ascii'))
self.assertEqual(self.test_auth.app.calls, 1)
def test_get_user_groups_not_found(self):
@ -4027,12 +4034,12 @@ class TestAuth(unittest.TestCase):
'x-auth-token': 'a' * MAX_TOKEN_LENGTH})
resp = req.get_response(self.test_auth)
self.assertEqual(resp.status_int, 401)
self.assertNotEqual(resp.body, 'Token exceeds maximum length.')
self.assertNotEqual(resp.body, b'Token exceeds maximum length.')
req = self._make_request('/v1/AUTH_account', headers={
'x-auth-token': 'a' * (MAX_TOKEN_LENGTH + 1)})
resp = req.get_response(self.test_auth)
self.assertEqual(resp.status_int, 400)
self.assertEqual(resp.body, 'Token exceeds maximum length.')
self.assertEqual(resp.body, b'Token exceeds maximum length.')
def test_s3_enabled_when_conditions_are_met(self):
# auth_type_salt needs to be set
@ -4091,7 +4098,7 @@ class TestAuth(unittest.TestCase):
self.test_auth.s3_support = True
self.test_auth.app = FakeApp(iter([
('200 Ok', {},
json.dumps({"auth": unicode("plaintext:key)"),
json.dumps({"auth": "plaintext:key)",
"groups": [{'name': "act:usr"}, {'name': "act"},
{'name': ".admin"}]})),
('204 Ok', {'X-Container-Meta-Account-Id': 'AUTH_act'}, '')]))
@ -4111,7 +4118,7 @@ class TestAuth(unittest.TestCase):
self.test_auth.s3_support = True
self.test_auth.app = FakeApp(iter([
('200 Ok', {},
json.dumps({"auth": unicode("plaintext:key)"),
json.dumps({"auth": "plaintext:key)",
"groups": [{'name': "act:usr"}, {'name': "act"},
{'name': ".admin"}]})),
('204 Ok', {'X-Container-Meta-Account-Id': 'AUTH_act'}, '')]))
@ -4132,7 +4139,7 @@ class TestAuth(unittest.TestCase):
self.test_auth.s3_support = True
key = 'dadada'
salt = 'zuck'
key_hash = hashlib.sha1('%s%s' % (salt, key)).hexdigest()
key_hash = hashlib.sha1((salt + key).encode('ascii')).hexdigest()
auth_stored = "sha1:%s$%s" % (salt, key_hash)
self.test_auth.app = FakeApp(iter([
('200 Ok', {},
@ -4150,15 +4157,17 @@ class TestAuth(unittest.TestCase):
'PATH_INFO': '/v1/AUTH_act/c1'}
token = 'not used'
mock_hmac_new = mock.MagicMock()
mock_hmac_new.return_value.digest.return_value = b'does not matter'
with mock.patch('hmac.new', mock_hmac_new):
self.test_auth.get_groups(env, token)
self.assertTrue(mock_hmac_new.called)
# Assert that string passed to hmac.new is only the hash
self.assertEqual(mock_hmac_new.call_args[0][0], key_hash)
self.assertEqual(mock_hmac_new.call_args[0][0],
key_hash.encode('ascii'))
def test_get_concealed_token(self):
auth.HASH_PATH_PREFIX = 'start'
auth.HASH_PATH_SUFFIX = 'end'
auth.HASH_PATH_PREFIX = b'start'
auth.HASH_PATH_SUFFIX = b'end'
token = 'token'
# Check sha512 of "start:token:end"
@ -4177,7 +4186,7 @@ class TestAuth(unittest.TestCase):
'f4259d')
# Check sha512 of "start2:token2:end"
auth.HASH_PATH_PREFIX = 'start2'
auth.HASH_PATH_PREFIX = b'start2'
hashed_token = self.test_auth._get_concealed_token(token)
self.assertEqual(hashed_token,
'ad594a69f44dd6e0aad54e360b01f15bd4833ccb4dcd9116d7aba0c25fb95'
@ -4185,7 +4194,7 @@ class TestAuth(unittest.TestCase):
'22bdde')
# Check sha512 of "start2:token2:end2"
auth.HASH_PATH_SUFFIX = 'end2'
auth.HASH_PATH_SUFFIX = b'end2'
hashed_token = self.test_auth._get_concealed_token(token)
self.assertEqual(hashed_token,
'446af2473ad6b28319a0fe02719a9d715b9941d12e0709851aedb4f53b890'

42
tox.ini
View File

@ -1,24 +1,57 @@
[tox]
minversion = 1.6
envlist = py27,pep8,cover
envlist = py27,py36,py37,pep8,cover,docs
skipsdist = True
[testenv]
basepython = python2.7
usedevelop = True
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_COVERAGE=1
NOSE_COVER_PACKAGE=swauth
NOSE_COVER_BRANCHES=1
NOSE_COVER_ERASE=1
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.15.1.tar.gz
# May as well point cover at latest release
https://tarballs.openstack.org/swift/swift-2.22.0.tar.gz
commands = nosetests {posargs:test/unit}
[testenv:py27-min]
basepython = python2.7
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.2.0.tar.gz
[testenv:py27-pike]
basepython = python2.7
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.15.1.tar.gz
[testenv:py27-queens]
basepython = python2.7
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.17.0.tar.gz
[testenv:py27-rocky]
basepython = python2.7
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.19.0.tar.gz
[testenv:py27-stein]
basepython = python2.7
deps =
-r{toxinidir}/test-requirements.txt
https://tarballs.openstack.org/swift/swift-2.21.0.tar.gz
[testenv:cover]
basepython = python2.7
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_COVERAGE=1
NOSE_COVER_PACKAGE=swauth
NOSE_COVER_BRANCHES=1
NOSE_COVER_HTML=1
NOSE_COVER_HTML_DIR={toxinidir}/cover
@ -26,12 +59,14 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_COVER_ERASE=1
[testenv:pep8]
basepython = python3
commands =
flake8 swauth test
flake8 --filename=swauth* bin
bandit -r swauth -s B303,B309
[testenv:bandit]
basepython = python3
# B303 Use of insecure hash function
# B309 Use of HTTPSConnection
commands = bandit -r swauth -s B303,B309
@ -40,6 +75,7 @@ commands = bandit -r swauth -s B303,B309
commands = {posargs}
[testenv:docs]
basepython = python3
commands = python setup.py build_sphinx
[flake8]