Fix Python 3 issues in nova.utils and nova.tests
* Fix sort(): dictionaries are no more comparable on Python 3, use a key function building a sorted list of the dictionary items * Replace __builtin__ with six.moves.builtins * get_hash_str() now accepts Unicode: Unicode is encoded to UTF-8 before hashing the string. Mention also the hash method (MD5) in the docstring. * LastBytesTestCase: use binary files instead of text files. Use also a context manager on the TemporaryFile to ensure that the temporary file is closed (even on error). * Functions: use the __code__ attribute instead of func_code, and __closure__ instead of func_closure. These attributes are also available on Python 2.6. * Replace unichr() with six.unichr() * SafeTruncateTestCase(): write directly the chinese Unicode character instead of using safe_decode() (which is not reliable, it depends on the locale encoding!) * sanitize_hostname(): On Python 3, decode the hostname from Latin1 to get back Unicode. Add also a comment explaining the purpose of the conversion to Latin1. * last_bytes(): replace the hardcoded constant 22 with errno.EINVAL and add a comment explaining how the function works. * tox.ini: add nova.tests.unit.test_utils to Python 3.4 Blueprint nova-python3 Change-Id: I96d9b0581ceaeccf9c35e0c9bada369e9d19fd15
This commit is contained in:
parent
3537513339
commit
0e5cefcf1d
22
nova/test.py
22
nova/test.py
|
@ -294,16 +294,14 @@ class TestCase(testtools.TestCase):
|
||||||
if isinstance(observed, six.string_types):
|
if isinstance(observed, six.string_types):
|
||||||
observed = jsonutils.loads(observed)
|
observed = jsonutils.loads(observed)
|
||||||
|
|
||||||
def sort(what):
|
def sort_key(x):
|
||||||
def get_key(item):
|
if isinstance(x, set) or isinstance(x, datetime.datetime):
|
||||||
if isinstance(item, (datetime.datetime, set)):
|
return str(x)
|
||||||
return str(item)
|
if isinstance(x, dict):
|
||||||
if six.PY3 and isinstance(item, dict):
|
items = ((sort_key(key), sort_key(value))
|
||||||
return str(sort(list(six.iterkeys(item)) +
|
for key, value in x.items())
|
||||||
list(six.itervalues(item))))
|
return sorted(items)
|
||||||
return str(item) if six.PY3 else item
|
return x
|
||||||
|
|
||||||
return sorted(what, key=get_key)
|
|
||||||
|
|
||||||
def inner(expected, observed):
|
def inner(expected, observed):
|
||||||
if isinstance(expected, dict) and isinstance(observed, dict):
|
if isinstance(expected, dict) and isinstance(observed, dict):
|
||||||
|
@ -318,8 +316,8 @@ class TestCase(testtools.TestCase):
|
||||||
isinstance(observed, (list, tuple, set))):
|
isinstance(observed, (list, tuple, set))):
|
||||||
self.assertEqual(len(expected), len(observed))
|
self.assertEqual(len(expected), len(observed))
|
||||||
|
|
||||||
expected_values_iter = iter(sort(expected))
|
expected_values_iter = iter(sorted(expected, key=sort_key))
|
||||||
observed_values_iter = iter(sort(observed))
|
observed_values_iter = iter(sorted(observed, key=sort_key))
|
||||||
|
|
||||||
for i in range(len(expected)):
|
for i in range(len(expected)):
|
||||||
inner(next(expected_values_iter),
|
inner(next(expected_values_iter),
|
||||||
|
|
|
@ -19,7 +19,6 @@ import importlib
|
||||||
import os
|
import os
|
||||||
import os.path
|
import os.path
|
||||||
import socket
|
import socket
|
||||||
import StringIO
|
|
||||||
import struct
|
import struct
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
@ -112,7 +111,7 @@ class GenericUtilsTestCase(test.NoDBTestCase):
|
||||||
|
|
||||||
fake_contents = "lorem ipsum"
|
fake_contents = "lorem ipsum"
|
||||||
m = mock.mock_open(read_data=fake_contents)
|
m = mock.mock_open(read_data=fake_contents)
|
||||||
with mock.patch("__builtin__.open", m, create=True):
|
with mock.patch("six.moves.builtins.open", m, create=True):
|
||||||
cache_data = {"data": 1123, "mtime": 1}
|
cache_data = {"data": 1123, "mtime": 1}
|
||||||
self.reload_called = False
|
self.reload_called = False
|
||||||
|
|
||||||
|
@ -209,10 +208,13 @@ class GenericUtilsTestCase(test.NoDBTestCase):
|
||||||
self.assertEqual("localhost", utils.safe_ip_format("localhost"))
|
self.assertEqual("localhost", utils.safe_ip_format("localhost"))
|
||||||
|
|
||||||
def test_get_hash_str(self):
|
def test_get_hash_str(self):
|
||||||
base_str = "foo"
|
base_str = b"foo"
|
||||||
|
base_unicode = u"foo"
|
||||||
value = hashlib.md5(base_str).hexdigest()
|
value = hashlib.md5(base_str).hexdigest()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
value, utils.get_hash_str(base_str))
|
value, utils.get_hash_str(base_str))
|
||||||
|
self.assertEqual(
|
||||||
|
value, utils.get_hash_str(base_unicode))
|
||||||
|
|
||||||
def test_use_rootwrap(self):
|
def test_use_rootwrap(self):
|
||||||
self.flags(disable_rootwrap=False, group='workarounds')
|
self.flags(disable_rootwrap=False, group='workarounds')
|
||||||
|
@ -554,26 +556,26 @@ class LastBytesTestCase(test.NoDBTestCase):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(LastBytesTestCase, self).setUp()
|
super(LastBytesTestCase, self).setUp()
|
||||||
self.f = StringIO.StringIO('1234567890')
|
self.f = six.BytesIO(b'1234567890')
|
||||||
|
|
||||||
def test_truncated(self):
|
def test_truncated(self):
|
||||||
self.f.seek(0, os.SEEK_SET)
|
self.f.seek(0, os.SEEK_SET)
|
||||||
out, remaining = utils.last_bytes(self.f, 5)
|
out, remaining = utils.last_bytes(self.f, 5)
|
||||||
self.assertEqual(out, '67890')
|
self.assertEqual(out, b'67890')
|
||||||
self.assertTrue(remaining > 0)
|
self.assertTrue(remaining > 0)
|
||||||
|
|
||||||
def test_read_all(self):
|
def test_read_all(self):
|
||||||
self.f.seek(0, os.SEEK_SET)
|
self.f.seek(0, os.SEEK_SET)
|
||||||
out, remaining = utils.last_bytes(self.f, 1000)
|
out, remaining = utils.last_bytes(self.f, 1000)
|
||||||
self.assertEqual(out, '1234567890')
|
self.assertEqual(out, b'1234567890')
|
||||||
self.assertFalse(remaining > 0)
|
self.assertFalse(remaining > 0)
|
||||||
|
|
||||||
def test_seek_too_far_real_file(self):
|
def test_seek_too_far_real_file(self):
|
||||||
# StringIO doesn't raise IOError if you see past the start of the file.
|
# StringIO doesn't raise IOError if you see past the start of the file.
|
||||||
flo = tempfile.TemporaryFile()
|
with tempfile.TemporaryFile() as flo:
|
||||||
content = '1234567890'
|
content = b'1234567890'
|
||||||
flo.write(content)
|
flo.write(content)
|
||||||
self.assertEqual((content, 0), utils.last_bytes(flo, 1000))
|
self.assertEqual((content, 0), utils.last_bytes(flo, 1000))
|
||||||
|
|
||||||
|
|
||||||
class MetadataToDictTestCase(test.NoDBTestCase):
|
class MetadataToDictTestCase(test.NoDBTestCase):
|
||||||
|
@ -587,11 +589,14 @@ class MetadataToDictTestCase(test.NoDBTestCase):
|
||||||
self.assertEqual(utils.metadata_to_dict([]), {})
|
self.assertEqual(utils.metadata_to_dict([]), {})
|
||||||
|
|
||||||
def test_dict_to_metadata(self):
|
def test_dict_to_metadata(self):
|
||||||
|
def sort_key(adict):
|
||||||
|
return sorted(adict.items())
|
||||||
|
|
||||||
|
metadata = utils.dict_to_metadata(dict(foo1='bar1', foo2='bar2'))
|
||||||
expected = [{'key': 'foo1', 'value': 'bar1'},
|
expected = [{'key': 'foo1', 'value': 'bar1'},
|
||||||
{'key': 'foo2', 'value': 'bar2'}]
|
{'key': 'foo2', 'value': 'bar2'}]
|
||||||
self.assertEqual(sorted(utils.dict_to_metadata(dict(foo1='bar1',
|
self.assertEqual(sorted(metadata, key=sort_key),
|
||||||
foo2='bar2'))),
|
sorted(expected, key=sort_key))
|
||||||
sorted(expected))
|
|
||||||
|
|
||||||
def test_dict_to_metadata_empty(self):
|
def test_dict_to_metadata_empty(self):
|
||||||
self.assertEqual(utils.dict_to_metadata({}), [])
|
self.assertEqual(utils.dict_to_metadata({}), [])
|
||||||
|
@ -612,7 +617,7 @@ class WrappedCodeTestCase(test.NoDBTestCase):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
func = utils.get_wrapped_function(wrapped)
|
func = utils.get_wrapped_function(wrapped)
|
||||||
func_code = func.func_code
|
func_code = func.__code__
|
||||||
self.assertEqual(4, len(func_code.co_varnames))
|
self.assertEqual(4, len(func_code.co_varnames))
|
||||||
self.assertIn('self', func_code.co_varnames)
|
self.assertIn('self', func_code.co_varnames)
|
||||||
self.assertIn('instance', func_code.co_varnames)
|
self.assertIn('instance', func_code.co_varnames)
|
||||||
|
@ -626,7 +631,7 @@ class WrappedCodeTestCase(test.NoDBTestCase):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
func = utils.get_wrapped_function(wrapped)
|
func = utils.get_wrapped_function(wrapped)
|
||||||
func_code = func.func_code
|
func_code = func.__code__
|
||||||
self.assertEqual(4, len(func_code.co_varnames))
|
self.assertEqual(4, len(func_code.co_varnames))
|
||||||
self.assertIn('self', func_code.co_varnames)
|
self.assertIn('self', func_code.co_varnames)
|
||||||
self.assertIn('instance', func_code.co_varnames)
|
self.assertIn('instance', func_code.co_varnames)
|
||||||
|
@ -641,7 +646,7 @@ class WrappedCodeTestCase(test.NoDBTestCase):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
func = utils.get_wrapped_function(wrapped)
|
func = utils.get_wrapped_function(wrapped)
|
||||||
func_code = func.func_code
|
func_code = func.__code__
|
||||||
self.assertEqual(4, len(func_code.co_varnames))
|
self.assertEqual(4, len(func_code.co_varnames))
|
||||||
self.assertIn('self', func_code.co_varnames)
|
self.assertIn('self', func_code.co_varnames)
|
||||||
self.assertIn('instance', func_code.co_varnames)
|
self.assertIn('instance', func_code.co_varnames)
|
||||||
|
@ -759,7 +764,7 @@ class ValidateIntegerTestCase(test.NoDBTestCase):
|
||||||
max_value=54)
|
max_value=54)
|
||||||
self.assertRaises(exception.InvalidInput,
|
self.assertRaises(exception.InvalidInput,
|
||||||
utils.validate_integer,
|
utils.validate_integer,
|
||||||
unichr(129), "UnicodeError",
|
six.unichr(129), "UnicodeError",
|
||||||
max_value=1000)
|
max_value=1000)
|
||||||
|
|
||||||
|
|
||||||
|
@ -1050,7 +1055,7 @@ class SafeTruncateTestCase(test.NoDBTestCase):
|
||||||
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
|
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
|
||||||
# character occupies 3 bytes. After truncating, the byte string length
|
# character occupies 3 bytes. After truncating, the byte string length
|
||||||
# should be 255.
|
# should be 255.
|
||||||
msg = encodeutils.safe_decode('\xe8\xb5\xb5' * 100)
|
msg = u'\u8d75' * 100
|
||||||
truncated_msg = utils.safe_truncate(msg, 255)
|
truncated_msg = utils.safe_truncate(msg, 255)
|
||||||
byte_message = encodeutils.safe_encode(truncated_msg)
|
byte_message = encodeutils.safe_encode(truncated_msg)
|
||||||
self.assertEqual(255, len(byte_message))
|
self.assertEqual(255, len(byte_message))
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import datetime
|
import datetime
|
||||||
|
import errno
|
||||||
import functools
|
import functools
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
|
@ -565,6 +566,12 @@ def monkey_patch():
|
||||||
# If CONF.monkey_patch is not True, this function do nothing.
|
# If CONF.monkey_patch is not True, this function do nothing.
|
||||||
if not CONF.monkey_patch:
|
if not CONF.monkey_patch:
|
||||||
return
|
return
|
||||||
|
if six.PY3:
|
||||||
|
def is_method(obj):
|
||||||
|
# Unbound methods became regular functions on Python 3
|
||||||
|
return inspect.ismethod(obj) or inspect.isfunction(obj)
|
||||||
|
else:
|
||||||
|
is_method = inspect.ismethod
|
||||||
# Get list of modules and decorators
|
# Get list of modules and decorators
|
||||||
for module_and_decorator in CONF.monkey_patch_modules:
|
for module_and_decorator in CONF.monkey_patch_modules:
|
||||||
module, decorator_name = module_and_decorator.split(':')
|
module, decorator_name = module_and_decorator.split(':')
|
||||||
|
@ -573,15 +580,15 @@ def monkey_patch():
|
||||||
__import__(module)
|
__import__(module)
|
||||||
# Retrieve module information using pyclbr
|
# Retrieve module information using pyclbr
|
||||||
module_data = pyclbr.readmodule_ex(module)
|
module_data = pyclbr.readmodule_ex(module)
|
||||||
for key in module_data.keys():
|
for key, value in module_data.items():
|
||||||
# set the decorator for the class methods
|
# set the decorator for the class methods
|
||||||
if isinstance(module_data[key], pyclbr.Class):
|
if isinstance(value, pyclbr.Class):
|
||||||
clz = importutils.import_class("%s.%s" % (module, key))
|
clz = importutils.import_class("%s.%s" % (module, key))
|
||||||
for method, func in inspect.getmembers(clz, inspect.ismethod):
|
for method, func in inspect.getmembers(clz, is_method):
|
||||||
setattr(clz, method,
|
setattr(clz, method,
|
||||||
decorator("%s.%s.%s" % (module, key, method), func))
|
decorator("%s.%s.%s" % (module, key, method), func))
|
||||||
# set the decorator for the function
|
# set the decorator for the function
|
||||||
if isinstance(module_data[key], pyclbr.Function):
|
if isinstance(value, pyclbr.Function):
|
||||||
func = importutils.import_class("%s.%s" % (module, key))
|
func = importutils.import_class("%s.%s" % (module, key))
|
||||||
setattr(sys.modules[module], key,
|
setattr(sys.modules[module], key,
|
||||||
decorator("%s.%s" % (module, key), func))
|
decorator("%s.%s" % (module, key), func))
|
||||||
|
@ -614,7 +621,10 @@ def make_dev_path(dev, partition=None, base='/dev'):
|
||||||
def sanitize_hostname(hostname):
|
def sanitize_hostname(hostname):
|
||||||
"""Return a hostname which conforms to RFC-952 and RFC-1123 specs."""
|
"""Return a hostname which conforms to RFC-952 and RFC-1123 specs."""
|
||||||
if isinstance(hostname, six.text_type):
|
if isinstance(hostname, six.text_type):
|
||||||
|
# Remove characters outside the Unicode range U+0000-U+00FF
|
||||||
hostname = hostname.encode('latin-1', 'ignore')
|
hostname = hostname.encode('latin-1', 'ignore')
|
||||||
|
if six.PY3:
|
||||||
|
hostname = hostname.decode('latin-1')
|
||||||
|
|
||||||
hostname = re.sub('[ _]', '-', hostname)
|
hostname = re.sub('[ _]', '-', hostname)
|
||||||
hostname = re.sub('[^\w.-]+', '', hostname)
|
hostname = re.sub('[^\w.-]+', '', hostname)
|
||||||
|
@ -830,7 +840,10 @@ def last_bytes(file_like_object, num):
|
||||||
try:
|
try:
|
||||||
file_like_object.seek(-num, os.SEEK_END)
|
file_like_object.seek(-num, os.SEEK_END)
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
if e.errno == 22:
|
# seek() fails with EINVAL when trying to go before the start of the
|
||||||
|
# file. It means that num is larger than the file size, so just
|
||||||
|
# go to the start.
|
||||||
|
if e.errno == errno.EINVAL:
|
||||||
file_like_object.seek(0, os.SEEK_SET)
|
file_like_object.seek(0, os.SEEK_SET)
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
@ -874,14 +887,14 @@ def instance_sys_meta(instance):
|
||||||
|
|
||||||
def get_wrapped_function(function):
|
def get_wrapped_function(function):
|
||||||
"""Get the method at the bottom of a stack of decorators."""
|
"""Get the method at the bottom of a stack of decorators."""
|
||||||
if not hasattr(function, 'func_closure') or not function.func_closure:
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
||||||
return function
|
return function
|
||||||
|
|
||||||
def _get_wrapped_function(function):
|
def _get_wrapped_function(function):
|
||||||
if not hasattr(function, 'func_closure') or not function.func_closure:
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
for closure in function.func_closure:
|
for closure in function.__closure__:
|
||||||
func = closure.cell_contents
|
func = closure.cell_contents
|
||||||
|
|
||||||
deeper_func = _get_wrapped_function(func)
|
deeper_func = _get_wrapped_function(func)
|
||||||
|
@ -1190,7 +1203,12 @@ def get_image_metadata_from_volume(volume):
|
||||||
|
|
||||||
|
|
||||||
def get_hash_str(base_str):
|
def get_hash_str(base_str):
|
||||||
"""returns string that represents hash of base_str (in hex format)."""
|
"""Returns string that represents MD5 hash of base_str (in hex format).
|
||||||
|
|
||||||
|
If base_str is a Unicode string, encode it to UTF-8.
|
||||||
|
"""
|
||||||
|
if isinstance(base_str, six.text_type):
|
||||||
|
base_str = base_str.encode('utf-8')
|
||||||
return hashlib.md5(base_str).hexdigest()
|
return hashlib.md5(base_str).hexdigest()
|
||||||
|
|
||||||
if hasattr(hmac, 'compare_digest'):
|
if hasattr(hmac, 'compare_digest'):
|
||||||
|
|
1
tox.ini
1
tox.ini
|
@ -78,6 +78,7 @@ commands =
|
||||||
nova.tests.unit.objects.test_virtual_interface \
|
nova.tests.unit.objects.test_virtual_interface \
|
||||||
nova.tests.unit.test_crypto \
|
nova.tests.unit.test_crypto \
|
||||||
nova.tests.unit.test_exception \
|
nova.tests.unit.test_exception \
|
||||||
|
nova.tests.unit.test_utils \
|
||||||
nova.tests.unit.test_versions
|
nova.tests.unit.test_versions
|
||||||
|
|
||||||
[testenv:functional]
|
[testenv:functional]
|
||||||
|
|
Loading…
Reference in New Issue