Fix UnicodeEncodeError in Ceilometer polling

In PY3, all strings are sequences of unicode characters. In PY2,
a string may be of type str or of type unicode. Six provides
simple utilities for wrapping over differences between Python 2
and Python 3.[1] So using six.text_type instead of str to be more
compatible.
Python 2.x automatically encode the unicode with sys.getdefaultencoding(),
which is usually 'ascii'. If there are some non-ASCII characters, it can
raise UnicodeeEncodeError. Add a unit test that use unicode string to test
PartitionCoordinator to check if there have the same error in the bug.

[1] https://pythonhosted.org/six/#six.text_type

Change-Id: Ic52c805b81e53a632a61ddcd3e8652b5849d913d
Closes-Bug: #1621305
(cherry picked from commit e3752c1e48)
This commit is contained in:
Hanxi 2016-09-11 17:25:01 +08:00 committed by Julien Danjou
parent 789d47d009
commit a88269820e
3 changed files with 27 additions and 4 deletions

View File

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
import uuid
from oslo_config import cfg
@ -224,9 +225,10 @@ class PartitionCoordinator(object):
hr = utils.HashRing(members)
iterable = list(iterable)
filtered = [v for v in iterable
if hr.get_node(str(v)) == self._my_id]
if hr.get_node(six.text_type(v)) == self._my_id]
LOG.debug('The universal set: %s, my subset: %s',
[str(f) for f in iterable], [str(f) for f in filtered])
[six.text_type(f) for f in iterable],
[six.text_type(f) for f in filtered])
return filtered
except tooz.coordination.ToozError:
LOG.exception(_LE('Error getting group membership info from '

View File

@ -281,3 +281,23 @@ class TestPartitioning(base.BaseTestCase):
coord.stop()
self.assertIsEmpty(coord._groups)
self.assertIsNone(coord._coordinator)
def test_partitioning_with_unicode(self):
all_resources = [u'\u0634\u0628\u06a9\u0647',
u'\u0627\u0647\u0644',
u'\u0645\u062d\u0628\u0627\u0646']
agents = ['agent_%s' % i for i in range(2)]
expected_resources = [list() for _ in range(len(agents))]
hr = utils.HashRing(agents)
for r in all_resources:
key = agents.index(hr.get_node(r))
expected_resources[key].append(r)
agents_kwargs = []
for i, agent in enumerate(agents):
agents_kwargs.append(dict(agent_id=agent,
group_id='group',
all_resources=all_resources,
expected_resources=expected_resources[i]))
self._usage_simulation(*agents_kwargs)

View File

@ -76,7 +76,7 @@ def decode_unicode(input):
# the tuple would become list. So we have to generate the value as
# list here.
return [decode_unicode(element) for element in input]
elif six.PY2 and isinstance(input, six.text_type):
elif isinstance(input, six.text_type):
return input.encode('utf-8')
elif six.PY3 and isinstance(input, six.binary_type):
return input.decode('utf-8')
@ -237,7 +237,8 @@ class HashRing(object):
@staticmethod
def _hash(key):
return struct.unpack_from('>I',
hashlib.md5(str(key).encode()).digest())[0]
hashlib.md5(decode_unicode(six
.text_type(key))).digest())[0]
def _get_position_on_ring(self, key):
hashed_key = self._hash(key)