From a88269820e25580eee28f181260c6509ae74d4ac Mon Sep 17 00:00:00 2001 From: Hanxi Date: Sun, 11 Sep 2016 17:25:01 +0800 Subject: [PATCH] Fix UnicodeEncodeError in Ceilometer polling In PY3, all strings are sequences of unicode characters. In PY2, a string may be of type str or of type unicode. Six provides simple utilities for wrapping over differences between Python 2 and Python 3.[1] So using six.text_type instead of str to be more compatible. Python 2.x automatically encode the unicode with sys.getdefaultencoding(), which is usually 'ascii'. If there are some non-ASCII characters, it can raise UnicodeeEncodeError. Add a unit test that use unicode string to test PartitionCoordinator to check if there have the same error in the bug. [1] https://pythonhosted.org/six/#six.text_type Change-Id: Ic52c805b81e53a632a61ddcd3e8652b5849d913d Closes-Bug: #1621305 (cherry picked from commit e3752c1e485f885d982725e3c6ece2eb6f2bdc36) --- ceilometer/coordination.py | 6 ++++-- ceilometer/tests/unit/test_coordination.py | 20 ++++++++++++++++++++ ceilometer/utils.py | 5 +++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/ceilometer/coordination.py b/ceilometer/coordination.py index 5961622056..89a84bc049 100644 --- a/ceilometer/coordination.py +++ b/ceilometer/coordination.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import six import uuid from oslo_config import cfg @@ -224,9 +225,10 @@ class PartitionCoordinator(object): hr = utils.HashRing(members) iterable = list(iterable) filtered = [v for v in iterable - if hr.get_node(str(v)) == self._my_id] + if hr.get_node(six.text_type(v)) == self._my_id] LOG.debug('The universal set: %s, my subset: %s', - [str(f) for f in iterable], [str(f) for f in filtered]) + [six.text_type(f) for f in iterable], + [six.text_type(f) for f in filtered]) return filtered except tooz.coordination.ToozError: LOG.exception(_LE('Error getting group membership info from ' diff --git a/ceilometer/tests/unit/test_coordination.py b/ceilometer/tests/unit/test_coordination.py index 966946b1ec..5f7684d326 100644 --- a/ceilometer/tests/unit/test_coordination.py +++ b/ceilometer/tests/unit/test_coordination.py @@ -281,3 +281,23 @@ class TestPartitioning(base.BaseTestCase): coord.stop() self.assertIsEmpty(coord._groups) self.assertIsNone(coord._coordinator) + + def test_partitioning_with_unicode(self): + all_resources = [u'\u0634\u0628\u06a9\u0647', + u'\u0627\u0647\u0644', + u'\u0645\u062d\u0628\u0627\u0646'] + agents = ['agent_%s' % i for i in range(2)] + + expected_resources = [list() for _ in range(len(agents))] + hr = utils.HashRing(agents) + for r in all_resources: + key = agents.index(hr.get_node(r)) + expected_resources[key].append(r) + + agents_kwargs = [] + for i, agent in enumerate(agents): + agents_kwargs.append(dict(agent_id=agent, + group_id='group', + all_resources=all_resources, + expected_resources=expected_resources[i])) + self._usage_simulation(*agents_kwargs) diff --git a/ceilometer/utils.py b/ceilometer/utils.py index 544d55233b..5fe3a93fbe 100644 --- a/ceilometer/utils.py +++ b/ceilometer/utils.py @@ -76,7 +76,7 @@ def decode_unicode(input): # the tuple would become list. So we have to generate the value as # list here. return [decode_unicode(element) for element in input] - elif six.PY2 and isinstance(input, six.text_type): + elif isinstance(input, six.text_type): return input.encode('utf-8') elif six.PY3 and isinstance(input, six.binary_type): return input.decode('utf-8') @@ -237,7 +237,8 @@ class HashRing(object): @staticmethod def _hash(key): return struct.unpack_from('>I', - hashlib.md5(str(key).encode()).digest())[0] + hashlib.md5(decode_unicode(six + .text_type(key))).digest())[0] def _get_position_on_ring(self, key): hashed_key = self._hash(key)