Read description file as utf-8

Currently pbr fails if the description file contains unicode
characters.  To fix this we need to open the description file as
utf-8 explicitly.  Since open() in Python 2 doesn't support an
encoding parameter, use io.open() which works on both 2 and 3.

Co-Authored-By: Hervé Beraud<hberaud@redhat.com>

Change-Id: I1bee502ac84b474cc9db5523d2437a8c0a861c00
Closes-Bug: 1704472

pbr/tests/test_util.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2015 Hewlett-Packard Development Company, L.P. (HP)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,6 +14,7 @@
 # under the License.
 
 import io
+import tempfile
 import textwrap
 
 import six
@@ -197,3 +199,21 @@ class TestDataFilesParsing(base.BaseTestCase):
 
         self.assertEqual(self.data_files,
                          list(kwargs['data_files']))
+
+
+class TestUTF8DescriptionFile(base.BaseTestCase):
+    def test_utf8_description_file(self):
+        _, path = tempfile.mkstemp()
+        ini_template = """
+        [metadata]
+        description_file = %s
+        """
+        # Two \n's because pbr strips the file content and adds \n\n
+        # This way we can use it directly as the assert comparison
+        unicode_description = u'UTF8 description: é"…-ʃŋ\'\n\n'
+        ini = ini_template % path
+        with io.open(path, 'w', encoding='utf8') as f:
+            f.write(unicode_description)
+        config = config_from_ini(ini)
+        kwargs = util.setup_cfg_to_setup_kwargs(config)
+        self.assertEqual(unicode_description, kwargs['long_description'])

pbr/util.py

@@ -62,6 +62,7 @@ except ImportError:
 import logging  # noqa
 
 from collections import defaultdict
+import io
 import os
 import re
 import shlex
@@ -320,7 +321,7 @@ def setup_cfg_to_setup_kwargs(config, script_args=()):
                     in_cfg_value = split_multiline(in_cfg_value)
                     value = ''
                     for filename in in_cfg_value:
-                        description_file = open(filename)
+                        description_file = io.open(filename, encoding='utf-8')
                         try:
                             value += description_file.read().strip() + '\n\n'
                         finally: