Fix default encoding issue with python 2.6
This change addresses issue #38: "fix unicode handling issues". The issue was originally reported against neutron client (https://bugs.launchpad.net/python-neutronclient/+bug/1189112) but was tracked down to the fact that python 2.6 does not set the default encoding for sys.stdout properly. A change to python 2.7 fixes the problem there and later (http://hg.python.org/cpython/rev/e60ef17561dc/), but since cliff supports python 2.6 it needs to handle the case explicitly. Change-Id: Id06507d78c7c82b25f39366ea4a6dfa4ef3a3a97
This commit is contained in:
parent
50de738446
commit
aa6bb0cfe3
25
cliff/app.py
25
cliff/app.py
|
@ -2,6 +2,8 @@
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import codecs
|
||||||
|
import locale
|
||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
import logging.handlers
|
||||||
import os
|
import os
|
||||||
|
@ -67,14 +69,31 @@ class App(object):
|
||||||
"""
|
"""
|
||||||
self.command_manager = command_manager
|
self.command_manager = command_manager
|
||||||
self.command_manager.add_command('help', HelpCommand)
|
self.command_manager.add_command('help', HelpCommand)
|
||||||
self.stdin = stdin or sys.stdin
|
self._set_streams(stdin, stdout, stderr)
|
||||||
self.stdout = stdout or sys.stdout
|
|
||||||
self.stderr = stderr or sys.stderr
|
|
||||||
self.interactive_app_factory = interactive_app_factory
|
self.interactive_app_factory = interactive_app_factory
|
||||||
self.parser = self.build_option_parser(description, version)
|
self.parser = self.build_option_parser(description, version)
|
||||||
self.interactive_mode = False
|
self.interactive_mode = False
|
||||||
self.interpreter = None
|
self.interpreter = None
|
||||||
|
|
||||||
|
def _set_streams(self, stdin, stdout, stderr):
|
||||||
|
locale.setlocale(locale.LC_ALL, '')
|
||||||
|
if sys.version_info[:2] == (2, 6):
|
||||||
|
# Configure the input and output streams. If a stream is
|
||||||
|
# provided, it must be configured correctly by the
|
||||||
|
# caller. If not, make sure the versions of the standard
|
||||||
|
# streams used by default are wrapped with encodings. This
|
||||||
|
# works around a problem with Python 2.6 fixed in 2.7 and
|
||||||
|
# later (http://hg.python.org/cpython/rev/e60ef17561dc/).
|
||||||
|
lang, encoding = locale.getdefaultlocale()
|
||||||
|
encoding = getattr(sys.stdout, 'encoding', None) or encoding
|
||||||
|
self.stdin = stdin or codecs.getreader(encoding)(sys.stdin)
|
||||||
|
self.stdout = stdout or codecs.getwriter(encoding)(sys.stdout)
|
||||||
|
self.stderr = stderr or codecs.getwriter(encoding)(sys.stderr)
|
||||||
|
else:
|
||||||
|
self.stdin = stdin or sys.stdin
|
||||||
|
self.stdout = stdout or sys.stdout
|
||||||
|
self.stderr = stderr or sys.stderr
|
||||||
|
|
||||||
def build_option_parser(self, description, version,
|
def build_option_parser(self, description, version,
|
||||||
argparse_kwargs=None):
|
argparse_kwargs=None):
|
||||||
"""Return an argparse option parser for this application.
|
"""Return an argparse option parser for this application.
|
||||||
|
|
|
@ -22,7 +22,10 @@ class TableFormatter(ListFormatter, SingleFormatter):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def emit_list(self, column_names, data, stdout, parsed_args):
|
def emit_list(self, column_names, data, stdout, parsed_args):
|
||||||
x = prettytable.PrettyTable(column_names, print_empty=False)
|
x = prettytable.PrettyTable(
|
||||||
|
column_names,
|
||||||
|
print_empty=False,
|
||||||
|
)
|
||||||
x.padding_width = 1
|
x.padding_width = 1
|
||||||
# Figure out the types of the columns in the
|
# Figure out the types of the columns in the
|
||||||
# first row and set the alignment of the
|
# first row and set the alignment of the
|
||||||
|
|
|
@ -1,11 +1,19 @@
|
||||||
|
# -*- encoding: utf-8 -*-
|
||||||
from argparse import ArgumentError
|
from argparse import ArgumentError
|
||||||
|
try:
|
||||||
|
from StringIO import StringIO
|
||||||
|
except ImportError:
|
||||||
|
# Probably python 3, that test won't be run so ignore the error
|
||||||
|
pass
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import nose
|
||||||
|
import mock
|
||||||
|
|
||||||
from cliff.app import App
|
from cliff.app import App
|
||||||
from cliff.command import Command
|
from cliff.command import Command
|
||||||
from cliff.commandmanager import CommandManager
|
from cliff.commandmanager import CommandManager
|
||||||
|
|
||||||
import mock
|
|
||||||
|
|
||||||
|
|
||||||
def make_app():
|
def make_app():
|
||||||
cmd_mgr = CommandManager('cliff.tests')
|
cmd_mgr = CommandManager('cliff.tests')
|
||||||
|
@ -227,3 +235,115 @@ def test_option_parser_conflicting_option_custom_arguments_should_not_throw():
|
||||||
)
|
)
|
||||||
|
|
||||||
MyApp()
|
MyApp()
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_encoding_default():
|
||||||
|
# The encoding should come from getdefaultlocale() because
|
||||||
|
# stdout has no encoding set.
|
||||||
|
if sys.version_info[:2] != (2, 6):
|
||||||
|
raise nose.SkipTest('only needed for python 2.6')
|
||||||
|
data = '\xc3\xa9'
|
||||||
|
u_data = data.decode('utf-8')
|
||||||
|
|
||||||
|
class MyApp(App):
|
||||||
|
def __init__(self):
|
||||||
|
super(MyApp, self).__init__(
|
||||||
|
description='testing',
|
||||||
|
version='0.1',
|
||||||
|
command_manager=CommandManager('tests'),
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout = StringIO()
|
||||||
|
getdefaultlocale = lambda: ('ignored', 'utf-8')
|
||||||
|
|
||||||
|
with mock.patch('sys.stdout', stdout):
|
||||||
|
with mock.patch('locale.getdefaultlocale', getdefaultlocale):
|
||||||
|
app = MyApp()
|
||||||
|
app.stdout.write(u_data)
|
||||||
|
actual = stdout.getvalue()
|
||||||
|
assert data == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_output_encoding_sys():
|
||||||
|
# The encoding should come from sys.stdout because it is set
|
||||||
|
# there.
|
||||||
|
if sys.version_info[:2] != (2, 6):
|
||||||
|
raise nose.SkipTest('only needed for python 2.6')
|
||||||
|
data = '\xc3\xa9'
|
||||||
|
u_data = data.decode('utf-8')
|
||||||
|
|
||||||
|
class MyApp(App):
|
||||||
|
def __init__(self):
|
||||||
|
super(MyApp, self).__init__(
|
||||||
|
description='testing',
|
||||||
|
version='0.1',
|
||||||
|
command_manager=CommandManager('tests'),
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout = StringIO()
|
||||||
|
stdout.encoding = 'utf-8'
|
||||||
|
getdefaultlocale = lambda: ('ignored', 'utf-16')
|
||||||
|
|
||||||
|
with mock.patch('sys.stdout', stdout):
|
||||||
|
with mock.patch('locale.getdefaultlocale', getdefaultlocale):
|
||||||
|
app = MyApp()
|
||||||
|
app.stdout.write(u_data)
|
||||||
|
actual = stdout.getvalue()
|
||||||
|
assert data == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_encoding_default():
|
||||||
|
# The encoding should come from getdefaultlocale() because
|
||||||
|
# stdout has no encoding set.
|
||||||
|
if sys.version_info[:2] != (2, 6):
|
||||||
|
raise nose.SkipTest('only needed for python 2.6')
|
||||||
|
data = '\xc3\xa9'
|
||||||
|
u_data = data.decode('utf-8')
|
||||||
|
|
||||||
|
class MyApp(App):
|
||||||
|
def __init__(self):
|
||||||
|
super(MyApp, self).__init__(
|
||||||
|
description='testing',
|
||||||
|
version='0.1',
|
||||||
|
command_manager=CommandManager('tests'),
|
||||||
|
)
|
||||||
|
|
||||||
|
stderr = StringIO()
|
||||||
|
getdefaultlocale = lambda: ('ignored', 'utf-8')
|
||||||
|
|
||||||
|
with mock.patch('sys.stderr', stderr):
|
||||||
|
with mock.patch('locale.getdefaultlocale', getdefaultlocale):
|
||||||
|
app = MyApp()
|
||||||
|
app.stderr.write(u_data)
|
||||||
|
actual = stderr.getvalue()
|
||||||
|
assert data == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_encoding_sys():
|
||||||
|
# The encoding should come from sys.stdout (not sys.stderr)
|
||||||
|
# because it is set there.
|
||||||
|
if sys.version_info[:2] != (2, 6):
|
||||||
|
raise nose.SkipTest('only needed for python 2.6')
|
||||||
|
data = '\xc3\xa9'
|
||||||
|
u_data = data.decode('utf-8')
|
||||||
|
|
||||||
|
class MyApp(App):
|
||||||
|
def __init__(self):
|
||||||
|
super(MyApp, self).__init__(
|
||||||
|
description='testing',
|
||||||
|
version='0.1',
|
||||||
|
command_manager=CommandManager('tests'),
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout = StringIO()
|
||||||
|
stdout.encoding = 'utf-8'
|
||||||
|
stderr = StringIO()
|
||||||
|
getdefaultlocale = lambda: ('ignored', 'utf-16')
|
||||||
|
|
||||||
|
with mock.patch('sys.stdout', stdout):
|
||||||
|
with mock.patch('sys.stderr', stderr):
|
||||||
|
with mock.patch('locale.getdefaultlocale', getdefaultlocale):
|
||||||
|
app = MyApp()
|
||||||
|
app.stderr.write(u_data)
|
||||||
|
actual = stderr.getvalue()
|
||||||
|
assert data == actual
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
# -*- encoding: utf-8 -*-
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from cliff.lister import Lister
|
||||||
|
|
||||||
|
|
||||||
|
class Encoding(Lister):
|
||||||
|
"""Show some unicode text
|
||||||
|
"""
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def take_action(self, parsed_args):
|
||||||
|
messages = [
|
||||||
|
u'pi: π',
|
||||||
|
u'GB18030:鼀丅㐀ٸཌྷᠧꌢ€',
|
||||||
|
]
|
||||||
|
return (
|
||||||
|
('UTF-8', 'Unicode'),
|
||||||
|
[(repr(t.encode('utf-8')), t)
|
||||||
|
for t in messages],
|
||||||
|
)
|
|
@ -68,6 +68,7 @@ setup(
|
||||||
'files = cliffdemo.list:Files',
|
'files = cliffdemo.list:Files',
|
||||||
'file = cliffdemo.show:File',
|
'file = cliffdemo.show:File',
|
||||||
'show file = cliffdemo.show:File',
|
'show file = cliffdemo.show:File',
|
||||||
|
'unicode = cliffdemo.encoding:Encoding',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue