diff -r 097b72d73594 Lib/pydoc.py --- a/Lib/pydoc.py Fri Apr 27 00:20:39 2012 -0700 +++ b/Lib/pydoc.py Fri Apr 27 13:52:05 2012 +0200 @@ -182,6 +182,24 @@ return name, kind, cls, value return map(fixup, inspect.classify_class_attrs(object)) +# ----------------------------------------------------- Unicode support helpers + +try: + unicode +except NameError: + _encoding = 'ascii' + def _encode(text, encoding='ascii'): + return text +else: + import locale + _encoding = locale.getpreferredencoding() + + def _encode(text, encoding=None): + if isinstance(text, unicode): + return text.encode(encoding or _encoding, 'xmlcharrefreplace') + else: + return text + # ----------------------------------------------------- module manipulation def ispackage(path): @@ -424,12 +442,12 @@ def page(self, title, contents): """Format an HTML page.""" - return ''' + return _encode(''' Python: %s %s -''' % (title, contents) +''' % (title, contents), 'ascii') def heading(self, title, fgcol, bgcol, extras=''): """Format a page heading.""" @@ -1375,7 +1393,7 @@ """Page through text by feeding it to another program.""" pipe = os.popen(cmd, 'w') try: - pipe.write(text) + pipe.write(_encode(text, _encoding)) pipe.close() except IOError: pass # Ignore broken pipes caused by quitting the pager program. @@ -1385,7 +1403,7 @@ import tempfile filename = tempfile.mktemp() file = open(filename, 'w') - file.write(text) + file.write(_encode(text, _encoding)) file.close() try: os.system(cmd + ' "' + filename + '"') @@ -1394,7 +1412,7 @@ def ttypager(text): """Page through text on a text terminal.""" - lines = split(plain(text), '\n') + lines = plain(_encode(plain(text), getattr(sys.stdout, 'encoding', _encoding))).split('\n') try: import tty fd = sys.stdin.fileno() @@ -1432,7 +1450,7 @@ def plainpager(text): """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(text)) + sys.stdout.write(_encode(plain(text), getattr(sys.stdout, 'encoding', _encoding))) def describe(thing): """Produce a short description of the given thing.""" diff -r 097b72d73594 Lib/test/test_pydoc.py --- a/Lib/test/test_pydoc.py Fri Apr 27 00:20:39 2012 -0700 +++ b/Lib/test/test_pydoc.py Fri Apr 27 13:52:05 2012 +0200 @@ -373,6 +373,90 @@ self.assertIn('_asdict', helptext) +class TestUnicode(unittest.TestCase): + + if test.test_support.have_unicode: + + def setUp(self): + # Better not to use unicode escapes in literals, lest the + # parser choke on it if Python has been built without + # unicode support. + self.Q = type('Q', (object,), + {'__doc__': 'Rational numbers: \xe2\x84\x9a'.decode('utf8')}) + self.assertIsInstance(self.Q.__doc__, unicode) + + def test_render_doc(self): + # render_doc is robust against unicode in docstrings + doc = pydoc.render_doc(self.Q) + self.assertIsInstance(doc, unicode) + + def test_encode(self): + # _encode is robust against characters out the specified encoding + self.assertEqual(pydoc._encode(self.Q.__doc__, 'ascii'), 'Rational numbers: ℚ') + + def test_pipepager(self): + # pipepager does not choke on unicode + doc = pydoc.render_doc(self.Q) + + saved, os.popen = os.popen, open + try: + with test.test_support.temp_cwd(): + pydoc.pipepager(doc, 'pipe') + self.assertTrue(open('pipe').read(), pydoc._encode(doc)) + finally: + os.popen = saved + + def test_tempfilepager(self): + # tempfilepager does not choke on unicode + doc = pydoc.render_doc(self.Q) + + output = {} + def mock_system(cmd): + import ast + output['content'] = open(ast.literal_eval(cmd.strip())).read() + saved, os.system = os.system, mock_system + try: + pydoc.tempfilepager(doc, '') + self.assertEqual(output['content'], pydoc._encode(doc)) + finally: + os.system = saved + + def test_plainpager(self): + # plainpager does not choke on unicode + doc = pydoc.render_doc(self.Q) + + # Note: captured_stdout is too permissive when it comes to + # unicode, and using it here would make the test always + # pass. + with test.test_support.temp_cwd(): + with open('output', 'w') as f: + saved, sys.stdout = sys.stdout, f + try: + pydoc.plainpager(doc) + finally: + sys.stdout = saved + self.assertIn('Rational numbers:', open('output').read()) + + def test_ttypager(self): + # ttypager does not choke on unicode + doc = pydoc.render_doc(self.Q) + # Test ttypager + with test.test_support.temp_cwd(), test.test_support.captured_stdin(): + with open('output', 'w') as f: + saved, sys.stdout = sys.stdout, f + try: + pydoc.ttypager(doc) + finally: + sys.stdout = saved + self.assertIn('Rational numbers:', open('output').read()) + + def test_htmlpage(self): + # html.page does not choke on unicode + with test.test_support.temp_cwd(): + with captured_stdout() as output: + pydoc.writedoc(self.Q) + self.assertEqual(output.getvalue(), 'wrote Q.html\n') + class TestHelper(unittest.TestCase): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), @@ -401,6 +485,7 @@ test.test_support.run_unittest(PyDocDocTest, PydocImportTest, TestDescriptions, + TestUnicode, TestHelper) finally: reap_children() diff -r 097b72d73594 Misc/ACKS --- a/Misc/ACKS Fri Apr 27 00:20:39 2012 -0700 +++ b/Misc/ACKS Fri Apr 27 13:52:05 2012 +0200 @@ -833,6 +833,7 @@ Geoff Talvola William Tanksley Christian Tanzer +Stefano Taschini Steven Taschuk Monty Taylor Amy Taylor