Weirdly, this patch fixes the issue, but I'm unable to say why:


diff -r cd1de1ff2657 Lib/tokenize.py
--- a/Lib/tokenize.py   Tue Nov 09 02:08:59 2010 +0100
+++ b/Lib/tokenize.py   Tue Nov 09 17:16:21 2010 +0100
@@ -24,12 +24,12 @@ __author__ = 'Ka-Ping Yee <ping@lfw.org>
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
+import io
 import re
 import sys
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
-from io import TextIOWrapper
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
@@ -336,16 +336,14 @@ def detect_encoding(readline):
     return default, [first, second]
 
 
-_builtin_open = open
-
 def open(filename):
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
-    buffer = _builtin_open(filename, 'rb')
+    buffer = io.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
-    text = TextIOWrapper(buffer, encoding, line_buffering=True)
+    text = io.TextIOWrapper(buffer, encoding, line_buffering=True)
     text.mode = 'r'
     return text