Tools/scripts/reindent.py fails on non-UTF-8 encodings

Tools/scripts/reindent.py

 #! /usr/bin/env python3
 
 # Released to the public domain, by Tim Peters, 03 October 2000.
 
 """reindent [-d][-r][-v] [ path ... ]
 
 -d (--dryrun)   Dry run.   Analyze, but don't make any changes to, files.
 -r (--recurse)  Recurse.   Search for all .py files in subdirectories too.
 -n (--nobackup) No backup. Does not make a ".bak" file before reindenting.
 -v (--verbose)  Verbose.   Print informative msgs; else no output.
@@ skipping 23 matching lines @@
 tabnanny.py, reindent should do a good job.
 
 The backup file is a copy of the one that is being reindented. The ".bak"
 file is generated with shutil.copy(), but some corner cases regarding
 user/group and permissions could leave the backup file more readable that
 you'd prefer. You can always use the --nobackup option to prevent this.
 """
 
 __version__ = "1"
 
-import tokenize
+import io
 import os
 import shutil
 import sys
+import tokenize
 
 verbose = False
 recurse = False
 dryrun = False
 makebackup = True
 
 
 def usage(msg=None):
     if msg is None:
         msg = __doc__
@@ skipping 20 matching lines @@
         elif o in ('-r', '--recurse'):
             recurse = True
         elif o in ('-n', '--nobackup'):
             makebackup = False
         elif o in ('-v', '--verbose'):
             verbose = True
         elif o in ('-h', '--help'):
             usage()
             return
     if not args:
-        r = Reindenter(sys.stdin)
+        content = sys.stdin.buffer.read()
+        raw = io.BytesIO(content)
+        buffer = io.BufferedReader(raw)
+        encoding, _ = tokenize.detect_encoding(buffer.readline)
+        buffer.seek(0)
+        text = io.TextIOWrapper(buffer, encoding)
+
+        r = Reindenter(text)
         r.run()
-        r.write(sys.stdout)
+        r.write(sys.stdout.buffer, encoding)
         return
     for arg in args:
         check(arg)
 
 
 def check(file):
     if os.path.isdir(file) and not os.path.islink(file):
         if verbose:
             print("listing directory", file)
         names = os.listdir(file)
@@ skipping 21 matching lines @@
         if verbose:
             print("changed.")
             if dryrun:
                 print("But this is a dry run, so leaving it alone.")
         if not dryrun:
             bak = file + ".bak"
             if makebackup:
                 shutil.copyfile(file, bak)
                 if verbose:
                     print("backed up", file, "to", bak)
-            with open(file, "w", encoding=encoding) as f:
-                r.write(f)
+            with open(file, "wb") as f:
+                r.write(f, encoding)
             if verbose:
                 print("wrote new", file)
         return True
     else:
         if verbose:
             print("unchanged.")
         return False
 
 
 def _rstrip(line, JUNK='\n \t'):
@@ skipping 97 matching lines @@
                     if diff > 0:
                         if line == "\n":
                             after.append(line)
                         else:
                             after.append(" " * diff + line)
                     else:
                         remove = min(getlspace(line), -diff)
                         after.append(line[remove:])
         return self.raw != self.after
 
-    def write(self, f):
-        f.writelines(self.after)
+    def write(self, f, encoding):

[storchaka, 2012/10/15 22:51:11]

This will not work with utf-16 and the like. Use io.TextIOWrapper to wrap
sys.stdout.buffer.

Also you can check sys.stdout.isatty() and if true then use sys.stdout without
wrapping.

+        for line in self.after:
+            line = line.encode(encoding)
+            f.write(line)
 
     # Line-getter for tokenize.
     def getline(self):
         if self.index >= len(self.lines):
             line = ""
         else:
             line = self.lines[self.index]
             self.index += 1
         return line
 
@@ skipping 40 matching lines @@
 # Count number of leading blanks.
 def getlspace(line):
     i, n = 0, len(line)
     while i < n and line[i] == " ":
         i += 1
     return i
 
 
 if __name__ == '__main__':
     main()