=== modified file 'lib2to3/refactor.py' --- lib2to3/refactor.py 2008-07-16 18:28:19 +0000 +++ lib2to3/refactor.py 2008-07-25 20:25:29 +0000 @@ -20,6 +20,8 @@ import logging from collections import defaultdict from itertools import chain +from processing import Process +from tempfile import TemporaryFile # Local imports from .pgen2 import driver @@ -48,6 +50,8 @@ help="Each FIX specifies a transformation; default all") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations (fixes/fix_*.py)") + parser.add_option("-n", "--num-processes", action="store", type="int", + default=1, help="Specifies number of processes to make") parser.add_option("-p", "--print-function", action="store_true", help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", @@ -243,6 +247,9 @@ Files and subdirectories starting with '.' are skipped. """ + # keep track of children if this is going to be multiprocess + if self.options.num_processes > 1: + children = [] for dirpath, dirnames, filenames in os.walk(arg): if self.options.verbose: self.log_message("Descending into %s", dirpath) @@ -251,9 +258,45 @@ for name in filenames: if not name.startswith(".") and name.endswith("py"): fullname = os.path.join(dirpath, name) - self.refactor_file(fullname) + if self.options.num_processes > 1: + children = self.refactor_parent(fullname, children) + else: + self.refactor_file(fullname) # Modify dirnames in-place to remove subdirs with leading dots dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] + # Print the results from all remaining children + if self.options.num_processes > 1: + self.print_child(children, 1) + + def print_child(self, children, max): + """ Joins and prints children until there are only max children + left in the children list. Returns this list.""" + while len(children) >= max: + for child, pipe, filename in children: + child.join(0.1) # is this a reasonable timeout? + if not child.isAlive(): + pipe.seek(0) + results = pipe.read() + if results: + print results + self.files.append(filename) + children.remove((child, pipe, filename)) + break + return children + + def refactor_parent(self, filename, children): + """ Parent process that farms refactoring out to child processes.""" + children = self.print_child(children, self.options.num_processes) + pipe = TemporaryFile() + child = Process(target=self.refactor_child, args=[filename, pipe]) + children.append((child, pipe, filename)) + child.start() + return children + + def refactor_child(self, filename, pipe): + """ Wrapper for refactor_file that redirects output to pipe.""" + sys.stdout = pipe + self.refactor_file(filename) def refactor_file(self, filename): """Refactors a file."""