diff -r 43820a4fcaba Doc/library/compileall.rst --- a/Doc/library/compileall.rst Mon Mar 10 19:31:52 2014 +0100 +++ b/Doc/library/compileall.rst Wed Mar 12 09:24:30 2014 +0200 @@ -65,9 +65,16 @@ is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. +.. cmdoption:: -j + + Use multiple processes to compile the given directory. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. +.. versionchanged:: 3.5 + Added the ``-j`` option. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already provides the option: :program:`python -O -m compileall`. @@ -75,7 +82,7 @@ Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1, processes=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. @@ -108,9 +115,15 @@ *optimize* specifies the optimization level for the compiler. It is passed to the built-in :func:`compile` function. + If *processes* is given, a number of ``processes`` workers will be used + for processing the directory tree concurrently. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. + .. versionchanged:: 3.5 + Added the *processes* parameter. + .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) diff -r 43820a4fcaba Lib/compileall.py --- a/Lib/compileall.py Mon Mar 10 19:31:52 2014 +0100 +++ b/Lib/compileall.py Wed Mar 12 09:24:30 2014 +0200 @@ -17,10 +17,37 @@ import py_compile import struct +from concurrent.futures import ProcessPoolExecutor +from functools import partial + __all__ = ["compile_dir","compile_file","compile_path"] +def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False): + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if ddir is not None: + dfile = os.path.join(ddir, name) + else: + dfile = None + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, ddir=dfile, + maxlevels=maxlevels - 1, quiet=quiet) + def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=False, legacy=False, optimize=-1): + quiet=False, legacy=False, optimize=-1, processes=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -33,33 +60,28 @@ quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter + processes: if given, it will be the number of workers which will + process the given directory. """ - if not quiet: - print('Listing {!r}...'.format(dir)) - try: - names = os.listdir(dir) - except OSError: - print("Can't list {!r}".format(dir)) - names = [] - names.sort() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, + ddir=ddir) success = 1 - for name in names: - if name == '__pycache__': - continue - fullname = os.path.join(dir, name) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None - if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, + if processes: + with ProcessPoolExecutor( + max_workers=processes) as executor: + results = executor.map(partial(compile_file, + ddir=ddir, force=force, + rx=rx, quiet=quiet, + legacy=legacy, + optimize=optimize), + files) + for result in results: + success = 0 if not result else 1 + else: + for file in files: + if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): success = 0 - elif (maxlevels > 0 and name != os.curdir and name != os.pardir and - os.path.isdir(fullname) and not os.path.islink(fullname)): - if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet, legacy, optimize): - success = 0 return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, @@ -193,8 +215,10 @@ help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' 'to the equivalent of -l sys.path')) + parser.add_argument('-j', '--processes', action='store', default=None, + type=int, help='Run compileall concurrently') + args = parser.parse_args() - compile_dests = args.compile_dest if (args.ddir and (len(compile_dests) != 1 @@ -225,7 +249,7 @@ else: if not compile_dir(dest, args.maxlevels, args.ddir, args.force, args.rx, args.quiet, - args.legacy): + args.legacy, processes=args.processes): success = False return success else: diff -r 43820a4fcaba Lib/test/test_compileall.py --- a/Lib/test/test_compileall.py Mon Mar 10 19:31:52 2014 +0100 +++ b/Lib/test/test_compileall.py Wed Mar 12 09:24:30 2014 +0200 @@ -10,6 +10,8 @@ import unittest import io +from unittest import mock + from test import support, script_helper class CompileallTests(unittest.TestCase): @@ -106,6 +108,12 @@ debug_override=not optimize) self.assertTrue(os.path.isfile(cached3)) + @mock.patch('compileall.ProcessPoolExecutor') + def test_compile_processes(self, pool_mock): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + compileall.compile_dir(self.directory, quiet=True, processes=5) + self.assertTrue(pool_mock.called) + class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors @@ -379,6 +387,16 @@ out = self.assertRunOK('badfilename') self.assertRegex(out, b"Can't list 'badfilename'") + def test_processes(self): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + for suffix in range(5): + pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) + os.mkdir(pkgdir) + fn = script_helper.make_script(pkgdir, '__init__', '') + bar2fn = script_helper.make_script(pkgdir, 'bar2', '') + self.assertRunOK(self.directory, '-j', '5') + self.assertCompiled(bar2fn) + if __name__ == "__main__": unittest.main()