diff -r c13398566409 Doc/library/compileall.rst --- a/Doc/library/compileall.rst Wed Mar 12 12:41:44 2014 +0100 +++ b/Doc/library/compileall.rst Thu Mar 13 19:40:33 2014 +0200 @@ -65,9 +65,18 @@ is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. +.. cmdoption:: -j N + + Use *N* processes to compile the given directory. + If ``0`` is used, then the result of :func:`os.cpu_count()` + will be used. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. +.. versionchanged:: 3.5 + Added the ``-j`` option. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already provides the option: :program:`python -O -m compileall`. @@ -75,7 +84,7 @@ Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1, processes=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. @@ -108,9 +117,17 @@ *optimize* specifies the optimization level for the compiler. It is passed to the built-in :func:`compile` function. + The optional argument *processes* gives the number of workers + used to compile files in parallel. If :mod:`multiprocessing` is + unavailable, then a :exc:`ValueError` will be raised in case that + *processes* is given. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. + .. versionchanged:: 3.5 + Added the *processes* parameter. + .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) diff -r c13398566409 Lib/compileall.py --- a/Lib/compileall.py Wed Mar 12 12:41:44 2014 +0100 +++ b/Lib/compileall.py Thu Mar 13 19:40:33 2014 +0200 @@ -17,10 +17,41 @@ import py_compile import struct +try: + from concurrent.futures import ProcessPoolExecutor + _have_multiprocessing = True +except ImportError: + _have_multiprocessing = False +from functools import partial + __all__ = ["compile_dir","compile_file","compile_path"] +def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False): + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if ddir is not None: + dfile = os.path.join(ddir, name) + else: + dfile = None + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, ddir=dfile, + maxlevels=maxlevels - 1, quiet=quiet) + def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=False, legacy=False, optimize=-1): + quiet=False, legacy=False, optimize=-1, processes=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -33,33 +64,30 @@ quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter + processes: if given, it will be the number of workers which will + process the given directory. """ - if not quiet: - print('Listing {!r}...'.format(dir)) - try: - names = os.listdir(dir) - except OSError: - print("Can't list {!r}".format(dir)) - names = [] - names.sort() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, + ddir=ddir) success = 1 - for name in names: - if name == '__pycache__': - continue - fullname = os.path.join(dir, name) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None - if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, + if processes is not None: + if not _have_multiprocessing: + raise ValueError('multiprocessing support not available') + with ProcessPoolExecutor( + max_workers=processes) as executor: + results = executor.map(partial(compile_file, + ddir=ddir, force=force, + rx=rx, quiet=quiet, + legacy=legacy, + optimize=optimize), + files) + for result in results: + success = 0 if not result else 1 + else: + for file in files: + if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): success = 0 - elif (maxlevels > 0 and name != os.curdir and name != os.pardir and - os.path.isdir(fullname) and not os.path.islink(fullname)): - if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet, legacy, optimize): - success = 0 return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, @@ -193,8 +221,10 @@ help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' 'to the equivalent of -l sys.path')) + parser.add_argument('-j', '--processes', action='store', default=None, + type=int, help='Run compileall concurrently') + args = parser.parse_args() - compile_dests = args.compile_dest if (args.ddir and (len(compile_dests) != 1 @@ -214,6 +244,10 @@ print("Error reading file list {}".format(args.flist)) return False + if args.processes is not None: + if args.processes <= 0: + args.processes = os.cpu_count() + success = True try: if compile_dests: @@ -225,7 +259,7 @@ else: if not compile_dir(dest, args.maxlevels, args.ddir, args.force, args.rx, args.quiet, - args.legacy): + args.legacy, processes=args.processes): success = False return success else: diff -r c13398566409 Lib/test/test_compileall.py --- a/Lib/test/test_compileall.py Wed Mar 12 12:41:44 2014 +0100 +++ b/Lib/test/test_compileall.py Thu Mar 13 19:40:33 2014 +0200 @@ -10,6 +10,8 @@ import unittest import io +from unittest import mock + from test import support, script_helper class CompileallTests(unittest.TestCase): @@ -106,6 +108,19 @@ debug_override=not optimize) self.assertTrue(os.path.isfile(cached3)) + @mock.patch('compileall.ProcessPoolExecutor') + def test_compile_processes(self, pool_mock): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + compileall.compile_dir(self.directory, quiet=True, processes=5) + self.assertTrue(pool_mock.called) + + @mock.patch('compileall._have_multiprocessing', False) + def test_compile_missing_multiprocessing(self): + with self.assertRaises(ValueError) as cm: + compileall.compile_dir(self.directory, quiet=True, processes=5) + self.assertEqual(str(cm.exception), + "multiprocessing support not available") + class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors @@ -379,6 +394,25 @@ out = self.assertRunOK('badfilename') self.assertRegex(out, b"Can't list 'badfilename'") + def test_processes(self): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + for suffix in range(5): + pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) + os.mkdir(pkgdir) + fn = script_helper.make_script(pkgdir, '__init__', '') + bar2fn = script_helper.make_script(pkgdir, 'bar2', '') + self.assertRunOK(self.directory, '-j', '0') + self.assertCompiled(bar2fn) + + @mock.patch('compileall.compile_dir') + def test_processes_available_cores(self, compile_dir): + with mock.patch("sys.argv", + new=[sys.executable, self.directory, "-j0"]): + compileall.main() + self.assertTrue(compile_dir.called) + self.assertEqual(compile_dir.call_args[-1]['processes'], + os.cpu_count()) + if __name__ == "__main__": unittest.main()