diff -r ed291f85434b Doc/library/compileall.rst --- a/Doc/library/compileall.rst Wed Apr 23 12:17:25 2014 -0500 +++ b/Doc/library/compileall.rst Thu Apr 24 09:18:13 2014 +0300 @@ -66,9 +66,18 @@ is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. +.. cmdoption:: -j N + + Use *N* processes to compile the given directory. + If ``0`` is used, then the result of :func:`os.cpu_count()` + will be used. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. +.. versionchanged:: 3.5 + Added the ``-j`` option. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already provides the option: :program:`python -O -m compileall`. @@ -76,7 +85,7 @@ Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1, processes=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. @@ -109,9 +118,17 @@ *optimize* specifies the optimization level for the compiler. It is passed to the built-in :func:`compile` function. + The argument *processes* gives the number of workers + used to compile files in parallel. If :mod:`multiprocessing` is + unavailable, then a :exc:`ValueError` will be raised in case that + *processes* is given. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. + .. versionchanged:: 3.5 + Added the *processes* parameter. + .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) diff -r ed291f85434b Lib/compileall.py --- a/Lib/compileall.py Wed Apr 23 12:17:25 2014 -0500 +++ b/Lib/compileall.py Thu Apr 24 09:18:13 2014 +0300 @@ -16,10 +16,41 @@ import py_compile import struct +try: + from concurrent.futures import ProcessPoolExecutor + _have_multiprocessing = True +except ImportError: + _have_multiprocessing = False +from functools import partial + __all__ = ["compile_dir","compile_file","compile_path"] +def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False): + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if ddir is not None: + dfile = os.path.join(ddir, name) + else: + dfile = None + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, ddir=dfile, + maxlevels=maxlevels - 1, quiet=quiet) + def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=False, legacy=False, optimize=-1): + quiet=False, legacy=False, optimize=-1, processes=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -32,33 +63,27 @@ quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter + processes: maximum number of parallel processes """ - if not quiet: - print('Listing {!r}...'.format(dir)) - try: - names = os.listdir(dir) - except OSError: - print("Can't list {!r}".format(dir)) - names = [] - names.sort() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, + ddir=ddir) success = 1 - for name in names: - if name == '__pycache__': - continue - fullname = os.path.join(dir, name) - if ddir is not None: - dfile = os.path.join(ddir, name) - else: - dfile = None - if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, + if processes is not None and processes != 1: + if not _have_multiprocessing: + raise ValueError('multiprocessing support not available') + with ProcessPoolExecutor(max_workers=processes) as executor: + results = executor.map(partial(compile_file, + ddir=ddir, force=force, + rx=rx, quiet=quiet, + legacy=legacy, + optimize=optimize), + files) + success = min(results, default=1) + else: + for file in files: + if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): success = 0 - elif (maxlevels > 0 and name != os.curdir and name != os.pardir and - os.path.isdir(fullname) and not os.path.islink(fullname)): - if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet, legacy, optimize): - success = 0 return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, @@ -192,8 +217,10 @@ help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' 'to the equivalent of -l sys.path')) + parser.add_argument('-j', '--processes', action='store', default=None, + type=int, help='Run compileall concurrently') + args = parser.parse_args() - compile_dests = args.compile_dest if (args.ddir and (len(compile_dests) != 1 @@ -213,6 +240,10 @@ print("Error reading file list {}".format(args.flist)) return False + if args.processes is not None: + if args.processes <= 0: + args.processes = os.cpu_count() + success = True try: if compile_dests: @@ -224,7 +255,7 @@ else: if not compile_dir(dest, args.maxlevels, args.ddir, args.force, args.rx, args.quiet, - args.legacy): + args.legacy, processes=args.processes): success = False return success else: diff -r ed291f85434b Lib/test/test_compileall.py --- a/Lib/test/test_compileall.py Wed Apr 23 12:17:25 2014 -0500 +++ b/Lib/test/test_compileall.py Thu Apr 24 09:18:13 2014 +0300 @@ -10,6 +10,13 @@ import unittest import io +from unittest import mock, skipUnless +try: + from concurrent.futures import ProcessPoolExecutor + _have_multiprocessing = True +except ImportError: + _have_multiprocessing = False + from test import support, script_helper class CompileallTests(unittest.TestCase): @@ -106,6 +113,31 @@ debug_override=not optimize) self.assertTrue(os.path.isfile(cached3)) + @skipUnless(_have_multiprocessing, "requires multiprocessing") + @mock.patch('compileall.ProcessPoolExecutor') + def test_compile_processes(self, pool_mock): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + compileall.compile_dir(self.directory, quiet=True, processes=5) + self.assertTrue(pool_mock.called) + + @skipUnless(_have_multiprocessing, "requires multiprocessing") + @mock.patch('compileall.ProcessPoolExecutor') + @mock.patch('compileall.compile_file') + def test_compile_processes_on_one(self, compile_mock, pool_mock): + # Test that `compile_dir` doesn't use ProcessPoolExecutor + # when passing processes=1 + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + compileall.compile_dir(self.directory, quiet=True, processes=1) + self.assertFalse(pool_mock.called) + self.assertTrue(compile_mock.called) + + @mock.patch('compileall._have_multiprocessing', False) + def test_compile_missing_multiprocessing(self): + with self.assertRaises(ValueError) as cm: + compileall.compile_dir(self.directory, quiet=True, processes=5) + self.assertEqual(str(cm.exception), + "multiprocessing support not available") + class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors @@ -379,6 +411,30 @@ out = self.assertRunOK('badfilename') self.assertRegex(out, b"Can't list 'badfilename'") + @skipUnless(_have_multiprocessing, "requires multiprocessing") + def test_processes(self): + bar2fn = script_helper.make_script(self.directory, 'bar2', '') + files = [] + for suffix in range(5): + pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) + os.mkdir(pkgdir) + fn = script_helper.make_script(pkgdir, '__init__', '') + files.append(script_helper.make_script(pkgdir, 'bar2', '')) + + self.assertRunOK(self.directory, '-j', '0') + self.assertCompiled(bar2fn) + for file in files: + self.assertCompiled(file) + + @mock.patch('compileall.compile_dir') + def test_processes_available_cores(self, compile_dir): + with mock.patch("sys.argv", + new=[sys.executable, self.directory, "-j0"]): + compileall.main() + self.assertTrue(compile_dir.called) + self.assertEqual(compile_dir.call_args[-1]['processes'], + os.cpu_count()) + if __name__ == "__main__": unittest.main()