Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(46987)

Side by Side Diff: Lib/compileall.py

Issue 16104: Use multiprocessing in compileall script
Patch Set: Created 5 years, 1 month ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Doc/library/compileall.rst ('k') | Lib/test/test_compileall.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files. 1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files.
2 2
3 When called as a script with arguments, this compiles the directories 3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from 4 given as arguments recursively; the -l option prevents it from
5 recursing into directories. 5 recursing into directories.
6 6
7 Without arguments, if compiles all modules on sys.path, without 7 Without arguments, if compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for 8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.) 9 packages -- for now, you'll have to deal with packages separately.)
10 10
11 See module py_compile for details of the actual byte-compilation. 11 See module py_compile for details of the actual byte-compilation.
12 """ 12 """
13 import os 13 import os
14 import sys 14 import sys
15 import importlib.util 15 import importlib.util
16 import py_compile 16 import py_compile
17 import struct 17 import struct
18 18
19 try:
20 from concurrent.futures import ProcessPoolExecutor
21 except ImportError:
22 ProcessPoolExecutor = None
23 from functools import partial
24
19 __all__ = ["compile_dir","compile_file","compile_path"] 25 __all__ = ["compile_dir","compile_file","compile_path"]
20 26
27 def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False):
28 if not quiet:
29 print('Listing {!r}...'.format(dir))
30 try:
31 names = os.listdir(dir)
32 except OSError:
33 print("Can't list {!r}".format(dir))
34 names = []
35 names.sort()
36 for name in names:
37 if name == '__pycache__':
38 continue
39 fullname = os.path.join(dir, name)
40 if ddir is not None:
41 dfile = os.path.join(ddir, name)
42 else:
43 dfile = None
44 if not os.path.isdir(fullname):
45 yield fullname
46 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
47 os.path.isdir(fullname) and not os.path.islink(fullname)):
48 yield from _walk_dir(fullname, ddir=dfile,
49 maxlevels=maxlevels - 1, quiet=quiet)
50
21 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 51 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
22 quiet=False, legacy=False, optimize=-1): 52 quiet=False, legacy=False, optimize=-1, workers=1):
23 """Byte-compile all modules in the given directory tree. 53 """Byte-compile all modules in the given directory tree.
24 54
25 Arguments (only dir is required): 55 Arguments (only dir is required):
26 56
27 dir: the directory to byte-compile 57 dir: the directory to byte-compile
28 maxlevels: maximum recursion level (default 10) 58 maxlevels: maximum recursion level (default 10)
29 ddir: the directory that will be prepended to the path to the 59 ddir: the directory that will be prepended to the path to the
30 file as it is compiled into each byte-code file. 60 file as it is compiled into each byte-code file.
31 force: if True, force compilation, even if timestamps are up-to-date 61 force: if True, force compilation, even if timestamps are up-to-date
32 quiet: if True, be quiet during compilation 62 quiet: if True, be quiet during compilation
33 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 63 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
34 optimize: optimization level or -1 for level of the interpreter 64 optimize: optimization level or -1 for level of the interpreter
65 workers: maximum number of parallel workers
35 """ 66 """
36 if not quiet: 67 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
37 print('Listing {!r}...'.format(dir)) 68 ddir=ddir)
38 try:
39 names = os.listdir(dir)
40 except OSError:
41 print("Can't list {!r}".format(dir))
42 names = []
43 names.sort()
44 success = 1 69 success = 1
45 for name in names: 70 if workers is not None and workers != 1:
46 if name == '__pycache__': 71 if workers < 0:
47 continue 72 raise ValueError('workers must be greater or equal to 0')
48 fullname = os.path.join(dir, name) 73 if ProcessPoolExecutor is None:
49 if ddir is not None: 74 raise NotImplementedError('multiprocessing support not available')
50 dfile = os.path.join(ddir, name) 75
51 else: 76 workers = workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `workers or None` as that will default
52 dfile = None 77 with ProcessPoolExecutor(max_workers=workers) as executor:
53 if not os.path.isdir(fullname): 78 results = executor.map(partial(compile_file,
54 if not compile_file(fullname, ddir, force, rx, quiet, 79 ddir=ddir, force=force,
80 rx=rx, quiet=quiet,
81 legacy=legacy,
82 optimize=optimize),
83 files)
84 success = min(results, default=1)
85 else:
86 for file in files:
87 if not compile_file(file, ddir, force, rx, quiet,
55 legacy, optimize): 88 legacy, optimize):
56 success = 0
57 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
58 os.path.isdir(fullname) and not os.path.islink(fullname)):
59 if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
60 quiet, legacy, optimize):
61 success = 0 89 success = 0
62 return success 90 return success
63 91
64 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, 92 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
65 legacy=False, optimize=-1): 93 legacy=False, optimize=-1):
66 """Byte-compile one file. 94 """Byte-compile one file.
67 95
68 Arguments (only fullname is required): 96 Arguments (only fullname is required):
69 97
70 fullname: the file to byte-compile 98 fullname: the file to byte-compile
(...skipping 118 matching lines...) Expand 10 before | Expand all | Expand 10 after
189 'the regexp is searched for in the full path ' 217 'the regexp is searched for in the full path '
190 'of each file considered for compilation')) 218 'of each file considered for compilation'))
191 parser.add_argument('-i', metavar='FILE', dest='flist', 219 parser.add_argument('-i', metavar='FILE', dest='flist',
192 help=('add all the files and directories listed in ' 220 help=('add all the files and directories listed in '
193 'FILE to the list considered for compilation; ' 221 'FILE to the list considered for compilation; '
194 'if "-", names are read from stdin')) 222 'if "-", names are read from stdin'))
195 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 223 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
196 help=('zero or more file and directory names ' 224 help=('zero or more file and directory names '
197 'to compile; if no arguments given, defaults ' 225 'to compile; if no arguments given, defaults '
198 'to the equivalent of -l sys.path')) 226 'to the equivalent of -l sys.path'))
227 parser.add_argument('-j', '--workers', default=1,
228 type=int, help='Run compileall concurrently')
229
199 args = parser.parse_args() 230 args = parser.parse_args()
200
201 compile_dests = args.compile_dest 231 compile_dests = args.compile_dest
202 232
203 if (args.ddir and (len(compile_dests) != 1 233 if (args.ddir and (len(compile_dests) != 1
204 or not os.path.isdir(compile_dests[0]))): 234 or not os.path.isdir(compile_dests[0]))):
205 parser.exit('-d destdir requires exactly one directory argument') 235 parser.exit('-d destdir requires exactly one directory argument')
206 if args.rx: 236 if args.rx:
207 import re 237 import re
208 args.rx = re.compile(args.rx) 238 args.rx = re.compile(args.rx)
209 239
210 240
211 if args.recursion is not None: 241 if args.recursion is not None:
212 maxlevels = args.recursion 242 maxlevels = args.recursion
213 else: 243 else:
214 maxlevels = args.maxlevels 244 maxlevels = args.maxlevels
215 245
216 # if flist is provided then load it 246 # if flist is provided then load it
217 if args.flist: 247 if args.flist:
218 try: 248 try:
219 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 249 with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
220 for line in f: 250 for line in f:
221 compile_dests.append(line.strip()) 251 compile_dests.append(line.strip())
222 except OSError: 252 except OSError:
223 print("Error reading file list {}".format(args.flist)) 253 print("Error reading file list {}".format(args.flist))
224 return False 254 return False
255
256 if args.workers is not None:
257 args.workers = args.workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `args.workers or None`.
225 258
226 success = True 259 success = True
227 try: 260 try:
228 if compile_dests: 261 if compile_dests:
229 for dest in compile_dests: 262 for dest in compile_dests:
230 if os.path.isfile(dest): 263 if os.path.isfile(dest):
231 if not compile_file(dest, args.ddir, args.force, args.rx, 264 if not compile_file(dest, args.ddir, args.force, args.rx,
232 args.quiet, args.legacy): 265 args.quiet, args.legacy):
233 success = False 266 success = False
234 else: 267 else:
235 if not compile_dir(dest, maxlevels, args.ddir, 268 if not compile_dir(dest, maxlevels, args.ddir,
236 args.force, args.rx, args.quiet, 269 args.force, args.rx, args.quiet,
237 args.legacy): 270 args.legacy, workers=args.workers):
238 success = False 271 success = False
239 return success 272 return success
240 else: 273 else:
241 return compile_path(legacy=args.legacy, force=args.force, 274 return compile_path(legacy=args.legacy, force=args.force,
242 quiet=args.quiet) 275 quiet=args.quiet)
243 except KeyboardInterrupt: 276 except KeyboardInterrupt:
244 print("\n[interrupted]") 277 print("\n[interrupted]")
245 return False 278 return False
246 return True 279 return True
247 280
248 281
249 if __name__ == '__main__': 282 if __name__ == '__main__':
250 exit_status = int(not main()) 283 exit_status = int(not main())
251 sys.exit(exit_status) 284 sys.exit(exit_status)
OLDNEW
« no previous file with comments | « Doc/library/compileall.rst ('k') | Lib/test/test_compileall.py » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+