Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(15106)

Delta Between Two Patch Sets: Lib/compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 4 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Doc/library/compileall.rst ('k') | Lib/test/test_compileall.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files. 1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files.
2 2
3 When called as a script with arguments, this compiles the directories 3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from 4 given as arguments recursively; the -l option prevents it from
5 recursing into directories. 5 recursing into directories.
6 6
7 Without arguments, if compiles all modules on sys.path, without 7 Without arguments, if compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for 8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.) 9 packages -- for now, you'll have to deal with packages separately.)
10 10
11 See module py_compile for details of the actual byte-compilation. 11 See module py_compile for details of the actual byte-compilation.
12 """ 12 """
13 import os 13 import os
14 import sys 14 import sys
15 import importlib.util 15 import importlib.util
16 import py_compile 16 import py_compile
17 import struct 17 import struct
18 18
19 try: 19 try:
20 from concurrent.futures import ProcessPoolExecutor 20 from concurrent.futures import ProcessPoolExecutor
21 _have_multiprocessing = True
Jim.J.Jewett 2014/04/25 23:42:22 This variable is only used once. Trying to elimin
22 except ImportError: 21 except ImportError:
23 _have_multiprocessing = False 22 ProcessPoolExecutor = None
24 from functools import partial 23 from functools import partial
25 24
26 __all__ = ["compile_dir","compile_file","compile_path"] 25 __all__ = ["compile_dir","compile_file","compile_path"]
27 26
28 def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False): 27 def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False):
29 if not quiet: 28 if not quiet:
30 print('Listing {!r}...'.format(dir)) 29 print('Listing {!r}...'.format(dir))
31 try: 30 try:
32 names = os.listdir(dir) 31 names = os.listdir(dir)
33 except OSError: 32 except OSError:
34 print("Can't list {!r}".format(dir)) 33 print("Can't list {!r}".format(dir))
35 names = [] 34 names = []
36 names.sort() 35 names.sort()
37 for name in names: 36 for name in names:
38 if name == '__pycache__': 37 if name == '__pycache__':
39 continue 38 continue
40 fullname = os.path.join(dir, name) 39 fullname = os.path.join(dir, name)
41 if ddir is not None: 40 if ddir is not None:
42 dfile = os.path.join(ddir, name) 41 dfile = os.path.join(ddir, name)
43 else: 42 else:
44 dfile = None 43 dfile = None
45 if not os.path.isdir(fullname): 44 if not os.path.isdir(fullname):
46 yield fullname 45 yield fullname
47 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and 46 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
48 os.path.isdir(fullname) and not os.path.islink(fullname)): 47 os.path.isdir(fullname) and not os.path.islink(fullname)):
49 yield from _walk_dir(fullname, ddir=dfile, 48 yield from _walk_dir(fullname, ddir=dfile,
50 maxlevels=maxlevels - 1, quiet=quiet) 49 maxlevels=maxlevels - 1, quiet=quiet)
51 50
52 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 51 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
53 quiet=False, legacy=False, optimize=-1, processes=None): 52 quiet=False, legacy=False, optimize=-1, workers=1):
54 """Byte-compile all modules in the given directory tree. 53 """Byte-compile all modules in the given directory tree.
55 54
56 Arguments (only dir is required): 55 Arguments (only dir is required):
57 56
58 dir: the directory to byte-compile 57 dir: the directory to byte-compile
59 maxlevels: maximum recursion level (default 10) 58 maxlevels: maximum recursion level (default 10)
Jim.J.Jewett 2014/04/25 23:42:22 Note that symbolic links to directories are ignore
Claudiu.Popa 2014/04/27 15:01:18 This should really go in another patch.
60 ddir: the directory that will be prepended to the path to the 59 ddir: the directory that will be prepended to the path to the
61 file as it is compiled into each byte-code file. 60 file as it is compiled into each byte-code file.
62 force: if True, force compilation, even if timestamps are up-to-date 61 force: if True, force compilation, even if timestamps are up-to-date
Jim.J.Jewett 2014/04/25 22:33:30 rx: a Regular Expression used to exclude file
Claudiu.Popa 2014/04/27 15:01:18 But doing it in another patch-build cycle is bette
63 quiet: if True, be quiet during compilation 62 quiet: if True, be quiet during compilation
64 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 63 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
65 optimize: optimization level or -1 for level of the interpreter 64 optimize: optimization level or -1 for level of the interpreter
66 processes: maximum number of parallel processes 65 workers: maximum number of parallel workers
67 """ 66 """
68 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, 67 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
69 ddir=ddir) 68 ddir=ddir)
70 success = 1 69 success = 1
71 if processes is not None and processes != 1: 70 if workers is not None and workers != 1:
Jim.J.Jewett 2014/04/25 22:33:30 Make that and processes > 1 If someone calls t
72 if not _have_multiprocessing: 71 if workers < 0:
73 raise ValueError('multiprocessing support not available') 72 raise ValueError('workers must be greater or equal to 0')
Jim.J.Jewett 2014/04/25 23:42:22 # multiprocessing explicitly requested, and not av
Claudiu.Popa 2014/04/27 15:01:18 Raising an ImportError is not the proper way. I co
74 with ProcessPoolExecutor(max_workers=processes) as executor: 73 if ProcessPoolExecutor is None:
74 raise NotImplementedError('multiprocessing support not available')
75
76 workers = workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `workers or None` as that will default
77 with ProcessPoolExecutor(max_workers=workers) as executor:
75 results = executor.map(partial(compile_file, 78 results = executor.map(partial(compile_file,
76 ddir=ddir, force=force, 79 ddir=ddir, force=force,
77 rx=rx, quiet=quiet, 80 rx=rx, quiet=quiet,
78 legacy=legacy, 81 legacy=legacy,
79 optimize=optimize), 82 optimize=optimize),
80 files) 83 files)
81 success = min(results, default=1) 84 success = min(results, default=1)
82 else: 85 else:
83 for file in files: 86 for file in files:
84 if not compile_file(file, ddir, force, rx, quiet, 87 if not compile_file(file, ddir, force, rx, quiet,
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after
187 190
188 def main(): 191 def main():
189 """Script main program.""" 192 """Script main program."""
190 import argparse 193 import argparse
191 194
192 parser = argparse.ArgumentParser( 195 parser = argparse.ArgumentParser(
193 description='Utilities to support installing Python libraries.') 196 description='Utilities to support installing Python libraries.')
194 parser.add_argument('-l', action='store_const', const=0, 197 parser.add_argument('-l', action='store_const', const=0,
195 default=10, dest='maxlevels', 198 default=10, dest='maxlevels',
196 help="don't recurse into subdirectories") 199 help="don't recurse into subdirectories")
200 parser.add_argument('-r', type=int, dest='recursion',
201 help=('control the maximum recursion level. '
202 'if `-l` and `-r` options are specified, '
203 'then `-r` takes precedence.'))
197 parser.add_argument('-f', action='store_true', dest='force', 204 parser.add_argument('-f', action='store_true', dest='force',
198 help='force rebuild even if timestamps are up to date') 205 help='force rebuild even if timestamps are up to date')
199 parser.add_argument('-q', action='store_true', dest='quiet', 206 parser.add_argument('-q', action='store_true', dest='quiet',
200 help='output only error messages') 207 help='output only error messages')
201 parser.add_argument('-b', action='store_true', dest='legacy', 208 parser.add_argument('-b', action='store_true', dest='legacy',
202 help='use legacy (pre-PEP3147) compiled file locations') 209 help='use legacy (pre-PEP3147) compiled file locations')
203 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, 210 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
204 help=('directory to prepend to file paths for use in ' 211 help=('directory to prepend to file paths for use in '
205 'compile-time tracebacks and in runtime ' 212 'compile-time tracebacks and in runtime '
206 'tracebacks in cases where the source file is ' 213 'tracebacks in cases where the source file is '
207 'unavailable')) 214 'unavailable'))
208 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, 215 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
209 help=('skip files matching the regular expression; ' 216 help=('skip files matching the regular expression; '
210 'the regexp is searched for in the full path ' 217 'the regexp is searched for in the full path '
211 'of each file considered for compilation')) 218 'of each file considered for compilation'))
212 parser.add_argument('-i', metavar='FILE', dest='flist', 219 parser.add_argument('-i', metavar='FILE', dest='flist',
213 help=('add all the files and directories listed in ' 220 help=('add all the files and directories listed in '
214 'FILE to the list considered for compilation; ' 221 'FILE to the list considered for compilation; '
215 'if "-", names are read from stdin')) 222 'if "-", names are read from stdin'))
216 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 223 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
217 help=('zero or more file and directory names ' 224 help=('zero or more file and directory names '
218 'to compile; if no arguments given, defaults ' 225 'to compile; if no arguments given, defaults '
219 'to the equivalent of -l sys.path')) 226 'to the equivalent of -l sys.path'))
220 parser.add_argument('-j', '--processes', action='store', default=None, 227 parser.add_argument('-j', '--workers', default=1,
221 type=int, help='Run compileall concurrently') 228 type=int, help='Run compileall concurrently')
222 229
223 args = parser.parse_args() 230 args = parser.parse_args()
224 compile_dests = args.compile_dest 231 compile_dests = args.compile_dest
225 232
226 if (args.ddir and (len(compile_dests) != 1 233 if (args.ddir and (len(compile_dests) != 1
227 or not os.path.isdir(compile_dests[0]))): 234 or not os.path.isdir(compile_dests[0]))):
228 parser.exit('-d destdir requires exactly one directory argument') 235 parser.exit('-d destdir requires exactly one directory argument')
229 if args.rx: 236 if args.rx:
230 import re 237 import re
231 args.rx = re.compile(args.rx) 238 args.rx = re.compile(args.rx)
239
240
241 if args.recursion is not None:
242 maxlevels = args.recursion
243 else:
244 maxlevels = args.maxlevels
232 245
233 # if flist is provided then load it 246 # if flist is provided then load it
234 if args.flist: 247 if args.flist:
235 try: 248 try:
236 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 249 with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
237 for line in f: 250 for line in f:
238 compile_dests.append(line.strip()) 251 compile_dests.append(line.strip())
239 except OSError: 252 except OSError:
240 print("Error reading file list {}".format(args.flist)) 253 print("Error reading file list {}".format(args.flist))
241 return False 254 return False
242 255
243 if args.processes is not None: 256 if args.workers is not None:
244 if args.processes <= 0: 257 args.workers = args.workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `args.workers or None`.
245 args.processes = os.cpu_count()
246 258
247 success = True 259 success = True
248 try: 260 try:
249 if compile_dests: 261 if compile_dests:
250 for dest in compile_dests: 262 for dest in compile_dests:
251 if os.path.isfile(dest): 263 if os.path.isfile(dest):
252 if not compile_file(dest, args.ddir, args.force, args.rx, 264 if not compile_file(dest, args.ddir, args.force, args.rx,
253 args.quiet, args.legacy): 265 args.quiet, args.legacy):
254 success = False 266 success = False
255 else: 267 else:
256 if not compile_dir(dest, args.maxlevels, args.ddir, 268 if not compile_dir(dest, maxlevels, args.ddir,
257 args.force, args.rx, args.quiet, 269 args.force, args.rx, args.quiet,
258 args.legacy, processes=args.processes): 270 args.legacy, workers=args.workers):
259 success = False 271 success = False
260 return success 272 return success
261 else: 273 else:
262 return compile_path(legacy=args.legacy, force=args.force, 274 return compile_path(legacy=args.legacy, force=args.force,
263 quiet=args.quiet) 275 quiet=args.quiet)
264 except KeyboardInterrupt: 276 except KeyboardInterrupt:
265 print("\n[interrupted]") 277 print("\n[interrupted]")
266 return False 278 return False
267 return True 279 return True
268 280
269 281
270 if __name__ == '__main__': 282 if __name__ == '__main__':
271 exit_status = int(not main()) 283 exit_status = int(not main())
272 sys.exit(exit_status) 284 sys.exit(exit_status)
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+