Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(28623)

Delta Between Two Patch Sets: Lib/compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 4 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Doc/library/compileall.rst ('k') | Lib/test/test_compileall.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files. 1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files.
2 2
3 When called as a script with arguments, this compiles the directories 3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from 4 given as arguments recursively; the -l option prevents it from
5 recursing into directories. 5 recursing into directories.
6 6
7 Without arguments, if compiles all modules on sys.path, without 7 Without arguments, if compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for 8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.) 9 packages -- for now, you'll have to deal with packages separately.)
10 10
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
66 """ 66 """
67 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, 67 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
68 ddir=ddir) 68 ddir=ddir)
69 success = 1 69 success = 1
70 if workers is not None and workers != 1: 70 if workers is not None and workers != 1:
71 if workers < 0: 71 if workers < 0:
72 raise ValueError('workers must be greater or equal to 0') 72 raise ValueError('workers must be greater or equal to 0')
73 if ProcessPoolExecutor is None: 73 if ProcessPoolExecutor is None:
74 raise NotImplementedError('multiprocessing support not available') 74 raise NotImplementedError('multiprocessing support not available')
75 75
76 workers = workers or os.cpu_count() 76 workers = workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `workers or None` as that will default
77 with ProcessPoolExecutor(max_workers=workers) as executor: 77 with ProcessPoolExecutor(max_workers=workers) as executor:
78 results = executor.map(partial(compile_file, 78 results = executor.map(partial(compile_file,
79 ddir=ddir, force=force, 79 ddir=ddir, force=force,
80 rx=rx, quiet=quiet, 80 rx=rx, quiet=quiet,
81 legacy=legacy, 81 legacy=legacy,
82 optimize=optimize), 82 optimize=optimize),
83 files) 83 files)
84 success = min(results, default=1) 84 success = min(results, default=1)
85 else: 85 else:
86 for file in files: 86 for file in files:
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after
190 190
191 def main(): 191 def main():
192 """Script main program.""" 192 """Script main program."""
193 import argparse 193 import argparse
194 194
195 parser = argparse.ArgumentParser( 195 parser = argparse.ArgumentParser(
196 description='Utilities to support installing Python libraries.') 196 description='Utilities to support installing Python libraries.')
197 parser.add_argument('-l', action='store_const', const=0, 197 parser.add_argument('-l', action='store_const', const=0,
198 default=10, dest='maxlevels', 198 default=10, dest='maxlevels',
199 help="don't recurse into subdirectories") 199 help="don't recurse into subdirectories")
200 parser.add_argument('-r', type=int, dest='recursion',
201 help=('control the maximum recursion level. '
202 'if `-l` and `-r` options are specified, '
203 'then `-r` takes precedence.'))
200 parser.add_argument('-f', action='store_true', dest='force', 204 parser.add_argument('-f', action='store_true', dest='force',
201 help='force rebuild even if timestamps are up to date') 205 help='force rebuild even if timestamps are up to date')
202 parser.add_argument('-q', action='store_true', dest='quiet', 206 parser.add_argument('-q', action='store_true', dest='quiet',
203 help='output only error messages') 207 help='output only error messages')
204 parser.add_argument('-b', action='store_true', dest='legacy', 208 parser.add_argument('-b', action='store_true', dest='legacy',
205 help='use legacy (pre-PEP3147) compiled file locations') 209 help='use legacy (pre-PEP3147) compiled file locations')
206 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, 210 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
207 help=('directory to prepend to file paths for use in ' 211 help=('directory to prepend to file paths for use in '
208 'compile-time tracebacks and in runtime ' 212 'compile-time tracebacks and in runtime '
209 'tracebacks in cases where the source file is ' 213 'tracebacks in cases where the source file is '
(...skipping 16 matching lines...) Expand all
226 args = parser.parse_args() 230 args = parser.parse_args()
227 compile_dests = args.compile_dest 231 compile_dests = args.compile_dest
228 232
229 if (args.ddir and (len(compile_dests) != 1 233 if (args.ddir and (len(compile_dests) != 1
230 or not os.path.isdir(compile_dests[0]))): 234 or not os.path.isdir(compile_dests[0]))):
231 parser.exit('-d destdir requires exactly one directory argument') 235 parser.exit('-d destdir requires exactly one directory argument')
232 if args.rx: 236 if args.rx:
233 import re 237 import re
234 args.rx = re.compile(args.rx) 238 args.rx = re.compile(args.rx)
235 239
240
241 if args.recursion is not None:
242 maxlevels = args.recursion
243 else:
244 maxlevels = args.maxlevels
245
236 # if flist is provided then load it 246 # if flist is provided then load it
237 if args.flist: 247 if args.flist:
238 try: 248 try:
239 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 249 with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
240 for line in f: 250 for line in f:
241 compile_dests.append(line.strip()) 251 compile_dests.append(line.strip())
242 except OSError: 252 except OSError:
243 print("Error reading file list {}".format(args.flist)) 253 print("Error reading file list {}".format(args.flist))
244 return False 254 return False
245 255
246 if args.workers is not None: 256 if args.workers is not None:
247 args.workers = args.workers or os.cpu_count() 257 args.workers = args.workers or os.cpu_count()
brett.cannon 2014/09/12 16:21:59 This can be `args.workers or None`.
248 258
249 success = True 259 success = True
250 try: 260 try:
251 if compile_dests: 261 if compile_dests:
252 for dest in compile_dests: 262 for dest in compile_dests:
253 if os.path.isfile(dest): 263 if os.path.isfile(dest):
254 if not compile_file(dest, args.ddir, args.force, args.rx, 264 if not compile_file(dest, args.ddir, args.force, args.rx,
255 args.quiet, args.legacy): 265 args.quiet, args.legacy):
256 success = False 266 success = False
257 else: 267 else:
258 if not compile_dir(dest, args.maxlevels, args.ddir, 268 if not compile_dir(dest, maxlevels, args.ddir,
259 args.force, args.rx, args.quiet, 269 args.force, args.rx, args.quiet,
260 args.legacy, workers=args.workers): 270 args.legacy, workers=args.workers):
261 success = False 271 success = False
262 return success 272 return success
263 else: 273 else:
264 return compile_path(legacy=args.legacy, force=args.force, 274 return compile_path(legacy=args.legacy, force=args.force,
265 quiet=args.quiet) 275 quiet=args.quiet)
266 except KeyboardInterrupt: 276 except KeyboardInterrupt:
267 print("\n[interrupted]") 277 print("\n[interrupted]")
268 return False 278 return False
269 return True 279 return True
270 280
271 281
272 if __name__ == '__main__': 282 if __name__ == '__main__':
273 exit_status = int(not main()) 283 exit_status = int(not main())
274 sys.exit(exit_status) 284 sys.exit(exit_status)
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+