Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(11368)

Side by Side Diff: Lib/compileall.py

Issue 16104: Use multiprocessing in compileall script
Patch Set: Created 5 years, 6 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files. 1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files.
2 2
3 When called as a script with arguments, this compiles the directories 3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from 4 given as arguments recursively; the -l option prevents it from
5 recursing into directories. 5 recursing into directories.
6 6
7 Without arguments, if compiles all modules on sys.path, without 7 Without arguments, if compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for 8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.) 9 packages -- for now, you'll have to deal with packages separately.)
10 10
11 See module py_compile for details of the actual byte-compilation. 11 See module py_compile for details of the actual byte-compilation.
12 """ 12 """
13 import os 13 import os
14 import sys 14 import sys
15 import errno 15 import errno
16 import importlib.util 16 import importlib.util
17 import py_compile 17 import py_compile
18 import struct 18 import struct
19 19
20 try:
21 from concurrent.futures import ProcessPoolExecutor
22 _have_multiprocessing = True
Jim.J.Jewett 2014/04/24 03:32:26 I personally see this as a good place to get the C
Claudiu.Popa 2014/04/24 08:05:18 2 doesn't seem a proper choice here. If the count
23 except ImportError:
24 _have_multiprocessing = False
Jim.J.Jewett 2014/04/24 03:32:26 _multiprocessing_maxworkers=0
25 from functools import partial
26
20 __all__ = ["compile_dir","compile_file","compile_path"] 27 __all__ = ["compile_dir","compile_file","compile_path"]
21 28
29 def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False):
30 if not quiet:
31 print('Listing {!r}...'.format(dir))
32 try:
33 names = os.listdir(dir)
34 except OSError:
35 print("Can't list {!r}".format(dir))
36 names = []
37 names.sort()
38 for name in names:
39 if name == '__pycache__':
40 continue
41 fullname = os.path.join(dir, name)
42 if ddir is not None:
43 dfile = os.path.join(ddir, name)
44 else:
45 dfile = None
46 if not os.path.isdir(fullname):
47 yield fullname
48 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
49 os.path.isdir(fullname) and not os.path.islink(fullname)):
Jim.J.Jewett 2014/04/24 03:32:26 Is the skipping of symbolic links documented? (and
Claudiu.Popa 2014/04/24 08:05:18 It's not documented, but it is not the subject of
50 yield from _walk_dir(fullname, ddir=dfile,
51 maxlevels=maxlevels - 1, quiet=quiet)
52
22 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 53 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
Jim.J.Jewett 2014/04/24 03:32:26 What is rx?
Claudiu.Popa 2014/04/24 08:05:18 A regex. Skip files matching it.
23 quiet=False, legacy=False, optimize=-1): 54 quiet=False, legacy=False, optimize=-1, processes=None):
Jim.J.Jewett 2014/04/24 03:32:26 processes=0 already has a special meaning. Giving
Claudiu.Popa 2014/04/24 08:05:18 For test.regrtest, 1 is the same as None, meaning
24 """Byte-compile all modules in the given directory tree. 55 """Byte-compile all modules in the given directory tree.
25 56
26 Arguments (only dir is required): 57 Arguments (only dir is required):
27 58
28 dir: the directory to byte-compile 59 dir: the directory to byte-compile
29 maxlevels: maximum recursion level (default 10) 60 maxlevels: maximum recursion level (default 10)
30 ddir: the directory that will be prepended to the path to the 61 ddir: the directory that will be prepended to the path to the
31 file as it is compiled into each byte-code file. 62 file as it is compiled into each byte-code file.
32 force: if True, force compilation, even if timestamps are up-to-date 63 force: if True, force compilation, even if timestamps are up-to-date
33 quiet: if True, be quiet during compilation 64 quiet: if True, be quiet during compilation
34 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 65 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
35 optimize: optimization level or -1 for level of the interpreter 66 optimize: optimization level or -1 for level of the interpreter
67 processes: if given, it will be the number of workers which will
68 process the given directory.
Jim.J.Jewett 2014/04/24 03:32:26 The wording is a bit awkward; force/quiet/legacy (
36 """ 69 """
37 if not quiet: 70 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
38 print('Listing {!r}...'.format(dir)) 71 ddir=ddir)
39 try:
40 names = os.listdir(dir)
41 except OSError:
42 print("Can't list {!r}".format(dir))
43 names = []
44 names.sort()
45 success = 1 72 success = 1
Jim.J.Jewett 2014/04/24 03:32:26 might as well use booleans now... that just shows
Claudiu.Popa 2014/04/24 08:05:18 Yes, but not in this patch. :-)
46 for name in names: 73 if processes is not None:
47 if name == '__pycache__': 74 if not _have_multiprocessing:
48 continue 75 raise ValueError('multiprocessing support not available')
Jim.J.Jewett 2014/04/24 03:32:26 Is this worth raising an error? There is a fine f
Claudiu.Popa 2014/04/24 08:05:18 Better to be explicit about it.
49 fullname = os.path.join(dir, name) 76 with ProcessPoolExecutor(
50 if ddir is not None: 77 max_workers=processes) as executor:
51 dfile = os.path.join(ddir, name) 78 results = executor.map(partial(compile_file,
52 else: 79 ddir=ddir, force=force,
53 dfile = None 80 rx=rx, quiet=quiet,
54 if not os.path.isdir(fullname): 81 legacy=legacy,
55 if not compile_file(fullname, ddir, force, rx, quiet, 82 optimize=optimize),
83 files)
84 for result in results:
85 success = 0 if not result else 1
Jim.J.Jewett 2014/04/24 03:32:26 This keeps only the last value, so an earlier fail
86 else:
87 for file in files:
88 if not compile_file(file, ddir, force, rx, quiet,
56 legacy, optimize): 89 legacy, optimize):
57 success = 0
58 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
59 os.path.isdir(fullname) and not os.path.islink(fullname)):
60 if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
61 quiet, legacy, optimize):
62 success = 0 90 success = 0
63 return success 91 return success
Jim.J.Jewett 2014/04/24 03:32:26 def compile_dir(dir, maxlevels=10, ddir=None, forc
Claudiu.Popa 2014/04/24 08:05:18 Using the same logic for non-multiprocessing branc
64 92
65 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, 93 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
66 legacy=False, optimize=-1): 94 legacy=False, optimize=-1):
67 """Byte-compile one file. 95 """Byte-compile one file.
68 96
69 Arguments (only fullname is required): 97 Arguments (only fullname is required):
70 98
71 fullname: the file to byte-compile 99 fullname: the file to byte-compile
72 ddir: if given, the directory name compiled in to the 100 ddir: if given, the directory name compiled in to the
73 byte-code file. 101 byte-code file.
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after
186 'the regexp is searched for in the full path ' 214 'the regexp is searched for in the full path '
187 'of each file considered for compilation')) 215 'of each file considered for compilation'))
188 parser.add_argument('-i', metavar='FILE', dest='flist', 216 parser.add_argument('-i', metavar='FILE', dest='flist',
189 help=('add all the files and directories listed in ' 217 help=('add all the files and directories listed in '
190 'FILE to the list considered for compilation; ' 218 'FILE to the list considered for compilation; '
191 'if "-", names are read from stdin')) 219 'if "-", names are read from stdin'))
192 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 220 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
193 help=('zero or more file and directory names ' 221 help=('zero or more file and directory names '
194 'to compile; if no arguments given, defaults ' 222 'to compile; if no arguments given, defaults '
195 'to the equivalent of -l sys.path')) 223 'to the equivalent of -l sys.path'))
224 parser.add_argument('-j', '--processes', action='store', default=None,
225 type=int, help='Run compileall concurrently')
226
Jim.J.Jewett 2014/04/24 03:32:26 so here, default=0
196 args = parser.parse_args() 227 args = parser.parse_args()
197
198 compile_dests = args.compile_dest 228 compile_dests = args.compile_dest
199 229
200 if (args.ddir and (len(compile_dests) != 1 230 if (args.ddir and (len(compile_dests) != 1
201 or not os.path.isdir(compile_dests[0]))): 231 or not os.path.isdir(compile_dests[0]))):
202 parser.exit('-d destdir requires exactly one directory argument') 232 parser.exit('-d destdir requires exactly one directory argument')
203 if args.rx: 233 if args.rx:
204 import re 234 import re
205 args.rx = re.compile(args.rx) 235 args.rx = re.compile(args.rx)
206 236
207 # if flist is provided then load it 237 # if flist is provided then load it
208 if args.flist: 238 if args.flist:
209 try: 239 try:
210 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 240 with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
211 for line in f: 241 for line in f:
212 compile_dests.append(line.strip()) 242 compile_dests.append(line.strip())
213 except OSError: 243 except OSError:
214 print("Error reading file list {}".format(args.flist)) 244 print("Error reading file list {}".format(args.flist))
215 return False 245 return False
246
247 if args.processes is not None:
248 if args.processes <= 0:
249 args.processes = os.cpu_count()
Jim.J.Jewett 2014/04/24 03:32:26 and do this logic in the function itself, so inter
Claudiu.Popa 2014/04/24 08:05:18 I tried to follow closely the behaviour used by re
216 250
217 success = True 251 success = True
218 try: 252 try:
219 if compile_dests: 253 if compile_dests:
220 for dest in compile_dests: 254 for dest in compile_dests:
221 if os.path.isfile(dest): 255 if os.path.isfile(dest):
222 if not compile_file(dest, args.ddir, args.force, args.rx, 256 if not compile_file(dest, args.ddir, args.force, args.rx,
223 args.quiet, args.legacy): 257 args.quiet, args.legacy):
224 success = False 258 success = False
225 else: 259 else:
226 if not compile_dir(dest, args.maxlevels, args.ddir, 260 if not compile_dir(dest, args.maxlevels, args.ddir,
227 args.force, args.rx, args.quiet, 261 args.force, args.rx, args.quiet,
228 args.legacy): 262 args.legacy, processes=args.processes):
229 success = False 263 success = False
230 return success 264 return success
231 else: 265 else:
232 return compile_path(legacy=args.legacy, force=args.force, 266 return compile_path(legacy=args.legacy, force=args.force,
233 quiet=args.quiet) 267 quiet=args.quiet)
234 except KeyboardInterrupt: 268 except KeyboardInterrupt:
235 print("\n[interrupted]") 269 print("\n[interrupted]")
236 return False 270 return False
237 return True 271 return True
238 272
239 273
240 if __name__ == '__main__': 274 if __name__ == '__main__':
241 exit_status = int(not main()) 275 exit_status = int(not main())
242 sys.exit(exit_status) 276 sys.exit(exit_status)
OLDNEW

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+