Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(14890)

Side by Side Diff: Lib/compileall.py

Issue 16104: Use multiprocessing in compileall script
Patch Set: Created 5 years, 4 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files. 1 """Module/script to byte-compile all .py files to .pyc (or .pyo) files.
2 2
3 When called as a script with arguments, this compiles the directories 3 When called as a script with arguments, this compiles the directories
4 given as arguments recursively; the -l option prevents it from 4 given as arguments recursively; the -l option prevents it from
5 recursing into directories. 5 recursing into directories.
6 6
7 Without arguments, if compiles all modules on sys.path, without 7 Without arguments, if compiles all modules on sys.path, without
8 recursing into subdirectories. (Even though it should do so for 8 recursing into subdirectories. (Even though it should do so for
9 packages -- for now, you'll have to deal with packages separately.) 9 packages -- for now, you'll have to deal with packages separately.)
10 10
11 See module py_compile for details of the actual byte-compilation. 11 See module py_compile for details of the actual byte-compilation.
12 """ 12 """
13 import os 13 import os
14 import sys 14 import sys
15 import importlib.util 15 import importlib.util
16 import py_compile 16 import py_compile
17 import struct 17 import struct
18 18
19 try:
20 from concurrent.futures import ProcessPoolExecutor
21 _have_multiprocessing = True
Jim.J.Jewett 2014/04/25 23:42:22 This variable is only used once. Trying to elimin
22 except ImportError:
23 _have_multiprocessing = False
24 from functools import partial
25
19 __all__ = ["compile_dir","compile_file","compile_path"] 26 __all__ = ["compile_dir","compile_file","compile_path"]
20 27
28 def _walk_dir(dir, ddir=None, maxlevels=10, quiet=False):
29 if not quiet:
30 print('Listing {!r}...'.format(dir))
31 try:
32 names = os.listdir(dir)
33 except OSError:
34 print("Can't list {!r}".format(dir))
35 names = []
36 names.sort()
37 for name in names:
38 if name == '__pycache__':
39 continue
40 fullname = os.path.join(dir, name)
41 if ddir is not None:
42 dfile = os.path.join(ddir, name)
43 else:
44 dfile = None
45 if not os.path.isdir(fullname):
46 yield fullname
47 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
48 os.path.isdir(fullname) and not os.path.islink(fullname)):
49 yield from _walk_dir(fullname, ddir=dfile,
50 maxlevels=maxlevels - 1, quiet=quiet)
51
21 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, 52 def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
22 quiet=False, legacy=False, optimize=-1): 53 quiet=False, legacy=False, optimize=-1, processes=None):
23 """Byte-compile all modules in the given directory tree. 54 """Byte-compile all modules in the given directory tree.
24 55
25 Arguments (only dir is required): 56 Arguments (only dir is required):
26 57
27 dir: the directory to byte-compile 58 dir: the directory to byte-compile
28 maxlevels: maximum recursion level (default 10) 59 maxlevels: maximum recursion level (default 10)
Jim.J.Jewett 2014/04/25 23:42:22 Note that symbolic links to directories are ignore
Claudiu.Popa 2014/04/27 15:01:18 This should really go in another patch.
29 ddir: the directory that will be prepended to the path to the 60 ddir: the directory that will be prepended to the path to the
30 file as it is compiled into each byte-code file. 61 file as it is compiled into each byte-code file.
31 force: if True, force compilation, even if timestamps are up-to-date 62 force: if True, force compilation, even if timestamps are up-to-date
Jim.J.Jewett 2014/04/25 22:33:30 rx: a Regular Expression used to exclude file
Claudiu.Popa 2014/04/27 15:01:18 But doing it in another patch-build cycle is bette
32 quiet: if True, be quiet during compilation 63 quiet: if True, be quiet during compilation
33 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths 64 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
34 optimize: optimization level or -1 for level of the interpreter 65 optimize: optimization level or -1 for level of the interpreter
66 processes: maximum number of parallel processes
35 """ 67 """
36 if not quiet: 68 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
37 print('Listing {!r}...'.format(dir)) 69 ddir=ddir)
38 try:
39 names = os.listdir(dir)
40 except OSError:
41 print("Can't list {!r}".format(dir))
42 names = []
43 names.sort()
44 success = 1 70 success = 1
45 for name in names: 71 if processes is not None and processes != 1:
Jim.J.Jewett 2014/04/25 22:33:30 Make that and processes > 1 If someone calls t
46 if name == '__pycache__': 72 if not _have_multiprocessing:
47 continue 73 raise ValueError('multiprocessing support not available')
Jim.J.Jewett 2014/04/25 23:42:22 # multiprocessing explicitly requested, and not av
Claudiu.Popa 2014/04/27 15:01:18 Raising an ImportError is not the proper way. I co
48 fullname = os.path.join(dir, name) 74 with ProcessPoolExecutor(max_workers=processes) as executor:
49 if ddir is not None: 75 results = executor.map(partial(compile_file,
50 dfile = os.path.join(ddir, name) 76 ddir=ddir, force=force,
51 else: 77 rx=rx, quiet=quiet,
52 dfile = None 78 legacy=legacy,
53 if not os.path.isdir(fullname): 79 optimize=optimize),
54 if not compile_file(fullname, ddir, force, rx, quiet, 80 files)
81 success = min(results, default=1)
82 else:
83 for file in files:
84 if not compile_file(file, ddir, force, rx, quiet,
55 legacy, optimize): 85 legacy, optimize):
56 success = 0
57 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
58 os.path.isdir(fullname) and not os.path.islink(fullname)):
59 if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
60 quiet, legacy, optimize):
61 success = 0 86 success = 0
62 return success 87 return success
63 88
64 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, 89 def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
65 legacy=False, optimize=-1): 90 legacy=False, optimize=-1):
66 """Byte-compile one file. 91 """Byte-compile one file.
67 92
68 Arguments (only fullname is required): 93 Arguments (only fullname is required):
69 94
70 fullname: the file to byte-compile 95 fullname: the file to byte-compile
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after
185 'the regexp is searched for in the full path ' 210 'the regexp is searched for in the full path '
186 'of each file considered for compilation')) 211 'of each file considered for compilation'))
187 parser.add_argument('-i', metavar='FILE', dest='flist', 212 parser.add_argument('-i', metavar='FILE', dest='flist',
188 help=('add all the files and directories listed in ' 213 help=('add all the files and directories listed in '
189 'FILE to the list considered for compilation; ' 214 'FILE to the list considered for compilation; '
190 'if "-", names are read from stdin')) 215 'if "-", names are read from stdin'))
191 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', 216 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
192 help=('zero or more file and directory names ' 217 help=('zero or more file and directory names '
193 'to compile; if no arguments given, defaults ' 218 'to compile; if no arguments given, defaults '
194 'to the equivalent of -l sys.path')) 219 'to the equivalent of -l sys.path'))
220 parser.add_argument('-j', '--processes', action='store', default=None,
221 type=int, help='Run compileall concurrently')
222
195 args = parser.parse_args() 223 args = parser.parse_args()
196
197 compile_dests = args.compile_dest 224 compile_dests = args.compile_dest
198 225
199 if (args.ddir and (len(compile_dests) != 1 226 if (args.ddir and (len(compile_dests) != 1
200 or not os.path.isdir(compile_dests[0]))): 227 or not os.path.isdir(compile_dests[0]))):
201 parser.exit('-d destdir requires exactly one directory argument') 228 parser.exit('-d destdir requires exactly one directory argument')
202 if args.rx: 229 if args.rx:
203 import re 230 import re
204 args.rx = re.compile(args.rx) 231 args.rx = re.compile(args.rx)
205 232
206 # if flist is provided then load it 233 # if flist is provided then load it
207 if args.flist: 234 if args.flist:
208 try: 235 try:
209 with (sys.stdin if args.flist=='-' else open(args.flist)) as f: 236 with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
210 for line in f: 237 for line in f:
211 compile_dests.append(line.strip()) 238 compile_dests.append(line.strip())
212 except OSError: 239 except OSError:
213 print("Error reading file list {}".format(args.flist)) 240 print("Error reading file list {}".format(args.flist))
214 return False 241 return False
242
243 if args.processes is not None:
244 if args.processes <= 0:
245 args.processes = os.cpu_count()
215 246
216 success = True 247 success = True
217 try: 248 try:
218 if compile_dests: 249 if compile_dests:
219 for dest in compile_dests: 250 for dest in compile_dests:
220 if os.path.isfile(dest): 251 if os.path.isfile(dest):
221 if not compile_file(dest, args.ddir, args.force, args.rx, 252 if not compile_file(dest, args.ddir, args.force, args.rx,
222 args.quiet, args.legacy): 253 args.quiet, args.legacy):
223 success = False 254 success = False
224 else: 255 else:
225 if not compile_dir(dest, args.maxlevels, args.ddir, 256 if not compile_dir(dest, args.maxlevels, args.ddir,
226 args.force, args.rx, args.quiet, 257 args.force, args.rx, args.quiet,
227 args.legacy): 258 args.legacy, processes=args.processes):
228 success = False 259 success = False
229 return success 260 return success
230 else: 261 else:
231 return compile_path(legacy=args.legacy, force=args.force, 262 return compile_path(legacy=args.legacy, force=args.force,
232 quiet=args.quiet) 263 quiet=args.quiet)
233 except KeyboardInterrupt: 264 except KeyboardInterrupt:
234 print("\n[interrupted]") 265 print("\n[interrupted]")
235 return False 266 return False
236 return True 267 return True
237 268
238 269
239 if __name__ == '__main__': 270 if __name__ == '__main__':
240 exit_status = int(not main()) 271 exit_status = int(not main())
241 sys.exit(exit_status) 272 sys.exit(exit_status)
OLDNEW

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+