Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(35372)

Delta Between Two Patch Sets: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 9 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import subprocess
9 import sys
10 import tempfile 8 import tempfile
11 import time 9 import time
12 import unittest 10 import unittest
13 from unittest import mock
14 from contextlib import ExitStack
15 import io 11 import io
12
13 from unittest import mock, skipUnless
14 try:
15 from concurrent.futures import ProcessPoolExecutor
16 _have_multiprocessing = True
17 except ImportError:
18 _have_multiprocessing = False
16 19
17 from test import support, script_helper 20 from test import support, script_helper
18 21
19 class CompileallTests(unittest.TestCase): 22 class CompileallTests(unittest.TestCase):
20 23
21 def setUp(self): 24 def setUp(self):
22 self.directory = tempfile.mkdtemp() 25 self.directory = tempfile.mkdtemp()
23 self.source_path = os.path.join(self.directory, '_test.py') 26 self.source_path = os.path.join(self.directory, '_test.py')
24 self.bc_path = importlib.util.cache_from_source(self.source_path) 27 self.bc_path = importlib.util.cache_from_source(self.source_path)
25 with open(self.source_path, 'w') as file: 28 with open(self.source_path, 'w') as file:
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
103 cached = importlib.util.cache_from_source(self.source_path, 106 cached = importlib.util.cache_from_source(self.source_path,
104 debug_override=not optimize) 107 debug_override=not optimize)
105 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
106 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
107 debug_override=not optimize) 110 debug_override=not optimize)
108 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
109 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
110 debug_override=not optimize) 113 debug_override=not optimize)
111 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
112 115
113 def test_compile_processes(self): 116 @mock.patch('compileall.ProcessPoolExecutor')
114 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 117 def test_compile_pool_called(self, pool_mock):
115 with ExitStack() as ctx: 118 compileall.compile_dir(self.directory, quiet=True, workers=5)
116 pool = ctx.enter_context( 119 self.assertTrue(pool_mock.called)
117 mock.patch('compileall.ProcessPoolExecutor', 120
118 new=mock.MagicMock())) 121 def test_compile_workers_non_positive(self):
119 as_completed = ctx.enter_context( 122 with self.assertRaisesRegex(ValueError,
120 mock.patch('compileall.as_completed')) 123 "workers must be greater or equal to 0"):
121 124 compileall.compile_dir(self.directory, workers=-1)
122 compileall.compile_dir(self.directory, quiet=True, processes=5) 125
123 self.assertTrue(pool.called) 126 @mock.patch('compileall.ProcessPoolExecutor')
124 127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
132 @mock.patch('compileall.ProcessPoolExecutor')
133 @mock.patch('compileall.compile_file')
134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
135 compileall.compile_dir(self.directory, quiet=True)
136 self.assertFalse(pool_mock.called)
137 self.assertTrue(compile_file_mock.called)
138
139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
140 def test_compile_missing_multiprocessing(self):
141 with self.assertRaisesRegex(NotImplementedError,
142 "multiprocessing support not available"):
143 compileall.compile_dir(self.directory, quiet=True, workers=5)
125 144
126 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
127 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
128 to stdout.""" 147 to stdout."""
129 148
130 def setUp(self): 149 def setUp(self):
131 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
132 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
133 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
134 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
187 def _cleanup(self): 206 def _cleanup(self):
188 support.rmtree(self.directory) 207 support.rmtree(self.directory)
189 208
190 def test_no_args_compiles_path(self): 209 def test_no_args_compiles_path(self):
191 # Note that -l is implied for the no args case. 210 # Note that -l is implied for the no args case.
192 bazfn = script_helper.make_script(self.directory, 'baz', '') 211 bazfn = script_helper.make_script(self.directory, 'baz', '')
193 self.assertRunOK(PYTHONPATH=self.directory) 212 self.assertRunOK(PYTHONPATH=self.directory)
194 self.assertCompiled(bazfn) 213 self.assertCompiled(bazfn)
195 self.assertNotCompiled(self.initfn) 214 self.assertNotCompiled(self.initfn)
196 self.assertNotCompiled(self.barfn) 215 self.assertNotCompiled(self.barfn)
216
217 def test_no_args_respects_force_flag(self):
218 bazfn = script_helper.make_script(self.directory, 'baz', '')
219 self.assertRunOK(PYTHONPATH=self.directory)
220 pycpath = importlib.util.cache_from_source(bazfn)
221 # Set atime/mtime backward to avoid file timestamp resolution issues
222 os.utime(pycpath, (time.time()-60,)*2)
223 mtime = os.stat(pycpath).st_mtime
224 # Without force, no recompilation
225 self.assertRunOK(PYTHONPATH=self.directory)
226 mtime2 = os.stat(pycpath).st_mtime
227 self.assertEqual(mtime, mtime2)
228 # Now force it.
229 self.assertRunOK('-f', PYTHONPATH=self.directory)
230 mtime2 = os.stat(pycpath).st_mtime
231 self.assertNotEqual(mtime, mtime2)
232
233 def test_no_args_respects_quiet_flag(self):
234 script_helper.make_script(self.directory, 'baz', '')
235 noisy = self.assertRunOK(PYTHONPATH=self.directory)
236 self.assertIn(b'Listing ', noisy)
237 quiet = self.assertRunOK('-q', PYTHONPATH=self.directory)
238 self.assertNotIn(b'Listing ', quiet)
197 239
198 # Ensure that the default behavior of compileall's CLI is to create 240 # Ensure that the default behavior of compileall's CLI is to create
199 # PEP 3147 pyc/pyo files. 241 # PEP 3147 pyc/pyo files.
200 for name, ext, switch in [ 242 for name, ext, switch in [
201 ('normal', 'pyc', []), 243 ('normal', 'pyc', []),
202 ('optimize', 'pyo', ['-O']), 244 ('optimize', 'pyo', ['-O']),
203 ('doubleoptimize', 'pyo', ['-OO']), 245 ('doubleoptimize', 'pyo', ['-OO']),
204 ]: 246 ]:
205 def f(self, ext=ext, switch=switch): 247 def f(self, ext=ext, switch=switch):
206 script_helper.assert_python_ok(*(switch + 248 script_helper.assert_python_ok(*(switch +
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
259 os.mkdir(subpackage) 301 os.mkdir(subpackage)
260 subinitfn = script_helper.make_script(subpackage, '__init__', '') 302 subinitfn = script_helper.make_script(subpackage, '__init__', '')
261 hamfn = script_helper.make_script(subpackage, 'ham', '') 303 hamfn = script_helper.make_script(subpackage, 'ham', '')
262 self.assertRunOK('-q', '-l', self.pkgdir) 304 self.assertRunOK('-q', '-l', self.pkgdir)
263 self.assertNotCompiled(subinitfn) 305 self.assertNotCompiled(subinitfn)
264 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) ) 306 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) )
265 self.assertRunOK('-q', self.pkgdir) 307 self.assertRunOK('-q', self.pkgdir)
266 self.assertCompiled(subinitfn) 308 self.assertCompiled(subinitfn)
267 self.assertCompiled(hamfn) 309 self.assertCompiled(hamfn)
268 310
311 def test_recursion_limit(self):
312 subpackage = os.path.join(self.pkgdir, 'spam')
313 subpackage2 = os.path.join(subpackage, 'ham')
314 subpackage3 = os.path.join(subpackage2, 'eggs')
315 for pkg in (subpackage, subpackage2, subpackage3):
316 script_helper.make_pkg(pkg)
317
318 subinitfn = os.path.join(subpackage, '__init__.py')
319 hamfn = script_helper.make_script(subpackage, 'ham', '')
320 spamfn = script_helper.make_script(subpackage2, 'spam', '')
321 eggfn = script_helper.make_script(subpackage3, 'egg', '')
322
323 self.assertRunOK('-q', '-r 0', self.pkgdir)
324 self.assertNotCompiled(subinitfn)
325 self.assertFalse(
326 os.path.exists(os.path.join(subpackage, '__pycache__')))
327
328 self.assertRunOK('-q', '-r 1', self.pkgdir)
329 self.assertCompiled(subinitfn)
330 self.assertCompiled(hamfn)
331 self.assertNotCompiled(spamfn)
332
333 self.assertRunOK('-q', '-r 2', self.pkgdir)
334 self.assertCompiled(subinitfn)
335 self.assertCompiled(hamfn)
336 self.assertCompiled(spamfn)
337 self.assertNotCompiled(eggfn)
338
339 self.assertRunOK('-q', '-r 5', self.pkgdir)
340 self.assertCompiled(subinitfn)
341 self.assertCompiled(hamfn)
342 self.assertCompiled(spamfn)
343 self.assertCompiled(eggfn)
344
269 def test_quiet(self): 345 def test_quiet(self):
270 noisy = self.assertRunOK(self.pkgdir) 346 noisy = self.assertRunOK(self.pkgdir)
271 quiet = self.assertRunOK('-q', self.pkgdir) 347 quiet = self.assertRunOK('-q', self.pkgdir)
272 self.assertNotEqual(b'', noisy) 348 self.assertNotEqual(b'', noisy)
273 self.assertEqual(b'', quiet) 349 self.assertEqual(b'', quiet)
274 350
275 def test_regexp(self): 351 def test_regexp(self):
276 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 352 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
277 self.assertNotCompiled(self.barfn) 353 self.assertNotCompiled(self.barfn)
278 self.assertCompiled(self.initfn) 354 self.assertCompiled(self.initfn)
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
365 bingfn, self.barfn) 441 bingfn, self.barfn)
366 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
367 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
368 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
369 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
370 446
371 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
372 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
373 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
374 450
375 def test_processes(self): 451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
452 def test_workers(self):
376 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
454 files = []
377 for suffix in range(5): 455 for suffix in range(5):
378 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
379 os.mkdir(pkgdir) 457 os.mkdir(pkgdir)
380 fn = script_helper.make_script(pkgdir, '__init__', '') 458 fn = script_helper.make_script(pkgdir, '__init__', '')
381 bar2fn = script_helper.make_script(pkgdir, 'bar2', '') 459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
382 self.assertRunOK(self.directory, '-j', '5') 460
461 self.assertRunOK(self.directory, '-j', '0')
383 self.assertCompiled(bar2fn) 462 self.assertCompiled(bar2fn)
384 463 for file in files:
464 self.assertCompiled(file)
465
466 @mock.patch('compileall.compile_dir')
467 def test_workers_available_cores(self, compile_dir):
468 with mock.patch("sys.argv",
469 new=[sys.executable, self.directory, "-j0"]):
470 compileall.main()
471 self.assertTrue(compile_dir.called)
472 self.assertEqual(compile_dir.call_args[-1]['workers'],
473 os.cpu_count())
474
385 475
386 if __name__ == "__main__": 476 if __name__ == "__main__":
387 unittest.main() 477 unittest.main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+