Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(29054)

Delta Between Two Patch Sets: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 6 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import tempfile 8 import tempfile
9 import time 9 import time
10 import unittest 10 import unittest
11 from unittest import mock
eric.araujo 2014/03/12 06:48:47 Minor: please keep all the “import X” imports grou
12 from contextlib import ExitStack
13 import io 11 import io
12
13 from unittest import mock, skipUnless
14 try:
15 from concurrent.futures import ProcessPoolExecutor
16 _have_multiprocessing = True
17 except ImportError:
18 _have_multiprocessing = False
14 19
15 from test import support, script_helper 20 from test import support, script_helper
16 21
17 class CompileallTests(unittest.TestCase): 22 class CompileallTests(unittest.TestCase):
18 23
19 def setUp(self): 24 def setUp(self):
20 self.directory = tempfile.mkdtemp() 25 self.directory = tempfile.mkdtemp()
21 self.source_path = os.path.join(self.directory, '_test.py') 26 self.source_path = os.path.join(self.directory, '_test.py')
22 self.bc_path = importlib.util.cache_from_source(self.source_path) 27 self.bc_path = importlib.util.cache_from_source(self.source_path)
23 with open(self.source_path, 'w') as file: 28 with open(self.source_path, 'w') as file:
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
101 cached = importlib.util.cache_from_source(self.source_path, 106 cached = importlib.util.cache_from_source(self.source_path,
102 debug_override=not optimize) 107 debug_override=not optimize)
103 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
104 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
105 debug_override=not optimize) 110 debug_override=not optimize)
106 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
107 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
108 debug_override=not optimize) 113 debug_override=not optimize)
109 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
110 115
111 def test_compile_processes(self): 116 @mock.patch('compileall.ProcessPoolExecutor')
112 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 117 def test_compile_pool_called(self, pool_mock):
113 with ExitStack() as ctx: 118 compileall.compile_dir(self.directory, quiet=True, workers=5)
eric.araujo 2014/03/12 06:48:47 I would use the decorator form of mock.patch: @
114 pool = ctx.enter_context( 119 self.assertTrue(pool_mock.called)
115 mock.patch('compileall.ProcessPoolExecutor', 120
116 new=mock.MagicMock())) 121 def test_compile_workers_non_positive(self):
117 as_completed = ctx.enter_context( 122 with self.assertRaisesRegex(ValueError,
118 mock.patch('compileall.as_completed')) 123 "workers must be greater or equal to 0"):
119 124 compileall.compile_dir(self.directory, workers=-1)
120 compileall.compile_dir(self.directory, quiet=True, processes=5) 125
121 self.assertTrue(pool.called) 126 @mock.patch('compileall.ProcessPoolExecutor')
122 127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
132 @mock.patch('compileall.ProcessPoolExecutor')
133 @mock.patch('compileall.compile_file')
134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
135 compileall.compile_dir(self.directory, quiet=True)
136 self.assertFalse(pool_mock.called)
137 self.assertTrue(compile_file_mock.called)
138
139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
140 def test_compile_missing_multiprocessing(self):
141 with self.assertRaisesRegex(NotImplementedError,
142 "multiprocessing support not available"):
143 compileall.compile_dir(self.directory, quiet=True, workers=5)
123 144
124 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
125 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
126 to stdout.""" 147 to stdout."""
127 148
128 def setUp(self): 149 def setUp(self):
129 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
130 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
131 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
132 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 147 matching lines...) Expand 10 before | Expand all | Expand 10 after
280 os.mkdir(subpackage) 301 os.mkdir(subpackage)
281 subinitfn = script_helper.make_script(subpackage, '__init__', '') 302 subinitfn = script_helper.make_script(subpackage, '__init__', '')
282 hamfn = script_helper.make_script(subpackage, 'ham', '') 303 hamfn = script_helper.make_script(subpackage, 'ham', '')
283 self.assertRunOK('-q', '-l', self.pkgdir) 304 self.assertRunOK('-q', '-l', self.pkgdir)
284 self.assertNotCompiled(subinitfn) 305 self.assertNotCompiled(subinitfn)
285 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) ) 306 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) )
286 self.assertRunOK('-q', self.pkgdir) 307 self.assertRunOK('-q', self.pkgdir)
287 self.assertCompiled(subinitfn) 308 self.assertCompiled(subinitfn)
288 self.assertCompiled(hamfn) 309 self.assertCompiled(hamfn)
289 310
311 def test_recursion_limit(self):
312 subpackage = os.path.join(self.pkgdir, 'spam')
313 subpackage2 = os.path.join(subpackage, 'ham')
314 subpackage3 = os.path.join(subpackage2, 'eggs')
315 for pkg in (subpackage, subpackage2, subpackage3):
316 script_helper.make_pkg(pkg)
317
318 subinitfn = os.path.join(subpackage, '__init__.py')
319 hamfn = script_helper.make_script(subpackage, 'ham', '')
320 spamfn = script_helper.make_script(subpackage2, 'spam', '')
321 eggfn = script_helper.make_script(subpackage3, 'egg', '')
322
323 self.assertRunOK('-q', '-r 0', self.pkgdir)
324 self.assertNotCompiled(subinitfn)
325 self.assertFalse(
326 os.path.exists(os.path.join(subpackage, '__pycache__')))
327
328 self.assertRunOK('-q', '-r 1', self.pkgdir)
329 self.assertCompiled(subinitfn)
330 self.assertCompiled(hamfn)
331 self.assertNotCompiled(spamfn)
332
333 self.assertRunOK('-q', '-r 2', self.pkgdir)
334 self.assertCompiled(subinitfn)
335 self.assertCompiled(hamfn)
336 self.assertCompiled(spamfn)
337 self.assertNotCompiled(eggfn)
338
339 self.assertRunOK('-q', '-r 5', self.pkgdir)
340 self.assertCompiled(subinitfn)
341 self.assertCompiled(hamfn)
342 self.assertCompiled(spamfn)
343 self.assertCompiled(eggfn)
344
290 def test_quiet(self): 345 def test_quiet(self):
291 noisy = self.assertRunOK(self.pkgdir) 346 noisy = self.assertRunOK(self.pkgdir)
292 quiet = self.assertRunOK('-q', self.pkgdir) 347 quiet = self.assertRunOK('-q', self.pkgdir)
293 self.assertNotEqual(b'', noisy) 348 self.assertNotEqual(b'', noisy)
294 self.assertEqual(b'', quiet) 349 self.assertEqual(b'', quiet)
295 350
296 def test_regexp(self): 351 def test_regexp(self):
297 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 352 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
298 self.assertNotCompiled(self.barfn) 353 self.assertNotCompiled(self.barfn)
299 self.assertCompiled(self.initfn) 354 self.assertCompiled(self.initfn)
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
386 bingfn, self.barfn) 441 bingfn, self.barfn)
387 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
388 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
389 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
390 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
391 446
392 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
393 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
394 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
395 450
396 def test_processes(self): 451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
452 def test_workers(self):
397 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
454 files = []
398 for suffix in range(5): 455 for suffix in range(5):
399 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
400 os.mkdir(pkgdir) 457 os.mkdir(pkgdir)
401 fn = script_helper.make_script(pkgdir, '__init__', '') 458 fn = script_helper.make_script(pkgdir, '__init__', '')
402 bar2fn = script_helper.make_script(pkgdir, 'bar2', '') 459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
403 self.assertRunOK(self.directory, '-j', '5') 460
461 self.assertRunOK(self.directory, '-j', '0')
404 self.assertCompiled(bar2fn) 462 self.assertCompiled(bar2fn)
463 for file in files:
464 self.assertCompiled(file)
465
466 @mock.patch('compileall.compile_dir')
467 def test_workers_available_cores(self, compile_dir):
468 with mock.patch("sys.argv",
469 new=[sys.executable, self.directory, "-j0"]):
470 compileall.main()
471 self.assertTrue(compile_dir.called)
472 self.assertEqual(compile_dir.call_args[-1]['workers'],
473 os.cpu_count())
405 474
406 475
407 if __name__ == "__main__": 476 if __name__ == "__main__":
408 unittest.main() 477 unittest.main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+