Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(29979)

Delta Between Two Patch Sets: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 4 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import tempfile 8 import tempfile
9 import time 9 import time
10 import unittest 10 import unittest
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after
106 cached = importlib.util.cache_from_source(self.source_path, 106 cached = importlib.util.cache_from_source(self.source_path,
107 debug_override=not optimize) 107 debug_override=not optimize)
108 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
109 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
110 debug_override=not optimize) 110 debug_override=not optimize)
111 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
112 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
113 debug_override=not optimize) 113 debug_override=not optimize)
114 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
115 115
116 @skipUnless(_have_multiprocessing, "requires multiprocessing")
117 @mock.patch('compileall.ProcessPoolExecutor') 116 @mock.patch('compileall.ProcessPoolExecutor')
118 def test_compile_processes(self, pool_mock): 117 def test_compile_pool_called(self, pool_mock):
119 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 118 compileall.compile_dir(self.directory, quiet=True, workers=5)
120 compileall.compile_dir(self.directory, quiet=True, processes=5)
121 self.assertTrue(pool_mock.called) 119 self.assertTrue(pool_mock.called)
Jim.J.Jewett 2014/04/25 22:33:30 Please add an XXX comment about verifying that mor
122 120
123 @skipUnless(_have_multiprocessing, "requires multiprocessing") 121 def test_compile_workers_non_positive(self):
122 with self.assertRaisesRegex(ValueError,
123 "workers must be greater or equal to 0"):
124 compileall.compile_dir(self.directory, workers=-1)
125
126 @mock.patch('compileall.ProcessPoolExecutor')
127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
124 @mock.patch('compileall.ProcessPoolExecutor') 132 @mock.patch('compileall.ProcessPoolExecutor')
125 @mock.patch('compileall.compile_file') 133 @mock.patch('compileall.compile_file')
126 def test_compile_processes_on_one(self, compile_mock, pool_mock): 134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
127 # Test that `compile_dir` doesn't use ProcessPoolExecutor 135 compileall.compile_dir(self.directory, quiet=True)
128 # when passing processes=1
129 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
130 compileall.compile_dir(self.directory, quiet=True, processes=1)
131 self.assertFalse(pool_mock.called) 136 self.assertFalse(pool_mock.called)
132 self.assertTrue(compile_mock.called) 137 self.assertTrue(compile_file_mock.called)
133 138
134 @mock.patch('compileall._have_multiprocessing', False) 139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
135 def test_compile_missing_multiprocessing(self): 140 def test_compile_missing_multiprocessing(self):
136 with self.assertRaises(ValueError) as cm: 141 with self.assertRaisesRegex(NotImplementedError,
137 compileall.compile_dir(self.directory, quiet=True, processes=5) 142 "multiprocessing support not available"):
138 self.assertEqual(str(cm.exception), 143 compileall.compile_dir(self.directory, quiet=True, workers=5)
139 "multiprocessing support not available")
Jim.J.Jewett 2014/04/25 22:33:30 Please also test 0 (because it has different logic
140
141 144
142 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
143 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
144 to stdout.""" 147 to stdout."""
145 148
146 def setUp(self): 149 def setUp(self):
147 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
148 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
149 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
150 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 147 matching lines...) Expand 10 before | Expand all | Expand 10 after
298 os.mkdir(subpackage) 301 os.mkdir(subpackage)
299 subinitfn = script_helper.make_script(subpackage, '__init__', '') 302 subinitfn = script_helper.make_script(subpackage, '__init__', '')
300 hamfn = script_helper.make_script(subpackage, 'ham', '') 303 hamfn = script_helper.make_script(subpackage, 'ham', '')
301 self.assertRunOK('-q', '-l', self.pkgdir) 304 self.assertRunOK('-q', '-l', self.pkgdir)
302 self.assertNotCompiled(subinitfn) 305 self.assertNotCompiled(subinitfn)
303 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) ) 306 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) )
304 self.assertRunOK('-q', self.pkgdir) 307 self.assertRunOK('-q', self.pkgdir)
305 self.assertCompiled(subinitfn) 308 self.assertCompiled(subinitfn)
306 self.assertCompiled(hamfn) 309 self.assertCompiled(hamfn)
307 310
311 def test_recursion_limit(self):
312 subpackage = os.path.join(self.pkgdir, 'spam')
313 subpackage2 = os.path.join(subpackage, 'ham')
314 subpackage3 = os.path.join(subpackage2, 'eggs')
315 for pkg in (subpackage, subpackage2, subpackage3):
316 script_helper.make_pkg(pkg)
317
318 subinitfn = os.path.join(subpackage, '__init__.py')
319 hamfn = script_helper.make_script(subpackage, 'ham', '')
320 spamfn = script_helper.make_script(subpackage2, 'spam', '')
321 eggfn = script_helper.make_script(subpackage3, 'egg', '')
322
323 self.assertRunOK('-q', '-r 0', self.pkgdir)
324 self.assertNotCompiled(subinitfn)
325 self.assertFalse(
326 os.path.exists(os.path.join(subpackage, '__pycache__')))
327
328 self.assertRunOK('-q', '-r 1', self.pkgdir)
329 self.assertCompiled(subinitfn)
330 self.assertCompiled(hamfn)
331 self.assertNotCompiled(spamfn)
332
333 self.assertRunOK('-q', '-r 2', self.pkgdir)
334 self.assertCompiled(subinitfn)
335 self.assertCompiled(hamfn)
336 self.assertCompiled(spamfn)
337 self.assertNotCompiled(eggfn)
338
339 self.assertRunOK('-q', '-r 5', self.pkgdir)
340 self.assertCompiled(subinitfn)
341 self.assertCompiled(hamfn)
342 self.assertCompiled(spamfn)
343 self.assertCompiled(eggfn)
344
308 def test_quiet(self): 345 def test_quiet(self):
309 noisy = self.assertRunOK(self.pkgdir) 346 noisy = self.assertRunOK(self.pkgdir)
310 quiet = self.assertRunOK('-q', self.pkgdir) 347 quiet = self.assertRunOK('-q', self.pkgdir)
311 self.assertNotEqual(b'', noisy) 348 self.assertNotEqual(b'', noisy)
312 self.assertEqual(b'', quiet) 349 self.assertEqual(b'', quiet)
313 350
314 def test_regexp(self): 351 def test_regexp(self):
315 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 352 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
316 self.assertNotCompiled(self.barfn) 353 self.assertNotCompiled(self.barfn)
317 self.assertCompiled(self.initfn) 354 self.assertCompiled(self.initfn)
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after
405 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
406 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
407 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
408 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
409 446
410 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
411 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
412 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
413 450
414 @skipUnless(_have_multiprocessing, "requires multiprocessing") 451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
415 def test_processes(self): 452 def test_workers(self):
416 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
417 files = [] 454 files = []
418 for suffix in range(5): 455 for suffix in range(5):
419 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
420 os.mkdir(pkgdir) 457 os.mkdir(pkgdir)
421 fn = script_helper.make_script(pkgdir, '__init__', '') 458 fn = script_helper.make_script(pkgdir, '__init__', '')
422 files.append(script_helper.make_script(pkgdir, 'bar2', '')) 459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
423 460
424 self.assertRunOK(self.directory, '-j', '0') 461 self.assertRunOK(self.directory, '-j', '0')
425 self.assertCompiled(bar2fn) 462 self.assertCompiled(bar2fn)
426 for file in files: 463 for file in files:
427 self.assertCompiled(file) 464 self.assertCompiled(file)
428 465
429 @mock.patch('compileall.compile_dir') 466 @mock.patch('compileall.compile_dir')
430 def test_processes_available_cores(self, compile_dir): 467 def test_workers_available_cores(self, compile_dir):
431 with mock.patch("sys.argv", 468 with mock.patch("sys.argv",
432 new=[sys.executable, self.directory, "-j0"]): 469 new=[sys.executable, self.directory, "-j0"]):
433 compileall.main() 470 compileall.main()
434 self.assertTrue(compile_dir.called) 471 self.assertTrue(compile_dir.called)
435 self.assertEqual(compile_dir.call_args[-1]['processes'], 472 self.assertEqual(compile_dir.call_args[-1]['workers'],
436 os.cpu_count()) 473 os.cpu_count())
437 474
438 475
439 if __name__ == "__main__": 476 if __name__ == "__main__":
440 unittest.main() 477 unittest.main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+