Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(50523)

Delta Between Two Patch Sets: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 6 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import tempfile 8 import tempfile
9 import time 9 import time
10 import unittest 10 import unittest
11 import io 11 import io
12 12
13 from unittest import mock 13 from unittest import mock, skipUnless
14 try:
15 from concurrent.futures import ProcessPoolExecutor
16 _have_multiprocessing = True
17 except ImportError:
18 _have_multiprocessing = False
14 19
15 from test import support, script_helper 20 from test import support, script_helper
16 21
17 class CompileallTests(unittest.TestCase): 22 class CompileallTests(unittest.TestCase):
18 23
19 def setUp(self): 24 def setUp(self):
20 self.directory = tempfile.mkdtemp() 25 self.directory = tempfile.mkdtemp()
21 self.source_path = os.path.join(self.directory, '_test.py') 26 self.source_path = os.path.join(self.directory, '_test.py')
22 self.bc_path = importlib.util.cache_from_source(self.source_path) 27 self.bc_path = importlib.util.cache_from_source(self.source_path)
23 with open(self.source_path, 'w') as file: 28 with open(self.source_path, 'w') as file:
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
102 debug_override=not optimize) 107 debug_override=not optimize)
103 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
104 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
105 debug_override=not optimize) 110 debug_override=not optimize)
106 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
107 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
108 debug_override=not optimize) 113 debug_override=not optimize)
109 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
110 115
111 @mock.patch('compileall.ProcessPoolExecutor') 116 @mock.patch('compileall.ProcessPoolExecutor')
112 def test_compile_processes(self, pool_mock): 117 def test_compile_pool_called(self, pool_mock):
113 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 118 compileall.compile_dir(self.directory, quiet=True, workers=5)
114 compileall.compile_dir(self.directory, quiet=True, processes=5)
115 self.assertTrue(pool_mock.called) 119 self.assertTrue(pool_mock.called)
116 120
117 @mock.patch('compileall._have_multiprocessing', False) 121 def test_compile_workers_non_positive(self):
122 with self.assertRaisesRegex(ValueError,
123 "workers must be greater or equal to 0"):
124 compileall.compile_dir(self.directory, workers=-1)
125
126 @mock.patch('compileall.ProcessPoolExecutor')
127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
132 @mock.patch('compileall.ProcessPoolExecutor')
133 @mock.patch('compileall.compile_file')
134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
135 compileall.compile_dir(self.directory, quiet=True)
136 self.assertFalse(pool_mock.called)
137 self.assertTrue(compile_file_mock.called)
138
139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
118 def test_compile_missing_multiprocessing(self): 140 def test_compile_missing_multiprocessing(self):
119 with self.assertRaises(ValueError) as cm: 141 with self.assertRaisesRegex(NotImplementedError,
120 compileall.compile_dir(self.directory, quiet=True, processes=5) 142 "multiprocessing support not available"):
121 self.assertEqual(str(cm.exception), 143 compileall.compile_dir(self.directory, quiet=True, workers=5)
122 "multiprocessing support not available")
123
124 144
125 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
126 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
127 to stdout.""" 147 to stdout."""
128 148
129 def setUp(self): 149 def setUp(self):
130 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
131 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
132 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
133 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 147 matching lines...) Expand 10 before | Expand all | Expand 10 after
281 os.mkdir(subpackage) 301 os.mkdir(subpackage)
282 subinitfn = script_helper.make_script(subpackage, '__init__', '') 302 subinitfn = script_helper.make_script(subpackage, '__init__', '')
283 hamfn = script_helper.make_script(subpackage, 'ham', '') 303 hamfn = script_helper.make_script(subpackage, 'ham', '')
284 self.assertRunOK('-q', '-l', self.pkgdir) 304 self.assertRunOK('-q', '-l', self.pkgdir)
285 self.assertNotCompiled(subinitfn) 305 self.assertNotCompiled(subinitfn)
286 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) ) 306 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) )
287 self.assertRunOK('-q', self.pkgdir) 307 self.assertRunOK('-q', self.pkgdir)
288 self.assertCompiled(subinitfn) 308 self.assertCompiled(subinitfn)
289 self.assertCompiled(hamfn) 309 self.assertCompiled(hamfn)
290 310
311 def test_recursion_limit(self):
312 subpackage = os.path.join(self.pkgdir, 'spam')
313 subpackage2 = os.path.join(subpackage, 'ham')
314 subpackage3 = os.path.join(subpackage2, 'eggs')
315 for pkg in (subpackage, subpackage2, subpackage3):
316 script_helper.make_pkg(pkg)
317
318 subinitfn = os.path.join(subpackage, '__init__.py')
319 hamfn = script_helper.make_script(subpackage, 'ham', '')
320 spamfn = script_helper.make_script(subpackage2, 'spam', '')
321 eggfn = script_helper.make_script(subpackage3, 'egg', '')
322
323 self.assertRunOK('-q', '-r 0', self.pkgdir)
324 self.assertNotCompiled(subinitfn)
325 self.assertFalse(
326 os.path.exists(os.path.join(subpackage, '__pycache__')))
327
328 self.assertRunOK('-q', '-r 1', self.pkgdir)
329 self.assertCompiled(subinitfn)
330 self.assertCompiled(hamfn)
331 self.assertNotCompiled(spamfn)
332
333 self.assertRunOK('-q', '-r 2', self.pkgdir)
334 self.assertCompiled(subinitfn)
335 self.assertCompiled(hamfn)
336 self.assertCompiled(spamfn)
337 self.assertNotCompiled(eggfn)
338
339 self.assertRunOK('-q', '-r 5', self.pkgdir)
340 self.assertCompiled(subinitfn)
341 self.assertCompiled(hamfn)
342 self.assertCompiled(spamfn)
343 self.assertCompiled(eggfn)
344
291 def test_quiet(self): 345 def test_quiet(self):
292 noisy = self.assertRunOK(self.pkgdir) 346 noisy = self.assertRunOK(self.pkgdir)
293 quiet = self.assertRunOK('-q', self.pkgdir) 347 quiet = self.assertRunOK('-q', self.pkgdir)
294 self.assertNotEqual(b'', noisy) 348 self.assertNotEqual(b'', noisy)
295 self.assertEqual(b'', quiet) 349 self.assertEqual(b'', quiet)
296 350
297 def test_regexp(self): 351 def test_regexp(self):
298 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 352 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
299 self.assertNotCompiled(self.barfn) 353 self.assertNotCompiled(self.barfn)
300 self.assertCompiled(self.initfn) 354 self.assertCompiled(self.initfn)
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
387 bingfn, self.barfn) 441 bingfn, self.barfn)
388 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
389 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
390 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
391 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
392 446
393 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
394 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
395 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
396 450
397 def test_processes(self): 451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
452 def test_workers(self):
398 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
454 files = []
399 for suffix in range(5): 455 for suffix in range(5):
400 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
401 os.mkdir(pkgdir) 457 os.mkdir(pkgdir)
402 fn = script_helper.make_script(pkgdir, '__init__', '') 458 fn = script_helper.make_script(pkgdir, '__init__', '')
403 bar2fn = script_helper.make_script(pkgdir, 'bar2', '') 459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
460
404 self.assertRunOK(self.directory, '-j', '0') 461 self.assertRunOK(self.directory, '-j', '0')
405 self.assertCompiled(bar2fn) 462 self.assertCompiled(bar2fn)
463 for file in files:
464 self.assertCompiled(file)
406 465
407 @mock.patch('compileall.compile_dir') 466 @mock.patch('compileall.compile_dir')
408 def test_processes_available_cores(self, compile_dir): 467 def test_workers_available_cores(self, compile_dir):
409 with mock.patch("sys.argv", 468 with mock.patch("sys.argv",
410 new=[sys.executable, self.directory, "-j0"]): 469 new=[sys.executable, self.directory, "-j0"]):
411 compileall.main() 470 compileall.main()
412 self.assertTrue(compile_dir.called) 471 self.assertTrue(compile_dir.called)
413 self.assertEqual(compile_dir.call_args[-1]['processes'], 472 self.assertEqual(compile_dir.call_args[-1]['workers'],
414 os.cpu_count()) 473 os.cpu_count())
415 474
416 475
417 if __name__ == "__main__": 476 if __name__ == "__main__":
418 unittest.main() 477 unittest.main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+