Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(13938)

Delta Between Two Patch Sets: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Left Patch Set: Created 5 years, 6 months ago
Right Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import tempfile 8 import tempfile
9 import time 9 import time
10 import unittest 10 import unittest
11 import io 11 import io
12 12
13 from unittest import mock 13 from unittest import mock, skipUnless
14 try:
15 from concurrent.futures import ProcessPoolExecutor
16 _have_multiprocessing = True
17 except ImportError:
18 _have_multiprocessing = False
14 19
15 from test import support, script_helper 20 from test import support, script_helper
16 21
17 class CompileallTests(unittest.TestCase): 22 class CompileallTests(unittest.TestCase):
18 23
19 def setUp(self): 24 def setUp(self):
20 self.directory = tempfile.mkdtemp() 25 self.directory = tempfile.mkdtemp()
21 self.source_path = os.path.join(self.directory, '_test.py') 26 self.source_path = os.path.join(self.directory, '_test.py')
22 self.bc_path = importlib.util.cache_from_source(self.source_path) 27 self.bc_path = importlib.util.cache_from_source(self.source_path)
23 with open(self.source_path, 'w') as file: 28 with open(self.source_path, 'w') as file:
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
102 debug_override=not optimize) 107 debug_override=not optimize)
103 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
104 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
105 debug_override=not optimize) 110 debug_override=not optimize)
106 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
107 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
108 debug_override=not optimize) 113 debug_override=not optimize)
109 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
110 115
111 @mock.patch('compileall.ProcessPoolExecutor') 116 @mock.patch('compileall.ProcessPoolExecutor')
112 def test_compile_processes(self, pool_mock): 117 def test_compile_pool_called(self, pool_mock):
113 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 118 compileall.compile_dir(self.directory, quiet=True, workers=5)
114 compileall.compile_dir(self.directory, quiet=True, processes=5)
115 self.assertTrue(pool_mock.called) 119 self.assertTrue(pool_mock.called)
116 120
121 def test_compile_workers_non_positive(self):
122 with self.assertRaisesRegex(ValueError,
123 "workers must be greater or equal to 0"):
124 compileall.compile_dir(self.directory, workers=-1)
125
126 @mock.patch('compileall.ProcessPoolExecutor')
127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
132 @mock.patch('compileall.ProcessPoolExecutor')
133 @mock.patch('compileall.compile_file')
134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
135 compileall.compile_dir(self.directory, quiet=True)
136 self.assertFalse(pool_mock.called)
137 self.assertTrue(compile_file_mock.called)
138
139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
140 def test_compile_missing_multiprocessing(self):
141 with self.assertRaisesRegex(NotImplementedError,
142 "multiprocessing support not available"):
143 compileall.compile_dir(self.directory, quiet=True, workers=5)
117 144
118 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
119 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
120 to stdout.""" 147 to stdout."""
121 148
122 def setUp(self): 149 def setUp(self):
123 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
124 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
125 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
126 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 147 matching lines...) Expand 10 before | Expand all | Expand 10 after
274 os.mkdir(subpackage) 301 os.mkdir(subpackage)
275 subinitfn = script_helper.make_script(subpackage, '__init__', '') 302 subinitfn = script_helper.make_script(subpackage, '__init__', '')
276 hamfn = script_helper.make_script(subpackage, 'ham', '') 303 hamfn = script_helper.make_script(subpackage, 'ham', '')
277 self.assertRunOK('-q', '-l', self.pkgdir) 304 self.assertRunOK('-q', '-l', self.pkgdir)
278 self.assertNotCompiled(subinitfn) 305 self.assertNotCompiled(subinitfn)
279 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) ) 306 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')) )
280 self.assertRunOK('-q', self.pkgdir) 307 self.assertRunOK('-q', self.pkgdir)
281 self.assertCompiled(subinitfn) 308 self.assertCompiled(subinitfn)
282 self.assertCompiled(hamfn) 309 self.assertCompiled(hamfn)
283 310
311 def test_recursion_limit(self):
312 subpackage = os.path.join(self.pkgdir, 'spam')
313 subpackage2 = os.path.join(subpackage, 'ham')
314 subpackage3 = os.path.join(subpackage2, 'eggs')
315 for pkg in (subpackage, subpackage2, subpackage3):
316 script_helper.make_pkg(pkg)
317
318 subinitfn = os.path.join(subpackage, '__init__.py')
319 hamfn = script_helper.make_script(subpackage, 'ham', '')
320 spamfn = script_helper.make_script(subpackage2, 'spam', '')
321 eggfn = script_helper.make_script(subpackage3, 'egg', '')
322
323 self.assertRunOK('-q', '-r 0', self.pkgdir)
324 self.assertNotCompiled(subinitfn)
325 self.assertFalse(
326 os.path.exists(os.path.join(subpackage, '__pycache__')))
327
328 self.assertRunOK('-q', '-r 1', self.pkgdir)
329 self.assertCompiled(subinitfn)
330 self.assertCompiled(hamfn)
331 self.assertNotCompiled(spamfn)
332
333 self.assertRunOK('-q', '-r 2', self.pkgdir)
334 self.assertCompiled(subinitfn)
335 self.assertCompiled(hamfn)
336 self.assertCompiled(spamfn)
337 self.assertNotCompiled(eggfn)
338
339 self.assertRunOK('-q', '-r 5', self.pkgdir)
340 self.assertCompiled(subinitfn)
341 self.assertCompiled(hamfn)
342 self.assertCompiled(spamfn)
343 self.assertCompiled(eggfn)
344
284 def test_quiet(self): 345 def test_quiet(self):
285 noisy = self.assertRunOK(self.pkgdir) 346 noisy = self.assertRunOK(self.pkgdir)
286 quiet = self.assertRunOK('-q', self.pkgdir) 347 quiet = self.assertRunOK('-q', self.pkgdir)
287 self.assertNotEqual(b'', noisy) 348 self.assertNotEqual(b'', noisy)
288 self.assertEqual(b'', quiet) 349 self.assertEqual(b'', quiet)
289 350
290 def test_regexp(self): 351 def test_regexp(self):
291 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 352 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir)
292 self.assertNotCompiled(self.barfn) 353 self.assertNotCompiled(self.barfn)
293 self.assertCompiled(self.initfn) 354 self.assertCompiled(self.initfn)
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
380 bingfn, self.barfn) 441 bingfn, self.barfn)
381 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
382 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
383 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
384 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
385 446
386 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
387 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
388 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
389 450
390 def test_processes(self): 451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
452 def test_workers(self):
391 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
454 files = []
392 for suffix in range(5): 455 for suffix in range(5):
393 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
394 os.mkdir(pkgdir) 457 os.mkdir(pkgdir)
395 fn = script_helper.make_script(pkgdir, '__init__', '') 458 fn = script_helper.make_script(pkgdir, '__init__', '')
396 bar2fn = script_helper.make_script(pkgdir, 'bar2', '') 459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
460
397 self.assertRunOK(self.directory, '-j', '0') 461 self.assertRunOK(self.directory, '-j', '0')
398 self.assertCompiled(bar2fn) 462 self.assertCompiled(bar2fn)
463 for file in files:
464 self.assertCompiled(file)
465
466 @mock.patch('compileall.compile_dir')
467 def test_workers_available_cores(self, compile_dir):
468 with mock.patch("sys.argv",
469 new=[sys.executable, self.directory, "-j0"]):
470 compileall.main()
471 self.assertTrue(compile_dir.called)
472 self.assertEqual(compile_dir.call_args[-1]['workers'],
473 os.cpu_count())
399 474
400 475
401 if __name__ == "__main__": 476 if __name__ == "__main__":
402 unittest.main() 477 unittest.main()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+