Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(16296)

Side by Side Diff: Lib/test/test_compileall.py

Issue 16104: Use multiprocessing in compileall script
Patch Set: Created 5 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 import sys 1 import sys
2 import compileall 2 import compileall
3 import importlib.util 3 import importlib.util
4 import os 4 import os
5 import py_compile 5 import py_compile
6 import shutil 6 import shutil
7 import struct 7 import struct
8 import tempfile 8 import tempfile
9 import time 9 import time
10 import unittest 10 import unittest
11 import io 11 import io
12
13 from unittest import mock, skipUnless
14 try:
15 from concurrent.futures import ProcessPoolExecutor
16 _have_multiprocessing = True
17 except ImportError:
18 _have_multiprocessing = False
12 19
13 from test import support, script_helper 20 from test import support, script_helper
14 21
15 class CompileallTests(unittest.TestCase): 22 class CompileallTests(unittest.TestCase):
16 23
17 def setUp(self): 24 def setUp(self):
18 self.directory = tempfile.mkdtemp() 25 self.directory = tempfile.mkdtemp()
19 self.source_path = os.path.join(self.directory, '_test.py') 26 self.source_path = os.path.join(self.directory, '_test.py')
20 self.bc_path = importlib.util.cache_from_source(self.source_path) 27 self.bc_path = importlib.util.cache_from_source(self.source_path)
21 with open(self.source_path, 'w') as file: 28 with open(self.source_path, 'w') as file:
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
99 cached = importlib.util.cache_from_source(self.source_path, 106 cached = importlib.util.cache_from_source(self.source_path,
100 debug_override=not optimize) 107 debug_override=not optimize)
101 self.assertTrue(os.path.isfile(cached)) 108 self.assertTrue(os.path.isfile(cached))
102 cached2 = importlib.util.cache_from_source(self.source_path2, 109 cached2 = importlib.util.cache_from_source(self.source_path2,
103 debug_override=not optimize) 110 debug_override=not optimize)
104 self.assertTrue(os.path.isfile(cached2)) 111 self.assertTrue(os.path.isfile(cached2))
105 cached3 = importlib.util.cache_from_source(self.source_path3, 112 cached3 = importlib.util.cache_from_source(self.source_path3,
106 debug_override=not optimize) 113 debug_override=not optimize)
107 self.assertTrue(os.path.isfile(cached3)) 114 self.assertTrue(os.path.isfile(cached3))
108 115
116 @mock.patch('compileall.ProcessPoolExecutor')
117 def test_compile_pool_called(self, pool_mock):
118 compileall.compile_dir(self.directory, quiet=True, workers=5)
119 self.assertTrue(pool_mock.called)
120
121 def test_compile_workers_non_positive(self):
122 with self.assertRaisesRegex(ValueError,
123 "workers must be greater or equal to 0"):
124 compileall.compile_dir(self.directory, workers=-1)
125
126 @mock.patch('compileall.ProcessPoolExecutor')
127 def test_compile_workers_cpu_count(self, pool_mock):
128 compileall.compile_dir(self.directory, quiet=True, workers=0)
129 self.assertEqual(pool_mock.call_args[1]['max_workers'],
130 os.cpu_count())
131
132 @mock.patch('compileall.ProcessPoolExecutor')
133 @mock.patch('compileall.compile_file')
134 def test_compile_one_worker(self, compile_file_mock, pool_mock):
135 compileall.compile_dir(self.directory, quiet=True)
136 self.assertFalse(pool_mock.called)
137 self.assertTrue(compile_file_mock.called)
138
139 @mock.patch('compileall.ProcessPoolExecutor', new=None)
140 def test_compile_missing_multiprocessing(self):
141 with self.assertRaisesRegex(NotImplementedError,
142 "multiprocessing support not available"):
143 compileall.compile_dir(self.directory, quiet=True, workers=5)
109 144
110 class EncodingTest(unittest.TestCase): 145 class EncodingTest(unittest.TestCase):
111 """Issue 6716: compileall should escape source code when printing errors 146 """Issue 6716: compileall should escape source code when printing errors
112 to stdout.""" 147 to stdout."""
113 148
114 def setUp(self): 149 def setUp(self):
115 self.directory = tempfile.mkdtemp() 150 self.directory = tempfile.mkdtemp()
116 self.source_path = os.path.join(self.directory, '_test.py') 151 self.source_path = os.path.join(self.directory, '_test.py')
117 with open(self.source_path, 'w', encoding='utf-8') as file: 152 with open(self.source_path, 'w', encoding='utf-8') as file:
118 file.write('# -*- coding: utf-8 -*-\n') 153 file.write('# -*- coding: utf-8 -*-\n')
(...skipping 286 matching lines...) Expand 10 before | Expand all | Expand 10 after
405 rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn, 440 rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn,
406 bingfn, self.barfn) 441 bingfn, self.barfn)
407 self.assertRegex(out, b'rror') 442 self.assertRegex(out, b'rror')
408 self.assertNotCompiled(bingfn) 443 self.assertNotCompiled(bingfn)
409 self.assertCompiled(self.initfn) 444 self.assertCompiled(self.initfn)
410 self.assertCompiled(self.barfn) 445 self.assertCompiled(self.barfn)
411 446
412 def test_invalid_arg_produces_message(self): 447 def test_invalid_arg_produces_message(self):
413 out = self.assertRunOK('badfilename') 448 out = self.assertRunOK('badfilename')
414 self.assertRegex(out, b"Can't list 'badfilename'") 449 self.assertRegex(out, b"Can't list 'badfilename'")
450
451 @skipUnless(_have_multiprocessing, "requires multiprocessing")
452 def test_workers(self):
453 bar2fn = script_helper.make_script(self.directory, 'bar2', '')
454 files = []
455 for suffix in range(5):
456 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix))
457 os.mkdir(pkgdir)
458 fn = script_helper.make_script(pkgdir, '__init__', '')
459 files.append(script_helper.make_script(pkgdir, 'bar2', ''))
460
461 self.assertRunOK(self.directory, '-j', '0')
462 self.assertCompiled(bar2fn)
463 for file in files:
464 self.assertCompiled(file)
465
466 @mock.patch('compileall.compile_dir')
467 def test_workers_available_cores(self, compile_dir):
468 with mock.patch("sys.argv",
469 new=[sys.executable, self.directory, "-j0"]):
470 compileall.main()
471 self.assertTrue(compile_dir.called)
472 self.assertEqual(compile_dir.call_args[-1]['workers'],
473 os.cpu_count())
415 474
416 475
417 if __name__ == "__main__": 476 if __name__ == "__main__":
418 unittest.main() 477 unittest.main()
OLDNEW
« Lib/compileall.py ('K') | « Lib/compileall.py ('k') | no next file » | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+