diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -4,7 +4,10 @@ Implements the Distutils 'build_ext' com modules (currently limited to C extensions, should accommodate C++ extensions ASAP).""" -import sys, os, re +import contextlib +import os +import re +import sys from distutils.core import Command from distutils.errors import * from distutils.sysconfig import customize_compiler, get_python_version @@ -133,6 +136,7 @@ class build_ext(Command): self.swig_cpp = None self.swig_opts = None self.user = None + self.parallel = False def finalize_options(self): from distutils import sysconfig @@ -451,15 +455,40 @@ class build_ext(Command): def build_extensions(self): # First, sanity-check the 'extensions' list self.check_extensions_list(self.extensions) + if self.parallel: + self._build_extensions_parallel() + else: + self._build_extensions_serial() + def _build_extensions_parallel(self): + try: + from os import cpu_count + from concurrent.futures import ThreadPoolExecutor, as_completed + except ImportError: + self._build_extensions_serial() + return + + with ThreadPoolExecutor(max_workers=cpu_count()) as executor: + futures = [executor.submit(self.build_extension, ext) + for ext in self.extensions] + for ext, fut in zip(self.extensions, futures): + with self._filter_build_errors(ext): + fut.result() + + def _build_extensions_serial(self): for ext in self.extensions: - try: + with self._filter_build_errors(ext): self.build_extension(ext) - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) + + @contextlib.contextmanager + def _filter_build_errors(self, ext): + try: + yield + except (CCompilerError, DistutilsError, CompileError) as e: + if not ext.optional: + raise + self.warn('building extension "%s" failed: %s' % + (ext.name, e)) def build_extension(self, ext): sources = ext.sources diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -38,6 +38,9 @@ class BuildExtTestCase(TempdirManager, from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + def build_ext(self, *args, **kwargs): + return build_ext(*args, **kwargs) + def test_build_ext(self): global ALREADY_TESTED copy_xxmodule_c(self.tmp_dir) @@ -45,7 +48,7 @@ class BuildExtTestCase(TempdirManager, xx_ext = Extension('xx', [xx_c]) dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -93,7 +96,7 @@ class BuildExtTestCase(TempdirManager, def test_solaris_enable_shared(self): dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) old = sys.platform sys.platform = 'sunos' # fooling finalize_options @@ -119,7 +122,7 @@ class BuildExtTestCase(TempdirManager, import site dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) # making sure the user option is there options = [name for name, short, lable in @@ -150,14 +153,14 @@ class BuildExtTestCase(TempdirManager, # with the optional argument. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertRaises((UnknownFileError, CompileError), cmd.run) # should raise an error modules = [Extension('foo', ['xxx'], optional=True)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() cmd.run() # should pass @@ -166,7 +169,7 @@ class BuildExtTestCase(TempdirManager, # etc.) are in the include search path. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() from distutils import sysconfig @@ -178,14 +181,14 @@ class BuildExtTestCase(TempdirManager, # make sure cmd.libraries is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.libraries = 'my_lib, other_lib lastlib' cmd.finalize_options() self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib']) # make sure cmd.library_dirs is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep cmd.finalize_options() self.assertIn('my_lib_dir', cmd.library_dirs) @@ -193,7 +196,7 @@ class BuildExtTestCase(TempdirManager, # make sure rpath is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.rpath = 'one%stwo' % os.pathsep cmd.finalize_options() self.assertEqual(cmd.rpath, ['one', 'two']) @@ -202,32 +205,32 @@ class BuildExtTestCase(TempdirManager, # make sure define is turned into 2-tuples # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.define = 'one,two' cmd.finalize_options() self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) # make sure undef is turned into a list of # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.undef = 'one,two' cmd.finalize_options() self.assertEqual(cmd.undef, ['one', 'two']) # make sure swig_opts is turned into a list - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = None cmd.finalize_options() self.assertEqual(cmd.swig_opts, []) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = '1 2' cmd.finalize_options() self.assertEqual(cmd.swig_opts, ['1', '2']) def test_check_extensions_list(self): dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() #'extensions' option must be a list of Extension instances @@ -276,7 +279,7 @@ class BuildExtTestCase(TempdirManager, def test_get_source_files(self): modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertEqual(cmd.get_source_files(), ['xxx']) @@ -285,7 +288,7 @@ class BuildExtTestCase(TempdirManager, # should not be overriden by a compiler instance # when the command is run dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.compiler = 'unix' cmd.ensure_finalized() cmd.run() @@ -298,7 +301,7 @@ class BuildExtTestCase(TempdirManager, ext = Extension('foo', [c_file], optional=False) dist = Distribution({'name': 'xx', 'ext_modules': [ext]}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.ensure_finalized() self.assertEqual(len(cmd.get_outputs()), 1) @@ -361,7 +364,7 @@ class BuildExtTestCase(TempdirManager, #etree_ext = Extension('lxml.etree', [etree_c]) #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.inplace = 1 cmd.distribution.package_dir = {'': 'src'} cmd.distribution.packages = ['lxml', 'lxml.html'] @@ -460,7 +463,7 @@ class BuildExtTestCase(TempdirManager, 'ext_modules': [deptarget_ext] }) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -479,8 +482,19 @@ class BuildExtTestCase(TempdirManager, self.fail("Wrong deployment target during compilation") +class ParallelBuildExtTestCase(BuildExtTestCase): + + def build_ext(self, *args, **kwargs): + build_ext = super().build_ext(*args, **kwargs) + build_ext.parallel = True + return build_ext + + def test_suite(): - return unittest.makeSuite(BuildExtTestCase) + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(BuildExtTestCase)) + suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase)) + return suite if __name__ == '__main__': - support.run_unittest(test_suite()) + support.run_unittest(__name__) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ class PyBuildExt(build_ext): def __init__(self, dist): build_ext.__init__(self, dist) self.failed = [] + self.parallel = True def build_extensions(self):