diff -r 13cd8ea4cafe Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py Fri Sep 05 11:00:56 2014 +0300 +++ b/Lib/distutils/command/build_ext.py Fri Sep 05 11:45:58 2014 +0200 @@ -4,7 +4,10 @@ modules (currently limited to C extensions, should accommodate C++ extensions ASAP).""" -import sys, os, re +import contextlib +import os +import re +import sys from distutils.core import Command from distutils.errors import * from distutils.sysconfig import customize_compiler, get_python_version @@ -13,6 +16,7 @@ from distutils.extension import Extension from distutils.util import get_platform from distutils import log +import traceback # this keeps compatibility from 2.3 to 2.5 if sys.version < "2.6": @@ -91,6 +95,8 @@ "forcibly build everything (ignore file timestamps)"), ('compiler=', 'c', "specify the compiler type"), + ('parallel=', 'j', + "parallel build jobs"), ('swig-cpp', None, "make SWIG create C++ files (default is C)"), ('swig-opts=', None, @@ -133,6 +139,7 @@ self.swig_cpp = None self.swig_opts = None self.user = None + self.parallel = False def finalize_options(self): from distutils import sysconfig @@ -268,6 +275,9 @@ if self.undef: self.undef = self.undef.split(',') + if isinstance(self.parallel, str): + self.parallel = int(self.parallel) + if self.swig_opts is None: self.swig_opts = [] else: @@ -451,15 +461,46 @@ def build_extensions(self): # First, sanity-check the 'extensions' list self.check_extensions_list(self.extensions) + if self.parallel: + self._build_extensions_parallel() + else: + self._build_extensions_serial() + def _build_extensions_parallel(self): + workers = self.parallel + if self.parallel is True: + workers = os.cpu_count() # may return None + try: + from concurrent.futures import ThreadPoolExecutor + except ImportError: + traceback.print_exc() + workers = None + + if workers is None: + self._build_extensions_serial() + return + + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(self.build_extension, ext) + for ext in self.extensions] + for ext, fut in zip(self.extensions, futures): + with self._filter_build_errors(ext): + fut.result() + + def _build_extensions_serial(self): for ext in self.extensions: - try: + with self._filter_build_errors(ext): self.build_extension(ext) - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) + + @contextlib.contextmanager + def _filter_build_errors(self, ext): + try: + yield + except (CCompilerError, DistutilsError, CompileError) as e: + if not ext.optional: + raise + self.warn('building extension "%s" failed: %s' % + (ext.name, e)) def build_extension(self, ext): sources = ext.sources diff -r 13cd8ea4cafe Lib/distutils/tests/test_build_ext.py --- a/Lib/distutils/tests/test_build_ext.py Fri Sep 05 11:00:56 2014 +0300 +++ b/Lib/distutils/tests/test_build_ext.py Fri Sep 05 11:45:58 2014 +0200 @@ -38,6 +38,9 @@ from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + def build_ext(self, *args, **kwargs): + return build_ext(*args, **kwargs) + def test_build_ext(self): global ALREADY_TESTED copy_xxmodule_c(self.tmp_dir) @@ -45,7 +48,7 @@ xx_ext = Extension('xx', [xx_c]) dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -93,7 +96,7 @@ def test_solaris_enable_shared(self): dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) old = sys.platform sys.platform = 'sunos' # fooling finalize_options @@ -115,7 +118,7 @@ def test_user_site(self): import site dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) # making sure the user option is there options = [name for name, short, lable in @@ -146,14 +149,14 @@ # with the optional argument. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertRaises((UnknownFileError, CompileError), cmd.run) # should raise an error modules = [Extension('foo', ['xxx'], optional=True)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() cmd.run() # should pass @@ -162,7 +165,7 @@ # etc.) are in the include search path. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() from distutils import sysconfig @@ -174,14 +177,14 @@ # make sure cmd.libraries is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.libraries = 'my_lib, other_lib lastlib' cmd.finalize_options() self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib']) # make sure cmd.library_dirs is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep cmd.finalize_options() self.assertIn('my_lib_dir', cmd.library_dirs) @@ -189,7 +192,7 @@ # make sure rpath is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.rpath = 'one%stwo' % os.pathsep cmd.finalize_options() self.assertEqual(cmd.rpath, ['one', 'two']) @@ -198,32 +201,32 @@ # make sure define is turned into 2-tuples # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.define = 'one,two' cmd.finalize_options() self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) # make sure undef is turned into a list of # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.undef = 'one,two' cmd.finalize_options() self.assertEqual(cmd.undef, ['one', 'two']) # make sure swig_opts is turned into a list - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = None cmd.finalize_options() self.assertEqual(cmd.swig_opts, []) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = '1 2' cmd.finalize_options() self.assertEqual(cmd.swig_opts, ['1', '2']) def test_check_extensions_list(self): dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() #'extensions' option must be a list of Extension instances @@ -272,7 +275,7 @@ def test_get_source_files(self): modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertEqual(cmd.get_source_files(), ['xxx']) @@ -281,7 +284,7 @@ # should not be overriden by a compiler instance # when the command is run dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.compiler = 'unix' cmd.ensure_finalized() cmd.run() @@ -294,7 +297,7 @@ ext = Extension('foo', [c_file], optional=False) dist = Distribution({'name': 'xx', 'ext_modules': [ext]}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.ensure_finalized() self.assertEqual(len(cmd.get_outputs()), 1) @@ -357,7 +360,7 @@ #etree_ext = Extension('lxml.etree', [etree_c]) #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.inplace = 1 cmd.distribution.package_dir = {'': 'src'} cmd.distribution.packages = ['lxml', 'lxml.html'] @@ -464,7 +467,7 @@ 'ext_modules': [deptarget_ext] }) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -483,8 +486,19 @@ self.fail("Wrong deployment target during compilation") +class ParallelBuildExtTestCase(BuildExtTestCase): + + def build_ext(self, *args, **kwargs): + build_ext = super().build_ext(*args, **kwargs) + build_ext.parallel = True + return build_ext + + def test_suite(): - return unittest.makeSuite(BuildExtTestCase) + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(BuildExtTestCase)) + suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase)) + return suite if __name__ == '__main__': - support.run_unittest(test_suite()) + support.run_unittest(__name__) diff -r 13cd8ea4cafe Modules/Setup.dist --- a/Modules/Setup.dist Fri Sep 05 11:00:56 2014 +0300 +++ b/Modules/Setup.dist Fri Sep 05 11:45:58 2014 +0200 @@ -118,6 +118,7 @@ itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _stat _stat.c # stat.h interface +time timemodule.c # time module # access to ISO C locale support _locale _localemodule.c # -lintl diff -r 13cd8ea4cafe setup.py --- a/setup.py Fri Sep 05 11:00:56 2014 +0300 +++ b/setup.py Fri Sep 05 11:45:58 2014 +0200 @@ -25,6 +25,11 @@ py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist +class Dummy: + """Hack for parallel build""" + ProcessPoolExecutor = None +sys.modules['concurrent.futures.process'] = Dummy + def get_platform(): # cross build if "_PYTHON_HOST_PLATFORM" in os.environ: @@ -173,6 +178,7 @@ def __init__(self, dist): build_ext.__init__(self, dist) self.failed = [] + self.parallel = True def build_extensions(self): @@ -252,6 +258,9 @@ build_ext.build_extensions(self) + for ext in self.extensions: + self.check_extension_import(ext) + longest = max([len(e.name) for e in self.extensions]) if self.failed: longest = max(longest, max([len(name) for name in self.failed])) @@ -295,6 +304,15 @@ (ext.name, sys.exc_info()[1])) self.failed.append(ext.name) return + + def check_extension_import(self, ext): + # Don't try to import an extension that has failed to compile + if ext.name in self.failed: + self.announce( + 'WARNING: skipping import check for failed build "%s"' % + ext.name, level=1) + return + # Workaround for Mac OS X: The Carbon-based modules cannot be # reliably imported into a command-line Python if 'Carbon' in ext.extra_link_args: