Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #5309: distutils' build and build_ext commands now accept a ``-j`` 

option to enable parallel building of extension modules.
This commit is contained in:
Antoine Pitrou 2014-09-26 23:31:59 +02:00
parent 7e23d82cec
commit 2c0a916061
6 changed files with 117 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Lib/distutils/command/build.py
View File

@ -36,6 +36,8 @@ class build(Command):
"(default: %s)" % get_platform()),
('compiler=', 'c',
"specify the compiler type"),
('parallel=', 'j',
"number of parallel build jobs"),
('debug', 'g',
"compile extensions and libraries with debugging information"),
('force', 'f',
@ -65,6 +67,7 @@ class build(Command):
self.debug = None
self.force = 0
self.executable = None
self.parallel = None
def finalize_options(self):
if self.plat_name is None:
@ -116,6 +119,12 @@ class build(Command):
if self.executable is None:
self.executable = os.path.normpath(sys.executable)
if isinstance(self.parallel, str):
try:
self.parallel = int(self.parallel)
except ValueError:
raise DistutilsOptionError("parallel should be an integer")
def run(self):
# Run all relevant sub-commands. This will be some subset of:
# - build_py - pure Python modules

57
Lib/distutils/command/build_ext.py
View File

@ -4,7 +4,10 @@ Implements the Distutils 'build_ext' command, for building extension
modules (currently limited to C extensions, should accommodate C++
extensions ASAP)."""
import sys, os, re
import contextlib
import os
import re
import sys
from distutils.core import Command
from distutils.errors import *
from distutils.sysconfig import customize_compiler, get_python_version
@ -85,6 +88,8 @@ class build_ext(Command):
"forcibly build everything (ignore file timestamps)"),
('compiler=', 'c',
"specify the compiler type"),
('parallel=', 'j',
"number of parallel build jobs"),
('swig-cpp', None,
"make SWIG create C++ files (default is C)"),
('swig-opts=', None,
@ -124,6 +129,7 @@ class build_ext(Command):
self.swig_cpp = None
self.swig_opts = None
self.user = None
self.parallel = None
def finalize_options(self):
from distutils import sysconfig
@ -134,6 +140,7 @@ class build_ext(Command):
('compiler', 'compiler'),
('debug', 'debug'),
('force', 'force'),
('parallel', 'parallel'),
('plat_name', 'plat_name'),
)
@ -274,6 +281,12 @@ class build_ext(Command):
self.library_dirs.append(user_lib)
self.rpath.append(user_lib)
if isinstance(self.parallel, str):
try:
self.parallel = int(self.parallel)
except ValueError:
raise DistutilsOptionError("parallel should be an integer")
def run(self):
from distutils.ccompiler import new_compiler
@ -442,15 +455,45 @@ class build_ext(Command):
def build_extensions(self):
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
if self.parallel:
self._build_extensions_parallel()
else:
self._build_extensions_serial()
def _build_extensions_parallel(self):
workers = self.parallel
if self.parallel is True:
workers = os.cpu_count() # may return None
try:
from concurrent.futures import ThreadPoolExecutor
except ImportError:
workers = None
if workers is None:
self._build_extensions_serial()
return
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = [executor.submit(self.build_extension, ext)
for ext in self.extensions]
for ext, fut in zip(self.extensions, futures):
with self._filter_build_errors(ext):
fut.result()
def _build_extensions_serial(self):
for ext in self.extensions:
try:
with self._filter_build_errors(ext):
self.build_extension(ext)
except (CCompilerError, DistutilsError, CompileError) as e:
if not ext.optional:
raise
self.warn('building extension "%s" failed: %s' %
(ext.name, e))
@contextlib.contextmanager
def _filter_build_errors(self, ext):
try:
yield
except (CCompilerError, DistutilsError, CompileError) as e:
if not ext.optional:
raise
self.warn('building extension "%s" failed: %s' %
(ext.name, e))
def build_extension(self, ext):
sources = ext.sources

56
Lib/distutils/tests/test_build_ext.py
View File

@ -37,6 +37,9 @@ class BuildExtTestCase(TempdirManager,
from distutils.command import build_ext
build_ext.USER_BASE = site.USER_BASE
def build_ext(self, *args, **kwargs):
return build_ext(*args, **kwargs)
def test_build_ext(self):
global ALREADY_TESTED
copy_xxmodule_c(self.tmp_dir)
@ -44,7 +47,7 @@ class BuildExtTestCase(TempdirManager,
xx_ext = Extension('xx', [xx_c])
dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]})
dist.package_dir = self.tmp_dir
cmd = build_ext(dist)
cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@ -91,7 +94,7 @@ class BuildExtTestCase(TempdirManager,
def test_solaris_enable_shared(self):
dist = Distribution({'name': 'xx'})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
old = sys.platform
sys.platform = 'sunos' # fooling finalize_options
@ -113,7 +116,7 @@ class BuildExtTestCase(TempdirManager,
def test_user_site(self):
import site
dist = Distribution({'name': 'xx'})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
# making sure the user option is there
options = [name for name, short, lable in
@ -144,14 +147,14 @@ class BuildExtTestCase(TempdirManager,
# with the optional argument.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertRaises((UnknownFileError, CompileError),
cmd.run) # should raise an error
modules = [Extension('foo', ['xxx'], optional=True)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.ensure_finalized()
cmd.run() # should pass
@ -160,7 +163,7 @@ class BuildExtTestCase(TempdirManager,
# etc.) are in the include search path.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.finalize_options()
from distutils import sysconfig
@ -172,14 +175,14 @@ class BuildExtTestCase(TempdirManager,
# make sure cmd.libraries is turned into a list
# if it's a string
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.libraries = 'my_lib, other_lib lastlib'
cmd.finalize_options()
self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib'])
# make sure cmd.library_dirs is turned into a list
# if it's a string
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep
cmd.finalize_options()
self.assertIn('my_lib_dir', cmd.library_dirs)
@ -187,7 +190,7 @@ class BuildExtTestCase(TempdirManager,
# make sure rpath is turned into a list
# if it's a string
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.rpath = 'one%stwo' % os.pathsep
cmd.finalize_options()
self.assertEqual(cmd.rpath, ['one', 'two'])
@ -196,32 +199,32 @@ class BuildExtTestCase(TempdirManager,
# make sure define is turned into 2-tuples
# strings if they are ','-separated strings
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.define = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.define, [('one', '1'), ('two', '1')])
# make sure undef is turned into a list of
# strings if they are ','-separated strings
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.undef = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.undef, ['one', 'two'])
# make sure swig_opts is turned into a list
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.swig_opts = None
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, [])
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.swig_opts = '1 2'
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, ['1', '2'])
def test_check_extensions_list(self):
dist = Distribution()
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.finalize_options()
#'extensions' option must be a list of Extension instances
@ -270,7 +273,7 @@ class BuildExtTestCase(TempdirManager,
def test_get_source_files(self):
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertEqual(cmd.get_source_files(), ['xxx'])
@ -279,7 +282,7 @@ class BuildExtTestCase(TempdirManager,
# should not be overriden by a compiler instance
# when the command is run
dist = Distribution()
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.compiler = 'unix'
cmd.ensure_finalized()
cmd.run()
@ -292,7 +295,7 @@ class BuildExtTestCase(TempdirManager,
ext = Extension('foo', [c_file], optional=False)
dist = Distribution({'name': 'xx',
'ext_modules': [ext]})
cmd = build_ext(dist)
cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.ensure_finalized()
self.assertEqual(len(cmd.get_outputs()), 1)
@ -355,7 +358,7 @@ class BuildExtTestCase(TempdirManager,
#etree_ext = Extension('lxml.etree', [etree_c])
#dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]})
dist = Distribution()
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.inplace = 1
cmd.distribution.package_dir = {'': 'src'}
cmd.distribution.packages = ['lxml', 'lxml.html']
@ -462,7 +465,7 @@ class BuildExtTestCase(TempdirManager,
'ext_modules': [deptarget_ext]
})
dist.package_dir = self.tmp_dir
cmd = build_ext(dist)
cmd = self.build_ext(dist)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@ -481,8 +484,19 @@ class BuildExtTestCase(TempdirManager,
self.fail("Wrong deployment target during compilation")
class ParallelBuildExtTestCase(BuildExtTestCase):
def build_ext(self, *args, **kwargs):
build_ext = super().build_ext(*args, **kwargs)
build_ext.parallel = True
return build_ext
def test_suite():
return unittest.makeSuite(BuildExtTestCase)
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(BuildExtTestCase))
suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase))
return suite
if __name__ == '__main__':
support.run_unittest(test_suite())
support.run_unittest(__name__)

3
Misc/NEWS
View File

@ -139,6 +139,9 @@ Core and Builtins
Library
-------
- Issue #5309: distutils' build and build_ext commands now accept a ``-j``
option to enable parallel building of extension modules.
- Issue #22448: Improve canceled timer handles cleanup to prevent
unbound memory usage. Patch by Joshua Moore-Oliva.

1
Modules/Setup.dist
View File

@ -118,6 +118,7 @@ _collections _collectionsmodule.c # Container types
itertools itertoolsmodule.c # Functions creating iterators for efficient looping
atexit atexitmodule.c # Register functions to be run at interpreter-shutdown
_stat _stat.c # stat.h interface
time timemodule.c # time module
# access to ISO C locale support
_locale _localemodule.c # -lintl

19
setup.py
View File

@ -25,6 +25,11 @@ cflags = sysconfig.get_config_var('CFLAGS')
py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist
class Dummy:
"""Hack for parallel build"""
ProcessPoolExecutor = None
sys.modules['concurrent.futures.process'] = Dummy
def get_platform():
# cross build
if "_PYTHON_HOST_PLATFORM" in os.environ:
@ -174,6 +179,8 @@ class PyBuildExt(build_ext):
build_ext.__init__(self, dist)
self.failed = []
self.failed_on_import = []
if '-j' in os.environ.get('MAKEFLAGS', ''):
self.parallel = True
def build_extensions(self):
@ -253,6 +260,9 @@ class PyBuildExt(build_ext):
build_ext.build_extensions(self)
for ext in self.extensions:
self.check_extension_import(ext)
longest = max([len(e.name) for e in self.extensions])
if self.failed or self.failed_on_import:
all_failed = self.failed + self.failed_on_import
@ -305,6 +315,15 @@ class PyBuildExt(build_ext):
(ext.name, sys.exc_info()[1]))
self.failed.append(ext.name)
return
def check_extension_import(self, ext):
# Don't try to import an extension that has failed to compile
if ext.name in self.failed:
self.announce(
'WARNING: skipping import check for failed build "%s"' %
ext.name, level=1)
return
# Workaround for Mac OS X: The Carbon-based modules cannot be
# reliably imported into a command-line Python
if 'Carbon' in ext.extra_link_args: