I am programming a C++ extension for Python and I am using distutils to compile the project. As the project grows, rebuilding it takes longer and longer. Is there a way to speed up the build process?
I read that parallel builds (as with make -j
) are not possible with distutils. Are there any good alternatives to distutils which might be faster?
I also noticed that it's recompiling all object files every time I call python setup.py build
, even when I only changed one source file. Should this be the case or might I be doing something wrong here?
In case it helps, here are some of the files which I try to compile: https://gist.github.com/2923577
Thanks!
distutils has been deprecated in NumPy 1.23. 0 . It will be removed for Python 3.12; for Python <= 3.11 it will not be removed until 2 years after the Python 3.12 release (Oct 2025).
The distutils package provides support for building and installing additional modules into a Python installation. The new modules may be either 100%-pure Python, or may be extension modules written in C, or may be collections of Python packages which include modules coded in both Python and C.
Try building with environment variable CC="ccache gcc"
, that will speed up build significantly when the source has not changed. (strangely, distutils uses CC
also for c++ source files). Install the ccache package, of course.
Since you have a single extension which is assembled from multiple compiled object files, you can monkey-patch distutils to compile those in parallel (they are independent) - put this into your setup.py (adjust the N=2
as you wish):
# monkey-patch for parallel compilation def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): # those lines are copied from distutils.ccompiler.CCompiler directly macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) # parallel code N=2 # number of parallel compilations import multiprocessing.pool def _single_compile(obj): try: src, ext = build[obj] except KeyError: return self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # convert to list, imap is evaluated on-demand list(multiprocessing.pool.ThreadPool(N).imap(_single_compile,objects)) return objects import distutils.ccompiler distutils.ccompiler.CCompiler.compile=parallelCCompile
For the sake of completeness, if you have multiple extensions, you can use the following solution:
import os import multiprocessing try: from concurrent.futures import ThreadPoolExecutor as Pool except ImportError: from multiprocessing.pool import ThreadPool as LegacyPool # To ensure the with statement works. Required for some older 2.7.x releases class Pool(LegacyPool): def __enter__(self): return self def __exit__(self, *args): self.close() self.join() def build_extensions(self): """Function to monkey-patch distutils.command.build_ext.build_ext.build_extensions """ self.check_extensions_list(self.extensions) try: num_jobs = os.cpu_count() except AttributeError: num_jobs = multiprocessing.cpu_count() with Pool(num_jobs) as pool: pool.map(self.build_extension, self.extensions) def compile( self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None, ): """Function to monkey-patch distutils.ccompiler.CCompiler""" macros, objects, extra_postargs, pp_opts, build = self._setup_compile( output_dir, macros, include_dirs, sources, depends, extra_postargs ) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) for obj in objects: try: src, ext = build[obj] except KeyError: continue self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # Return *all* object filenames, not just the ones we just built. return objects from distutils.ccompiler import CCompiler from distutils.command.build_ext import build_ext build_ext.build_extensions = build_extensions CCompiler.compile = compile
I've got this working on Windows with clcache, derived from eudoxos's answer:
# Python modules import datetime import distutils import distutils.ccompiler import distutils.sysconfig import multiprocessing import multiprocessing.pool import os import sys from distutils.core import setup from distutils.core import Extension from distutils.errors import CompileError from distutils.errors import DistutilsExecError now = datetime.datetime.now ON_LINUX = "linux" in sys.platform N_JOBS = 4 #------------------------------------------------------------------------------ # Enable ccache to speed up builds if ON_LINUX: os.environ['CC'] = 'ccache gcc' # Windows else: # Using clcache.exe, see: https://github.com/frerich/clcache # Insert path to clcache.exe into the path. prefix = os.path.dirname(os.path.abspath(__file__)) path = os.path.join(prefix, "bin") print "Adding %s to the system path." % path os.environ['PATH'] = '%s;%s' % (path, os.environ['PATH']) clcache_exe = os.path.join(path, "clcache.exe") #------------------------------------------------------------------------------ # Parallel Compile # # Reference: # # http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # def linux_parallel_cpp_compile( self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): # Copied from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = self._setup_compile( output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) def _single_compile(obj): try: src, ext = build[obj] except KeyError: return self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # convert to list, imap is evaluated on-demand list(multiprocessing.pool.ThreadPool(N_JOBS).imap( _single_compile, objects)) return objects def windows_parallel_cpp_compile( self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): # Copied from distutils.msvc9compiler.MSVCCompiler if not self.initialized: self.initialize() macros, objects, extra_postargs, pp_opts, build = self._setup_compile( output_dir, macros, include_dirs, sources, depends, extra_postargs) compile_opts = extra_preargs or [] compile_opts.append('/c') if debug: compile_opts.extend(self.compile_options_debug) else: compile_opts.extend(self.compile_options) def _single_compile(obj): try: src, ext = build[obj] except KeyError: return input_opt = "/Tp" + src output_opt = "/Fo" + obj try: self.spawn( [clcache_exe] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) except DistutilsExecError, msg: raise CompileError(msg) # convert to list, imap is evaluated on-demand list(multiprocessing.pool.ThreadPool(N_JOBS).imap( _single_compile, objects)) return objects #------------------------------------------------------------------------------ # Only enable parallel compile on 2.7 Python if sys.version_info[1] == 7: if ON_LINUX: distutils.ccompiler.CCompiler.compile = linux_parallel_cpp_compile else: import distutils.msvccompiler import distutils.msvc9compiler distutils.msvccompiler.MSVCCompiler.compile = windows_parallel_cpp_compile distutils.msvc9compiler.MSVCCompiler.compile = windows_parallel_cpp_compile # ... call setup() as usual
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With