""" This module provides helpers for C++11+ projects using pybind11. LICENSE: Copyright (c) 2016 Wenzel Jakob , All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ # IMPORTANT: If you change this file in the pybind11 repo, also review # setup_helpers.pyi for matching changes. # # If you copy this file in, you don't # need the .pyi file; it's just an interface file for static type checkers. import contextlib import os import platform import shlex import shutil import sys import sysconfig import tempfile import threading import warnings from functools import lru_cache from pathlib import Path from typing import ( Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, TypeVar, Union, ) try: from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: from distutils.command.build_ext import build_ext as _build_ext from distutils.extension import Extension as _Extension import distutils.ccompiler import distutils.errors WIN = sys.platform.startswith("win32") and "mingw" not in sysconfig.get_platform() MACOS = sys.platform.startswith("darwin") STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" # It is recommended to use PEP 518 builds if using this module. However, this # file explicitly supports being copied into a user's project directory # standalone, and pulling pybind11 with the deprecated setup_requires feature. # If you copy the file, remember to add it to your MANIFEST.in, and add the current # directory into your path if it sits beside your setup.py. class Pybind11Extension(_Extension): # type: ignore[misc] """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you can further modify the options yourself. The customizations are: * ``/EHsc`` and ``/bigobj`` on Windows * ``stdlib=libc++`` on macOS * ``visibility=hidden`` and ``-g0`` on Unix Finally, you can set ``cxx_std`` via constructor or afterwards to enable flags for C++ std, and a few extra helper flags related to the C++ standard level. It is _highly_ recommended you either set this, or use the provided ``build_ext``, which will search for the highest supported extension for you if the ``cxx_std`` property is not set. Do not set the ``cxx_std`` property more than once, as flags are added when you set it. Set the property to None to disable the addition of C++ standard flags. If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. """ # flags are prepended, so that they can be further overridden, e.g. by # ``extra_compile_args=["-g"]``. def _add_cflags(self, flags: List[str]) -> None: self.extra_compile_args[:0] = flags def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) if "language" not in kwargs: kwargs["language"] = "c++" include_pybind11 = kwargs.pop("include_pybind11", True) super().__init__(*args, **kwargs) # Include the installed package pybind11 headers if include_pybind11: # If using setup_requires, this fails the first time - that's okay try: import pybind11 pyinc = pybind11.get_include() if pyinc not in self.include_dirs: self.include_dirs.append(pyinc) except ModuleNotFoundError: pass self.cxx_std = cxx_std cflags = [] ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: cflags += ["-fvisibility=hidden"] env_cflags = os.environ.get("CFLAGS", "") env_cppflags = os.environ.get("CPPFLAGS", "") c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] if MACOS: cflags += ["-stdlib=libc++"] ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) self._add_ldflags(ldflags) @property def cxx_std(self) -> int: """ The CXX standard level. If set, will add the required flags. If left at 0, it will trigger an automatic search when pybind11's build_ext is used. If None, will have no effect. Besides just the flags, this may add a macos-min 10.9 or 10.14 flag if MACOSX_DEPLOYMENT_TARGET is unset. """ return self._cxx_level @cxx_std.setter def cxx_std(self, level: int) -> None: if self._cxx_level: warnings.warn( "You cannot safely change the cxx_level after setting it!", stacklevel=2 ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. if WIN and level == 11: level = 14 self._cxx_level = level if not level: return cflags = [STD_TMPL.format(level)] ldflags = [] if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ: # C++17 requires a higher min version of macOS. An earlier version # (10.12 or 10.13) can be set manually via environment variable if # you are careful in your feature usage, but 10.14 is the safest # setting for general use. However, never set higher than the # current macOS version! current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2]) desired_macos = (10, 9) if level < 17 else (10, 14) macos_string = ".".join(str(x) for x in min(current_macos, desired_macos)) macosx_min = f"-mmacosx-version-min={macos_string}" cflags += [macosx_min] ldflags += [macosx_min] self._add_cflags(cflags) self._add_ldflags(ldflags) # Just in case someone clever tries to multithread tmp_chdir_lock = threading.Lock() @contextlib.contextmanager def tmp_chdir() -> Iterator[str]: "Prepare and enter a temporary directory, cleanup when done" # Threadsafe with tmp_chdir_lock: olddir = os.getcwd() try: tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) yield tmpdir finally: os.chdir(olddir) shutil.rmtree(tmpdir) # cf http://bugs.python.org/issue26689 def has_flag(compiler: Any, flag: str) -> bool: """ Return the flag if a flag name is supported on the specified compiler, otherwise None (can be used as a boolean). If multiple flags are passed, return the first that matches. """ with tmp_chdir(): fname = Path("flagcheck.cpp") # Don't trigger -Wunused-parameter. fname.write_text("int main (int, char **) { return 0; }", encoding="utf-8") try: compiler.compile([str(fname)], extra_postargs=[flag]) except distutils.errors.CompileError: return False return True # Every call will cache the result cpp_flag_cache = None @lru_cache() def auto_cpp_level(compiler: Any) -> Union[str, int]: """ Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows. """ if WIN: return "latest" levels = [17, 14, 11] for level in levels: if has_flag(compiler, STD_TMPL.format(level)): return level msg = "Unsupported compiler -- at least C++11 support is needed!" raise RuntimeError(msg) class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search for now, and is completely optional otherwise. """ def build_extensions(self) -> None: """ Build extensions, injecting C++ std for Pybind11Extension if needed. """ for ext in self.extensions: if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: ext.cxx_std = auto_cpp_level(self.compiler) super().build_extensions() def intree_extensions( paths: Iterable[str], package_dir: Optional[Dict[str, str]] = None ) -> List[Pybind11Extension]: """ Generate Pybind11Extensions from source files directly located in a Python source tree. ``package_dir`` behaves as in ``setuptools.setup``. If unset, the Python package root parent is determined as the first parent directory that does not contain an ``__init__.py`` file. """ exts = [] if package_dir is None: for path in paths: parent, _ = os.path.split(path) while os.path.exists(os.path.join(parent, "__init__.py")): parent, _ = os.path.split(parent) relname, _ = os.path.splitext(os.path.relpath(path, parent)) qualified_name = relname.replace(os.path.sep, ".") exts.append(Pybind11Extension(qualified_name, [path])) return exts for path in paths: for prefix, parent in package_dir.items(): if path.startswith(parent): relname, _ = os.path.splitext(os.path.relpath(path, parent)) qualified_name = relname.replace(os.path.sep, ".") if prefix: qualified_name = prefix + "." + qualified_name exts.append(Pybind11Extension(qualified_name, [path])) break else: msg = ( f"path {path} is not a child of any of the directories listed " f"in 'package_dir' ({package_dir})" ) raise ValueError(msg) return exts def naive_recompile(obj: str, src: str) -> bool: """ This will recompile only if the source file changes. It does not check header files, so a more advanced function or Ccache is better if you have editable header files in your package. """ return os.stat(obj).st_mtime < os.stat(src).st_mtime def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument """ This is the safest but slowest choice (and is the default) - will always recompile sources. """ return True S = TypeVar("S", bound="ParallelCompile") CCompilerMethod = Callable[ [ distutils.ccompiler.CCompiler, List[str], Optional[str], Optional[Union[Tuple[str], Tuple[str, Optional[str]]]], Optional[List[str]], bool, Optional[List[str]], Optional[List[str]], Optional[List[str]], ], List[str], ] # Optional parallel compile utility # inspired by: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # and: https://github.com/tbenthompson/cppimport/blob/stable/cppimport/build_module.py # and NumPy's parallel distutils module: # https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py class ParallelCompile: """ Make a parallel compile function. Inspired by numpy.distutils.ccompiler.CCompiler.compile and cppimport. This takes several arguments that allow you to customize the compile function created: envvar: Set an environment variable to control the compilation threads, like NPY_NUM_BUILD_JOBS default: 0 will automatically multithread, or 1 will only multithread if the envvar is set. max: The limit for automatic multithreading if non-zero needs_recompile: A function of (obj, src) that returns True when recompile is needed. No effect in isolated mode; use ccache instead, see https://github.com/matplotlib/matplotlib/issues/1507/ To use:: ParallelCompile("NPY_NUM_BUILD_JOBS").install() or:: with ParallelCompile("NPY_NUM_BUILD_JOBS"): setup(...) By default, this assumes all files need to be recompiled. A smarter function can be provided via needs_recompile. If the output has not yet been generated, the compile will always run, and this function is not called. """ __slots__ = ("envvar", "default", "max", "_old", "needs_recompile") def __init__( self, envvar: Optional[str] = None, default: int = 0, max: int = 0, # pylint: disable=redefined-builtin needs_recompile: Callable[[str, str], bool] = no_recompile, ) -> None: self.envvar = envvar self.default = default self.max = max self.needs_recompile = needs_recompile self._old: List[CCompilerMethod] = [] def function(self) -> CCompilerMethod: """ Builds a function object usable as distutils.ccompiler.CCompiler.compile. """ def compile_function( compiler: distutils.ccompiler.CCompiler, sources: List[str], output_dir: Optional[str] = None, macros: Optional[Union[Tuple[str], Tuple[str, Optional[str]]]] = None, include_dirs: Optional[List[str]] = None, debug: bool = False, extra_preargs: Optional[List[str]] = None, extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs ) cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) # type: ignore[attr-defined] # The number of threads; start with default. threads = self.default # Determine the number of compilation threads, unless set by an environment variable. if self.envvar is not None: threads = int(os.environ.get(self.envvar, self.default)) def _single_compile(obj: Any) -> None: try: src, ext = build[obj] except KeyError: return if not os.path.exists(obj) or self.needs_recompile(obj, src): compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # type: ignore[attr-defined] try: # Importing .synchronize checks for platforms that have some multiprocessing # capabilities but lack semaphores, such as AWS Lambda and Android Termux. import multiprocessing.synchronize from multiprocessing.pool import ThreadPool except ImportError: threads = 1 if threads == 0: try: threads = multiprocessing.cpu_count() threads = self.max if self.max and self.max < threads else threads except NotImplementedError: threads = 1 if threads > 1: with ThreadPool(threads) as pool: for _ in pool.imap_unordered(_single_compile, objects): pass else: for ob in objects: _single_compile(ob) return objects return compile_function def install(self: S) -> S: """ Installs the compile function into distutils.ccompiler.CCompiler.compile. """ distutils.ccompiler.CCompiler.compile = self.function() # type: ignore[assignment] return self def __enter__(self: S) -> S: self._old.append(distutils.ccompiler.CCompiler.compile) return self.install() def __exit__(self, *args: Any) -> None: distutils.ccompiler.CCompiler.compile = self._old.pop() # type: ignore[assignment]