From 80852b24228fed05ff8913ece6a941b77ef4931f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 6 Aug 2026 23:20:07 -0400 Subject: [PATCH] feat: expose the precompiled-mode sources to non-CMake builds Adds pybind11.get_source_dir() / python -m pybind11 --srcdir, a srcdir variable in pybind11.pc, and Pybind11Extension(precompile=True) which compiles src/pybind11_combined.cpp into the extension and defines PYBIND11_PRECOMPILED (hard error if the sources are missing). Assisted-by: ClaudeCode:claude-fable-5 --- CMakeLists.txt | 1 + pybind11/__init__.py | 3 +- pybind11/__main__.py | 9 +++ pybind11/commands.py | 18 ++++++ pybind11/setup_helpers.py | 27 ++++++++ tests/extra_python_package/test_files.py | 1 + tests/extra_setuptools/test_setuphelper.py | 72 ++++++++++++++++++++++ tools/pybind11.pc.in | 1 + 8 files changed, 131 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ea1b3a157b..3fd6d77fe7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -410,6 +410,7 @@ if(PYBIND11_INSTALL) endif() endif() join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + join_paths(srcdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 3882b2b17b..1d66e119d6 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -8,12 +8,13 @@ from ._version import __version__, version_info -from .commands import get_cmake_dir, get_include, get_pkgconfig_dir +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir, get_source_dir __all__ = ( "__version__", "get_cmake_dir", "get_include", "get_pkgconfig_dir", + "get_source_dir", "version_info", ) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index ce597c781a..98e9184807 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -17,6 +17,7 @@ get_include_dirs, get_ldflags, get_pkgconfig_dir, + get_source_dir, ) @@ -50,6 +51,12 @@ def main() -> None: action="store_true", help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", ) + parser.add_argument( + "--srcdir", + action="store_true", + help="Print the directory containing the library sources for the optional" + " precompiled mode.", + ) parser.add_argument( "--extension-suffix", action="store_true", @@ -101,6 +108,8 @@ def main() -> None: print(quote(get_cmake_dir())) if args.pkgconfigdir: print(quote(get_pkgconfig_dir())) + if args.srcdir: + print(quote(get_source_dir())) if args.extension_suffix: print(ext_suffix) diff --git a/pybind11/commands.py b/pybind11/commands.py index 8bd0a9bf13..573b7cf4ac 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -52,6 +52,24 @@ def get_include(user: bool = False) -> str: # noqa: ARG001 return installed_path if os.path.exists(installed_path) else source_path +def get_source_dir() -> str: + """ + Return the path to the pybind11 library sources, for the optional + precompiled mode. Compile ``pybind11_combined.cpp`` (or the individual + ``.cpp`` files) with ``PYBIND11_PRECOMPILED`` defined, and define that + macro for every translation unit that includes pybind11. + """ + installed_path = os.path.join(DIR, "share", "pybind11", "src") + source_path = os.path.join(os.path.dirname(DIR), "src") + if os.path.exists(installed_path): + return installed_path + if os.path.exists(source_path): + return source_path + + msg = "pybind11 library sources not found (pybind11 not installed?)" + raise ImportError(msg) + + def get_cmake_dir() -> str: """ Return the path to the pybind11 CMake module directory. diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 66a3eec0d4..43b0e5e5fc 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -108,6 +108,11 @@ class Pybind11Extension(_Extension): If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. + + Set ``precompile=True`` to compile the pybind11 library sources into the + extension (one extra translation unit) instead of instantiating everything + inline in every file; this usually builds faster. Requires an installed + pybind11 package that ships the library sources. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -127,6 +132,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: kwargs["language"] = "c++" include_pybind11 = kwargs.pop("include_pybind11", True) + precompile = kwargs.pop("precompile", False) super().__init__(*args, **kwargs) @@ -143,6 +149,27 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: except ModuleNotFoundError: pass + if precompile: + # No silent fallback: failing to precompile would quietly rebuild + # everything inline, so a missing source tree is an error. + try: + import pybind11 + + combined = os.path.join( + pybind11.get_source_dir(), "pybind11_combined.cpp" + ) + except (ImportError, AttributeError) as err: + msg = ( + "precompile=True requires an installed pybind11 package " + "that provides the library sources" + ) + raise ValueError(msg) from err + if not os.path.exists(combined): + msg = f"pybind11 library sources not found: {combined}" + raise ValueError(msg) + self.sources.append(combined) + self.define_macros.append(("PYBIND11_PRECOMPILED", None)) + self.cxx_std = cxx_std cflags = [] diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 58dbca4bc1..a14f057a68 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -33,6 +33,7 @@ PKGCONFIG = """\ prefix=${{pcfiledir}}/../../ includedir=${{prefix}}/include +srcdir=${{prefix}}/share/pybind11/src Name: pybind11 Description: Seamless operability between C++11 and Python diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 22c0c76f5d..7742834c8a 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -110,6 +110,78 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): ) +def test_precompile_setup_py(monkeypatch, tmpdir): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + f"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup + from pybind11.setup_helpers import Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "precompile_setup", + sorted(["main.cpp"]), + cxx_std=17, + precompile=True, + ), + ] + + setup( + name="precompile_setup_package", + ext_modules=ext_modules, + ) + """ + ), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + """\ + #include + + #ifndef PYBIND11_PRECOMPILED + # error "expected PYBIND11_PRECOMPILED to be defined" + #endif + + int f(int x) { + return x * 3; + } + PYBIND11_MODULE(precompile_setup, m, pybind11::mod_gil_used()) { + m.def("f", &f); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + (tmpdir / "test.py").write_text( + dedent( + """\ + import precompile_setup + assert precompile_setup.f(3) == 9 + """ + ), + encoding="ascii", + ) + + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) diff --git a/tools/pybind11.pc.in b/tools/pybind11.pc.in index 402f0b357d..8b5af0a991 100644 --- a/tools/pybind11.pc.in +++ b/tools/pybind11.pc.in @@ -1,5 +1,6 @@ prefix=@prefix_for_pc_file@ includedir=@includedir_for_pc_file@ +srcdir=@srcdir_for_pc_file@ Name: @PROJECT_NAME@ Description: Seamless operability between C++11 and Python