[feat/precompile-pybind11-h] feat: expose the precompiled-mode sources to non-CMake builds

CMakeLists.txt

@@ -410,6 +410,7 @@ if(PYBIND11_INSTALL)
     endif()
   endif()
   join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+  join_paths(srcdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
   configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"
                  "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"

pybind11/__init__.py

@@ -8,12 +8,13 @@
 
 
 from ._version import __version__, version_info
-from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
+from .commands import get_cmake_dir, get_include, get_pkgconfig_dir, get_source_dir
 
 __all__ = (
     "__version__",
     "get_cmake_dir",
     "get_include",
     "get_pkgconfig_dir",
+    "get_source_dir",
     "version_info",
 )

pybind11/__main__.py

@@ -17,6 +17,7 @@
     get_include_dirs,
     get_ldflags,
     get_pkgconfig_dir,
+    get_source_dir,
 )
 
 
@@ -50,6 +51,12 @@ def main() -> None:
         action="store_true",
         help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
     )
+    parser.add_argument(
+        "--srcdir",
+        action="store_true",
+        help="Print the directory containing the library sources for the optional"
+        " precompiled mode.",
+    )
     parser.add_argument(
         "--extension-suffix",
         action="store_true",
@@ -101,6 +108,8 @@ def main() -> None:
         print(quote(get_cmake_dir()))
     if args.pkgconfigdir:
         print(quote(get_pkgconfig_dir()))
+    if args.srcdir:
+        print(quote(get_source_dir()))
     if args.extension_suffix:
         print(ext_suffix)
 

pybind11/commands.py

@@ -52,6 +52,24 @@ def get_include(user: bool = False) -> str:  # noqa: ARG001
     return installed_path if os.path.exists(installed_path) else source_path
 
 
+def get_source_dir() -> str:
+    """
+    Return the path to the pybind11 library sources, for the optional
+    precompiled mode. Compile ``pybind11_combined.cpp`` (or the individual
+    ``.cpp`` files) with ``PYBIND11_PRECOMPILED`` defined, and define that
+    macro for every translation unit that includes pybind11.
+    """
+    installed_path = os.path.join(DIR, "share", "pybind11", "src")
+    source_path = os.path.join(os.path.dirname(DIR), "src")
+    if os.path.exists(installed_path):
+        return installed_path
+    if os.path.exists(source_path):
+        return source_path
+
+    msg = "pybind11 library sources not found (pybind11 not installed?)"
+    raise ImportError(msg)
+
+
 def get_cmake_dir() -> str:
     """
     Return the path to the pybind11 CMake module directory.

pybind11/setup_helpers.py

@@ -108,6 +108,11 @@ class Pybind11Extension(_Extension):
 
     If you want to add pybind11 headers manually, for example for an exact
     git checkout, then set ``include_pybind11=False``.
+
+    Set ``precompile=True`` to compile the pybind11 library sources into the
+    extension (one extra translation unit) instead of instantiating everything
+    inline in every file; this usually builds faster. Requires an installed
+    pybind11 package that ships the library sources.
     """
 
     # flags are prepended, so that they can be further overridden, e.g. by
@@ -127,6 +132,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
             kwargs["language"] = "c++"
 
         include_pybind11 = kwargs.pop("include_pybind11", True)
+        precompile = kwargs.pop("precompile", False)
 
         super().__init__(*args, **kwargs)
 
@@ -143,6 +149,27 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
             except ModuleNotFoundError:
                 pass
 
+        if precompile:
+            # No silent fallback: failing to precompile would quietly rebuild
+            # everything inline, so a missing source tree is an error.
+            try:
+                import pybind11
+
+                combined = os.path.join(
+                    pybind11.get_source_dir(), "pybind11_combined.cpp"
+                )
+            except (ImportError, AttributeError) as err:
+                msg = (
+                    "precompile=True requires an installed pybind11 package "
+                    "that provides the library sources"
+                )
+                raise ValueError(msg) from err
+            if not os.path.exists(combined):
+                msg = f"pybind11 library sources not found: {combined}"
+                raise ValueError(msg)
+            self.sources.append(combined)
+            self.define_macros.append(("PYBIND11_PRECOMPILED", None))
+
         self.cxx_std = cxx_std
 
         cflags = []

tests/extra_python_package/test_files.py

@@ -33,6 +33,7 @@
 PKGCONFIG = """\
 prefix=${{pcfiledir}}/../../
 includedir=${{prefix}}/include
+srcdir=${{prefix}}/share/pybind11/src
 
 Name: pybind11
 Description: Seamless operability between C++11 and Python

tests/extra_setuptools/test_setuphelper.py

@@ -110,6 +110,78 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std):
     )
 
 
+def test_precompile_setup_py(monkeypatch, tmpdir):
+    monkeypatch.chdir(tmpdir)
+    monkeypatch.syspath_prepend(MAIN_DIR)
+
+    (tmpdir / "setup.py").write_text(
+        dedent(
+            f"""\
+            import sys
+            sys.path.append({MAIN_DIR!r})
+
+            from setuptools import setup
+            from pybind11.setup_helpers import Pybind11Extension
+
+            ext_modules = [
+                Pybind11Extension(
+                    "precompile_setup",
+                    sorted(["main.cpp"]),
+                    cxx_std=17,
+                    precompile=True,
+                ),
+            ]
+
+            setup(
+                name="precompile_setup_package",
+                ext_modules=ext_modules,
+            )
+            """
+        ),
+        encoding="ascii",
+    )
+
+    (tmpdir / "main.cpp").write_text(
+        dedent(
+            """\
+            #include <pybind11/pybind11.h>
+
+            #ifndef PYBIND11_PRECOMPILED
+            #    error "expected PYBIND11_PRECOMPILED to be defined"
+            #endif
+
+            int f(int x) {
+                return x * 3;
+            }
+            PYBIND11_MODULE(precompile_setup, m, pybind11::mod_gil_used()) {
+                m.def("f", &f);
+            }
+            """
+        ),
+        encoding="ascii",
+    )
+
+    subprocess.check_call(
+        [sys.executable, "setup.py", "build_ext", "--inplace"],
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+    )
+
+    (tmpdir / "test.py").write_text(
+        dedent(
+            """\
+            import precompile_setup
+            assert precompile_setup.f(3) == 9
+            """
+        ),
+        encoding="ascii",
+    )
+
+    subprocess.check_call(
+        [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr
+    )
+
+
 def test_intree_extensions(monkeypatch, tmpdir):
     monkeypatch.syspath_prepend(MAIN_DIR)
 

tools/pybind11.pc.in

@@ -1,5 +1,6 @@
 prefix=@prefix_for_pc_file@
 includedir=@includedir_for_pc_file@
+srcdir=@srcdir_for_pc_file@
 
 Name: @PROJECT_NAME@
 Description: Seamless operability between C++11 and Python