Merge branch 'v2.13' into stable

.github/workflows/ci.yml

@@ -243,7 +243,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }} (deadsnakes)
-      uses: deadsnakes/action@v3.1.0
+      uses: deadsnakes/action@v3.2.0
       with:
         python-version: ${{ matrix.python-version }}
         debug: ${{ matrix.python-debug }}

.github/workflows/pip.yml

@@ -91,18 +91,19 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
     needs: [packaging]
-    environment: pypi
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pybind11
     permissions:
       id-token: write
       attestations: write
-      contents: read
 
     steps:
     # Downloads all to directories matching the artifact names
     - uses: actions/download-artifact@v4
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@310b0a4a3b0b78ef57ecda988ee04b132db73ef8 # v1.4.1
+      uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
       with:
         subject-path: "*/pybind11*"
 
@@ -110,8 +111,10 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: standard/
+        attestations: true
 
     - name: Publish global package
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: global/
+        attestations: true

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.3
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -40,7 +40,7 @@ repos:
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v1.11.1"
+  rev: "v1.11.2"
   hooks:
   - id: mypy
     args: []
@@ -93,7 +93,7 @@ repos:
 
 # Avoid directional quotes
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.6.6"
+  rev: "0.6.7"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
@@ -142,14 +142,14 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.2.6"
+  rev: "v3.2.7"
   hooks:
   - id: pylint
     files: ^pybind11
 
 # Check schemas on some of our YAML files
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-readthedocs
   - id: check-github-workflows

CMakeLists.txt

@@ -149,12 +149,14 @@ endif()
 set(PYBIND11_HEADERS
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/cpp_conduit.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
     include/pybind11/detail/value_and_holder.h
+    include/pybind11/detail/exception_translation.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
     include/pybind11/cast.h

docs/changelog.rst

@@ -31,6 +31,37 @@ New Features:
 * The ``array_caster`` in pybind11/stl.h was enhanced to support value types that are not default-constructible.
   `#5305 <https://github.com/pybind/pybind11/pull/5305>`_
 
+* Added ``py::warnings`` namespace with ``py::warnings::warn`` and ``py::warnings::new_warning_type`` that provides the interface for Python warnings.
+  `#5291 <https://github.com/pybind/pybind11/pull/5291>`_
+
+Version 2.13.6 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
+
+Bug fixes:
+
+* Using ``__cpp_nontype_template_args`` instead of ``__cpp_nontype_template_parameter_class``.
+  `#5330 <https://github.com/pybind/pybind11/pull/5330>`_
+
+* Properly translate C++ exception to Python exception when creating Python buffer from wrapped object.
+  `#5324 <https://github.com/pybind/pybind11/pull/5324>`_
+
+
+Documentation:
+
+* Adds an answer (FAQ) for "What is a highly conclusive and simple way to find memory leaks?".
+  `#5340 <https://github.com/pybind/pybind11/pull/5340>`_
+
+
 Version 2.13.5 (August 22, 2024)
 --------------------------------
 
@@ -238,6 +269,18 @@ Other:
 * Update docs and noxfile.
   `#5071 <https://github.com/pybind/pybind11/pull/5071>`_
 
+Version 2.12.1 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
 
 Version 2.12.0 (March 27, 2024)
 -------------------------------
@@ -413,6 +456,18 @@ Other:
 * An ``assert()`` was added to help Coverty avoid generating a false positive.
   `#4817 <https://github.com/pybind/pybind11/pull/4817>`_
 
+Version 2.11.2 (September 13, 2024)
+-----------------------------------
+
+New Features:
+
+* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all
+  ``py::class_``-wrapped types, to enable type-safe interoperability between
+  different independent Python/C++ bindings systems, including pybind11
+  versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on
+  pybind11 2.11.2, 2.12.1, and 2.13.6+.
+  `#5296 <https://github.com/pybind/pybind11/pull/5296>`_
+
 
 Version 2.11.1 (July 17, 2023)
 ------------------------------

docs/faq.rst

@@ -247,6 +247,50 @@ been received, you must either explicitly interrupt execution by throwing
         });
     }
 
+What is a highly conclusive and simple way to find memory leaks (e.g. in pybind11 bindings)?
+============================================================================================
+
+Use ``while True`` & ``top`` (Linux, macOS).
+
+For example, locally change tests/test_type_caster_pyobject_ptr.py like this:
+
+.. code-block:: diff
+
+     def test_return_list_pyobject_ptr_reference():
+    +  while True:
+         vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder)
+         assert [e.value for e in vec_obj] == [93, 186]
+         # Commenting out the next `assert` will leak the Python references.
+         # An easy way to see evidence of the leaks:
+         # Insert `while True:` as the first line of this function and monitor the
+         # process RES (Resident Memory Size) with the Unix top command.
+    -    assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+    +    # assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+
+Then run the test as you would normally do, which will go into the infinite loop.
+
+**In another shell, but on the same machine** run:
+
+.. code-block:: bash
+
+    top
+
+This will show:
+
+.. code-block::
+
+        PID USER      PR  NI    VIRT    RES    SHR S  %CPU  %MEM     TIME+ COMMAND
+    1266095 rwgk      20   0 5207496 611372  45696 R 100.0   0.3   0:08.01 test_type_caste
+
+Look for the number under ``RES`` there. You'll see it going up very quickly.
+
+**Don't forget to Ctrl-C the test command** before your machine becomes
+unresponsive due to swapping.
+
+This method only takes a couple minutes of effort and is very conclusive.
+What you want to see is that the ``RES`` number is stable after a couple
+seconds.
+
 CMake doesn't detect the right Python version
 =============================================
 

include/pybind11/detail/class.h

@@ -12,6 +12,8 @@
 #include <pybind11/attr.h>
 #include <pybind11/options.h>
 
+#include "exception_translation.h"
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
@@ -591,7 +593,18 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla
         return -1;
     }
     std::memset(view, 0, sizeof(Py_buffer));
-    buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data);
+    buffer_info *info = nullptr;
+    try {
+        info = tinfo->get_buffer(obj, tinfo->get_buffer_data);
+    } catch (...) {
+        try_translate_exceptions();
+        raise_from(PyExc_BufferError, "Error getting buffer");
+        return -1;
+    }
+    if (info == nullptr) {
+        pybind11_fail("FATAL UNEXPECTED SITUATION: tinfo->get_buffer() returned nullptr.");
+    }
+
     if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) {
         delete info;
         // view->obj = nullptr;  // Was just memset to 0, so not necessary

include/pybind11/detail/common.h

@@ -11,11 +11,11 @@
 
 #define PYBIND11_VERSION_MAJOR 2
 #define PYBIND11_VERSION_MINOR 13
-#define PYBIND11_VERSION_PATCH 5
+#define PYBIND11_VERSION_PATCH 6
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x020D0500
+#define PYBIND11_VERSION_HEX 0x020D0600
 
 // Define some generic pybind11 helper macros for warning management.
 //

include/pybind11/detail/cpp_conduit.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2024 The pybind Community.
+
+#pragma once
+
+#include <pybind11/pytypes.h>
+
+#include "common.h"
+#include "internals.h"
+
+#include <typeinfo>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Forward declaration needed here: Refactoring opportunity.
+extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *);
+
+inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) {
+#if defined(PYPY_VERSION)
+    auto &internals = get_internals();
+    return bool(internals.registered_types_py.find(type_obj)
+                != internals.registered_types_py.end());
+#else
+    return bool(type_obj->tp_new == pybind11_object_new);
+#endif
+}
+
+inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) {
+    PyObject *descr = _PyType_Lookup(type_obj, attr_name);
+    return bool((descr != nullptr) && PyInstanceMethod_Check(descr));
+}
+
+inline object try_get_cpp_conduit_method(PyObject *obj) {
+    if (PyType_Check(obj)) {
+        return object();
+    }
+    PyTypeObject *type_obj = Py_TYPE(obj);
+    str attr_name("_pybind11_conduit_v1_");
+    bool assumed_to_be_callable = false;
+    if (type_is_managed_by_our_internals(type_obj)) {
+        if (!is_instance_method_of_type(type_obj, attr_name.ptr())) {
+            return object();
+        }
+        assumed_to_be_callable = true;
+    }
+    PyObject *method = PyObject_GetAttr(obj, attr_name.ptr());
+    if (method == nullptr) {
+        PyErr_Clear();
+        return object();
+    }
+    if (!assumed_to_be_callable && PyCallable_Check(method) == 0) {
+        Py_DECREF(method);
+        return object();
+    }
+    return reinterpret_steal<object>(method);
+}
+
+inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src,
+                                                        const std::type_info *cpp_type_info) {
+    object method = try_get_cpp_conduit_method(src.ptr());
+    if (method) {
+        capsule cpp_type_info_capsule(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
+                                      typeid(std::type_info).name());
+        object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID),
+                                    cpp_type_info_capsule,
+                                    bytes("raw_pointer_ephemeral"));
+        if (isinstance<capsule>(cpp_conduit)) {
+            return reinterpret_borrow<capsule>(cpp_conduit).get_pointer();
+        }
+    }
+    return nullptr;
+}
+
+#define PYBIND11_HAS_CPP_CONDUIT 1
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/exception_translation.h

@@ -0,0 +1,71 @@
+/*
+    pybind11/detail/exception_translation.h: means to translate C++ exceptions to Python exceptions
+
+    Copyright (c) 2024 The Pybind Development Team.
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "common.h"
+#include "internals.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Apply all the extensions translators from a list
+// Return true if one of the translators completed without raising an exception
+// itself. Return of false indicates that if there are other translators
+// available, they should be tried.
+inline bool apply_exception_translators(std::forward_list<ExceptionTranslator> &translators) {
+    auto last_exception = std::current_exception();
+
+    for (auto &translator : translators) {
+        try {
+            translator(last_exception);
+            return true;
+        } catch (...) {
+            last_exception = std::current_exception();
+        }
+    }
+    return false;
+}
+
+inline void try_translate_exceptions() {
+    /* When an exception is caught, give each registered exception
+        translator a chance to translate it to a Python exception. First
+        all module-local translators will be tried in reverse order of
+        registration. If none of the module-locale translators handle
+        the exception (or there are no module-locale translators) then
+        the global translators will be tried, also in reverse order of
+        registration.
+
+        A translator may choose to do one of the following:
+
+        - catch the exception and call py::set_error()
+            to set a standard (or custom) Python exception, or
+        - do nothing and let the exception fall through to the next translator, or
+        - delegate translation to the next translator by throwing a new type of exception.
+        */
+
+    bool handled = with_internals([&](internals &internals) {
+        auto &local_exception_translators = get_local_internals().registered_exception_translators;
+        if (detail::apply_exception_translators(local_exception_translators)) {
+            return true;
+        }
+        auto &exception_translators = internals.registered_exception_translators;
+        if (detail::apply_exception_translators(exception_translators)) {
+            return true;
+        }
+        return false;
+    });
+
+    if (!handled) {
+        set_error(PyExc_SystemError, "Exception escaped from default exception translator!");
+    }
+}
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/internals.h

@@ -321,15 +321,17 @@ struct type_info {
 #    define PYBIND11_INTERNALS_KIND ""
 #endif
 
+#define PYBIND11_PLATFORM_ABI_ID                                                                  \
+    PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI             \
+        PYBIND11_BUILD_TYPE
+
 #define PYBIND11_INTERNALS_ID                                                                     \
     "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION)                        \
-        PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB                            \
+        PYBIND11_PLATFORM_ABI_ID "__"
-            PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__"
 
 #define PYBIND11_MODULE_LOCAL_ID                                                                  \
     "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION)                     \
-        PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB                            \
+        PYBIND11_PLATFORM_ABI_ID "__"
-            PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__"
 
 /// Each module locally stores a pointer to the `internals` data. The data
 /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`.

include/pybind11/detail/type_caster_base.h

@@ -12,14 +12,17 @@
 #include <pybind11/pytypes.h>
 
 #include "common.h"
+#include "cpp_conduit.h"
 #include "descr.h"
 #include "internals.h"
 #include "typeid.h"
 #include "value_and_holder.h"
 
 #include <cstdint>
+#include <cstring>
 #include <iterator>
 #include <new>
+#include <stdexcept>
 #include <string>
 #include <type_traits>
 #include <typeindex>
@@ -611,6 +614,13 @@ public:
         }
         return false;
     }
+    bool try_cpp_conduit(handle src) {
+        value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype);
+        if (value != nullptr) {
+            return true;
+        }
+        return false;
+    }
     void check_holder_compat() {}
 
     PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) {
@@ -742,6 +752,10 @@ public:
             return true;
         }
 
+        if (convert && cpptype && this_.try_cpp_conduit(src)) {
+            return true;
+        }
+
         return false;
     }
 
@@ -769,6 +783,32 @@ public:
     void *value = nullptr;
 };
 
+inline object cpp_conduit_method(handle self,
+                                 const bytes &pybind11_platform_abi_id,
+                                 const capsule &cpp_type_info_capsule,
+                                 const bytes &pointer_kind) {
+#ifdef PYBIND11_HAS_STRING_VIEW
+    using cpp_str = std::string_view;
+#else
+    using cpp_str = std::string;
+#endif
+    if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) {
+        return none();
+    }
+    if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) {
+        return none();
+    }
+    if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") {
+        throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\"");
+    }
+    const auto *cpp_type_info = cpp_type_info_capsule.get_pointer<const std::type_info>();
+    type_caster_generic caster(*cpp_type_info);
+    if (!caster.load(self, false)) {
+        return none();
+    }
+    return capsule(caster.value, cpp_type_info->name());
+}
+
 /**
  * Determine suitable casting operator for pointer-or-lvalue-casting type casters.  The type caster
  * needs to provide `operator T*()` and `operator T&()` operators.

include/pybind11/pybind11.h

@@ -9,8 +9,8 @@
 */
 
 #pragma once
-
 #include "detail/class.h"
+#include "detail/exception_translation.h"
 #include "detail/init.h"
 #include "attr.h"
 #include "gil.h"
@@ -95,24 +95,6 @@ inline std::string replace_newlines_and_squash(const char *text) {
     return result.substr(str_begin, str_range);
 }
 
-// Apply all the extensions translators from a list
-// Return true if one of the translators completed without raising an exception
-// itself. Return of false indicates that if there are other translators
-// available, they should be tried.
-inline bool apply_exception_translators(std::forward_list<ExceptionTranslator> &translators) {
-    auto last_exception = std::current_exception();
-
-    for (auto &translator : translators) {
-        try {
-            translator(last_exception);
-            return true;
-        } catch (...) {
-            last_exception = std::current_exception();
-        }
-    }
-    return false;
-}
-
 #if defined(_MSC_VER)
 #    define PYBIND11_COMPAT_STRDUP _strdup
 #else
@@ -610,7 +592,8 @@ protected:
         int index = 0;
         /* Create a nice pydoc rec including all signatures and
            docstrings of the functions in the overload chain */
-        if (chain && options::show_function_signatures()) {
+        if (chain && options::show_function_signatures()
+            && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
             // First a generic signature
             signatures += rec->name;
             signatures += "(*args, **kwargs)\n";
@@ -619,7 +602,8 @@ protected:
         // Then specific overload signatures
         bool first_user_def = true;
         for (auto *it = chain_start; it != nullptr; it = it->next) {
-            if (options::show_function_signatures()) {
+            if (options::show_function_signatures()
+                && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) {
                 if (index > 0) {
                     signatures += '\n';
                 }
@@ -1038,40 +1022,7 @@ protected:
             throw;
 #endif
         } catch (...) {
-            /* When an exception is caught, give each registered exception
+            try_translate_exceptions();
-               translator a chance to translate it to a Python exception. First
-               all module-local translators will be tried in reverse order of
-               registration. If none of the module-locale translators handle
-               the exception (or there are no module-locale translators) then
-               the global translators will be tried, also in reverse order of
-               registration.
-
-               A translator may choose to do one of the following:
-
-                - catch the exception and call py::set_error()
-                  to set a standard (or custom) Python exception, or
-                - do nothing and let the exception fall through to the next translator, or
-                - delegate translation to the next translator by throwing a new type of exception.
-             */
-
-            bool handled = with_internals([&](internals &internals) {
-                auto &local_exception_translators
-                    = get_local_internals().registered_exception_translators;
-                if (detail::apply_exception_translators(local_exception_translators)) {
-                    return true;
-                }
-                auto &exception_translators = internals.registered_exception_translators;
-                if (detail::apply_exception_translators(exception_translators)) {
-                    return true;
-                }
-                return false;
-            });
-
-            if (handled) {
-                return nullptr;
-            }
-
-            set_error(PyExc_SystemError, "Exception escaped from default exception translator!");
             return nullptr;
         }
 
@@ -1652,6 +1603,7 @@ public:
                     = instances[std::type_index(typeid(type))];
             });
         }
+        def("_pybind11_conduit_v1_", cpp_conduit_method);
     }
 
     template <typename Base, detail::enable_if_t<is_base<Base>::value, int> = 0>

include/pybind11/typing.h

@@ -100,9 +100,7 @@ class Never : public none {
     using none::none;
 };
 
-#if defined(__cpp_nontype_template_parameter_class)                                               \
+#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L
-    && (/* See #5201 */ !defined(__GNUC__)                                                        \
-        || (__GNUC__ > 10 || (__GNUC__ == 10 && __GNUC_MINOR__ >= 3)))
 #    define PYBIND11_TYPING_H_HAS_STRING_LITERAL
 template <size_t N>
 struct StringLiteral {

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s: str) -> int | str:
         return s
 
 
-__version__ = "2.13.5"
+__version__ = "2.13.6"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

tests/CMakeLists.txt

@@ -127,6 +127,7 @@ set(PYBIND11_TEST_FILES
     test_const_name
     test_constants_and_functions
     test_copy_move
+    test_cpp_conduit
     test_custom_type_casters
     test_custom_type_setup
     test_docstring_options
@@ -226,6 +227,8 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_
 # And add additional targets for other tests.
 tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set")
 tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils")
+tests_extra_targets("test_cpp_conduit.py"
+                    "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler")
 
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"

tests/conftest.py

@@ -136,7 +136,7 @@ class Capture:
         return Output(self.err)
 
 
-@pytest.fixture()
+@pytest.fixture
 def capture(capsys):
     """Extended `capsys` with context manager and custom equality operators"""
     return Capture(capsys)
@@ -172,7 +172,7 @@ def _sanitize_docstring(thing):
     return _sanitize_general(s)
 
 
-@pytest.fixture()
+@pytest.fixture
 def doc():
     """Sanitize docstrings and add custom failure explanation"""
     return SanitizedString(_sanitize_docstring)
@@ -184,7 +184,7 @@ def _sanitize_message(thing):
     return _hexadecimal.sub("0", s)
 
 
-@pytest.fixture()
+@pytest.fixture
 def msg():
     """Sanitize messages and add custom failure explanation"""
     return SanitizedString(_sanitize_message)

tests/exo_planet_c_api.cpp

@@ -0,0 +1,103 @@
+// Copyright (c) 2024 The pybind Community.
+
+// THIS MUST STAY AT THE TOP!
+#include <pybind11/pybind11.h> // EXCLUSIVELY for PYBIND11_PLATFORM_ABI_ID
+// Potential future direction to maximize reusability:
+// (e.g. for use from SWIG, Cython, PyCLIF, nanobind):
+//     #include <pybind11/compat/platform_abi_id.h>
+// This would only depend on:
+//     1. A C++ compiler, WITHOUT requiring -fexceptions.
+//     2. Python.h
+
+#include "test_cpp_conduit_traveler_types.h"
+
+#include <Python.h>
+#include <typeinfo>
+
+namespace {
+
+void *get_cpp_conduit_void_ptr(PyObject *py_obj, const std::type_info *cpp_type_info) {
+    PyObject *cpp_type_info_capsule
+        = PyCapsule_New(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
+                        typeid(std::type_info).name(),
+                        nullptr);
+    if (cpp_type_info_capsule == nullptr) {
+        return nullptr;
+    }
+    PyObject *cpp_conduit = PyObject_CallMethod(py_obj,
+                                                "_pybind11_conduit_v1_",
+                                                "yOy",
+                                                PYBIND11_PLATFORM_ABI_ID,
+                                                cpp_type_info_capsule,
+                                                "raw_pointer_ephemeral");
+    Py_DECREF(cpp_type_info_capsule);
+    if (cpp_conduit == nullptr) {
+        return nullptr;
+    }
+    void *void_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name());
+    Py_DECREF(cpp_conduit);
+    if (PyErr_Occurred()) {
+        return nullptr;
+    }
+    return void_ptr;
+}
+
+template <typename T>
+T *get_cpp_conduit_type_ptr(PyObject *py_obj) {
+    void *void_ptr = get_cpp_conduit_void_ptr(py_obj, &typeid(T));
+    if (void_ptr == nullptr) {
+        return nullptr;
+    }
+    return static_cast<T *>(void_ptr);
+}
+
+extern "C" PyObject *wrapGetLuggage(PyObject * /*self*/, PyObject *traveler) {
+    const auto *cpp_traveler
+        = get_cpp_conduit_type_ptr<pybind11_tests::test_cpp_conduit::Traveler>(traveler);
+    if (cpp_traveler == nullptr) {
+        return nullptr;
+    }
+    return PyUnicode_FromString(cpp_traveler->luggage.c_str());
+}
+
+extern "C" PyObject *wrapGetPoints(PyObject * /*self*/, PyObject *premium_traveler) {
+    const auto *cpp_premium_traveler
+        = get_cpp_conduit_type_ptr<pybind11_tests::test_cpp_conduit::PremiumTraveler>(
+            premium_traveler);
+    if (cpp_premium_traveler == nullptr) {
+        return nullptr;
+    }
+    return PyLong_FromLong(static_cast<long>(cpp_premium_traveler->points));
+}
+
+PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr},
+                               {"GetPoints", wrapGetPoints, METH_O, nullptr},
+                               {nullptr, nullptr, 0, nullptr}};
+
+struct PyModuleDef ThisModuleDef = {
+    PyModuleDef_HEAD_INIT, // m_base
+    "exo_planet_c_api",    // m_name
+    nullptr,               // m_doc
+    -1,                    // m_size
+    ThisMethodDef,         // m_methods
+    nullptr,               // m_slots
+    nullptr,               // m_traverse
+    nullptr,               // m_clear
+    nullptr                // m_free
+};
+
+} // namespace
+
+#if defined(WIN32) || defined(_WIN32)
+#    define EXO_PLANET_C_API_EXPORT __declspec(dllexport)
+#else
+#    define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default")))
+#endif
+
+extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() {
+    PyObject *m = PyModule_Create(&ThisModuleDef);
+    if (m == nullptr) {
+        return nullptr;
+    }
+    return m;
+}

tests/exo_planet_pybind11.cpp

@@ -0,0 +1,19 @@
+// Copyright (c) 2024 The pybind Community.
+
+#if defined(PYBIND11_INTERNALS_VERSION)
+#    undef PYBIND11_INTERNALS_VERSION
+#endif
+#define PYBIND11_INTERNALS_VERSION 900000001
+
+#include "test_cpp_conduit_traveler_bindings.h"
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+PYBIND11_MODULE(exo_planet_pybind11, m) {
+    wrap_traveler(m);
+    m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

tests/extra_python_package/test_files.py

@@ -53,12 +53,14 @@ main_headers = {
 detail_headers = {
     "include/pybind11/detail/class.h",
     "include/pybind11/detail/common.h",
+    "include/pybind11/detail/cpp_conduit.h",
     "include/pybind11/detail/descr.h",
     "include/pybind11/detail/init.h",
     "include/pybind11/detail/internals.h",
     "include/pybind11/detail/type_caster_base.h",
     "include/pybind11/detail/typeid.h",
     "include/pybind11/detail/value_and_holder.h",
+    "include/pybind11/detail/exception_translation.h",
 }
 
 eigen_headers = {

tests/home_planet_very_lonely_traveler.cpp

@@ -0,0 +1,13 @@
+// Copyright (c) 2024 The pybind Community.
+
+#include "test_cpp_conduit_traveler_bindings.h"
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+PYBIND11_MODULE(home_planet_very_lonely_traveler, m) {
+    m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); });
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

tests/test_async.py

@@ -11,7 +11,7 @@ if sys.platform.startswith("emscripten"):
     pytest.skip("Can't run a new event_loop in pyodide", allow_module_level=True)
 
 
-@pytest.fixture()
+@pytest.fixture
 def event_loop():
     loop = asyncio.new_event_loop()
     yield loop

tests/test_buffers.cpp

@@ -167,6 +167,18 @@ TEST_SUBMODULE(buffers, m) {
                  sizeof(float)});
         });
 
+    class BrokenMatrix : public Matrix {
+    public:
+        BrokenMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(rows, cols) {}
+        void throw_runtime_error() { throw std::runtime_error("See PR #5324 for context."); }
+    };
+    py::class_<BrokenMatrix>(m, "BrokenMatrix", py::buffer_protocol())
+        .def(py::init<py::ssize_t, py::ssize_t>())
+        .def_buffer([](BrokenMatrix &m) {
+            m.throw_runtime_error();
+            return py::buffer_info();
+        });
+
     // test_inherited_protocol
     class SquareMatrix : public Matrix {
     public:

tests/test_buffers.py

@@ -228,3 +228,10 @@ def test_buffer_docstring():
         m.get_buffer_info.__doc__.strip()
         == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info"
     )
+
+
+def test_buffer_exception():
+    with pytest.raises(BufferError, match="Error getting buffer") as excinfo:
+        memoryview(m.BrokenMatrix(1, 1))
+    assert isinstance(excinfo.value.__cause__, RuntimeError)
+    assert "for context" in str(excinfo.value.__cause__)

tests/test_cpp_conduit.cpp

@@ -0,0 +1,22 @@
+// Copyright (c) 2024 The pybind Community.
+
+#include "pybind11_tests.h"
+#include "test_cpp_conduit_traveler_bindings.h"
+
+#include <typeinfo>
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+TEST_SUBMODULE(cpp_conduit, m) {
+    m.attr("PYBIND11_PLATFORM_ABI_ID") = py::bytes(PYBIND11_PLATFORM_ABI_ID);
+    m.attr("cpp_type_info_capsule_Traveler")
+        = py::capsule(&typeid(Traveler), typeid(std::type_info).name());
+    m.attr("cpp_type_info_capsule_int") = py::capsule(&typeid(int), typeid(std::type_info).name());
+
+    wrap_traveler(m);
+    wrap_lonely_traveler(m);
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

tests/test_cpp_conduit.py

@@ -0,0 +1,162 @@
+# Copyright (c) 2024 The pybind Community.
+
+from __future__ import annotations
+
+import exo_planet_c_api
+import exo_planet_pybind11
+import home_planet_very_lonely_traveler
+import pytest
+
+from pybind11_tests import cpp_conduit as home_planet
+
+
+def test_traveler_getattr_actually_exists():
+    t_h = home_planet.Traveler("home")
+    assert t_h.any_name == "Traveler GetAttr: any_name luggage: home"
+
+
+def test_premium_traveler_getattr_actually_exists():
+    t_h = home_planet.PremiumTraveler("home", 7)
+    assert t_h.secret_name == "PremiumTraveler GetAttr: secret_name points: 7"
+
+
+def test_call_cpp_conduit_success():
+    t_h = home_planet.Traveler("home")
+    cap = t_h._pybind11_conduit_v1_(
+        home_planet.PYBIND11_PLATFORM_ABI_ID,
+        home_planet.cpp_type_info_capsule_Traveler,
+        b"raw_pointer_ephemeral",
+    )
+    assert cap.__class__.__name__ == "PyCapsule"
+
+
+def test_call_cpp_conduit_platform_abi_id_mismatch():
+    t_h = home_planet.Traveler("home")
+    cap = t_h._pybind11_conduit_v1_(
+        home_planet.PYBIND11_PLATFORM_ABI_ID + b"MISMATCH",
+        home_planet.cpp_type_info_capsule_Traveler,
+        b"raw_pointer_ephemeral",
+    )
+    assert cap is None
+
+
+def test_call_cpp_conduit_cpp_type_info_capsule_mismatch():
+    t_h = home_planet.Traveler("home")
+    cap = t_h._pybind11_conduit_v1_(
+        home_planet.PYBIND11_PLATFORM_ABI_ID,
+        home_planet.cpp_type_info_capsule_int,
+        b"raw_pointer_ephemeral",
+    )
+    assert cap is None
+
+
+def test_call_cpp_conduit_pointer_kind_invalid():
+    t_h = home_planet.Traveler("home")
+    with pytest.raises(
+        RuntimeError, match='^Invalid pointer_kind: "raw_pointer_ephemreal"$'
+    ):
+        t_h._pybind11_conduit_v1_(
+            home_planet.PYBIND11_PLATFORM_ABI_ID,
+            home_planet.cpp_type_info_capsule_Traveler,
+            b"raw_pointer_ephemreal",
+        )
+
+
+def test_home_only_basic():
+    t_h = home_planet.Traveler("home")
+    assert t_h.luggage == "home"
+    assert home_planet.get_luggage(t_h) == "home"
+
+
+def test_home_only_premium():
+    p_h = home_planet.PremiumTraveler("home", 2)
+    assert p_h.luggage == "home"
+    assert home_planet.get_luggage(p_h) == "home"
+    assert home_planet.get_points(p_h) == 2
+
+
+def test_exo_only_basic():
+    t_e = exo_planet_pybind11.Traveler("exo")
+    assert t_e.luggage == "exo"
+    assert exo_planet_pybind11.get_luggage(t_e) == "exo"
+
+
+def test_exo_only_premium():
+    p_e = exo_planet_pybind11.PremiumTraveler("exo", 3)
+    assert p_e.luggage == "exo"
+    assert exo_planet_pybind11.get_luggage(p_e) == "exo"
+    assert exo_planet_pybind11.get_points(p_e) == 3
+
+
+def test_home_passed_to_exo_basic():
+    t_h = home_planet.Traveler("home")
+    assert exo_planet_pybind11.get_luggage(t_h) == "home"
+
+
+def test_exo_passed_to_home_basic():
+    t_e = exo_planet_pybind11.Traveler("exo")
+    assert home_planet.get_luggage(t_e) == "exo"
+
+
+def test_home_passed_to_exo_premium():
+    p_h = home_planet.PremiumTraveler("home", 2)
+    assert exo_planet_pybind11.get_luggage(p_h) == "home"
+    assert exo_planet_pybind11.get_points(p_h) == 2
+
+
+def test_exo_passed_to_home_premium():
+    p_e = exo_planet_pybind11.PremiumTraveler("exo", 3)
+    assert home_planet.get_luggage(p_e) == "exo"
+    assert home_planet.get_points(p_e) == 3
+
+
+@pytest.mark.parametrize(
+    "traveler_type", [home_planet.Traveler, exo_planet_pybind11.Traveler]
+)
+def test_exo_planet_c_api_traveler(traveler_type):
+    t = traveler_type("socks")
+    assert exo_planet_c_api.GetLuggage(t) == "socks"
+
+
+@pytest.mark.parametrize(
+    "premium_traveler_type",
+    [home_planet.PremiumTraveler, exo_planet_pybind11.PremiumTraveler],
+)
+def test_exo_planet_c_api_premium_traveler(premium_traveler_type):
+    pt = premium_traveler_type("gucci", 5)
+    assert exo_planet_c_api.GetLuggage(pt) == "gucci"
+    assert exo_planet_c_api.GetPoints(pt) == 5
+
+
+def test_home_planet_wrap_very_lonely_traveler():
+    # This does not exercise the cpp_conduit feature, but is here to
+    # demonstrate that the cpp_conduit feature does not solve all
+    # cross-extension interoperability issues.
+    # Here is the proof that the following works for extensions with
+    # matching `PYBIND11_INTERNALS_ID`s:
+    #     test_cpp_conduit.cpp:
+    #         py::class_<LonelyTraveler>
+    #     home_planet_very_lonely_traveler.cpp:
+    #         py::class_<VeryLonelyTraveler, LonelyTraveler>
+    # See test_exo_planet_pybind11_wrap_very_lonely_traveler() for the negative
+    # test.
+    assert home_planet.LonelyTraveler is not None  # Verify that the base class exists.
+    home_planet_very_lonely_traveler.wrap_very_lonely_traveler()
+    # Ensure that the derived class exists.
+    assert home_planet_very_lonely_traveler.VeryLonelyTraveler is not None
+
+
+def test_exo_planet_pybind11_wrap_very_lonely_traveler():
+    # See comment under test_home_planet_wrap_very_lonely_traveler() first.
+    # Here the `PYBIND11_INTERNALS_ID`s don't match between:
+    #     test_cpp_conduit.cpp:
+    #         py::class_<LonelyTraveler>
+    #     exo_planet_pybind11.cpp:
+    #         py::class_<VeryLonelyTraveler, LonelyTraveler>
+    assert home_planet.LonelyTraveler is not None  # Verify that the base class exists.
+    with pytest.raises(
+        RuntimeError,
+        match='^generic_type: type "VeryLonelyTraveler" referenced unknown base type '
+        '"pybind11_tests::test_cpp_conduit::LonelyTraveler"$',
+    ):
+        exo_planet_pybind11.wrap_very_lonely_traveler()

tests/test_cpp_conduit_traveler_bindings.h

@@ -0,0 +1,47 @@
+// Copyright (c) 2024 The pybind Community.
+
+#pragma once
+
+#include <pybind11/pybind11.h>
+
+#include "test_cpp_conduit_traveler_types.h"
+
+#include <string>
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+namespace py = pybind11;
+
+inline void wrap_traveler(py::module_ m) {
+    py::class_<Traveler>(m, "Traveler")
+        .def(py::init<std::string>())
+        .def_readwrite("luggage", &Traveler::luggage)
+        // See issue #3788:
+        .def("__getattr__", [](const Traveler &self, const std::string &key) {
+            return "Traveler GetAttr: " + key + " luggage: " + self.luggage;
+        });
+
+    m.def("get_luggage", [](const Traveler &person) { return person.luggage; });
+
+    py::class_<PremiumTraveler, Traveler>(m, "PremiumTraveler")
+        .def(py::init<std::string, int>())
+        .def_readwrite("points", &PremiumTraveler::points)
+        // See issue #3788:
+        .def("__getattr__", [](const PremiumTraveler &self, const std::string &key) {
+            return "PremiumTraveler GetAttr: " + key + " points: " + std::to_string(self.points);
+        });
+
+    m.def("get_points", [](const PremiumTraveler &person) { return person.points; });
+}
+
+inline void wrap_lonely_traveler(py::module_ m) {
+    py::class_<LonelyTraveler>(std::move(m), "LonelyTraveler");
+}
+
+inline void wrap_very_lonely_traveler(py::module_ m) {
+    py::class_<VeryLonelyTraveler, LonelyTraveler>(std::move(m), "VeryLonelyTraveler");
+}
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

tests/test_cpp_conduit_traveler_types.h

@@ -0,0 +1,25 @@
+// Copyright (c) 2024 The pybind Community.
+
+#pragma once
+
+#include <string>
+
+namespace pybind11_tests {
+namespace test_cpp_conduit {
+
+struct Traveler {
+    explicit Traveler(const std::string &luggage) : luggage(luggage) {}
+    std::string luggage;
+};
+
+struct PremiumTraveler : Traveler {
+    explicit PremiumTraveler(const std::string &luggage, int points)
+        : Traveler(luggage), points(points) {}
+    int points;
+};
+
+struct LonelyTraveler {};
+struct VeryLonelyTraveler : LonelyTraveler {};
+
+} // namespace test_cpp_conduit
+} // namespace pybind11_tests

tests/test_custom_type_setup.py

@@ -9,7 +9,7 @@ import env  # noqa: F401
 from pybind11_tests import custom_type_setup as m
 
 
-@pytest.fixture()
+@pytest.fixture
 def gc_tester():
     """Tests that an object is garbage collected.
 

tests/test_numpy_array.py

@@ -24,7 +24,7 @@ def test_dtypes():
             )
 
 
-@pytest.fixture()
+@pytest.fixture
 def arr():
     return np.array([[1, 2, 3], [4, 5, 6]], "=u2")
 

tests/test_pytypes.py

@@ -1026,7 +1026,7 @@ def test_optional_object_annotations(doc):
 
 @pytest.mark.skipif(
     not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL,
-    reason="C++20 feature not available.",
+    reason="C++20 non-type template args feature not available.",
 )
 def test_literal(doc):
     assert (
@@ -1037,7 +1037,7 @@ def test_literal(doc):
 
 @pytest.mark.skipif(
     not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL,
-    reason="C++20 feature not available.",
+    reason="C++20 non-type template args feature not available.",
 )
 def test_typevar(doc):
     assert (