self.__cpp_transporter__() proof of concept: Enable passing C++ pointers across extensions even if the PYBIND11_INTERNALS_VERSIONs do not match.

CMakeLists.txt

@@ -149,6 +149,7 @@ endif()
 set(PYBIND11_HEADERS
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/cpp_transporter.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h

include/pybind11/detail/cpp_transporter.h

@@ -0,0 +1,71 @@
+// Copyright (c) 2024 The pybind Community.
+
+#pragma once
+
+#include "../pytypes.h"
+#include "common.h"
+#include "typeid.h"
+
+#include <string>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+// Forward declaration needed here: Refactoring opportunity.
+extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *);
+
+inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) {
+#if defined(PYPY_VERSION)
+    auto &internals = get_internals();
+    return bool(internals.registered_types_py.find(type_obj)
+                != internals.registered_types_py.end());
+#else
+    return bool(type_obj->tp_new == pybind11_object_new);
+#endif
+}
+
+inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) {
+    PyObject *descr = _PyType_Lookup(type_obj, attr_name);
+    return bool((descr != nullptr) && PyInstanceMethod_Check(descr));
+}
+
+inline object try_get_cpp_transporter_method(PyObject *obj) {
+    if (PyType_Check(obj)) {
+        return object();
+    }
+    PyTypeObject *type_obj = Py_TYPE(obj);
+    str attr_name("__cpp_transporter__");
+    bool assumed_to_be_callable = false;
+    if (type_is_managed_by_our_internals(type_obj)) {
+        if (!is_instance_method_of_type(type_obj, attr_name.ptr())) {
+            return object();
+        }
+        assumed_to_be_callable = true;
+    }
+    PyObject *method = PyObject_GetAttr(obj, attr_name.ptr());
+    if (method == nullptr) {
+        PyErr_Clear();
+        return object();
+    }
+    if (!assumed_to_be_callable && PyCallable_Check(method) == 0) {
+        Py_DECREF(method);
+        return object();
+    }
+    return reinterpret_steal<object>(method);
+}
+
+inline void *try_raw_pointer_ephemeral_from_cpp_transporter(handle src, const char *typeid_name) {
+    object method = try_get_cpp_transporter_method(src.ptr());
+    if (method) {
+        object cpp_transporter = method("cpp_abi_code", typeid_name, "raw_pointer_ephemeral");
+        if (isinstance<capsule>(cpp_transporter)) {
+            return reinterpret_borrow<capsule>(cpp_transporter).get_pointer();
+        }
+    }
+    return nullptr;
+}
+
+#define PYBIND11_HAS_CPP_TRANSPORTER
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/type_caster_base.h

@@ -11,6 +11,7 @@
 
 #include "../pytypes.h"
 #include "common.h"
+#include "cpp_transporter.h"
 #include "descr.h"
 #include "internals.h"
 #include "typeid.h"
@@ -610,6 +611,13 @@ public:
         }
         return false;
     }
+    bool try_cpp_transporter(handle src) {
+        value = try_raw_pointer_ephemeral_from_cpp_transporter(src, cpptype->name());
+        if (value != nullptr) {
+            return true;
+        }
+        return false;
+    }
     void check_holder_compat() {}
 
     PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) {
@@ -741,6 +749,10 @@ public:
             return true;
         }
 
+        if (convert && cpptype && this_.try_cpp_transporter(src)) {
+            return true;
+        }
+
         return false;
     }
 

tests/CMakeLists.txt

@@ -157,6 +157,7 @@ set(PYBIND11_TEST_FILES
     test_stl_binders
     test_tagbased_polymorphic
     test_thread
+    test_cpp_transporter
     test_type_caster_pyobject_ptr
     test_type_caster_std_function_specializations
     test_union
@@ -226,6 +227,7 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_
 # And add additional targets for other tests.
 tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set")
 tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils")
+tests_extra_targets("test_cpp_transporter.py" "exo_planet")
 
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"

tests/exo_planet.cpp

@@ -0,0 +1,8 @@
+#if defined(PYBIND11_INTERNALS_VERSION)
+#    undef PYBIND11_INTERNALS_VERSION
+#endif
+#define PYBIND11_INTERNALS_VERSION 900000001
+
+#include "test_cpp_transporter_traveler_bindings.h"
+
+PYBIND11_MODULE(exo_planet, m) { pybind11_tests::test_cpp_transporter::wrap_traveler(m); }

tests/extra_python_package/test_files.py

@@ -53,6 +53,7 @@ main_headers = {
 detail_headers = {
     "include/pybind11/detail/class.h",
     "include/pybind11/detail/common.h",
+    "include/pybind11/detail/cpp_transporter.h",
     "include/pybind11/detail/descr.h",
     "include/pybind11/detail/init.h",
     "include/pybind11/detail/internals.h",

tests/test_cpp_transporter.cpp

@@ -0,0 +1,4 @@
+#include "pybind11_tests.h"
+#include "test_cpp_transporter_traveler_bindings.h"
+
+TEST_SUBMODULE(cpp_transporter, m) { pybind11_tests::test_cpp_transporter::wrap_traveler(m); }

tests/test_cpp_transporter.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import exo_planet
+
+from pybind11_tests import cpp_transporter as home_planet
+
+
+def test_home_only():
+    t_h = home_planet.Traveler("home")
+    assert t_h.luggage == "home"
+    assert home_planet.get_luggage(t_h) == "home"
+
+
+def test_exo_only():
+    t_e = exo_planet.Traveler("exo")
+    assert t_e.luggage == "exo"
+    assert exo_planet.get_luggage(t_e) == "exo"
+
+
+def test_home_passed_to_exo():
+    t_h = home_planet.Traveler("home")
+    assert exo_planet.get_luggage(t_h) == "home"
+
+
+def test_exo_passed_to_home():
+    t_e = exo_planet.Traveler("exo")
+    assert home_planet.get_luggage(t_e) == "exo"
+
+
+def test_call_cpp_transporter():
+    t_h = home_planet.Traveler("home")
+    assert (
+        t_h.__cpp_transporter__(
+            "cpp_abi_code", "cpp_typeid_name", "raw_pointer_ephemeral"
+        )
+        is not None
+    )

tests/test_cpp_transporter_traveler_bindings.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <pybind11/pybind11.h>
+
+#include "test_cpp_transporter_traveler_type.h"
+
+#include <string>
+
+namespace pybind11_tests {
+namespace test_cpp_transporter {
+
+namespace py = pybind11;
+
+inline void wrap_traveler(py::module_ m) {
+    py::class_<Traveler>(m, "Traveler")
+        .def(py::init<std::string>())
+        .def("__cpp_transporter__",
+             [](py::handle self,
+                py::str /*cpp_abi_code*/,
+                py::str /*cpp_typeid_name*/,
+                py::str pointer_kind) {
+                 auto pointer_kind_cpp = pointer_kind.cast<std::string>();
+                 if (pointer_kind_cpp != "raw_pointer_ephemeral") {
+                     throw std::runtime_error("Unknown pointer_kind: \"" + pointer_kind_cpp
+                                              + "\"");
+                 }
+                 auto *self_cpp_ptr = py::cast<Traveler *>(self);
+                 return py::capsule(static_cast<void *>(self_cpp_ptr), typeid(Traveler).name());
+             })
+        .def_readwrite("luggage", &Traveler::luggage);
+
+    m.def("get_luggage", [](const Traveler &person) { return person.luggage; });
+};
+
+} // namespace test_cpp_transporter
+} // namespace pybind11_tests

tests/test_cpp_transporter_traveler_type.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+
+namespace pybind11_tests {
+namespace test_cpp_transporter {
+
+struct Traveler {
+    explicit Traveler(const std::string &luggage) : luggage(luggage) {}
+    std::string luggage;
+};
+
+} // namespace test_cpp_transporter
+} // namespace pybind11_tests