From 3f522e5a92c2f082f3a95258054604dbfc0772e3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 11 Aug 2024 09:19:51 -0700 Subject: [PATCH] `self.__cpp_transporter__()` proof of concept: Enable passing C++ pointers across extensions even if the `PYBIND11_INTERNALS_VERSION`s do not match. --- CMakeLists.txt | 1 + include/pybind11/detail/cpp_transporter.h | 71 +++++++++++++++++++ include/pybind11/detail/type_caster_base.h | 12 ++++ tests/CMakeLists.txt | 2 + tests/exo_planet.cpp | 8 +++ tests/extra_python_package/test_files.py | 1 + tests/test_cpp_transporter.cpp | 4 ++ tests/test_cpp_transporter.py | 37 ++++++++++ .../test_cpp_transporter_traveler_bindings.h | 36 ++++++++++ tests/test_cpp_transporter_traveler_type.h | 14 ++++ 10 files changed, 186 insertions(+) create mode 100644 include/pybind11/detail/cpp_transporter.h create mode 100644 tests/exo_planet.cpp create mode 100644 tests/test_cpp_transporter.cpp create mode 100644 tests/test_cpp_transporter.py create mode 100644 tests/test_cpp_transporter_traveler_bindings.h create mode 100644 tests/test_cpp_transporter_traveler_type.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e5b8c8f3..b1396d7d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,7 @@ endif() set(PYBIND11_HEADERS include/pybind11/detail/class.h include/pybind11/detail/common.h + include/pybind11/detail/cpp_transporter.h include/pybind11/detail/descr.h include/pybind11/detail/init.h include/pybind11/detail/internals.h diff --git a/include/pybind11/detail/cpp_transporter.h b/include/pybind11/detail/cpp_transporter.h new file mode 100644 index 000000000..d068a4977 --- /dev/null +++ b/include/pybind11/detail/cpp_transporter.h @@ -0,0 +1,71 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include "../pytypes.h" +#include "common.h" +#include "typeid.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Forward declaration needed here: Refactoring opportunity. +extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *); + +inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { +#if defined(PYPY_VERSION) + auto &internals = get_internals(); + return bool(internals.registered_types_py.find(type_obj) + != internals.registered_types_py.end()); +#else + return bool(type_obj->tp_new == pybind11_object_new); +#endif +} + +inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { + PyObject *descr = _PyType_Lookup(type_obj, attr_name); + return bool((descr != nullptr) && PyInstanceMethod_Check(descr)); +} + +inline object try_get_cpp_transporter_method(PyObject *obj) { + if (PyType_Check(obj)) { + return object(); + } + PyTypeObject *type_obj = Py_TYPE(obj); + str attr_name("__cpp_transporter__"); + bool assumed_to_be_callable = false; + if (type_is_managed_by_our_internals(type_obj)) { + if (!is_instance_method_of_type(type_obj, attr_name.ptr())) { + return object(); + } + assumed_to_be_callable = true; + } + PyObject *method = PyObject_GetAttr(obj, attr_name.ptr()); + if (method == nullptr) { + PyErr_Clear(); + return object(); + } + if (!assumed_to_be_callable && PyCallable_Check(method) == 0) { + Py_DECREF(method); + return object(); + } + return reinterpret_steal(method); +} + +inline void *try_raw_pointer_ephemeral_from_cpp_transporter(handle src, const char *typeid_name) { + object method = try_get_cpp_transporter_method(src.ptr()); + if (method) { + object cpp_transporter = method("cpp_abi_code", typeid_name, "raw_pointer_ephemeral"); + if (isinstance(cpp_transporter)) { + return reinterpret_borrow(cpp_transporter).get_pointer(); + } + } + return nullptr; +} + +#define PYBIND11_HAS_CPP_TRANSPORTER + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 481b7c783..5ef9ca616 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -11,6 +11,7 @@ #include "../pytypes.h" #include "common.h" +#include "cpp_transporter.h" #include "descr.h" #include "internals.h" #include "typeid.h" @@ -610,6 +611,13 @@ public: } return false; } + bool try_cpp_transporter(handle src) { + value = try_raw_pointer_ephemeral_from_cpp_transporter(src, cpptype->name()); + if (value != nullptr) { + return true; + } + return false; + } void check_holder_compat() {} PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { @@ -741,6 +749,10 @@ public: return true; } + if (convert && cpptype && this_.try_cpp_transporter(src)) { + return true; + } + return false; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b7e3f801..fb67f29a3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -157,6 +157,7 @@ set(PYBIND11_TEST_FILES test_stl_binders test_tagbased_polymorphic test_thread + test_cpp_transporter test_type_caster_pyobject_ptr test_type_caster_std_function_specializations test_union @@ -226,6 +227,7 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_ # And add additional targets for other tests. tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set") tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") +tests_extra_targets("test_cpp_transporter.py" "exo_planet") set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" diff --git a/tests/exo_planet.cpp b/tests/exo_planet.cpp new file mode 100644 index 000000000..e1fefda3c --- /dev/null +++ b/tests/exo_planet.cpp @@ -0,0 +1,8 @@ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 900000001 + +#include "test_cpp_transporter_traveler_bindings.h" + +PYBIND11_MODULE(exo_planet, m) { pybind11_tests::test_cpp_transporter::wrap_traveler(m); } diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index aedbdf1c1..4f737a261 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -53,6 +53,7 @@ main_headers = { detail_headers = { "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", + "include/pybind11/detail/cpp_transporter.h", "include/pybind11/detail/descr.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", diff --git a/tests/test_cpp_transporter.cpp b/tests/test_cpp_transporter.cpp new file mode 100644 index 000000000..d13e649e7 --- /dev/null +++ b/tests/test_cpp_transporter.cpp @@ -0,0 +1,4 @@ +#include "pybind11_tests.h" +#include "test_cpp_transporter_traveler_bindings.h" + +TEST_SUBMODULE(cpp_transporter, m) { pybind11_tests::test_cpp_transporter::wrap_traveler(m); } diff --git a/tests/test_cpp_transporter.py b/tests/test_cpp_transporter.py new file mode 100644 index 000000000..984b977d2 --- /dev/null +++ b/tests/test_cpp_transporter.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import exo_planet + +from pybind11_tests import cpp_transporter as home_planet + + +def test_home_only(): + t_h = home_planet.Traveler("home") + assert t_h.luggage == "home" + assert home_planet.get_luggage(t_h) == "home" + + +def test_exo_only(): + t_e = exo_planet.Traveler("exo") + assert t_e.luggage == "exo" + assert exo_planet.get_luggage(t_e) == "exo" + + +def test_home_passed_to_exo(): + t_h = home_planet.Traveler("home") + assert exo_planet.get_luggage(t_h) == "home" + + +def test_exo_passed_to_home(): + t_e = exo_planet.Traveler("exo") + assert home_planet.get_luggage(t_e) == "exo" + + +def test_call_cpp_transporter(): + t_h = home_planet.Traveler("home") + assert ( + t_h.__cpp_transporter__( + "cpp_abi_code", "cpp_typeid_name", "raw_pointer_ephemeral" + ) + is not None + ) diff --git a/tests/test_cpp_transporter_traveler_bindings.h b/tests/test_cpp_transporter_traveler_bindings.h new file mode 100644 index 000000000..39cb36879 --- /dev/null +++ b/tests/test_cpp_transporter_traveler_bindings.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include "test_cpp_transporter_traveler_type.h" + +#include + +namespace pybind11_tests { +namespace test_cpp_transporter { + +namespace py = pybind11; + +inline void wrap_traveler(py::module_ m) { + py::class_(m, "Traveler") + .def(py::init()) + .def("__cpp_transporter__", + [](py::handle self, + py::str /*cpp_abi_code*/, + py::str /*cpp_typeid_name*/, + py::str pointer_kind) { + auto pointer_kind_cpp = pointer_kind.cast(); + if (pointer_kind_cpp != "raw_pointer_ephemeral") { + throw std::runtime_error("Unknown pointer_kind: \"" + pointer_kind_cpp + + "\""); + } + auto *self_cpp_ptr = py::cast(self); + return py::capsule(static_cast(self_cpp_ptr), typeid(Traveler).name()); + }) + .def_readwrite("luggage", &Traveler::luggage); + + m.def("get_luggage", [](const Traveler &person) { return person.luggage; }); +}; + +} // namespace test_cpp_transporter +} // namespace pybind11_tests diff --git a/tests/test_cpp_transporter_traveler_type.h b/tests/test_cpp_transporter_traveler_type.h new file mode 100644 index 000000000..8d40c917e --- /dev/null +++ b/tests/test_cpp_transporter_traveler_type.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace pybind11_tests { +namespace test_cpp_transporter { + +struct Traveler { + explicit Traveler(const std::string &luggage) : luggage(luggage) {} + std::string luggage; +}; + +} // namespace test_cpp_transporter +} // namespace pybind11_tests