diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06c53bf10..3f7e8a6c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,7 +243,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v3.1.0 + uses: deadsnakes/action@v3.2.0 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 3edfa612d..371353737 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -91,18 +91,19 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' needs: [packaging] - environment: pypi + environment: + name: pypi + url: https://pypi.org/p/pybind11 permissions: id-token: write attestations: write - contents: read steps: # Downloads all to directories matching the artifact names - uses: actions/download-artifact@v4 - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@310b0a4a3b0b78ef57ecda988ee04b132db73ef8 # v1.4.1 + uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 with: subject-path: "*/pybind11*" @@ -110,8 +111,10 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: standard/ + attestations: true - name: Publish global package uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: global/ + attestations: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ecac1cbaf..a8190df44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.3 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -40,7 +40,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.11.1" + rev: "v1.11.2" hooks: - id: mypy args: [] @@ -93,7 +93,7 @@ repos: # Avoid directional quotes - repo: https://github.com/sirosen/texthooks - rev: "0.6.6" + rev: "0.6.7" hooks: - id: fix-ligatures - id: fix-smartquotes @@ -142,14 +142,14 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.2.6" + rev: "v3.2.7" hooks: - id: pylint files: ^pybind11 # Check schemas on some of our YAML files - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.1 + rev: 0.29.2 hooks: - id: check-readthedocs - id: check-github-workflows diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e5b8c8f3..f53aa2098 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,12 +149,14 @@ endif() set(PYBIND11_HEADERS include/pybind11/detail/class.h include/pybind11/detail/common.h + include/pybind11/detail/cpp_conduit.h include/pybind11/detail/descr.h include/pybind11/detail/init.h include/pybind11/detail/internals.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h include/pybind11/detail/value_and_holder.h + include/pybind11/detail/exception_translation.h include/pybind11/attr.h include/pybind11/buffer_info.h include/pybind11/cast.h diff --git a/docs/changelog.rst b/docs/changelog.rst index 014531774..a91082113 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,6 +31,37 @@ New Features: * The ``array_caster`` in pybind11/stl.h was enhanced to support value types that are not default-constructible. `#5305 `_ +* Added ``py::warnings`` namespace with ``py::warnings::warn`` and ``py::warnings::new_warning_type`` that provides the interface for Python warnings. + `#5291 `_ + +Version 2.13.6 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + + +Bug fixes: + +* Using ``__cpp_nontype_template_args`` instead of ``__cpp_nontype_template_parameter_class``. + `#5330 `_ + +* Properly translate C++ exception to Python exception when creating Python buffer from wrapped object. + `#5324 `_ + + +Documentation: + +* Adds an answer (FAQ) for "What is a highly conclusive and simple way to find memory leaks?". + `#5340 `_ + + Version 2.13.5 (August 22, 2024) -------------------------------- @@ -238,6 +269,18 @@ Other: * Update docs and noxfile. `#5071 `_ +Version 2.12.1 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + Version 2.12.0 (March 27, 2024) ------------------------------- @@ -413,6 +456,18 @@ Other: * An ``assert()`` was added to help Coverty avoid generating a false positive. `#4817 `_ +Version 2.11.2 (September 13, 2024) +----------------------------------- + +New Features: + +* A new ``self._pybind11_conduit_v1_()`` method is automatically added to all + ``py::class_``-wrapped types, to enable type-safe interoperability between + different independent Python/C++ bindings systems, including pybind11 + versions with different ``PYBIND11_INTERNALS_VERSION``'s. Supported on + pybind11 2.11.2, 2.12.1, and 2.13.6+. + `#5296 `_ + Version 2.11.1 (July 17, 2023) ------------------------------ diff --git a/docs/faq.rst b/docs/faq.rst index 1eb00efad..9b688b308 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -247,6 +247,50 @@ been received, you must either explicitly interrupt execution by throwing }); } +What is a highly conclusive and simple way to find memory leaks (e.g. in pybind11 bindings)? +============================================================================================ + +Use ``while True`` & ``top`` (Linux, macOS). + +For example, locally change tests/test_type_caster_pyobject_ptr.py like this: + +.. code-block:: diff + + def test_return_list_pyobject_ptr_reference(): + + while True: + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + - assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + # assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + +Then run the test as you would normally do, which will go into the infinite loop. + +**In another shell, but on the same machine** run: + +.. code-block:: bash + + top + +This will show: + +.. code-block:: + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1266095 rwgk 20 0 5207496 611372 45696 R 100.0 0.3 0:08.01 test_type_caste + +Look for the number under ``RES`` there. You'll see it going up very quickly. + +**Don't forget to Ctrl-C the test command** before your machine becomes +unresponsive due to swapping. + +This method only takes a couple minutes of effort and is very conclusive. +What you want to see is that the ``RES`` number is stable after a couple +seconds. + CMake doesn't detect the right Python version ============================================= diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 7c8966cee..b990507d6 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -12,6 +12,8 @@ #include #include +#include "exception_translation.h" + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -591,7 +593,18 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla return -1; } std::memset(view, 0, sizeof(Py_buffer)); - buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + buffer_info *info = nullptr; + try { + info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + } catch (...) { + try_translate_exceptions(); + raise_from(PyExc_BufferError, "Error getting buffer"); + return -1; + } + if (info == nullptr) { + pybind11_fail("FATAL UNEXPECTED SITUATION: tinfo->get_buffer() returned nullptr."); + } + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 4a3e1d14a..c51d1d60b 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 13 -#define PYBIND11_VERSION_PATCH 5 +#define PYBIND11_VERSION_PATCH 6 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020D0500 +#define PYBIND11_VERSION_HEX 0x020D0600 // Define some generic pybind11 helper macros for warning management. // diff --git a/include/pybind11/detail/cpp_conduit.h b/include/pybind11/detail/cpp_conduit.h new file mode 100644 index 000000000..b66c2d39c --- /dev/null +++ b/include/pybind11/detail/cpp_conduit.h @@ -0,0 +1,77 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +#include "common.h" +#include "internals.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Forward declaration needed here: Refactoring opportunity. +extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *); + +inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { +#if defined(PYPY_VERSION) + auto &internals = get_internals(); + return bool(internals.registered_types_py.find(type_obj) + != internals.registered_types_py.end()); +#else + return bool(type_obj->tp_new == pybind11_object_new); +#endif +} + +inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { + PyObject *descr = _PyType_Lookup(type_obj, attr_name); + return bool((descr != nullptr) && PyInstanceMethod_Check(descr)); +} + +inline object try_get_cpp_conduit_method(PyObject *obj) { + if (PyType_Check(obj)) { + return object(); + } + PyTypeObject *type_obj = Py_TYPE(obj); + str attr_name("_pybind11_conduit_v1_"); + bool assumed_to_be_callable = false; + if (type_is_managed_by_our_internals(type_obj)) { + if (!is_instance_method_of_type(type_obj, attr_name.ptr())) { + return object(); + } + assumed_to_be_callable = true; + } + PyObject *method = PyObject_GetAttr(obj, attr_name.ptr()); + if (method == nullptr) { + PyErr_Clear(); + return object(); + } + if (!assumed_to_be_callable && PyCallable_Check(method) == 0) { + Py_DECREF(method); + return object(); + } + return reinterpret_steal(method); +} + +inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src, + const std::type_info *cpp_type_info) { + object method = try_get_cpp_conduit_method(src.ptr()); + if (method) { + capsule cpp_type_info_capsule(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name()); + object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID), + cpp_type_info_capsule, + bytes("raw_pointer_ephemeral")); + if (isinstance(cpp_conduit)) { + return reinterpret_borrow(cpp_conduit).get_pointer(); + } + } + return nullptr; +} + +#define PYBIND11_HAS_CPP_CONDUIT 1 + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/exception_translation.h b/include/pybind11/detail/exception_translation.h new file mode 100644 index 000000000..2764180bb --- /dev/null +++ b/include/pybind11/detail/exception_translation.h @@ -0,0 +1,71 @@ +/* + pybind11/detail/exception_translation.h: means to translate C++ exceptions to Python exceptions + + Copyright (c) 2024 The Pybind Development Team. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "common.h" +#include "internals.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Apply all the extensions translators from a list +// Return true if one of the translators completed without raising an exception +// itself. Return of false indicates that if there are other translators +// available, they should be tried. +inline bool apply_exception_translators(std::forward_list &translators) { + auto last_exception = std::current_exception(); + + for (auto &translator : translators) { + try { + translator(last_exception); + return true; + } catch (...) { + last_exception = std::current_exception(); + } + } + return false; +} + +inline void try_translate_exceptions() { + /* When an exception is caught, give each registered exception + translator a chance to translate it to a Python exception. First + all module-local translators will be tried in reverse order of + registration. If none of the module-locale translators handle + the exception (or there are no module-locale translators) then + the global translators will be tried, also in reverse order of + registration. + + A translator may choose to do one of the following: + + - catch the exception and call py::set_error() + to set a standard (or custom) Python exception, or + - do nothing and let the exception fall through to the next translator, or + - delegate translation to the next translator by throwing a new type of exception. + */ + + bool handled = with_internals([&](internals &internals) { + auto &local_exception_translators = get_local_internals().registered_exception_translators; + if (detail::apply_exception_translators(local_exception_translators)) { + return true; + } + auto &exception_translators = internals.registered_exception_translators; + if (detail::apply_exception_translators(exception_translators)) { + return true; + } + return false; + }); + + if (!handled) { + set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); + } +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 72be79d8d..232bc32d8 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -321,15 +321,17 @@ struct type_info { # define PYBIND11_INTERNALS_KIND "" #endif +#define PYBIND11_PLATFORM_ABI_ID \ + PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ + PYBIND11_BUILD_TYPE + #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB \ - PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" + PYBIND11_PLATFORM_ABI_ID "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB \ - PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" + PYBIND11_PLATFORM_ABI_ID "__" /// Each module locally stores a pointer to the `internals` data. The data /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index bc42252f9..e40e44ba6 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -12,14 +12,17 @@ #include #include "common.h" +#include "cpp_conduit.h" #include "descr.h" #include "internals.h" #include "typeid.h" #include "value_and_holder.h" #include +#include #include #include +#include #include #include #include @@ -611,6 +614,13 @@ public: } return false; } + bool try_cpp_conduit(handle src) { + value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype); + if (value != nullptr) { + return true; + } + return false; + } void check_holder_compat() {} PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { @@ -742,6 +752,10 @@ public: return true; } + if (convert && cpptype && this_.try_cpp_conduit(src)) { + return true; + } + return false; } @@ -769,6 +783,32 @@ public: void *value = nullptr; }; +inline object cpp_conduit_method(handle self, + const bytes &pybind11_platform_abi_id, + const capsule &cpp_type_info_capsule, + const bytes &pointer_kind) { +#ifdef PYBIND11_HAS_STRING_VIEW + using cpp_str = std::string_view; +#else + using cpp_str = std::string; +#endif + if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) { + return none(); + } + if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) { + return none(); + } + if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") { + throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\""); + } + const auto *cpp_type_info = cpp_type_info_capsule.get_pointer(); + type_caster_generic caster(*cpp_type_info); + if (!caster.load(self, false)) { + return none(); + } + return capsule(caster.value, cpp_type_info->name()); +} + /** * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster * needs to provide `operator T*()` and `operator T&()` operators. diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 74919a7d5..5219c0ff8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -9,8 +9,8 @@ */ #pragma once - #include "detail/class.h" +#include "detail/exception_translation.h" #include "detail/init.h" #include "attr.h" #include "gil.h" @@ -95,24 +95,6 @@ inline std::string replace_newlines_and_squash(const char *text) { return result.substr(str_begin, str_range); } -// Apply all the extensions translators from a list -// Return true if one of the translators completed without raising an exception -// itself. Return of false indicates that if there are other translators -// available, they should be tried. -inline bool apply_exception_translators(std::forward_list &translators) { - auto last_exception = std::current_exception(); - - for (auto &translator : translators) { - try { - translator(last_exception); - return true; - } catch (...) { - last_exception = std::current_exception(); - } - } - return false; -} - #if defined(_MSC_VER) # define PYBIND11_COMPAT_STRDUP _strdup #else @@ -610,7 +592,8 @@ protected: int index = 0; /* Create a nice pydoc rec including all signatures and docstrings of the functions in the overload chain */ - if (chain && options::show_function_signatures()) { + if (chain && options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { // First a generic signature signatures += rec->name; signatures += "(*args, **kwargs)\n"; @@ -619,7 +602,8 @@ protected: // Then specific overload signatures bool first_user_def = true; for (auto *it = chain_start; it != nullptr; it = it->next) { - if (options::show_function_signatures()) { + if (options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { if (index > 0) { signatures += '\n'; } @@ -1038,40 +1022,7 @@ protected: throw; #endif } catch (...) { - /* When an exception is caught, give each registered exception - translator a chance to translate it to a Python exception. First - all module-local translators will be tried in reverse order of - registration. If none of the module-locale translators handle - the exception (or there are no module-locale translators) then - the global translators will be tried, also in reverse order of - registration. - - A translator may choose to do one of the following: - - - catch the exception and call py::set_error() - to set a standard (or custom) Python exception, or - - do nothing and let the exception fall through to the next translator, or - - delegate translation to the next translator by throwing a new type of exception. - */ - - bool handled = with_internals([&](internals &internals) { - auto &local_exception_translators - = get_local_internals().registered_exception_translators; - if (detail::apply_exception_translators(local_exception_translators)) { - return true; - } - auto &exception_translators = internals.registered_exception_translators; - if (detail::apply_exception_translators(exception_translators)) { - return true; - } - return false; - }); - - if (handled) { - return nullptr; - } - - set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); + try_translate_exceptions(); return nullptr; } @@ -1652,6 +1603,7 @@ public: = instances[std::type_index(typeid(type))]; }); } + def("_pybind11_conduit_v1_", cpp_conduit_method); } template ::value, int> = 0> diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index b0feb9464..84aaf9f70 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -100,9 +100,7 @@ class Never : public none { using none::none; }; -#if defined(__cpp_nontype_template_parameter_class) \ - && (/* See #5201 */ !defined(__GNUC__) \ - || (__GNUC__ > 10 || (__GNUC__ == 10 && __GNUC_MINOR__ >= 3))) +#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L # define PYBIND11_TYPING_H_HAS_STRING_LITERAL template struct StringLiteral { diff --git a/pybind11/_version.py b/pybind11/_version.py index 4a29f8e07..c298836bc 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> int | str: return s -__version__ = "2.13.5" +__version__ = "2.13.6" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7ae2cd1f0..81ec65821 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -127,6 +127,7 @@ set(PYBIND11_TEST_FILES test_const_name test_constants_and_functions test_copy_move + test_cpp_conduit test_custom_type_casters test_custom_type_setup test_docstring_options @@ -226,6 +227,8 @@ tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_ # And add additional targets for other tests. tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set") tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") +tests_extra_targets("test_cpp_conduit.py" + "exo_planet_pybind11;exo_planet_c_api;home_planet_very_lonely_traveler") set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" diff --git a/tests/conftest.py b/tests/conftest.py index 7de6c2ace..c40b11221 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -136,7 +136,7 @@ class Capture: return Output(self.err) -@pytest.fixture() +@pytest.fixture def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) @@ -172,7 +172,7 @@ def _sanitize_docstring(thing): return _sanitize_general(s) -@pytest.fixture() +@pytest.fixture def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -184,7 +184,7 @@ def _sanitize_message(thing): return _hexadecimal.sub("0", s) -@pytest.fixture() +@pytest.fixture def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) diff --git a/tests/exo_planet_c_api.cpp b/tests/exo_planet_c_api.cpp new file mode 100644 index 000000000..3bde0b27b --- /dev/null +++ b/tests/exo_planet_c_api.cpp @@ -0,0 +1,103 @@ +// Copyright (c) 2024 The pybind Community. + +// THIS MUST STAY AT THE TOP! +#include // EXCLUSIVELY for PYBIND11_PLATFORM_ABI_ID +// Potential future direction to maximize reusability: +// (e.g. for use from SWIG, Cython, PyCLIF, nanobind): +// #include +// This would only depend on: +// 1. A C++ compiler, WITHOUT requiring -fexceptions. +// 2. Python.h + +#include "test_cpp_conduit_traveler_types.h" + +#include +#include + +namespace { + +void *get_cpp_conduit_void_ptr(PyObject *py_obj, const std::type_info *cpp_type_info) { + PyObject *cpp_type_info_capsule + = PyCapsule_New(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name(), + nullptr); + if (cpp_type_info_capsule == nullptr) { + return nullptr; + } + PyObject *cpp_conduit = PyObject_CallMethod(py_obj, + "_pybind11_conduit_v1_", + "yOy", + PYBIND11_PLATFORM_ABI_ID, + cpp_type_info_capsule, + "raw_pointer_ephemeral"); + Py_DECREF(cpp_type_info_capsule); + if (cpp_conduit == nullptr) { + return nullptr; + } + void *void_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name()); + Py_DECREF(cpp_conduit); + if (PyErr_Occurred()) { + return nullptr; + } + return void_ptr; +} + +template +T *get_cpp_conduit_type_ptr(PyObject *py_obj) { + void *void_ptr = get_cpp_conduit_void_ptr(py_obj, &typeid(T)); + if (void_ptr == nullptr) { + return nullptr; + } + return static_cast(void_ptr); +} + +extern "C" PyObject *wrapGetLuggage(PyObject * /*self*/, PyObject *traveler) { + const auto *cpp_traveler + = get_cpp_conduit_type_ptr(traveler); + if (cpp_traveler == nullptr) { + return nullptr; + } + return PyUnicode_FromString(cpp_traveler->luggage.c_str()); +} + +extern "C" PyObject *wrapGetPoints(PyObject * /*self*/, PyObject *premium_traveler) { + const auto *cpp_premium_traveler + = get_cpp_conduit_type_ptr( + premium_traveler); + if (cpp_premium_traveler == nullptr) { + return nullptr; + } + return PyLong_FromLong(static_cast(cpp_premium_traveler->points)); +} + +PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr}, + {"GetPoints", wrapGetPoints, METH_O, nullptr}, + {nullptr, nullptr, 0, nullptr}}; + +struct PyModuleDef ThisModuleDef = { + PyModuleDef_HEAD_INIT, // m_base + "exo_planet_c_api", // m_name + nullptr, // m_doc + -1, // m_size + ThisMethodDef, // m_methods + nullptr, // m_slots + nullptr, // m_traverse + nullptr, // m_clear + nullptr // m_free +}; + +} // namespace + +#if defined(WIN32) || defined(_WIN32) +# define EXO_PLANET_C_API_EXPORT __declspec(dllexport) +#else +# define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default"))) +#endif + +extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { + PyObject *m = PyModule_Create(&ThisModuleDef); + if (m == nullptr) { + return nullptr; + } + return m; +} diff --git a/tests/exo_planet_pybind11.cpp b/tests/exo_planet_pybind11.cpp new file mode 100644 index 000000000..9d1a2b84b --- /dev/null +++ b/tests/exo_planet_pybind11.cpp @@ -0,0 +1,19 @@ +// Copyright (c) 2024 The pybind Community. + +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 900000001 + +#include "test_cpp_conduit_traveler_bindings.h" + +namespace pybind11_tests { +namespace test_cpp_conduit { + +PYBIND11_MODULE(exo_planet_pybind11, m) { + wrap_traveler(m); + m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); }); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index aedbdf1c1..7737f24ad 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -53,12 +53,14 @@ main_headers = { detail_headers = { "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", + "include/pybind11/detail/cpp_conduit.h", "include/pybind11/detail/descr.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", "include/pybind11/detail/value_and_holder.h", + "include/pybind11/detail/exception_translation.h", } eigen_headers = { diff --git a/tests/home_planet_very_lonely_traveler.cpp b/tests/home_planet_very_lonely_traveler.cpp new file mode 100644 index 000000000..78d50cff5 --- /dev/null +++ b/tests/home_planet_very_lonely_traveler.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2024 The pybind Community. + +#include "test_cpp_conduit_traveler_bindings.h" + +namespace pybind11_tests { +namespace test_cpp_conduit { + +PYBIND11_MODULE(home_planet_very_lonely_traveler, m) { + m.def("wrap_very_lonely_traveler", [m]() { wrap_very_lonely_traveler(m); }); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/tests/test_async.py b/tests/test_async.py index 1705196d1..64f4d6a77 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -11,7 +11,7 @@ if sys.platform.startswith("emscripten"): pytest.skip("Can't run a new event_loop in pyodide", allow_module_level=True) -@pytest.fixture() +@pytest.fixture def event_loop(): loop = asyncio.new_event_loop() yield loop diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index b5b8c355b..a6c527c10 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -167,6 +167,18 @@ TEST_SUBMODULE(buffers, m) { sizeof(float)}); }); + class BrokenMatrix : public Matrix { + public: + BrokenMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(rows, cols) {} + void throw_runtime_error() { throw std::runtime_error("See PR #5324 for context."); } + }; + py::class_(m, "BrokenMatrix", py::buffer_protocol()) + .def(py::init()) + .def_buffer([](BrokenMatrix &m) { + m.throw_runtime_error(); + return py::buffer_info(); + }); + // test_inherited_protocol class SquareMatrix : public Matrix { public: diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 84a301e25..535f33c2d 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -228,3 +228,10 @@ def test_buffer_docstring(): m.get_buffer_info.__doc__.strip() == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info" ) + + +def test_buffer_exception(): + with pytest.raises(BufferError, match="Error getting buffer") as excinfo: + memoryview(m.BrokenMatrix(1, 1)) + assert isinstance(excinfo.value.__cause__, RuntimeError) + assert "for context" in str(excinfo.value.__cause__) diff --git a/tests/test_cpp_conduit.cpp b/tests/test_cpp_conduit.cpp new file mode 100644 index 000000000..4ee4f0690 --- /dev/null +++ b/tests/test_cpp_conduit.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2024 The pybind Community. + +#include "pybind11_tests.h" +#include "test_cpp_conduit_traveler_bindings.h" + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +TEST_SUBMODULE(cpp_conduit, m) { + m.attr("PYBIND11_PLATFORM_ABI_ID") = py::bytes(PYBIND11_PLATFORM_ABI_ID); + m.attr("cpp_type_info_capsule_Traveler") + = py::capsule(&typeid(Traveler), typeid(std::type_info).name()); + m.attr("cpp_type_info_capsule_int") = py::capsule(&typeid(int), typeid(std::type_info).name()); + + wrap_traveler(m); + wrap_lonely_traveler(m); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/tests/test_cpp_conduit.py b/tests/test_cpp_conduit.py new file mode 100644 index 000000000..51fcf6936 --- /dev/null +++ b/tests/test_cpp_conduit.py @@ -0,0 +1,162 @@ +# Copyright (c) 2024 The pybind Community. + +from __future__ import annotations + +import exo_planet_c_api +import exo_planet_pybind11 +import home_planet_very_lonely_traveler +import pytest + +from pybind11_tests import cpp_conduit as home_planet + + +def test_traveler_getattr_actually_exists(): + t_h = home_planet.Traveler("home") + assert t_h.any_name == "Traveler GetAttr: any_name luggage: home" + + +def test_premium_traveler_getattr_actually_exists(): + t_h = home_planet.PremiumTraveler("home", 7) + assert t_h.secret_name == "PremiumTraveler GetAttr: secret_name points: 7" + + +def test_call_cpp_conduit_success(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemeral", + ) + assert cap.__class__.__name__ == "PyCapsule" + + +def test_call_cpp_conduit_platform_abi_id_mismatch(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID + b"MISMATCH", + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemeral", + ) + assert cap is None + + +def test_call_cpp_conduit_cpp_type_info_capsule_mismatch(): + t_h = home_planet.Traveler("home") + cap = t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_int, + b"raw_pointer_ephemeral", + ) + assert cap is None + + +def test_call_cpp_conduit_pointer_kind_invalid(): + t_h = home_planet.Traveler("home") + with pytest.raises( + RuntimeError, match='^Invalid pointer_kind: "raw_pointer_ephemreal"$' + ): + t_h._pybind11_conduit_v1_( + home_planet.PYBIND11_PLATFORM_ABI_ID, + home_planet.cpp_type_info_capsule_Traveler, + b"raw_pointer_ephemreal", + ) + + +def test_home_only_basic(): + t_h = home_planet.Traveler("home") + assert t_h.luggage == "home" + assert home_planet.get_luggage(t_h) == "home" + + +def test_home_only_premium(): + p_h = home_planet.PremiumTraveler("home", 2) + assert p_h.luggage == "home" + assert home_planet.get_luggage(p_h) == "home" + assert home_planet.get_points(p_h) == 2 + + +def test_exo_only_basic(): + t_e = exo_planet_pybind11.Traveler("exo") + assert t_e.luggage == "exo" + assert exo_planet_pybind11.get_luggage(t_e) == "exo" + + +def test_exo_only_premium(): + p_e = exo_planet_pybind11.PremiumTraveler("exo", 3) + assert p_e.luggage == "exo" + assert exo_planet_pybind11.get_luggage(p_e) == "exo" + assert exo_planet_pybind11.get_points(p_e) == 3 + + +def test_home_passed_to_exo_basic(): + t_h = home_planet.Traveler("home") + assert exo_planet_pybind11.get_luggage(t_h) == "home" + + +def test_exo_passed_to_home_basic(): + t_e = exo_planet_pybind11.Traveler("exo") + assert home_planet.get_luggage(t_e) == "exo" + + +def test_home_passed_to_exo_premium(): + p_h = home_planet.PremiumTraveler("home", 2) + assert exo_planet_pybind11.get_luggage(p_h) == "home" + assert exo_planet_pybind11.get_points(p_h) == 2 + + +def test_exo_passed_to_home_premium(): + p_e = exo_planet_pybind11.PremiumTraveler("exo", 3) + assert home_planet.get_luggage(p_e) == "exo" + assert home_planet.get_points(p_e) == 3 + + +@pytest.mark.parametrize( + "traveler_type", [home_planet.Traveler, exo_planet_pybind11.Traveler] +) +def test_exo_planet_c_api_traveler(traveler_type): + t = traveler_type("socks") + assert exo_planet_c_api.GetLuggage(t) == "socks" + + +@pytest.mark.parametrize( + "premium_traveler_type", + [home_planet.PremiumTraveler, exo_planet_pybind11.PremiumTraveler], +) +def test_exo_planet_c_api_premium_traveler(premium_traveler_type): + pt = premium_traveler_type("gucci", 5) + assert exo_planet_c_api.GetLuggage(pt) == "gucci" + assert exo_planet_c_api.GetPoints(pt) == 5 + + +def test_home_planet_wrap_very_lonely_traveler(): + # This does not exercise the cpp_conduit feature, but is here to + # demonstrate that the cpp_conduit feature does not solve all + # cross-extension interoperability issues. + # Here is the proof that the following works for extensions with + # matching `PYBIND11_INTERNALS_ID`s: + # test_cpp_conduit.cpp: + # py::class_ + # home_planet_very_lonely_traveler.cpp: + # py::class_ + # See test_exo_planet_pybind11_wrap_very_lonely_traveler() for the negative + # test. + assert home_planet.LonelyTraveler is not None # Verify that the base class exists. + home_planet_very_lonely_traveler.wrap_very_lonely_traveler() + # Ensure that the derived class exists. + assert home_planet_very_lonely_traveler.VeryLonelyTraveler is not None + + +def test_exo_planet_pybind11_wrap_very_lonely_traveler(): + # See comment under test_home_planet_wrap_very_lonely_traveler() first. + # Here the `PYBIND11_INTERNALS_ID`s don't match between: + # test_cpp_conduit.cpp: + # py::class_ + # exo_planet_pybind11.cpp: + # py::class_ + assert home_planet.LonelyTraveler is not None # Verify that the base class exists. + with pytest.raises( + RuntimeError, + match='^generic_type: type "VeryLonelyTraveler" referenced unknown base type ' + '"pybind11_tests::test_cpp_conduit::LonelyTraveler"$', + ): + exo_planet_pybind11.wrap_very_lonely_traveler() diff --git a/tests/test_cpp_conduit_traveler_bindings.h b/tests/test_cpp_conduit_traveler_bindings.h new file mode 100644 index 000000000..4e52c90c1 --- /dev/null +++ b/tests/test_cpp_conduit_traveler_bindings.h @@ -0,0 +1,47 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +#include "test_cpp_conduit_traveler_types.h" + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +namespace py = pybind11; + +inline void wrap_traveler(py::module_ m) { + py::class_(m, "Traveler") + .def(py::init()) + .def_readwrite("luggage", &Traveler::luggage) + // See issue #3788: + .def("__getattr__", [](const Traveler &self, const std::string &key) { + return "Traveler GetAttr: " + key + " luggage: " + self.luggage; + }); + + m.def("get_luggage", [](const Traveler &person) { return person.luggage; }); + + py::class_(m, "PremiumTraveler") + .def(py::init()) + .def_readwrite("points", &PremiumTraveler::points) + // See issue #3788: + .def("__getattr__", [](const PremiumTraveler &self, const std::string &key) { + return "PremiumTraveler GetAttr: " + key + " points: " + std::to_string(self.points); + }); + + m.def("get_points", [](const PremiumTraveler &person) { return person.points; }); +} + +inline void wrap_lonely_traveler(py::module_ m) { + py::class_(std::move(m), "LonelyTraveler"); +} + +inline void wrap_very_lonely_traveler(py::module_ m) { + py::class_(std::move(m), "VeryLonelyTraveler"); +} + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/tests/test_cpp_conduit_traveler_types.h b/tests/test_cpp_conduit_traveler_types.h new file mode 100644 index 000000000..b8e6a5a77 --- /dev/null +++ b/tests/test_cpp_conduit_traveler_types.h @@ -0,0 +1,25 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +namespace pybind11_tests { +namespace test_cpp_conduit { + +struct Traveler { + explicit Traveler(const std::string &luggage) : luggage(luggage) {} + std::string luggage; +}; + +struct PremiumTraveler : Traveler { + explicit PremiumTraveler(const std::string &luggage, int points) + : Traveler(luggage), points(points) {} + int points; +}; + +struct LonelyTraveler {}; +struct VeryLonelyTraveler : LonelyTraveler {}; + +} // namespace test_cpp_conduit +} // namespace pybind11_tests diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 56922c6dd..ca3340bd5 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -9,7 +9,7 @@ import env # noqa: F401 from pybind11_tests import custom_type_setup as m -@pytest.fixture() +@pytest.fixture def gc_tester(): """Tests that an object is garbage collected. diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 4726a8e73..bc7b3d555 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -24,7 +24,7 @@ def test_dtypes(): ) -@pytest.fixture() +@pytest.fixture def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 1c6335f75..6f015eec8 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1026,7 +1026,7 @@ def test_optional_object_annotations(doc): @pytest.mark.skipif( not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL, - reason="C++20 feature not available.", + reason="C++20 non-type template args feature not available.", ) def test_literal(doc): assert ( @@ -1037,7 +1037,7 @@ def test_literal(doc): @pytest.mark.skipif( not m.defined_PYBIND11_TYPING_H_HAS_STRING_LITERAL, - reason="C++20 feature not available.", + reason="C++20 non-type template args feature not available.", ) def test_typevar(doc): assert (