From 7f23e9f3a47cb6b6373467c18962c16dfdd82fe7 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 27 Dec 2022 15:14:10 -0500 Subject: [PATCH 01/83] chore: update clang-tidy to 15 (#4387) * chore: update clang-tidy to 15 * Add git * Add NOLINTNEXTLINE for assignment in if * Update CONTRIBUTING.md * Add NOLINTNEXTLINE where needed * Add one more NOLINTNEXTLINE * stl_bind: make more readable * Another missing NOLINTNEXTLINE * Match style elsewhere * Apply reviewer suggestion. Mark false positive --- .github/CONTRIBUTING.md | 4 ++-- .github/workflows/format.yml | 4 ++-- include/pybind11/detail/internals.h | 3 +++ include/pybind11/stl.h | 1 + include/pybind11/stl_bind.h | 14 +++++++++----- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 00b1fea4c..ad7974395 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -235,8 +235,8 @@ directory inside your pybind11 git clone. Files will be modified in place, so you can use git to monitor the changes. ```bash -docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:13 -apt-get update && apt-get install -y python3-dev python3-pytest +docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:15-bullseye +apt-get update && apt-get install -y git python3-dev python3-pytest cmake -S /mounted_pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 cmake --build build -j 2 ``` diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b18474bc3..46489feb3 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -38,12 +38,12 @@ jobs: # in .github/CONTRIBUTING.md and update as needed. name: Clang-Tidy runs-on: ubuntu-latest - container: silkeh/clang:13 + container: silkeh/clang:15-bullseye steps: - uses: actions/checkout@v3 - name: Install requirements - run: apt-get update && apt-get install -y python3-dev python3-pytest + run: apt-get update && apt-get install -y git python3-dev python3-pytest - name: Configure run: > diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index ef1849fbe..05e36ad18 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -469,12 +469,14 @@ PYBIND11_NOINLINE internals &get_internals() { #if defined(WITH_THREAD) PyThreadState *tstate = PyThreadState_Get(); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); # if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); @@ -514,6 +516,7 @@ struct local_internals { struct shared_loader_life_support_data { PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { pybind11_fail("local_internals: could not successfully initialize the " "loader_life_support TLS key!"); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 2d144b598..84c517108 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -316,6 +316,7 @@ struct optional_caster { if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return value_conv::cast(*std::forward(src), policy, parent); } diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 0c634597e..7cfd996dd 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -355,13 +355,17 @@ void vector_accessor(enable_if_t::value, Class_> &cl) using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; cl.def("__getitem__", [](const Vector &v, DiffType i) -> T { - if (i < 0 && (i += v.size()) < 0) { + if (i < 0) { + i += v.size(); + if (i < 0) { + throw index_error(); + } + } + auto i_st = static_cast(i); + if (i_st >= v.size()) { throw index_error(); } - if ((SizeType) i >= v.size()) { - throw index_error(); - } - return v[(SizeType) i]; + return v[i_st]; }); cl.def( From 70af9873c26ea46cf97d66444f4d80c4af79dd7a Mon Sep 17 00:00:00 2001 From: kajananchinniah Date: Wed, 28 Dec 2022 12:49:57 -0500 Subject: [PATCH 02/83] docs: fixed typo in spelling of first (#4428) --- include/pybind11/detail/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 068ae0137..421cbe192 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -436,7 +436,7 @@ PYBIND11_WARNING_POP /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. From 60f02f5f663829c9fd2d7d52c03d4064129e38df Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Fri, 30 Dec 2022 10:46:55 -0800 Subject: [PATCH 03/83] fix: improve the error reporting for inc_ref GIL failures (#4427) * First * Fixs * Improve * Additional assertions comment * Improve docs --- docs/advanced/misc.rst | 28 ++++++++++++++++++++++++++++ include/pybind11/pytypes.h | 30 +++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 35a6ebcd6..805ec838f 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -118,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy m.def("call_go", &call_go, py::call_guard()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 56e0423ac..3660c180d 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -250,9 +250,9 @@ public: #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); #endif -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::inc_ref()"); } #endif Py_XINCREF(m_ptr); @@ -265,9 +265,9 @@ public: this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { -#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF if (m_ptr != nullptr && !PyGILState_Check()) { - throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure."); + throw_gilstate_error("pybind11::handle::dec_ref()"); } #endif Py_XDECREF(m_ptr); @@ -296,8 +296,28 @@ public: protected: PyObject *m_ptr = nullptr; -#ifdef PYBIND11_HANDLE_REF_DEBUG private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG static std::size_t inc_ref_counter(std::size_t add) { thread_local std::size_t counter = 0; counter += add; From f12e098f1d68d9e7445f84d9bf94ce7d0908c531 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 2 Jan 2023 03:46:17 -0800 Subject: [PATCH 04/83] Fix `detail::obj_class_name()` to work correctly for meta classes. (#4436) * Fix `detail::obj_class_name()` to work correctly for meta classes. * Adjust expected name for PyPy --- include/pybind11/pytypes.h | 2 +- tests/test_class.cpp | 2 ++ tests/test_class.py | 11 ++++++++++- tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 6 ++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 3660c180d..a2824a0e0 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -463,7 +463,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline const char *obj_class_name(PyObject *obj) { - if (Py_TYPE(obj) == &PyType_Type) { + if (PyType_Check(obj)) { return reinterpret_cast(obj)->tp_name; } return Py_TYPE(obj)->tp_name; diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 9ea10fae1..ca925917e 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -55,6 +55,8 @@ void bind_empty0(py::module_ &m) { } // namespace test_class TEST_SUBMODULE(class_, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + // test_instance struct NoConstructor { NoConstructor() = default; diff --git a/tests/test_class.py b/tests/test_class.py index 7c1ed2060..9c964e001 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,10 +1,19 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import class_ as m +def test_obj_class_name(): + if env.PYPY: + expected_name = "UserType" + else: + expected_name = "pybind11_tests.UserType" + assert m.obj_class_name(UserType(1)) == expected_name + assert m.obj_class_name(UserType) == expected_name + + def test_repr(): assert "pybind11_type" in repr(type(UserType)) assert "UserType" in repr(UserType) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index ea8d03958..1028bb58e 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -99,6 +99,8 @@ void m_defs(py::module_ &m) { } // namespace handle_from_move_only_type_with_operator_PyObject TEST_SUBMODULE(pytypes, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + handle_from_move_only_type_with_operator_PyObject::m_defs(m); // test_bool diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 079ee7ca5..8f9f2987e 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -9,6 +9,12 @@ from pybind11_tests import detailed_error_messages_enabled from pybind11_tests import pytypes as m +def test_obj_class_name(): + assert m.obj_class_name(None) == "NoneType" + assert m.obj_class_name(list) == "list" + assert m.obj_class_name([]) == "list" + + def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() From 6da268a5c58b76c0577d0d7b223a78edd6a34bea Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 3 Jan 2023 05:46:55 -0800 Subject: [PATCH 05/83] ci: remove clang 10 C++20 (it broke recently) (#4438) * Remove clang 10 C++20 (it broke recently), add clang 15 C++20 while we are at it. * No luck trying clang15: Error response from daemon: manifest for silkeh/clang:15 not found: manifest unknown: manifest unknown. Giving up, this needs to be a separate PR. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af5a29d62..b36bbfe1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,8 +283,6 @@ jobs: include: - clang: 5 std: 14 - - clang: 10 - std: 20 - clang: 10 std: 17 - clang: 11 From e3e24f3f656841142ef5897e33a0aabc2b34fd07 Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 3 Jan 2023 23:20:39 +0800 Subject: [PATCH 06/83] fix: issuses detected by static analyzer (#4440) * fix incorrect variable check * remove duplicated check * remove unneeded const cast --- include/pybind11/attr.h | 2 +- include/pybind11/eigen/tensor.h | 2 +- include/pybind11/pytypes.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index db7cd8eff..b5e3b7b22 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -399,7 +399,7 @@ struct process_attribute : process_attribute_default { template <> struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute : process_attribute {}; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 568c641a2..0877da895 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -176,7 +176,7 @@ struct type_caster::ValidType> { return false; } - if (!convert && !temp.dtype().is(dtype::of())) { + if (!temp.dtype().is(dtype::of())) { return false; } } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index a2824a0e0..f11ed5da7 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,7 +501,7 @@ struct error_fetch_and_normalize { "active exception."); } const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { + if (exc_type_name_norm == nullptr) { pybind11_fail("Internal error: " + std::string(called) + " failed to obtain the name " "of the normalized active exception type."); From 769fd3b889fef6cddb060f2a0be26aee62b4da05 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:21:05 -0500 Subject: [PATCH 07/83] chore(deps): update pre-commit hooks (#4439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.3.0 → v3.3.1](https://github.com/asottile/pyupgrade/compare/v3.3.0...v3.3.1) - [github.com/PyCQA/isort: 5.10.1 → 5.11.4](https://github.com/PyCQA/isort/compare/5.10.1...5.11.4) - [github.com/psf/black: 22.10.0 → 22.12.0](https://github.com/psf/black/compare/22.10.0...22.12.0) - [github.com/PyCQA/pylint: v2.15.8 → v2.15.9](https://github.com/PyCQA/pylint/compare/v2.15.8...v2.15.9) - [github.com/shellcheck-py/shellcheck-py: v0.8.0.4 → v0.9.0.2](https://github.com/shellcheck-py/shellcheck-py/compare/v0.8.0.4...v0.9.0.2) - [github.com/pre-commit/mirrors-clang-format: v15.0.4 → v15.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.4...v15.0.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e80fbad1..d625d5726 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,20 +41,20 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v3.3.0" + rev: "v3.3.1" hooks: - id: pyupgrade args: [--py36-plus] # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.10.1" + rev: "5.11.4" hooks: - id: isort # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.10.0" # Keep in sync with blacken-docs + rev: "22.12.0" # Keep in sync with blacken-docs hooks: - id: black @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.8" + rev: "v2.15.9" hooks: - id: pylint files: ^pybind11 @@ -160,7 +160,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.8.0.4" + rev: "v0.9.0.2" hooks: - id: shellcheck @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.4" + rev: "v15.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] From b2d7ad72c24ee8d2acc22a4ce39fa38a1e9d77b8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jan 2023 11:34:22 -0500 Subject: [PATCH 08/83] chore: prepare for 2.10.3 (#4437) * docs: update changelog for v2.10.3 Signed-off-by: Henry Schreiner * chore: bump versions for 2.10.3 Signed-off-by: Henry Schreiner * chore: fix make changelog script with entry is empty Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 35 +++++++++++++++++++++++++++++++- include/pybind11/detail/common.h | 4 ++-- pybind11/_version.py | 2 +- tools/make_changelog.py | 9 ++++---- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ff7281bc..bb111c5f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,39 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 `_ + +Build system improvements: + +* Update clang-tidy to 15 in CI. + `#4387 `_ + + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 `_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 `_ + `#4436 `_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 `_ + + Version 2.10.2 (Dec 20, 2022) ----------------------------- @@ -30,7 +63,7 @@ Changes: * ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if - ``NDEBUG`` is not defined. + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) `#4246 `_ * Add option for enable/disable enum members in docstring. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 421cbe192..fc7e50776 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 2 +#define PYBIND11_VERSION_PATCH 3 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0200 +#define PYBIND11_VERSION_HEX 0x020A0300 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 4c814edfa..63078bbe6 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.2" +__version__ = "2.10.3" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/tools/make_changelog.py b/tools/make_changelog.py index 839040a93..b5bd83294 100755 --- a/tools/make_changelog.py +++ b/tools/make_changelog.py @@ -31,8 +31,10 @@ issues = (issue for page in issues_pages for issue in page) missing = [] for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog if not msg.startswith("* "): msg = "* " + msg @@ -44,9 +46,6 @@ for issue in issues: print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) From a34596bfe1947b4a6b0bcc4218e1f72d0c2e9b4c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Jan 2023 14:23:36 -0500 Subject: [PATCH 09/83] chore: back to work Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index fc7e50776..0fc81e053 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 3 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0300 +#define PYBIND11_VERSION_HEX 0x020B00D1 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 63078bbe6..1cb51fc5c 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.3" +__version__ = "2.11.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) From d821788bb62fa911bb9797550b8d06ca41462455 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 12 Jan 2023 17:50:28 -0800 Subject: [PATCH 10/83] Add clang15 C++20 job (#4443) --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b36bbfe1b..7c5fcb43a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,6 +280,8 @@ jobs: - dev std: - 11 + container_suffix: + - "" include: - clang: 5 std: 14 @@ -293,9 +295,12 @@ jobs: std: 20 - clang: 14 std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" - container: "silkeh/clang:${{ matrix.clang }}" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 From e53d58af6c0ade85fa6edce6a228ad2aeace55ee Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 14 Jan 2023 13:47:56 -0800 Subject: [PATCH 11/83] Ensure `import pybind11_tests` traceback is shown. (#4455) --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 402fd4b25..c24d51d4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,11 +11,17 @@ import multiprocessing import os import re import textwrap +import traceback import pytest # Early diagnostic for failed imports -import pybind11_tests +try: + import pybind11_tests +except Exception: + # pytest does not show the traceback without this. + traceback.print_exc() + raise @pytest.fixture(scope="session", autouse=True) From c709d2a83e06aaec06987aa9761d0ebbfefbecd7 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 18 Jan 2023 21:11:26 +0100 Subject: [PATCH 12/83] Make sure to properly untrack gc objects before freeing them (#4461) * Make sure to properly untrack gc objects before freeing them * style: pre-commit fixes * Fix lint * Add comment about where the original track comes from Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/class.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 528e716f7..bc2b40c50 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -445,9 +445,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 From a500f439d06d220ee2c680cdd2c8828eac8e7dfc Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 19 Jan 2023 10:48:46 -0800 Subject: [PATCH 13/83] Resolve new flake8 error (#4462) * Resolve flake8 error by replacing `pytest.raises(Exception)` with `SystemError` * Also remove the obsolete comment. * Tweak comment instead of removing it. --- tests/test_modules.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index e11d68e78..c3e8cea6e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -107,11 +107,10 @@ def test_def_submodule_failures(): sm_name_orig = sm.__name__ sm.__name__ = malformed_utf8 try: - with pytest.raises(Exception): - # Seen with Python 3.9: SystemError: nameless module - # But we do not want to exercise the internals of PyModule_GetName(), which could - # change in future versions of Python, but a bad __name__ is very likely to cause - # some kind of failure indefinitely. + # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise + # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that + # changes in future Python versions, simply add the new expected exception types here. + with pytest.raises(SystemError): m.def_submodule(sm, b"SubSubModuleName") finally: # Clean up to ensure nothing gets upset by a module with an invalid __name__. From c71e3af73f80140023ee399a982678a8b968b5c5 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 31 Jan 2023 22:44:18 -0800 Subject: [PATCH 14/83] Bump isort version to 5.12.0 (#4480) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d625d5726..9ec31ae99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Nicely sort includes - repo: https://github.com/PyCQA/isort - rev: "5.11.4" + rev: "5.12.0" hooks: - id: isort From 44e936822252df9e6420b42dc2520d622f430bc3 Mon Sep 17 00:00:00 2001 From: Daniel Jacobs <44678615+danielcjacobs@users.noreply.github.com> Date: Wed, 1 Feb 2023 02:42:05 -0500 Subject: [PATCH 15/83] Use PyConfig_InitPythonConfig instead of PyConfig_InitIsolatedConfig (#4473) * Use PyConfig_InitPythonConfig instead of PyConfig_InitIsolatedConfig * add unit test for default python configuration --------- Co-authored-by: Daniel Jacobs --- include/pybind11/embed.h | 7 ++++--- tests/test_embed/test_interpreter.cpp | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 749c75beb..5a175b134 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -198,9 +198,10 @@ inline void initialize_interpreter(bool init_signal_handlers = true, init_signal_handlers, argc, argv, add_program_dir_to_path); #else PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 10b20f371..e54e82270 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -184,7 +184,7 @@ TEST_CASE("Custom PyConfig") { py::initialize_interpreter(); } -TEST_CASE("Custom PyConfig with argv") { +TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") { py::finalize_interpreter(); { PyConfig config; @@ -199,6 +199,26 @@ TEST_CASE("Custom PyConfig with argv") { } py::initialize_interpreter(); } + +TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + + // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by + // changing it to `0`. This test exercises `scoped_interpreter` with the default config. + char *argv[] = {strdup("a.out"), strdup("arg1")}; + py::scoped_interpreter argv_scope(&config, 2, argv); + std::free(argv[0]); + std::free(argv[1]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "arg1"); + } + py::initialize_interpreter(); +} #endif TEST_CASE("Add program dir to path pre-PyConfig") { From 3efe9d4cb5d7314faee722205e560b8b932aed9e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 1 Feb 2023 14:23:37 -0500 Subject: [PATCH 16/83] chore: update to black 23 (#4482) Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++++++++--------- docs/conf.py | 1 - pybind11/__main__.py | 1 - pybind11/setup_helpers.py | 3 --- tests/extra_python_package/test_files.py | 2 -- tests/test_chrono.py | 4 ---- tests/test_class.py | 1 - tests/test_custom_type_casters.py | 3 ++- tests/test_eigen_tensor.py | 6 ------ tests/test_local_bindings.py | 3 ++- tests/test_operator_overloading.py | 1 - 11 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ec31ae99..363ac71c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,40 +54,40 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.12.0" # Keep in sync with blacken-docs + rev: "23.1.0" # Keep in sync with blacken-docs hooks: - id: black # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "v1.12.1" + rev: "1.13.0" hooks: - id: blacken-docs additional_dependencies: - - black==22.10.0 # keep in sync with black hook + - black==23.1.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.1" + rev: "v1.4.2" hooks: - id: remove-tabs - repo: https://github.com/sirosen/texthooks - rev: "0.4.0" + rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.1.2" + rev: "v2.1.3" hooks: - id: pycln stages: [manual] # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: python-check-blanket-noqa - id: python-check-blanket-type-ignore @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.9" + rev: "v2.16.0" hooks: - id: pylint files: ^pybind11 @@ -175,7 +175,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.6" + rev: "v15.0.7" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/docs/conf.py b/docs/conf.py index 2da6773f4..cdb9f6306 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -358,7 +358,6 @@ def clean_up(app, exception): def setup(app): - # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 8c8953384..e87b16beb 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -24,7 +24,6 @@ def print_includes() -> None: def main() -> None: - parser = argparse.ArgumentParser() parser.add_argument( "--includes", diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1fd04b915..1db479532 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -118,7 +118,6 @@ class Pybind11Extension(_Extension): # type: ignore[misc] self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -174,7 +173,6 @@ class Pybind11Extension(_Extension): # type: ignore[misc] @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: warnings.warn("You cannot safely change the cxx_level after setting it!") @@ -439,7 +437,6 @@ class ParallelCompile: extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 9a9bb1556..dd6393bf0 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -135,7 +135,6 @@ def normalize_line_endings(value: bytes) -> bytes: def test_build_sdist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) subprocess.run( @@ -186,7 +185,6 @@ def test_build_sdist(monkeypatch, tmpdir): def test_build_global_dist(monkeypatch, tmpdir): - monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.run( diff --git a/tests/test_chrono.py b/tests/test_chrono.py index 7f47b37a2..a29316c38 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -7,7 +7,6 @@ from pybind11_tests import chrono as m def test_chrono_system_clock(): - # Get the time from both c++ and datetime date0 = datetime.datetime.today() date1 = m.test_chrono1() @@ -122,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): def test_chrono_duration_roundtrip(): - # Get the difference between two times (a timedelta) date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -143,7 +141,6 @@ def test_chrono_duration_roundtrip(): def test_chrono_duration_subtraction_equivalence(): - date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -154,7 +151,6 @@ def test_chrono_duration_subtraction_equivalence(): def test_chrono_duration_subtraction_equivalence_date(): - date1 = datetime.date.today() date2 = datetime.date.today() diff --git a/tests/test_class.py b/tests/test_class.py index 9c964e001..64abac133 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -185,7 +185,6 @@ def test_inheritance(msg): def test_inheritance_init(msg): - # Single base class Python(m.Pet): def __init__(self): diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index adfa6cf86..731380e46 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -94,7 +94,8 @@ def test_noconvert_args(msg): def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets - destroyed when the function has py::return_value_policy::take_ownership policy applied.""" + destroyed when the function has py::return_value_policy::take_ownership policy applied. + """ cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index dc8aa4643..cd54efa06 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -59,7 +59,6 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) def test_reference_internal(m, member_name): - if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") foo = m.CustomExample() @@ -108,7 +107,6 @@ def test_convert_tensor_to_py(m, func_name): @pytest.mark.parametrize("m", submodules) def test_bad_cpp_to_python_casts(m): - with pytest.raises( RuntimeError, match="Cannot use reference internal when there is no parent" ): @@ -131,7 +129,6 @@ def test_bad_cpp_to_python_casts(m): @pytest.mark.parametrize("m", submodules) def test_bad_python_to_cpp_casts(m): - with pytest.raises( TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" ): @@ -194,7 +191,6 @@ def test_bad_python_to_cpp_casts(m): @pytest.mark.parametrize("m", submodules) def test_references_actually_refer(m): - a = m.reference_tensor() temp = a[indices] a[indices] = 100 @@ -211,7 +207,6 @@ def test_references_actually_refer(m): @pytest.mark.parametrize("m", submodules) def test_round_trip(m): - assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) with pytest.raises(TypeError, match="^Cannot cast array data from"): @@ -260,7 +255,6 @@ def test_round_trip(m): @pytest.mark.parametrize("m", submodules) def test_round_trip_references_actually_refer(m): - # Need to create a copy that matches the type on the C side copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) a = m.round_trip_view_tensor(copy) diff --git a/tests/test_local_bindings.py b/tests/test_local_bindings.py index 654d96d49..d64187739 100644 --- a/tests/test_local_bindings.py +++ b/tests/test_local_bindings.py @@ -130,7 +130,8 @@ def test_stl_bind_global(): def test_mixed_local_global(): """Local types take precedence over globally registered types: a module with a `module_local` type can be registered even if the type is already registered globally. With the module, - casting will go to the local type; outside the module casting goes to the global type.""" + casting will go to the local type; outside the module casting goes to the global type. + """ import pybind11_cross_module_tests as cm m.register_mixed_global() diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index b228da3cc..9fde305a0 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -130,7 +130,6 @@ def test_nested(): def test_overriding_eq_reset_hash(): - assert m.Comparable(15) is not m.Comparable(15) assert m.Comparable(15) == m.Comparable(15) From 08a89fac3a11c5cb5498d76cfde3ba3988368d3a Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 4 Feb 2023 13:40:13 -0500 Subject: [PATCH 17/83] bugfix: delete proper ctors in gil.h (#4490) --- include/pybind11/gil.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index cb0028d50..570a5581d 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -152,8 +152,8 @@ public: } } - gil_scoped_release(const gil_scoped_acquire &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter @@ -203,7 +203,7 @@ class gil_scoped_release { public: gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; @@ -230,7 +230,7 @@ public: (void) (this != (this + 1)); } gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; From 9ef65cee0e780e6443fa0813122712c9ae4bdcc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 00:16:52 -0500 Subject: [PATCH 18/83] chore(deps): bump ilammy/msvc-dev-cmd from 1.12.0 to 1.12.1 (#4493) Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.12.0 to 1.12.1. - [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases) - [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: ilammy/msvc-dev-cmd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c5fcb43a..8befccdbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -766,7 +766,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 @@ -819,7 +819,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.12.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 From b2c1978caaa6627095e3326f7ecba84b43037869 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 6 Feb 2023 11:36:05 -0500 Subject: [PATCH 19/83] bugfix: Keep registered types until after Py_Finalize(). Fix #4459 (#4486) * Keep registered types until after Py_Finalize(). Fix #4459 * Address reviewer comments --- include/pybind11/embed.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5a175b134..8450d5358 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -254,13 +254,16 @@ inline void finalize_interpreter() { if (builtins.contains(id) && isinstance(builtins[id])) { internals_ptr_ptr = capsule(builtins[id]); } - // Local internals contains data managed by the current interpreter, so we must clear them to - // avoid undefined behaviors when initializing another interpreter - detail::get_local_internals().registered_types_cpp.clear(); - detail::get_local_internals().registered_exception_translators.clear(); Py_Finalize(); + // Local internals contains data managed by the current interpreter, so we must clear them to + // avoid undefined behaviors when initializing another interpreter + // Must be cleared only after Py_Finalize() so atexit and other hooks can still use + // registered_types + detail::get_local_internals().registered_types_cpp.clear(); + detail::get_local_internals().registered_exception_translators.clear(); + if (internals_ptr_ptr) { delete *internals_ptr_ptr; *internals_ptr_ptr = nullptr; From 8a90b36772f36da577c485bd450130c8318c8111 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 01:16:25 -0500 Subject: [PATCH 20/83] chore(deps): update pre-commit hooks (#4495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/pylint: v2.16.0 → v2.16.1](https://github.com/PyCQA/pylint/compare/v2.16.0...v2.16.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 363ac71c6..27474f2f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.0" + rev: "v2.16.1" hooks: - id: pylint files: ^pybind11 From f8713ec43e6833033b43126c8b55de97242ab4dd Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 7 Feb 2023 16:55:00 -0800 Subject: [PATCH 21/83] Revert "bugfix: Keep registered types until after Py_Finalize(). Fix #4459 (#4486)" (#4501) This reverts commit b2c1978caaa6627095e3326f7ecba84b43037869. See #4500 for background. --- include/pybind11/embed.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 8450d5358..5a175b134 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -254,16 +254,13 @@ inline void finalize_interpreter() { if (builtins.contains(id) && isinstance(builtins[id])) { internals_ptr_ptr = capsule(builtins[id]); } - - Py_Finalize(); - // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter - // Must be cleared only after Py_Finalize() so atexit and other hooks can still use - // registered_types detail::get_local_internals().registered_types_cpp.clear(); detail::get_local_internals().registered_exception_translators.clear(); + Py_Finalize(); + if (internals_ptr_ptr) { delete *internals_ptr_ptr; *internals_ptr_ptr = nullptr; From b8f28551cc3a98ea9fbfc15c05b513c8f2d23e84 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 7 Feb 2023 20:19:33 -0800 Subject: [PATCH 22/83] Go back to CMake 3.25.2 (#4496) --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8befccdbc..5d93d42b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,9 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 + # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: + with: + cmake-version: '3.25.2' - name: Cache wheels if: runner.os == 'macOS' @@ -1071,6 +1074,9 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 + # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: + with: + cmake-version: '3.25.2' - name: Run pip installs run: | From 531144dddc8abf05612523a83ff575f6834fd611 Mon Sep 17 00:00:00 2001 From: Mike Essenmacher <112431871+mikeessen@users.noreply.github.com> Date: Thu, 9 Feb 2023 13:11:01 -0500 Subject: [PATCH 23/83] Replace "whitelist" with "allowlist" (#4506) --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9beb268ed..b1cb222b4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -169,7 +169,7 @@ if(PYBIND11_TEST_OVERRIDE) # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT - )# If not in the whitelist, add to be filtered out. + )# If not in the allowlist, add to be filtered out. list(APPEND PYBIND11_TEST_FILTER ${test_name}) endif() endforeach() From 8dcced29ae4c7654c25bc50742a72556ee3ae36c Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Fri, 10 Feb 2023 00:21:17 -0500 Subject: [PATCH 24/83] Always display python type information in cast errors (#4463) * Always display python type information in cast errors * Address comments * Update comment --- include/pybind11/cast.h | 33 ++++++++++++++++++-------------- include/pybind11/detail/common.h | 5 +++-- tests/test_callbacks.py | 14 ++++++++++++-- tests/test_exceptions.cpp | 5 +++++ tests/test_exceptions.py | 9 +++++++++ tests/test_pytypes.py | 2 +- 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3a4046027..9b013bc39 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1017,11 +1017,14 @@ type_caster &load_type(type_caster &conv, const handle &ha "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error("Unable to cast Python instance to C++ type (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast() + " to C++ type '" + type_id() + "'"); #endif } @@ -1085,12 +1088,13 @@ detail::enable_if_t::value, T> move(object &&obj) { if (obj.ref_count() > 1) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast() + " instance to C++ " + + type_id() + " instance: instance has multiple references"); #endif } @@ -1195,9 +1199,10 @@ PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error("Unable to convert call argument to Python object (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1220,7 +1225,7 @@ tuple make_tuple(Args &&...args_) { for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array argtypes{{type_id()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1510,7 +1515,7 @@ private: detail::make_caster::cast(std::forward(x), policy, {})); if (!o) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), type_id()); @@ -1542,7 +1547,7 @@ private: } if (!a.value) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 0fc81e053..1032293d2 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1225,8 +1225,9 @@ constexpr #endif // Pybind offers detailed error messages by default for all builts that are debug (through the -// negation of ndebug). This can also be manually enabled by users, for any builds, through -// defining PYBIND11_DETAILED_ERROR_MESSAGES. +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) # define PYBIND11_DETAILED_ERROR_MESSAGES #endif diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 57b659988..4a652f53e 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -5,6 +5,7 @@ import pytest import env # noqa: F401 from pybind11_tests import callbacks as m +from pybind11_tests import detailed_error_messages_enabled def test_callbacks(): @@ -70,11 +71,20 @@ def test_keyword_args_and_generalized_unpacking(): with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error1(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'1' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error2(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'expected_name' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'expected_name' to Python object " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) def test_lambda_closure_cleanup(): diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index f57e09506..854c7e6f7 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -339,4 +339,9 @@ TEST_SUBMODULE(exceptions, m) { } return py::str("UNEXPECTED"); }); + + m.def("test_fn_cast_int", [](const py::function &fn) { + // function returns None instead of int, should give a useful error message + fn().cast(); + }); } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 0d2c80814..8bcefe3de 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -381,3 +381,12 @@ def test_pypy_oserror_normalization(): # https://github.com/pybind/pybind11/issues/4075 what = m.test_pypy_oserror_normalization() assert "this_filename_must_not_exist" in what + + +def test_fn_cast_int_exception(): + with pytest.raises(RuntimeError) as excinfo: + m.test_fn_cast_int(lambda: None) + + assert str(excinfo.value).startswith( + "Unable to cast Python instance of type to C++ type" + ) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 8f9f2987e..a1a80a293 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -536,7 +536,7 @@ def test_print(capture): assert str(excinfo.value) == "Unable to convert call argument " + ( "'1' of type 'UnregisteredType' to Python object" if detailed_error_messages_enabled - else "to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" ) From 535f81a1c0f60f39aaf0f7661f2a1ac2a3a0af16 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Feb 2023 06:54:27 -0800 Subject: [PATCH 25/83] fix: tests dir has started to show up in packaging (#4510) Signed-off-by: Henry Schreiner --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 033303a74..31632acc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed From 08a4a47a6c0a8e8ca99a86c23c9b6ac843837395 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 16 Feb 2023 07:54:57 -0800 Subject: [PATCH 26/83] Revert "Go back to CMake 3.25.2 (#4496)" (#4503) * Revert "Go back to CMake 3.25.2 (#4496)" This reverts commit b8f28551cc3a98ea9fbfc15c05b513c8f2d23e84. * Apply patch provided by @bradking (with pre-commit cmake-format auto fixes). https://github.com/pybind/pybind11/pull/4503#issuecomment-1424768172 --- .github/workflows/ci.yml | 6 ------ tools/FindCatch.cmake | 8 ++++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d93d42b1..8befccdbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,9 +83,6 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: - with: - cmake-version: '3.25.2' - name: Cache wheels if: runner.os == 'macOS' @@ -1074,9 +1071,6 @@ jobs: - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - # TEMPORARILY pin version because 3.26.0-rc1 is failing under macOS: - with: - cmake-version: '3.25.2' - name: Run pip installs run: | diff --git a/tools/FindCatch.cmake b/tools/FindCatch.cmake index 57bba58b3..5d3fcbfb1 100644 --- a/tools/FindCatch.cmake +++ b/tools/FindCatch.cmake @@ -36,10 +36,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" From d1956eabb50d93428097a8a564cef4fc8b0ded82 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:31:08 -0800 Subject: [PATCH 27/83] Appease new flake8 B028 error: (#4513) ``` flake8...................................................................Failed - hook id: flake8 - exit code: 1 pybind11/setup_helpers.py:177:13: B028 No explicit stacklevel keyword argument found. The warn method from the warnings module uses a stacklevel of 1 by default. This will only show a stack trace for the line on which the warn method is called. It is therefore recommended to use a stacklevel of 2 or greater to provide more information to the user. warnings.warn("You cannot safely change the cxx_level after setting it!") ^ ``` --- pybind11/setup_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 1db479532..4c14498f6 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -174,7 +174,9 @@ class Pybind11Extension(_Extension): # type: ignore[misc] @cxx_std.setter def cxx_std(self, level: int) -> None: if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=1 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. From 6a5e6007cdfa9441fc05a589d7b819c6c37eb489 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 17 Feb 2023 12:58:35 -0800 Subject: [PATCH 28/83] Make warning suppressions MINGW-specific again. (#4515) Background: https://github.com/pybind/pybind11/pull/4285#issuecomment-1435066554 --- include/pybind11/eigen/matrix.h | 2 ++ include/pybind11/eigen/tensor.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index 34fe329a8..bd533bed3 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -19,7 +19,9 @@ PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include #include diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 0877da895..de7dcba89 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -17,7 +17,9 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0 PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(4554) PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include From 68211d41c1ef94fa65ef27ea2a6cdd8677996635 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 17 Feb 2023 14:13:29 -0800 Subject: [PATCH 29/83] fix: nicer stack level for warning (#4516) Signed-off-by: Henry Schreiner --- pybind11/setup_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 4c14498f6..b9b433b65 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -175,7 +175,7 @@ class Pybind11Extension(_Extension): # type: ignore[misc] def cxx_std(self, level: int) -> None: if self._cxx_level: warnings.warn( - "You cannot safely change the cxx_level after setting it!", stacklevel=1 + "You cannot safely change the cxx_level after setting it!", stacklevel=2 ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so From a19daeac16685877fd0b5ffcadce4b9f60d90f7b Mon Sep 17 00:00:00 2001 From: xkszltl Date: Tue, 21 Feb 2023 06:58:37 +0800 Subject: [PATCH 30/83] Inconsistent comments between 2 templates of `unchecked()`. (#4519) This comment is unrelated to having T in tparam or not. Probably a typo. Copied the correct one here. Resolve https://github.com/pybind/pybind11/issues/4518 --- include/pybind11/numpy.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8f072af26..854d6e87f 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1121,10 +1121,10 @@ public: /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { From 438034c5b8a6a8bf82dda7a195e3c00cafe575e0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 22 Feb 2023 06:18:55 -0800 Subject: [PATCH 31/83] chore: move to Ruff and add rules (#4483) --- .pre-commit-config.yaml | 44 ++-------- docs/conf.py | 2 +- pybind11/__init__.py | 2 +- pybind11/commands.py | 2 +- pybind11/setup_helpers.py | 2 +- pyproject.toml | 46 ++++++++-- setup.cfg | 8 -- tests/conftest.py | 50 ++++------- tests/env.py | 3 +- tests/test_async.py | 4 +- tests/test_buffers.py | 3 +- tests/test_builtin_casters.py | 10 ++- tests/test_class.py | 11 +-- tests/test_const_name.py | 4 +- tests/test_custom_type_casters.py | 3 +- tests/test_custom_type_setup.py | 2 +- tests/test_eigen_matrix.py | 93 +++++++++++++-------- tests/test_eigen_tensor.py | 20 ++--- tests/test_enum.py | 2 + tests/test_exceptions.py | 18 ++-- tests/test_factory_constructors.py | 2 +- tests/test_gil_scoped.py | 8 +- tests/test_iostream.py | 20 ++--- tests/test_kwargs_and_defaults.py | 31 ++++--- tests/test_methods_and_attributes.py | 9 +- tests/test_modules.py | 9 +- tests/test_numpy_array.py | 18 ++-- tests/test_numpy_dtypes.py | 28 +++---- tests/test_numpy_vectorize.py | 2 +- tests/test_pytypes.py | 48 +++++------ tests/test_sequences_and_iterators.py | 7 +- tests/test_stl.py | 7 +- tests/test_stl_binders.py | 12 +-- tests/test_virtual_functions.py | 7 +- tools/codespell_ignore_lines_from_errors.py | 8 +- 35 files changed, 266 insertions(+), 279 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27474f2f7..2255f2e4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,19 +39,6 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Upgrade old Python syntax -- repo: https://github.com/asottile/pyupgrade - rev: "v3.3.1" - hooks: - - id: pyupgrade - args: [--py36-plus] - -# Nicely sort includes -- repo: https://github.com/PyCQA/isort - rev: "5.12.0" - hooks: - - id: isort - # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black rev: "23.1.0" # Keep in sync with blacken-docs @@ -72,47 +59,28 @@ repos: hooks: - id: remove-tabs +# Avoid directional quotes - repo: https://github.com/sirosen/texthooks rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: "v2.1.3" +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.251 hooks: - - id: pycln - stages: [manual] + - id: ruff + args: ["--fix", "--show-fixes"] # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks rev: "v1.10.0" hooks: - - id: python-check-blanket-noqa - - id: python-check-blanket-type-ignore - - id: python-no-log-warn - - id: python-use-type-annotations - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal -# Automatically remove noqa that are not used -- repo: https://github.com/asottile/yesqa - rev: "v1.4.0" - hooks: - - id: yesqa - additional_dependencies: &flake8_dependencies - - flake8-bugbear - - pep8-naming - -# Flake8 also supports pre-commit natively (same author) -- repo: https://github.com/PyCQA/flake8 - rev: "6.0.0" - hooks: - - id: flake8 - exclude: ^(docs/.*|tools/.*)$ - additional_dependencies: *flake8_dependencies # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint diff --git a/docs/conf.py b/docs/conf.py index cdb9f6306..6e24751e9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -353,7 +353,7 @@ def prepare(app): f.write(contents) -def clean_up(app, exception): +def clean_up(app, exception): # noqa: ARG001 (DIR / "readme.rst").unlink() diff --git a/pybind11/__init__.py b/pybind11/__init__.py index 4fbb17079..7c10b3057 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,6 +1,6 @@ import sys -if sys.version_info < (3, 6): +if sys.version_info < (3, 6): # noqa: UP036 msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." raise ImportError(msg) diff --git a/pybind11/commands.py b/pybind11/commands.py index 152fa20ce..b11690f46 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -3,7 +3,7 @@ import os DIR = os.path.abspath(os.path.dirname(__file__)) -def get_include(user: bool = False) -> str: # pylint: disable=unused-argument +def get_include(user: bool = False) -> str: # noqa: ARG001 """ Return the path to the pybind11 include directory. The historical "user" argument is unused, and may be removed. diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index b9b433b65..0fb4679e4 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -341,7 +341,7 @@ def naive_recompile(obj: str, src: str) -> bool: return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument +def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. diff --git a/pyproject.toml b/pyproject.toml index 3ba1b4b22..96cf84265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,6 @@ ignore = [ "noxfile.py", ] -[tool.isort] -# Needs the compiled .so modules and env.py from tests -known_first_party = "env,pybind11_cross_module_tests,pybind11_tests," -# For black compatibility -profile = "black" - [tool.mypy] files = ["pybind11"] python_version = "3.6" @@ -58,4 +52,44 @@ messages_control.disable = [ "invalid-name", "protected-access", "missing-module-docstring", + "unused-argument", # covered by Ruff ARG ] + +[tool.ruff] +select = [ + "E", "F", "W", # flake8 + "B", "B904", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 +] +ignore = [ + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118",# iter(x) is not always the same as iter(x.keys()) +] +target-version = "py37" +typing-modules = ["scikit_build_core._compat.typing"] +src = ["src"] +unfixable = ["T20"] +exclude = [] +line-length = 120 +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] + +[tool.ruff.per-file-ignores] +"tests/**" = ["EM", "N"] diff --git a/setup.cfg b/setup.cfg index 8b3361981..c35dd07ae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,11 +40,3 @@ project_urls = [options] python_requires = >=3.6 zip_safe = False - - -[flake8] -max-line-length = 120 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722 -extend-select = B902, B904 diff --git a/tests/conftest.py b/tests/conftest.py index c24d51d4d..b60d3c956 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,9 +83,8 @@ class Output: b = _strip_and_dedent(other).splitlines() if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Unordered(Output): @@ -96,9 +95,8 @@ class Unordered(Output): b = _split_and_sort(other) if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Capture: @@ -119,9 +117,8 @@ class Capture: b = other if a == b: return True - else: - self.explanation = a.explanation - return False + self.explanation = a.explanation + return False def __str__(self): return self.out @@ -138,7 +135,7 @@ class Capture: return Output(self.err) -@pytest.fixture +@pytest.fixture() def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) @@ -159,25 +156,22 @@ class SanitizedString: b = _strip_and_dedent(other) if a == b: return True - else: - self.explanation = _make_explanation(a.splitlines(), b.splitlines()) - return False + self.explanation = _make_explanation(a.splitlines(), b.splitlines()) + return False def _sanitize_general(s): s = s.strip() s = s.replace("pybind11_tests.", "m.") - s = _long_marker.sub(r"\1", s) - return s + return _long_marker.sub(r"\1", s) def _sanitize_docstring(thing): s = thing.__doc__ - s = _sanitize_general(s) - return s + return _sanitize_general(s) -@pytest.fixture +@pytest.fixture() def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -186,30 +180,20 @@ def doc(): def _sanitize_message(thing): s = str(thing) s = _sanitize_general(s) - s = _hexadecimal.sub("0", s) - return s + return _hexadecimal.sub("0", s) -@pytest.fixture +@pytest.fixture() def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) -# noinspection PyUnusedLocal -def pytest_assertrepr_compare(op, left, right): +def pytest_assertrepr_compare(op, left, right): # noqa: ARG001 """Hook to insert custom failure explanation""" if hasattr(left, "explanation"): return left.explanation - - -@contextlib.contextmanager -def suppress(exception): - """Suppress the desired exception""" - try: - yield - except exception: - pass + return None def gc_collect(): @@ -220,7 +204,7 @@ def gc_collect(): def pytest_configure(): - pytest.suppress = suppress + pytest.suppress = contextlib.suppress pytest.gc_collect = gc_collect diff --git a/tests/env.py b/tests/env.py index 0345df65d..7eea5a3b3 100644 --- a/tests/env.py +++ b/tests/env.py @@ -24,5 +24,4 @@ def deprecated_call(): pytest_major_minor = (int(pieces[0]), int(pieces[1])) if pytest_major_minor < (3, 9): return pytest.warns((DeprecationWarning, PendingDeprecationWarning)) - else: - return pytest.deprecated_call() + return pytest.deprecated_call() diff --git a/tests/test_async.py b/tests/test_async.py index b9ff9514d..83a4c5036 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -4,7 +4,7 @@ asyncio = pytest.importorskip("asyncio") m = pytest.importorskip("pybind11_tests.async_module") -@pytest.fixture +@pytest.fixture() def event_loop(): loop = asyncio.new_event_loop() yield loop @@ -16,7 +16,7 @@ async def get_await_result(x): def test_await(event_loop): - assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync())) + assert event_loop.run_until_complete(get_await_result(m.SupportsAsync())) == 5 def test_await_missing(event_loop): diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 8354b68cd..eb58c4675 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -54,7 +54,8 @@ def test_to_python(): mat2 = np.array(mat, copy=False) assert mat2.shape == (5, 4) assert abs(mat2).sum() == 11 - assert mat2[2, 3] == 4 and mat2[3, 2] == 7 + assert mat2[2, 3] == 4 + assert mat2[3, 2] == 7 mat2[2, 3] = 5 assert mat2[2, 3] == 5 diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index d38ae6802..b1f57bdd9 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -126,8 +126,8 @@ def test_bytes_to_string(): assert m.strlen(b"hi") == 2 assert m.string_length(b"world") == 5 - assert m.string_length("a\x00b".encode()) == 3 - assert m.strlen("a\x00b".encode()) == 1 # C-string limitation + assert m.string_length(b"a\x00b") == 3 + assert m.strlen(b"a\x00b") == 1 # C-string limitation # passing in a utf8 encoded string should work assert m.string_length("💩".encode()) == 4 @@ -421,13 +421,15 @@ def test_reference_wrapper(): a2 = m.refwrap_list(copy=True) assert [x.value for x in a1] == [2, 3] assert [x.value for x in a2] == [2, 3] - assert not a1[0] is a2[0] and not a1[1] is a2[1] + assert a1[0] is not a2[0] + assert a1[1] is not a2[1] b1 = m.refwrap_list(copy=False) b2 = m.refwrap_list(copy=False) assert [x.value for x in b1] == [1, 2] assert [x.value for x in b2] == [1, 2] - assert b1[0] is b2[0] and b1[1] is b2[1] + assert b1[0] is b2[0] + assert b1[1] is b2[1] assert m.refwrap_iiw(IncType(5)) == 5 assert m.refwrap_call_iiw(IncType(10), m.refwrap_iiw) == [10, 10, 10, 10] diff --git a/tests/test_class.py b/tests/test_class.py index 64abac133..ee7467cf8 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -6,10 +6,7 @@ from pybind11_tests import class_ as m def test_obj_class_name(): - if env.PYPY: - expected_name = "UserType" - else: - expected_name = "pybind11_tests.UserType" + expected_name = "UserType" if env.PYPY else "pybind11_tests.UserType" assert m.obj_class_name(UserType(1)) == expected_name assert m.obj_class_name(UserType) == expected_name @@ -32,7 +29,7 @@ def test_instance(msg): assert cstats.alive() == 0 -def test_instance_new(msg): +def test_instance_new(): instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) cstats = ConstructorStats.get(m.NoConstructorNew) assert cstats.alive() == 1 @@ -221,7 +218,7 @@ def test_automatic_upcasting(): def test_isinstance(): - objects = [tuple(), dict(), m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 + objects = [(), {}, m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 expected = (True, True, True, True, True, False, False) assert m.check_instances(objects) == expected @@ -427,7 +424,7 @@ def test_exception_rvalue_abort(): # https://github.com/pybind/pybind11/issues/1568 -def test_multiple_instances_with_same_pointer(capture): +def test_multiple_instances_with_same_pointer(): n = 100 instances = [m.SamePointer() for _ in range(n)] for i in range(n): diff --git a/tests/test_const_name.py b/tests/test_const_name.py index 10b0caee2..a145f0bbb 100644 --- a/tests/test_const_name.py +++ b/tests/test_const_name.py @@ -3,9 +3,9 @@ import pytest from pybind11_tests import const_name as m -@pytest.mark.parametrize("func", (m.const_name_tests, m.underscore_tests)) +@pytest.mark.parametrize("func", [m.const_name_tests, m.underscore_tests]) @pytest.mark.parametrize( - "selector, expected", + ("selector", "expected"), enumerate( ( "", diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index 731380e46..3a00ea964 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -100,7 +100,8 @@ def test_custom_caster_destruction(): cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: z = m.custom_caster_no_destroy() - assert cstats.alive() == 1 and cstats.default_constructions == 1 + assert cstats.alive() == 1 + assert cstats.default_constructions == 1 assert z # take_ownership applied: this constructs a new object, casts it, then destroys it: diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 19b44c9de..e63ff5758 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -7,7 +7,7 @@ import env # noqa: F401 from pybind11_tests import custom_type_setup as m -@pytest.fixture +@pytest.fixture() def gc_tester(): """Tests that an object is garbage collected. diff --git a/tests/test_eigen_matrix.py b/tests/test_eigen_matrix.py index 4407fa6ae..b2e76740b 100644 --- a/tests/test_eigen_matrix.py +++ b/tests/test_eigen_matrix.py @@ -263,79 +263,96 @@ def test_eigen_return_references(): primary = np.ones((10, 10)) a = m.ReturnTester() a_get1 = a.get() - assert not a_get1.flags.owndata and a_get1.flags.writeable + assert not a_get1.flags.owndata + assert a_get1.flags.writeable assign_both(a_get1, primary, 3, 3, 5) a_get2 = a.get_ptr() - assert not a_get2.flags.owndata and a_get2.flags.writeable + assert not a_get2.flags.owndata + assert a_get2.flags.writeable assign_both(a_get1, primary, 2, 3, 6) a_view1 = a.view() - assert not a_view1.flags.owndata and not a_view1.flags.writeable + assert not a_view1.flags.owndata + assert not a_view1.flags.writeable with pytest.raises(ValueError): a_view1[2, 3] = 4 a_view2 = a.view_ptr() - assert not a_view2.flags.owndata and not a_view2.flags.writeable + assert not a_view2.flags.owndata + assert not a_view2.flags.writeable with pytest.raises(ValueError): a_view2[2, 3] = 4 a_copy1 = a.copy_get() - assert a_copy1.flags.owndata and a_copy1.flags.writeable + assert a_copy1.flags.owndata + assert a_copy1.flags.writeable np.testing.assert_array_equal(a_copy1, primary) a_copy1[7, 7] = -44 # Shouldn't affect anything else c1want = array_copy_but_one(primary, 7, 7, -44) a_copy2 = a.copy_view() - assert a_copy2.flags.owndata and a_copy2.flags.writeable + assert a_copy2.flags.owndata + assert a_copy2.flags.writeable np.testing.assert_array_equal(a_copy2, primary) a_copy2[4, 4] = -22 # Shouldn't affect anything else c2want = array_copy_but_one(primary, 4, 4, -22) a_ref1 = a.ref() - assert not a_ref1.flags.owndata and a_ref1.flags.writeable + assert not a_ref1.flags.owndata + assert a_ref1.flags.writeable assign_both(a_ref1, primary, 1, 1, 15) a_ref2 = a.ref_const() - assert not a_ref2.flags.owndata and not a_ref2.flags.writeable + assert not a_ref2.flags.owndata + assert not a_ref2.flags.writeable with pytest.raises(ValueError): a_ref2[5, 5] = 33 a_ref3 = a.ref_safe() - assert not a_ref3.flags.owndata and a_ref3.flags.writeable + assert not a_ref3.flags.owndata + assert a_ref3.flags.writeable assign_both(a_ref3, primary, 0, 7, 99) a_ref4 = a.ref_const_safe() - assert not a_ref4.flags.owndata and not a_ref4.flags.writeable + assert not a_ref4.flags.owndata + assert not a_ref4.flags.writeable with pytest.raises(ValueError): a_ref4[7, 0] = 987654321 a_copy3 = a.copy_ref() - assert a_copy3.flags.owndata and a_copy3.flags.writeable + assert a_copy3.flags.owndata + assert a_copy3.flags.writeable np.testing.assert_array_equal(a_copy3, primary) a_copy3[8, 1] = 11 c3want = array_copy_but_one(primary, 8, 1, 11) a_copy4 = a.copy_ref_const() - assert a_copy4.flags.owndata and a_copy4.flags.writeable + assert a_copy4.flags.owndata + assert a_copy4.flags.writeable np.testing.assert_array_equal(a_copy4, primary) a_copy4[8, 4] = 88 c4want = array_copy_but_one(primary, 8, 4, 88) a_block1 = a.block(3, 3, 2, 2) - assert not a_block1.flags.owndata and a_block1.flags.writeable + assert not a_block1.flags.owndata + assert a_block1.flags.writeable a_block1[0, 0] = 55 primary[3, 3] = 55 a_block2 = a.block_safe(2, 2, 3, 2) - assert not a_block2.flags.owndata and a_block2.flags.writeable + assert not a_block2.flags.owndata + assert a_block2.flags.writeable a_block2[2, 1] = -123 primary[4, 3] = -123 a_block3 = a.block_const(6, 7, 4, 3) - assert not a_block3.flags.owndata and not a_block3.flags.writeable + assert not a_block3.flags.owndata + assert not a_block3.flags.writeable with pytest.raises(ValueError): a_block3[2, 2] = -44444 a_copy5 = a.copy_block(2, 2, 2, 3) - assert a_copy5.flags.owndata and a_copy5.flags.writeable + assert a_copy5.flags.owndata + assert a_copy5.flags.writeable np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5]) a_copy5[1, 1] = 777 c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777) a_corn1 = a.corners() - assert not a_corn1.flags.owndata and a_corn1.flags.writeable + assert not a_corn1.flags.owndata + assert a_corn1.flags.writeable a_corn1 *= 50 a_corn1[1, 1] = 999 primary[0, 0] = 50 @@ -343,7 +360,8 @@ def test_eigen_return_references(): primary[9, 0] = 50 primary[9, 9] = 999 a_corn2 = a.corners_const() - assert not a_corn2.flags.owndata and not a_corn2.flags.writeable + assert not a_corn2.flags.owndata + assert not a_corn2.flags.writeable with pytest.raises(ValueError): a_corn2[1, 0] = 51 @@ -503,10 +521,14 @@ def test_numpy_ref_mutators(): assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4 - assert not zc.flags.owndata and zc.flags.writeable - assert not zr.flags.owndata and zr.flags.writeable - assert not zcro.flags.owndata and not zcro.flags.writeable - assert not zrro.flags.owndata and not zrro.flags.writeable + assert not zc.flags.owndata + assert zc.flags.writeable + assert not zr.flags.owndata + assert zr.flags.writeable + assert not zcro.flags.owndata + assert not zcro.flags.writeable + assert not zrro.flags.owndata + assert not zrro.flags.writeable zc[1, 2] = 99 expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]]) @@ -530,7 +552,8 @@ def test_numpy_ref_mutators(): # the const should drop away) y1 = np.array(m.get_cm_const_ref()) - assert y1.flags.owndata and y1.flags.writeable + assert y1.flags.owndata + assert y1.flags.writeable # We should get copies of the eigen data, which was modified above: assert y1[1, 2] == 99 y1[1, 2] += 12 @@ -603,38 +626,38 @@ def test_nocopy_wrapper(): # All but the second should fail with m.get_elem_nocopy: with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_colmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(dbl_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) # For the row-major test, we take a long matrix in row-major, so only the third is allowed: with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(int_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_rowmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) def test_eigen_ref_life_support(): diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index cd54efa06..3e7ee6b7f 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -11,14 +11,15 @@ try: submodules += [avoid.c_style, avoid.f_style] except ImportError as e: # Ensure config, build, toolchain, etc. issues are not masked here: - raise RuntimeError( + msg = ( "import eigen_tensor_avoid_stl_array FAILED, while " "import pybind11_tests.eigen_tensor succeeded. " "Please ensure that " "test_eigen_tensor.cpp & " "eigen_tensor_avoid_stl_array.cpp " "are built together (or both are not built if Eigen is not available)." - ) from e + ) + raise RuntimeError(msg) from e tensor_ref = np.empty((3, 5, 2), dtype=np.int64) @@ -147,10 +148,7 @@ def test_bad_python_to_cpp_casts(m): m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64)) ) - if m.needed_options == "F": - bad_options = "C" - else: - bad_options = "F" + bad_options = "C" if m.needed_options == "F" else "F" # Shape, dtype and the order need to be correct for a TensorMap cast with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" @@ -173,19 +171,19 @@ def test_bad_python_to_cpp_casts(m): np.zeros((3, 5), dtype=np.float64, order=m.needed_options) ) + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" ): - temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) m.round_trip_view_tensor( temp[:, ::-1, :], ) + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + temp.setflags(write=False) with pytest.raises( TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" ): - temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) - temp.setflags(write=False) m.round_trip_view_tensor(temp) @@ -282,9 +280,9 @@ def test_doc_string(m, doc): order_flag = f"flags.{m.needed_options.lower()}_contiguous" assert doc(m.round_trip_view_tensor) == ( f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" - + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" ) assert doc(m.round_trip_const_view_tensor) == ( f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" - + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" ) diff --git a/tests/test_enum.py b/tests/test_enum.py index f14a72398..4e85d29c3 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -1,3 +1,5 @@ +# ruff: noqa: SIM201 SIM300 SIM202 + import pytest from pybind11_tests import enums as m diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 8bcefe3de..7f80a5da5 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -94,8 +94,7 @@ def ignore_pytest_unraisable_warning(f): if hasattr(pytest, unraisable): # Python >= 3.8 and pytest >= 6 dec = pytest.mark.filterwarnings(f"ignore::pytest.{unraisable}") return dec(f) - else: - return f + return f # TODO: find out why this fails on PyPy, https://foss.heptapod.net/pypy/pypy/-/issues/3583 @@ -183,7 +182,7 @@ def test_custom(msg): m.throws5_1() assert msg(excinfo.value) == "MyException5 subclass" - with pytest.raises(m.MyException5) as excinfo: + with pytest.raises(m.MyException5) as excinfo: # noqa: PT012 try: m.throws5() except m.MyException5_1 as err: @@ -212,7 +211,7 @@ def test_nested_throws(capture): m.try_catch(m.MyException5, throw_myex) assert str(excinfo.value) == "nested error" - def pycatch(exctype, f, *args): + def pycatch(exctype, f, *args): # noqa: ARG001 try: f(*args) except m.MyException as e: @@ -303,12 +302,12 @@ class FlakyException(Exception): @pytest.mark.parametrize( - "exc_type, exc_value, expected_what", - ( + ("exc_type", "exc_value", "expected_what"), + [ (ValueError, "plain_str", "ValueError: plain_str"), (ValueError, ("tuple_elem",), "ValueError: tuple_elem"), (FlakyException, ("happy",), "FlakyException: FlakyException.__str__"), - ), + ], ) def test_error_already_set_what_with_happy_exceptions( exc_type, exc_value, expected_what @@ -342,10 +341,7 @@ def test_flaky_exception_failure_point_str(): ) assert not py_err_set_after_what lines = what.splitlines() - if env.PYPY and len(lines) == 3: - n = 3 # Traceback is missing. - else: - n = 5 + n = 3 if env.PYPY and len(lines) == 3 else 5 assert ( lines[:n] == [ diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 120a587c4..04df80260 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -96,7 +96,7 @@ def test_init_factory_signature(msg): 3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 6af6a472d..fc8af9b77 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -148,10 +148,7 @@ ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_dead def _run_in_process(target, *args, **kwargs): - if len(args) == 0: - test_fn = target - else: - test_fn = args[0] + test_fn = target if len(args) == 0 else args[0] # Do not need to wait much, 10s should be more than enough. timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) @@ -178,7 +175,8 @@ def _run_in_process(target, *args, **kwargs): elif test_fn is _intentional_deadlock: assert process.exitcode is None return 0 - elif process.exitcode is None: + + if process.exitcode is None: assert t_delta > 0.9 * timeout msg = "DEADLOCK, most likely, exactly what this test is meant to detect." if env.PYPY and env.WIN: diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 5bbdf6955..79b2a2b8c 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -224,9 +224,8 @@ def test_redirect(capfd): assert stream.getvalue() == "" stream = StringIO() - with redirect_stdout(stream): - with m.ostream_redirect(): - m.raw_output(msg) + with redirect_stdout(stream), m.ostream_redirect(): + m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == "" assert stream.getvalue() == msg @@ -244,10 +243,9 @@ def test_redirect_err(capfd): msg2 = "StdErr" stream = StringIO() - with redirect_stderr(stream): - with m.ostream_redirect(stdout=False): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stderr(stream), m.ostream_redirect(stdout=False): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg assert stderr == "" @@ -260,11 +258,9 @@ def test_redirect_both(capfd): stream = StringIO() stream2 = StringIO() - with redirect_stdout(stream): - with redirect_stderr(stream2): - with m.ostream_redirect(): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stdout(stream), redirect_stderr(stream2), m.ostream_redirect(): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == "" assert stderr == "" diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index ab7017886..7174726fc 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -25,7 +25,7 @@ def test_function_signatures(doc): ) -def test_named_arguments(msg): +def test_named_arguments(): assert m.kw_func0(5, 10) == "x=5, y=10" assert m.kw_func1(5, 10) == "x=5, y=10" @@ -43,8 +43,7 @@ def test_named_arguments(msg): # noinspection PyArgumentList m.kw_func2(x=5, y=10, z=12) assert excinfo.match( - r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))" - + "{3}$" + r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$)){3}$" ) assert m.kw_func4() == "{13 17}" @@ -59,7 +58,7 @@ def test_arg_and_kwargs(): assert m.args_function(*args) == args args = "a1", "a2" - kwargs = dict(arg3="a3", arg4=4) + kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) @@ -177,7 +176,7 @@ def test_mixed_args_and_kwargs(msg): assert ( m.args_kwonly_kwargs_defaults.__doc__ - == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {}) assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {}) @@ -233,15 +232,15 @@ def test_keyword_only_args(msg): x.method(i=1, j=2) assert ( m.first_arg_kw_only.__init__.__doc__ - == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" # noqa: E501 line too long + == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" ) assert ( m.first_arg_kw_only.method.__doc__ - == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" # noqa: E501 line too long + == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" ) -def test_positional_only_args(msg): +def test_positional_only_args(): assert m.pos_only_all(1, 2) == (1, 2) assert m.pos_only_all(2, 1) == (2, 1) @@ -283,7 +282,7 @@ def test_positional_only_args(msg): # Mix it with args and kwargs: assert ( m.args_kwonly_full_monty.__doc__ - == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {}) assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {}) @@ -326,18 +325,18 @@ def test_positional_only_args(msg): # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 assert ( m.first_arg_kw_only.pos_only.__doc__ - == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" # noqa: E501 line too long + == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" ) def test_signatures(): - assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ - assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ - assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ - assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert m.kw_only_all.__doc__ == "kw_only_all(*, i: int, j: int) -> tuple\n" + assert m.kw_only_mixed.__doc__ == "kw_only_mixed(i: int, *, j: int) -> tuple\n" + assert m.pos_only_all.__doc__ == "pos_only_all(i: int, j: int, /) -> tuple\n" + assert m.pos_only_mix.__doc__ == "pos_only_mix(i: int, /, j: int) -> tuple\n" assert ( - "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" - == m.pos_kw_only_mix.__doc__ + m.pos_kw_only_mix.__doc__ + == "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" ) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 0a2ae1239..a85468575 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -183,9 +183,9 @@ def test_static_properties(): # Only static attributes can be deleted del m.TestPropertiesOverride.def_readonly_static + assert hasattr(m.TestPropertiesOverride, "def_readonly_static") assert ( - hasattr(m.TestPropertiesOverride, "def_readonly_static") - and m.TestPropertiesOverride.def_readonly_static + m.TestPropertiesOverride.def_readonly_static is m.TestProperties.def_readonly_static ) assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__ @@ -256,10 +256,7 @@ def test_no_mixed_overloads(): @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) def test_property_return_value_policies(access): - if not access.startswith("static"): - obj = m.TestPropRVP() - else: - obj = m.TestPropRVP + obj = m.TestPropRVP() if not access.startswith("static") else m.TestPropRVP ref = getattr(obj, access + "_ref") assert ref.value == 1 diff --git a/tests/test_modules.py b/tests/test_modules.py index c3e8cea6e..3d73e3fbd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,3 +1,5 @@ +import builtins + import pytest import env @@ -86,12 +88,7 @@ def test_builtin_key_type(): Previous versions of pybind11 would add a unicode key in python 2. """ - if hasattr(__builtins__, "keys"): - keys = __builtins__.keys() - else: # this is to make pypy happy since builtins is different there. - keys = __builtins__.__dict__.keys() - - assert {type(k) for k in keys} == {str} + assert all(type(k) == str for k in dir(builtins)) @pytest.mark.xfail("env.PYPY", reason="PyModule_GetName()") diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index cdec9ad60..070813d3a 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -22,7 +22,7 @@ def test_dtypes(): ) -@pytest.fixture(scope="function") +@pytest.fixture() def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") @@ -67,7 +67,7 @@ def test_array_attributes(): @pytest.mark.parametrize( - "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] + ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] ) def test_index_offset(arr, args, ret): assert m.index_at(arr, *args) == ret @@ -93,7 +93,7 @@ def test_dim_check_fail(arr): @pytest.mark.parametrize( - "args, ret", + ("args", "ret"), [ ([], [1, 2, 3, 4, 5, 6]), ([1], [4, 5, 6]), @@ -211,12 +211,14 @@ def test_wrap(): assert b[0, 0] == 1234 a1 = np.array([1, 2], dtype=np.int16) - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F") - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) @@ -451,13 +453,15 @@ def test_array_resize(): try: m.array_resize3(a, 3, True) except ValueError as e: - assert str(e).startswith("cannot resize an array") + assert str(e).startswith("cannot resize an array") # noqa: PT017 # transposed array doesn't own data b = a.transpose() try: m.array_resize3(b, 3, False) except ValueError as e: - assert str(e).startswith("cannot resize this array: it does not own its data") + assert str(e).startswith( # noqa: PT017 + "cannot resize this array: it does not own its data" + ) # ... but reshape should be fine m.array_reshape2(b) assert b.shape == (8, 8) diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index fcfd587b1..d10457eeb 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -130,14 +130,10 @@ def test_dtype(simple_dtype): partial_nested_fmt(), "[('a','S3'),('b','S3')]", ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('" - + e - + "f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e), + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ), "[('e1','" + e + "i8'),('e2','u1')]", "[('x','i1'),('y','" + e + "u8')]", "[('cflt','" + e + "c8'),('cdbl','" + e + "c16')]", @@ -291,19 +287,17 @@ def test_array_array(): arr = m.create_array_array(3) assert str(arr.dtype).replace(" ", "") == ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e) + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ) assert m.print_array_array(arr) == [ "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," - + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," - + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," - + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", ] assert arr["a"].tolist() == [ [b"ABCD", b"KLMN", b"UVWX"], diff --git a/tests/test_numpy_vectorize.py b/tests/test_numpy_vectorize.py index 7e8c015c4..f1e8b6254 100644 --- a/tests/test_numpy_vectorize.py +++ b/tests/test_numpy_vectorize.py @@ -149,7 +149,7 @@ def test_docs(doc): doc(m.vectorized_func) == """ vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index a1a80a293..946b94140 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -15,7 +15,7 @@ def test_obj_class_name(): assert m.obj_class_name([]) == "list" -def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 +def test_handle_from_move_only_type_with_operator_PyObject(): assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() @@ -33,7 +33,7 @@ def test_iterator(doc): @pytest.mark.parametrize( - "pytype, from_iter_func", + ("pytype", "from_iter_func"), [ (frozenset, m.get_frozenset_from_iterable), (list, m.get_list_from_iterable), @@ -87,7 +87,7 @@ def test_list(capture, doc): assert doc(m.print_list) == "print_list(arg0: list) -> None" -def test_none(capture, doc): +def test_none(doc): assert doc(m.get_none) == "get_none() -> None" assert doc(m.print_none) == "print_none(arg0: None) -> None" @@ -182,10 +182,10 @@ class CustomContains: @pytest.mark.parametrize( - "arg,func", + ("arg", "func"), [ (set(), m.anyset_contains), - (dict(), m.dict_contains), + ({}, m.dict_contains), (CustomContains(), m.obj_contains), ], ) @@ -273,7 +273,7 @@ def test_bytes(doc): assert doc(m.bytes_from_str) == "bytes_from_str() -> bytes" -def test_bytearray(doc): +def test_bytearray(): assert m.bytearray_from_char_ssize_t().decode() == "$%" assert m.bytearray_from_char_size_t().decode() == "@$!" assert m.bytearray_from_string().decode() == "foo" @@ -385,7 +385,7 @@ def test_accessors(): assert d["implicit_list"] == [1, 2, 3] assert all(x in TestObject.__dict__ for x in d["implicit_dict"]) - assert m.tuple_accessor(tuple()) == (0, 1, 2) + assert m.tuple_accessor(()) == (0, 1, 2) d = m.accessor_assignment() assert d["get"] == 0 @@ -593,7 +593,7 @@ def test_issue2361(): @pytest.mark.parametrize( - "method, args, fmt, expected_view", + ("method", "args", "fmt", "expected_view"), [ (m.test_memoryview_object, (b"red",), "B", b"red"), (m.test_memoryview_buffer_info, (b"green",), "B", b"green"), @@ -651,7 +651,7 @@ def test_memoryview_from_memory(): def test_builtin_functions(): - assert m.get_len([i for i in range(42)]) == 42 + assert m.get_len(list(range(42))) == 42 with pytest.raises(TypeError) as exc_info: m.get_len(i for i in range(42)) assert str(exc_info.value) in [ @@ -695,7 +695,7 @@ def test_pass_bytes_or_unicode_to_string_types(): @pytest.mark.parametrize( - "create_weakref, create_weakref_with_callback", + ("create_weakref", "create_weakref_with_callback"), [ (m.weakref_from_handle, m.weakref_from_handle_and_function), (m.weakref_from_object, m.weakref_from_object_and_function), @@ -710,7 +710,7 @@ def test_weakref(create_weakref, create_weakref_with_callback): callback_called = False - def callback(wr): + def callback(_): nonlocal callback_called callback_called = True @@ -730,7 +730,7 @@ def test_weakref(create_weakref, create_weakref_with_callback): @pytest.mark.parametrize( - "create_weakref, has_callback", + ("create_weakref", "has_callback"), [ (m.weakref_from_handle, False), (m.weakref_from_object, False), @@ -748,10 +748,7 @@ def test_weakref_err(create_weakref, has_callback): ob = C() # Should raise TypeError on CPython with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): - if has_callback: - _ = create_weakref(ob, callback) - else: - _ = create_weakref(ob) + _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) def test_cpp_iterators(): @@ -814,33 +811,36 @@ def test_populate_obj_str_attrs(): @pytest.mark.parametrize( - "a,b", [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))] + ("a", "b"), + [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))], ) def test_inplace_append(a, b): expected = a + b assert m.inplace_append(a, b) == expected -@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))]) +@pytest.mark.parametrize( + ("a", "b"), [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))] +) def test_inplace_subtract(a, b): expected = a - b assert m.inplace_subtract(a, b) == expected -@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), ([1], 3)]) +@pytest.mark.parametrize(("a", "b"), [(3, 2), (3.0, 2.0), ([1], 3)]) def test_inplace_multiply(a, b): expected = a * b assert m.inplace_multiply(a, b) == expected -@pytest.mark.parametrize("a,b", [(6, 3), (6.0, 3.0)]) +@pytest.mark.parametrize(("a", "b"), [(6, 3), (6.0, 3.0)]) def test_inplace_divide(a, b): expected = a / b assert m.inplace_divide(a, b) == expected @pytest.mark.parametrize( - "a,b", + ("a", "b"), [ (False, True), ( @@ -857,7 +857,7 @@ def test_inplace_or(a, b): @pytest.mark.parametrize( - "a,b", + ("a", "b"), [ (True, False), ( @@ -873,13 +873,13 @@ def test_inplace_and(a, b): assert m.inplace_and(a, b) == expected -@pytest.mark.parametrize("a,b", [(8, 1), (-3, 2)]) +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-3, 2)]) def test_inplace_lshift(a, b): expected = a << b assert m.inplace_lshift(a, b) == expected -@pytest.mark.parametrize("a,b", [(8, 1), (-2, 2)]) +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-2, 2)]) def test_inplace_rshift(a, b): expected = a >> b assert m.inplace_rshift(a, b) == expected diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index de486e3e8..dc129f2bf 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -1,5 +1,5 @@ import pytest -from pytest import approx +from pytest import approx # noqa: PT013 from pybind11_tests import ConstructorStats from pybind11_tests import sequences_and_iterators as m @@ -103,7 +103,8 @@ def test_sequence(): assert "Sequence" in repr(s) assert len(s) == 5 - assert s[0] == 0 and s[3] == 0 + assert s[0] == 0 + assert s[3] == 0 assert 12.34 not in s s[0], s[3] = 12.34, 56.78 assert 12.34 in s @@ -245,7 +246,7 @@ def test_iterator_rvp(): def test_carray_iterator(): """#4100: Check for proper iterator overload with C-Arrays""" - args_gt = list(float(i) for i in range(3)) + args_gt = [float(i) for i in range(3)] arr_h = m.CArrayHolder(*args_gt) args = list(arr_h) assert args_gt == args diff --git a/tests/test_stl.py b/tests/test_stl.py index d30c38211..b7a0d1c0d 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -14,7 +14,7 @@ def test_vector(doc): assert m.cast_bool_vector() == [True, False] assert m.load_bool_vector([True, False]) - assert m.load_bool_vector(tuple([True, False])) + assert m.load_bool_vector((True, False)) assert doc(m.cast_vector) == "cast_vector() -> List[int]" assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool" @@ -23,7 +23,7 @@ def test_vector(doc): assert m.cast_ptr_vector() == ["lvalue", "lvalue"] -def test_deque(doc): +def test_deque(): """std::deque <-> list""" lst = m.cast_deque() assert lst == [1] @@ -95,7 +95,8 @@ def test_recursive_casting(): # Issue #853 test case: z = m.cast_unique_ptr_vector() - assert z[0].value == 7 and z[1].value == 42 + assert z[0].value == 7 + assert z[1].value == 42 def test_move_out_container(): diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 9eb906f06..7dca742a9 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -186,9 +186,9 @@ def test_map_string_double(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] + assert sorted(um) == ["ua", "ub"] assert list(um.keys()) == list(um) - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] assert list(zip(um.keys(), um.values())) == list(um.items()) assert "UnorderedMapStringDouble" in str(um) @@ -304,11 +304,11 @@ def test_map_delitem(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um) == ["ua", "ub"] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] del um["ua"] - assert sorted(list(um)) == ["ub"] - assert sorted(list(um.items())) == [("ub", 2.6)] + assert sorted(um) == ["ub"] + assert sorted(um.items()) == [("ub", 2.6)] def test_map_view_types(): diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index 4d00d3690..c17af7df5 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -192,8 +192,7 @@ def test_move_support(): class NCVirtExt(m.NCVirt): def get_noncopyable(self, a, b): # Constructs and returns a new instance: - nc = m.NonCopyable(a * a, b * b) - return nc + return m.NonCopyable(a * a, b * b) def get_movable(self, a, b): # Return a referenced copy @@ -256,7 +255,7 @@ def test_dispatch_issue(msg): assert m.dispatch_issue_go(b) == "Yay.." -def test_recursive_dispatch_issue(msg): +def test_recursive_dispatch_issue(): """#3357: Recursive dispatch fails to find python function override""" class Data(m.Data): @@ -269,7 +268,7 @@ def test_recursive_dispatch_issue(msg): # lambda is a workaround, which adds extra frame to the # current CPython thread. Removing lambda reveals the bug # [https://github.com/pybind/pybind11/issues/3357] - (lambda: visitor(Data(first.value + second.value)))() + (lambda: visitor(Data(first.value + second.value)))() # noqa: PLC3002 class StoreResultVisitor: def __init__(self): diff --git a/tools/codespell_ignore_lines_from_errors.py b/tools/codespell_ignore_lines_from_errors.py index 5403ec3ad..4ec9add12 100644 --- a/tools/codespell_ignore_lines_from_errors.py +++ b/tools/codespell_ignore_lines_from_errors.py @@ -17,14 +17,18 @@ def run(args: List[str]) -> None: assert len(args) == 1, "codespell_errors.txt" cache = {} done = set() - for line in sorted(open(args[0]).read().splitlines()): + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): i = line.find(" ==> ") if i > 0: flds = line[:i].split(":") if len(flds) >= 2: filename, line_num = flds[:2] if filename not in cache: - cache[filename] = open(filename).read().splitlines() + with open(filename) as f: + cache[filename] = f.read().splitlines() supp = cache[filename][int(line_num) - 1] if supp not in done: print(supp) From 3cc7e4258c15a6a19ba5e0b62a220b1a6196d4eb Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 23 Feb 2023 00:59:14 -0500 Subject: [PATCH 32/83] add --version option to pybind11-config (#4526) Without this, it's impossible to get feature parity between detection mechanisms. Both the pkg-config file and the cmake config set their versions, but the python probe script didn't provide an option for this. So you could print the compiler flags for using it, but you could not check what you got. --- pybind11/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index e87b16beb..180665c23 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -4,6 +4,7 @@ import argparse import sys import sysconfig +from ._version import __version__ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir @@ -25,6 +26,12 @@ def print_includes() -> None: def main() -> None: parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", From cbb876cc7b02c5f57e715cbc2c46ead3d1fbcf79 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 00:34:41 -0500 Subject: [PATCH 33/83] chore(deps): update pre-commit hooks (#4552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.251 → v0.0.254](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.251...v0.0.254) - [github.com/PyCQA/pylint: v2.16.1 → v2.16.4](https://github.com/PyCQA/pylint/compare/v2.16.1...v2.16.4) - [github.com/pre-commit/mirrors-mypy: v0.991 → v1.0.1](https://github.com/pre-commit/mirrors-mypy/compare/v0.991...v1.0.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2255f2e4d..7c4316192 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.251 + rev: v0.0.254 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.1" + rev: "v2.16.4" hooks: - id: pylint files: ^pybind11 @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.991" + rev: "v1.0.1" hooks: - id: mypy args: [] From 442261da585536521ff459b1457b2904895f23b4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 9 Mar 2023 13:02:34 -0800 Subject: [PATCH 34/83] Remove test code that does not exercise anything in pybind11, but breaks between Python 3.12alpha3 (still working) and 3.12alpha6 (broken): (#4559) ``` str(OrderedDict([(1, "a"), (2, "b")])) ``` Old: ``` OrderedDict([(1, 'a'), (2, 'b')]) ``` New: ``` OrderedDict({1: 'a', 2: 'b'}) ``` See also: https://github.com/python/cpython/issues/101446 --- tests/test_modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 3d73e3fbd..2f6d825b7 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -63,7 +63,6 @@ def test_importing(): from pybind11_tests.modules import OD assert OD is OrderedDict - assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])" def test_pydoc(): From 04ef4e422980ce1406a76358cbb74e7e3fd23a21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 18:18:44 +0000 Subject: [PATCH 35/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.6.4 to 1.8.1 (#4576) --- .github/workflows/pip.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7c6fc67a3..3d4d2c654 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password }} - packages_dir: standard/ + packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.8.1 with: password: ${{ secrets.pypi_password_global }} - packages_dir: global/ + packages-dir: global/ From c4c15d4b109b7870319818d0f17fc0b4357ff631 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 16 Mar 2023 13:01:28 -0700 Subject: [PATCH 36/83] docs: changelog for 2.10.4 (#4532) * docs: changelog for 2.10.4 Signed-off-by: Henry Schreiner * Apply suggestions from code review * Update docs/changelog.rst --------- Signed-off-by: Henry Schreiner --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb111c5f2..58cc40983 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,11 +21,47 @@ Changes: ``dec_ref()`` are now enabled by default again. `#4246 `_ +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 `_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 `_ + + Build system improvements: * Update clang-tidy to 15 in CI. `#4387 `_ +* Moved the linting framework over to Ruff. + `#4483 `_ + +Version 2.10.4 (Mar 16, 2023) +---------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 `_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 `_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 `_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 `_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 `_ Version 2.10.3 (Jan 3, 2023) ---------------------------- From cf7d2e6f15be7622515a77466e84bd9414737680 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 16 Mar 2023 14:33:34 -0700 Subject: [PATCH 37/83] Change `always_forkserver_on_unix()` to `use_multiprocessing_forkserver_on_linux()` (#4577) --- tests/conftest.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b60d3c956..ad5b47b4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,8 @@ import contextlib import difflib import gc import multiprocessing -import os import re +import sys import textwrap import traceback @@ -25,8 +25,9 @@ except Exception: @pytest.fixture(scope="session", autouse=True) -def always_forkserver_on_unix(): - if os.name == "nt": +def use_multiprocessing_forkserver_on_linux(): + if sys.platform != "linux": + # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. return # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 @@ -34,8 +35,6 @@ def always_forkserver_on_unix(): # It is actually a well-known pitfall, unfortunately without guard rails. # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, # visit the issuecomment link above for details). - # Windows does not have fork() and the associated pitfall, therefore it is best left - # running with defaults. multiprocessing.set_start_method("forkserver") From 286873ecf72f03df3a177a45d9dd4e9c678b2bdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:23:52 -0400 Subject: [PATCH 38/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.1 to 1.8.3 (#4584) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.1 to 1.8.3. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.1...v1.8.3) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 3d4d2c654..d5427c796 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.3 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.1 + uses: pypa/gh-action-pypi-publish@v1.8.3 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 5bbcba548a27baf8d6bfb7993e814da24b432dbf Mon Sep 17 00:00:00 2001 From: Konstantin Bespalov Date: Fri, 24 Mar 2023 06:01:09 +0100 Subject: [PATCH 39/83] use C++17 syntax to get rid of recursive template instantiations for concatenating type signatures (#4587) --- include/pybind11/detail/descr.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index e7a5e2c14..85259d488 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -143,11 +143,24 @@ constexpr descr concat(const descr &descr) { return descr; } +#if defined(PYBIND11_CPP17) +template +constexpr descr operator,(const descr &a, + const descr &b) { + return a + const_name(", ") + b; +} + +template +constexpr auto concat(const descr &d, const Args &...args) { + return (d, ..., args); +} +#else template constexpr auto concat(const descr &d, const Args &...args) -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +#endif template constexpr descr type_descr(const descr &descr) { From 66f12df03b33a3ea32c82b0d0af7c303cf17d50e Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 27 Mar 2023 10:59:56 -0400 Subject: [PATCH 40/83] chore: make #4587 use proper cpp17 feature macro (#4592) --- include/pybind11/detail/descr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 85259d488..635614b0d 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -143,7 +143,7 @@ constexpr descr concat(const descr &descr) { return descr; } -#if defined(PYBIND11_CPP17) +#ifdef __cpp_fold_expressions template constexpr descr operator,(const descr &a, const descr &b) { From 1e8b52a9acea1aa7a322fe6a87049c4828e439f9 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 27 Mar 2023 20:21:06 -0400 Subject: [PATCH 41/83] bugfix: allow noexcept lambdas in C++17. Fix #4565 (#4593) * bugfix: allow noexcept lambdas in CPP17. Fix #4565 * Remove unused code from test case * Fix clang-tidy error * Address reviewer comment --- include/pybind11/detail/common.h | 11 ++++++++++- tests/test_constants_and_functions.cpp | 3 +++ tests/test_constants_and_functions.py | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1032293d2..129039252 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -754,7 +754,16 @@ template struct remove_class { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template +struct remove_class { + using type = R(A...); +}; +template +struct remove_class { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template struct intrinsic_type { diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index 922375c5e..312edca9e 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -148,4 +148,7 @@ TEST_SUBMODULE(constants_and_functions, m) { py::arg_v("y", 42, ""), py::arg_v("z", default_value)); }); + + // test noexcept(true) lambda (#4565) + m.def("l1", []() noexcept(true) { return 0; }); } diff --git a/tests/test_constants_and_functions.py b/tests/test_constants_and_functions.py index 5da0b84b8..a1142461c 100644 --- a/tests/test_constants_and_functions.py +++ b/tests/test_constants_and_functions.py @@ -50,3 +50,7 @@ def test_function_record_leaks(): m.register_large_capture_with_invalid_arguments(m) with pytest.raises(RuntimeError): m.register_with_raising_repr(m, RaisingRepr()) + + +def test_noexcept_lambda(): + assert m.l1() == 0 From 654fe92652e6dc0eec80b1877b531aaab3a3e56c Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 27 Mar 2023 17:52:57 -0700 Subject: [PATCH 42/83] Introduce `get_python_state_dict()` for Python 3.12 compatibility. (#4570) * Introduce `get_python_state_dict()` * Conditional version bump for Python 3.12+ * Shuffle subexpressions to make the condition easier to understand (no change to logic). * Make pybind11 ABI version 5 the minimum for Python 3.12+ (as suggested by @Lalaland) * Add back condition for PYPY_VERSION, but keep it open for future PyPy versions. * Fall back to simple `|| defined(PYPY_VERSION)`. `PY_VERSION_HEX` does not appear to be meaningful with PyPy. --- include/pybind11/detail/internals.h | 57 +++++++++++++++++++++++---- include/pybind11/embed.h | 10 ++--- tests/test_embed/test_interpreter.cpp | 22 +++++------ 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 05e36ad18..8364ebd1e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -34,9 +34,18 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# if PY_VERSION_HEX >= 0x030C0000 +// Version bump for Python 3.12+, before first 3.12 beta release. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif #endif +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); @@ -421,6 +430,38 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + } + return static_cast(raw_ptr); +} + /// Return a reference to the current `internals` data PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); @@ -445,12 +486,12 @@ PYBIND11_NOINLINE internals &get_internals() { #endif error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_pp = static_cast(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // @@ -484,7 +525,7 @@ PYBIND11_NOINLINE internals &get_internals() { # endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5a175b134..caa14f4a0 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -243,16 +243,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr_ptr = capsule(builtins[id]); + // It could also be stashed in state_dict, so look there too: + if (object internals_obj + = get_internals_obj_from_state_dict(detail::get_python_state_dict())) { + internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj); } // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index e54e82270..c6c8a22d9 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -255,10 +255,10 @@ TEST_CASE("Add program dir to path using PyConfig") { } #endif -bool has_pybind11_internals_builtin() { - auto builtins = py::handle(PyEval_GetBuiltins()); - return builtins.contains(PYBIND11_INTERNALS_ID); -}; +bool has_state_dict_internals_obj() { + return bool( + py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict())); +} bool has_pybind11_internals_static() { auto **&ipp = py::detail::get_internals_pp(); @@ -268,7 +268,7 @@ bool has_pybind11_internals_static() { TEST_CASE("Restart the interpreter") { // Verify pre-restart state. REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast() == 3); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast() == 123); @@ -285,10 +285,10 @@ TEST_CASE("Restart the interpreter") { REQUIRE(Py_IsInitialized() == 1); // Internals are deleted after a restart. - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); pybind11::detail::get_internals(); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(reinterpret_cast(*py::detail::get_internals_pp()) == py::module_::import("external_module").attr("internals_at")().cast()); @@ -303,13 +303,13 @@ TEST_CASE("Restart the interpreter") { py::detail::get_internals(); *static_cast(ran) = true; }); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); REQUIRE_FALSE(ran); py::finalize_interpreter(); REQUIRE(ran); py::initialize_interpreter(); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); // C++ modules can be reloaded. @@ -331,7 +331,7 @@ TEST_CASE("Subinterpreter") { REQUIRE(m.attr("add")(1, 2).cast() == 3); } - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); /// Create and switch to a subinterpreter. @@ -341,7 +341,7 @@ TEST_CASE("Subinterpreter") { // Subinterpreters get their own copy of builtins. detail::get_internals() still // works by returning from the static variable, i.e. all interpreters share a single // global pybind11::internals; - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); // Modules tags should be gone. From 4ce05175d51a4685232452bdc1e9cbb13a240a65 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 30 Mar 2023 07:29:25 -0700 Subject: [PATCH 43/83] ci: Python 3.12 optional test job (#4575) * ci: Python 3.12 optional testing Signed-off-by: Henry Schreiner * Skip test_flaky_exception_failure_point_init() for Python 3.12.0a6 (similar to https://github.com/google/pywrapcc/commit/af5c6536aba77b1db99bb16995d3a684351efb93#diff-f46006e3f43ffb1dd5d6862005427f6620f4dcfb1fa2f883d8482550069eeecc). * Disable tests/test_embed/test_interpreter.cpp (broken with Python 3.12.0alpha6) ``` free(): invalid pointer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ test_embed is a Catch v2.13.9 host application. Run with -? for options ------------------------------------------------------------------------------- Custom PyConfig ------------------------------------------------------------------------------- /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:175 ............................................................................... /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:179: FAILED: {Unknown expression after the reported line} due to a fatal error condition: SIGABRT - Abort (abnormal termination) signal =============================================================================== test cases: 6 | 5 passed | 1 failed assertions: 1518 | 1517 passed | 1 failed ``` --------- Signed-off-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/upstream.yml | 60 ++++++++++++++++------------------ tests/test_exceptions.py | 5 +++ 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index a15861ee4..b7e14f72e 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -10,103 +10,101 @@ concurrency: cancel-in-progress: true env: - PIP_ONLY_BINARY: numpy + PIP_ONLY_BINARY: ":all:" # For cmake: VERBOSE: 1 jobs: standard: - name: "🐍 3.11 latest internals • ubuntu-latest • x64" + name: "🐍 3.12 latest • ubuntu-latest • x64" runs-on: ubuntu-latest + # Only runs when the 'python dev' label is selected if: "contains(github.event.pull_request.labels.*.name, 'python dev')" steps: - uses: actions/checkout@v3 - - name: Setup Python 3.11 + - name: Setup Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.11-dev" + python-version: "3.12-dev" - - name: Setup Boost (Linux) - if: runner.os == 'Linux' + - name: Setup Boost run: sudo apt-get install libboost-dev - name: Update CMake uses: jwlawson/actions-setup-cmake@v1.13 - - name: Prepare env + - name: Run pip installs run: | + python -m pip install --upgrade pip python -m pip install -r tests/requirements.txt - - name: Setup annotations on Linux - if: runner.os == 'Linux' - run: python -m pip install pytest-github-actions-annotate-failures + - name: Show platform info + run: | + python -m platform + cmake --version + pip list # First build - C++11 mode and inplace - name: Configure C++11 run: > - cmake -S . -B . + cmake -S . -B build11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 + -DCMAKE_BUILD_TYPE=Debug - name: Build C++11 - run: cmake --build . -j 2 + run: cmake --build build11 -j 2 - name: Python tests C++11 - run: cmake --build . --target pytest -j 2 + run: cmake --build build11 --target pytest -j 2 - - name: C++11 tests - run: cmake --build . --target cpptest -j 2 + # - name: C++11 tests + # run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 - run: cmake --build . --target test_cmake_build - - - name: Clean directory - run: git clean -fdx + run: cmake --build build11 --target test_cmake_build # Second build - C++17 mode and in a build directory - name: Configure C++17 run: > - cmake -S . -B build2 + cmake -S . -B build17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - ${{ matrix.args }} - ${{ matrix.args2 }} - name: Build - run: cmake --build build2 -j 2 + run: cmake --build build17 -j 2 - name: Python tests - run: cmake --build build2 --target pytest + run: cmake --build build17 --target pytest - - name: C++ tests - run: cmake --build build2 --target cpptest + # - name: C++ tests + # run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) run: > - cmake -S . -B build3 + cmake -S . -B build17max -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - ${{ matrix.args }} - name: Build (unstable ABI) - run: cmake --build build3 -j 2 + run: cmake --build build17max -j 2 - name: Python tests (unstable ABI) - run: cmake --build build3 --target pytest + run: cmake --build build17max --target pytest - name: Interface test - run: cmake --build build3 --target test_cmake_build + run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 7f80a5da5..1b4c89add 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -317,6 +317,11 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what +@pytest.mark.skipif( + # Intentionally very specific: + "sys.version_info == (3, 12, 0, 'alpha', 6)", + reason="WIP: https://github.com/python/cpython/issues/102594", +) @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") def test_flaky_exception_failure_point_init(): with pytest.raises(RuntimeError) as excinfo: From 7ab88d2e4fbb7fb21473f87b17b68a1f0e51cb4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 10:18:53 -0400 Subject: [PATCH 44/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.3 to 1.8.4 (#4602) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.3 to 1.8.4. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.3...v1.8.4) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index d5427c796..8ca6fd69e 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.8.4 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.3 + uses: pypa/gh-action-pypi-publish@v1.8.4 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From ed466da571fbc1d711351eb818e4bf82adb99eca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 15:52:41 -0400 Subject: [PATCH 45/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.4 to 1.8.5 (#4604) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.4 to 1.8.5. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.4...v1.8.5) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 8ca6fd69e..7f686b13f 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 071f35ab85aa5d8d91ee3c5735a601fd3221d783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:32:35 -0400 Subject: [PATCH 46/83] chore(deps): bump jwlawson/actions-setup-cmake from 1.13 to 1.14 (#4632) Bumps [jwlawson/actions-setup-cmake](https://github.com/jwlawson/actions-setup-cmake) from 1.13 to 1.14. - [Release notes](https://github.com/jwlawson/actions-setup-cmake/releases) - [Commits](https://github.com/jwlawson/actions-setup-cmake/compare/v1.13...v1.14) --- updated-dependencies: - dependency-name: jwlawson/actions-setup-cmake dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/configure.yml | 2 +- .github/workflows/upstream.yml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8befccdbc..a6a9a16fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: run: brew install boost - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Cache wheels if: runner.os == 'macOS' @@ -208,7 +208,7 @@ jobs: debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Valgrind cache if: matrix.valgrind @@ -474,7 +474,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure shell: bash @@ -763,7 +763,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.12.1 @@ -816,7 +816,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.12.1 @@ -867,7 +867,7 @@ jobs: python3 -m pip install -r tests/requirements.txt - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure C++20 run: > @@ -1000,7 +1000,7 @@ jobs: python-version: ${{ matrix.python }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Install ninja-build tool uses: seanmiddleditch/gha-setup-ninja@v3 @@ -1070,7 +1070,7 @@ jobs: run: clang++ --version - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Run pip installs run: | diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 29b041168..b469a69d1 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -55,7 +55,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: ${{ matrix.cmake }} diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index b7e14f72e..be3cd4050 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -33,7 +33,7 @@ jobs: run: sudo apt-get install libboost-dev - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.13 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Run pip installs run: | From 07725c28c0b1d68e044ccbc7820920c15c1f21e3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Apr 2023 00:19:21 -0700 Subject: [PATCH 47/83] Introduce `pybind11::detail::is_move_constructible` (#4631) To support the use case captured in the new test_vector_unique_ptr_member.cpp --- include/pybind11/cast.h | 4 +- include/pybind11/detail/init.h | 4 +- include/pybind11/detail/type_caster_base.h | 5 +- tests/CMakeLists.txt | 1 + tests/test_vector_unique_ptr_member.cpp | 56 ++++++++++++++++++++++ tests/test_vector_unique_ptr_member.py | 14 ++++++ 6 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 tests/test_vector_unique_ptr_member.cpp create mode 100644 tests/test_vector_unique_ptr_member.py diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9b013bc39..9d1ce15d4 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -964,7 +964,7 @@ struct move_always< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -975,7 +975,7 @@ struct move_if_unreferenced< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 9f71278c2..e21171688 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -175,7 +175,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { template void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible>::value, + static_assert(is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); @@ -190,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { template void construct(value_and_holder &v_h, Alias &&result, bool) { static_assert( - std::is_move_constructible>::value, + is_move_constructible>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias(std::move(result)); } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 0b710d7e4..34dcd26ce 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -827,6 +827,9 @@ using movable_cast_op_type template struct is_copy_constructible : std::is_copy_constructible {}; +template +struct is_move_constructible : std::is_move_constructible {}; + // Specialization for types that appear to be copy constructible but also look like stl containers // (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if // so, copy constructability depends on whether the value_type is copy constructible. @@ -994,7 +997,7 @@ protected: return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } - template ::value>> + template ::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b1cb222b4..1d0eb27c0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,7 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_union + test_vector_unique_ptr_member test_virtual_functions) # Invoking cmake with something like: diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp new file mode 100644 index 000000000..657743e4d --- /dev/null +++ b/tests/test_vector_unique_ptr_member.cpp @@ -0,0 +1,56 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace vector_unique_ptr_member { + +struct DataType {}; + +// Reduced from a use case in the wild. +struct VectorOwner { + static std::unique_ptr Create(std::size_t num_elems) { + return std::unique_ptr( + new VectorOwner(std::vector>(num_elems))); + } + + std::size_t data_size() const { return data_.size(); } + +private: + explicit VectorOwner(std::vector> data) : data_(std::move(data)) {} + + const std::vector> data_; +}; + +} // namespace vector_unique_ptr_member +} // namespace pybind11_tests + +namespace pybind11 { +namespace detail { + +template <> +struct is_copy_constructible + : std::false_type {}; + +template <> +struct is_move_constructible + : std::false_type {}; + +} // namespace detail +} // namespace pybind11 + +using namespace pybind11_tests::vector_unique_ptr_member; + +py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } + +// PYBIND11_SMART_HOLDER_TYPE_CASTERS(VectorOwner) + +TEST_SUBMODULE(vector_unique_ptr_member, m) { + py::class_(m, "VectorOwner") + .def_static("Create", &VectorOwner::Create) + .def("data_size", &VectorOwner::data_size); + + m.def("py_cast_VectorOwner_ptr", py_cast_VectorOwner_ptr); +} diff --git a/tests/test_vector_unique_ptr_member.py b/tests/test_vector_unique_ptr_member.py new file mode 100644 index 000000000..2da3d97c3 --- /dev/null +++ b/tests/test_vector_unique_ptr_member.py @@ -0,0 +1,14 @@ +import pytest + +from pybind11_tests import vector_unique_ptr_member as m + + +@pytest.mark.parametrize("num_elems", range(3)) +def test_create(num_elems): + vo = m.VectorOwner.Create(num_elems) + assert vo.data_size() == num_elems + + +def test_cast(): + vo = m.VectorOwner.Create(0) + assert m.py_cast_VectorOwner_ptr(vo) is vo From ff7f5dfca32224b65524a9cebe3daa8f2d328a35 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Apr 2023 10:25:57 -0700 Subject: [PATCH 48/83] 1. Fully test unstable ABI (#4635) 2. Selectively exercise cmake `-DPYBIND11_TEST_OVERRIDE`: ubuntu, macos, windows Extra work added to quick jobs, based on timings below, to not increase the GHA start-to-last-job-finished time. ``` Duration ^ Number of pytest runs ^ ^ Job identifier ^ ^ ^ 0:03:48.024227 1 1___3___Clang_3.6___C++11___x64.txt 0:03:58.992814 1 2___3___Clang_3.7___C++11___x64.txt 0:04:25.758942 1 1___3.7___Debian___x86____Install.txt 0:04:50.148276 1 4___3___Clang_7___C++11___x64.txt 0:04:55.784558 1 13___3___Clang_15___C++20___x64.txt 0:04:57.048754 1 6___3___Clang_dev___C++11___x64.txt 0:05:00.485181 1 7___3___Clang_5___C++14___x64.txt 0:05:03.744964 1 2___3___almalinux8___x64.txt 0:05:06.222752 1 5___3___Clang_9___C++11___x64.txt 0:05:11.767022 1 2___3___GCC_7___C++17__x64.txt 0:05:18.634930 1 2___3.11__deadsnakes____x64.txt 0:05:22.810995 1 1___3___GCC_7___C++11__x64.txt 0:05:25.275317 1 12___3___Clang_14___C++20___x64.txt 0:05:32.058174 1 5___3___GCC_10___C++17__x64.txt 0:05:39.381351 1 7___3___GCC_12___C++20__x64.txt 0:05:40.502252 1 8___3___Clang_10___C++17___x64.txt 0:05:59.344905 1 3___3___Clang_3.9___C++11___x64.txt 0:06:10.825147 1 6___3___GCC_11___C++20__x64.txt 0:06:20.655443 1 3___3___almalinux9___x64.txt 0:06:22.472061 1 3___3___GCC_8___C++14__x64.txt 0:06:42.647406 1 11___3___Clang_13___C++20___x64.txt 0:06:53.352720 1 1___3.10___CUDA_11.7___Ubuntu_22.04.txt 0:07:07.357801 1 2___3.7___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=14.txt 0:07:09.057603 1 1___3___centos7___x64.txt 0:07:15.546282 1 1___3.8___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=17.txt 0:07:22.566022 1 4___3___GCC_8___C++17__x64.txt 0:08:13.592674 1 2___3.9___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=20.txt 0:08:16.422768 1 9___3___Clang_11___C++20___x64.txt 0:08:21.168457 1 3___3.8___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=17.txt 0:08:27.129468 1 10___3___Clang_12___C++20___x64.txt 0:09:35.045470 1 1___3.10___windows-latest___clang-latest.txt 0:09:57.361843 1 1___3.9___MSVC_2022_C++20___x64.txt 0:10:35.187767 1 1___3.6___MSVC_2019___x86.txt 0:11:14.691200 4 2___3.9___ubuntu-20.04___x64.txt 0:11:37.701167 1 1_macos-latest___brew_install_llvm.txt 0:11:38.688299 4 4___3.11___ubuntu-20.04___x64.txt 0:11:52.720216 1 4___3.9___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=20.txt 0:13:23.456591 4 6___pypy-3.8___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:13:25.863592 2 1___3___ICC_latest___x64.txt 0:13:32.411758 3 9___3.9___windows-2022___x64.txt 0:13:45.473377 4 3___3.10___ubuntu-20.04___x64.txt 0:13:55.366447 4 5___pypy-3.7___ubuntu-20.04___x64.txt 0:13:57.969502 3 10___3.10___windows-2022___x64.txt 0:14:19.837475 3 11___3.11___windows-2022___x64.txt 0:14:33.316770 4 1___3.6___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON_-DCMA.txt 0:15:34.449278 4 22___3.6___windows-2019___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:16:25.189055 2 1___3.9-dbg__deadsnakes____Valgrind___x64.txt 0:17:20.956667 4 15___3.6___macos-latest___x64.txt 0:17:27.513891 4 23___3.9___windows-2019___x64.txt 0:17:58.783286 3 8___3.6___windows-2022___x64.txt 0:18:25.917828 4 7___pypy-3.9___ubuntu-20.04___x64.txt 0:19:17.399820 3 13___pypy-3.8___windows-2022___x64.txt 0:19:45.002122 3 12___pypy-3.7___windows-2022___x64.txt 0:20:03.201926 4 16___3.9___macos-latest___x64.txt 0:20:15.415178 4 17___3.10___macos-latest___x64.txt 0:20:20.263216 4 20___pypy-3.8___macos-latest___x64.txt 0:20:31.998226 3 1___3___windows-latest___mingw64.txt 0:20:40.812286 4 18___3.11___macos-latest___x64.txt 0:22:47.714749 4 19___pypy-3.7___macos-latest___x64.txt 0:23:04.435859 3 2___3___windows-latest___mingw32.txt 0:25:48.719597 3 14___pypy-3.9___windows-2022___x64.txt 0:26:01.211688 4 21___pypy-3.9___macos-latest___x64.txt 0:28:19.971015 1 1___3___CentOS7__PGI_22.9___x64.txt ``` --- .github/workflows/ci.yml | 52 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6a9a16fa..c88f0797f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" ${{ matrix.args }} - name: Build (unstable ABI) @@ -497,6 +496,24 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + shell: bash + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial --target pytest # Testing on ICC using the oneAPI apt repo icc: @@ -889,6 +906,21 @@ jobs: - name: Interface test C++20 run: cmake --build build --target test_cmake_build + - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest + mingw: name: "🐍 3 • windows-latest • ${{ matrix.sys }}" runs-on: windows-latest @@ -1105,5 +1137,23 @@ jobs: - name: Interface test run: cmake --build . --target test_cmake_build -j 2 + - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest -j 2 + - name: Clean directory run: git clean -fdx From 6de6191a0c56a9b0d31ac1c32778081a9e2b0311 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Apr 2023 14:03:24 -0700 Subject: [PATCH 49/83] Use `std::hash`, `std::equal_to` everywhere **except when libc++ is in use** (#4319) * Try using `std::hash`, `std::equal_to` everywhere. From PR #4316 we know that types in the unnamed namespace in different translation units do not compare equal, as desired. But do types in named namespaces compare equal, as desired? * Revert "Try using `std::hash`, `std::equal_to` everywhere." This reverts commit a06949a9265014b3c581396c4f37c45ccc03dea6. * Use "our own name-based hash and equality functions" for `std::type_index` only under macOS, based on results shown under https://github.com/pybind/pybind11/pull/4316#issuecomment-1305097879 * Patch in PR #4313: Minimal reproducer for clash when binding types defined in the unnamed namespace. * test_unnamed_namespace_b xfail for clang * `PYBIND11_INTERNALS_VERSION 5` * Add a note to docs/classes.rst * For compatibility with Google-internal testing, test_unnamed_namespace_a & test_unnamed_namespace_b need to work when imported in any order. * Trying "__GLIBCXX__ or Windows", based on observations from Google-internal testing. * Try _LIBCPP_VERSION * Account for libc++ behavior in tests and documentation. * Adjust expectations for Windows Clang (and make code less redundant). * Add WindowsClang to ci.yml Added block transferred from PR #4321 * Add clang-latest to name that appears in the GitHub Actions web view. * Tweak the note in classes.rst again. * Add `pip install --upgrade pip`, Show env, cosmetic changes Already tested under PR #4321 * Add macos_brew_install_llvm to ci.yml Added block transferred from PR #4324 * `test_cross_module_exception_translator` xfail 'Homebrew Clang' * Revert back to base version of .github/workflows/ci.yml (the ci.yml changes were merged under #4323 and #4326) * Fixes for ruff * Make updated condition in internals.h dependent on ABI version. * Remove PYBIND11_TEST_OVERRIDE when testing with PYBIND11_INTERNALS_VERSION=10000000 * Selectively exercise cmake `-DPYBIND11_TEST_OVERRIDE`: ubuntu, macos, windows Extra work added to quick jobs, based on timings below, to not increase the GHA start-to-last-job-finished time. ``` Duration ^ Number of pytest runs ^ ^ Job identifier ^ ^ ^ 0:03:48.024227 1 1___3___Clang_3.6___C++11___x64.txt 0:03:58.992814 1 2___3___Clang_3.7___C++11___x64.txt 0:04:25.758942 1 1___3.7___Debian___x86____Install.txt 0:04:50.148276 1 4___3___Clang_7___C++11___x64.txt 0:04:55.784558 1 13___3___Clang_15___C++20___x64.txt 0:04:57.048754 1 6___3___Clang_dev___C++11___x64.txt 0:05:00.485181 1 7___3___Clang_5___C++14___x64.txt 0:05:03.744964 1 2___3___almalinux8___x64.txt 0:05:06.222752 1 5___3___Clang_9___C++11___x64.txt 0:05:11.767022 1 2___3___GCC_7___C++17__x64.txt 0:05:18.634930 1 2___3.11__deadsnakes____x64.txt 0:05:22.810995 1 1___3___GCC_7___C++11__x64.txt 0:05:25.275317 1 12___3___Clang_14___C++20___x64.txt 0:05:32.058174 1 5___3___GCC_10___C++17__x64.txt 0:05:39.381351 1 7___3___GCC_12___C++20__x64.txt 0:05:40.502252 1 8___3___Clang_10___C++17___x64.txt 0:05:59.344905 1 3___3___Clang_3.9___C++11___x64.txt 0:06:10.825147 1 6___3___GCC_11___C++20__x64.txt 0:06:20.655443 1 3___3___almalinux9___x64.txt 0:06:22.472061 1 3___3___GCC_8___C++14__x64.txt 0:06:42.647406 1 11___3___Clang_13___C++20___x64.txt 0:06:53.352720 1 1___3.10___CUDA_11.7___Ubuntu_22.04.txt 0:07:07.357801 1 2___3.7___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=14.txt 0:07:09.057603 1 1___3___centos7___x64.txt 0:07:15.546282 1 1___3.8___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=17.txt 0:07:22.566022 1 4___3___GCC_8___C++17__x64.txt 0:08:13.592674 1 2___3.9___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=20.txt 0:08:16.422768 1 9___3___Clang_11___C++20___x64.txt 0:08:21.168457 1 3___3.8___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=17.txt 0:08:27.129468 1 10___3___Clang_12___C++20___x64.txt 0:09:35.045470 1 1___3.10___windows-latest___clang-latest.txt 0:09:57.361843 1 1___3.9___MSVC_2022_C++20___x64.txt 0:10:35.187767 1 1___3.6___MSVC_2019___x86.txt 0:11:14.691200 4 2___3.9___ubuntu-20.04___x64.txt 0:11:37.701167 1 1_macos-latest___brew_install_llvm.txt 0:11:38.688299 4 4___3.11___ubuntu-20.04___x64.txt 0:11:52.720216 1 4___3.9___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=20.txt 0:13:23.456591 4 6___pypy-3.8___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:13:25.863592 2 1___3___ICC_latest___x64.txt 0:13:32.411758 3 9___3.9___windows-2022___x64.txt 0:13:45.473377 4 3___3.10___ubuntu-20.04___x64.txt 0:13:55.366447 4 5___pypy-3.7___ubuntu-20.04___x64.txt 0:13:57.969502 3 10___3.10___windows-2022___x64.txt 0:14:19.837475 3 11___3.11___windows-2022___x64.txt 0:14:33.316770 4 1___3.6___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON_-DCMA.txt 0:15:34.449278 4 22___3.6___windows-2019___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:16:25.189055 2 1___3.9-dbg__deadsnakes____Valgrind___x64.txt 0:17:20.956667 4 15___3.6___macos-latest___x64.txt 0:17:27.513891 4 23___3.9___windows-2019___x64.txt 0:17:58.783286 3 8___3.6___windows-2022___x64.txt 0:18:25.917828 4 7___pypy-3.9___ubuntu-20.04___x64.txt 0:19:17.399820 3 13___pypy-3.8___windows-2022___x64.txt 0:19:45.002122 3 12___pypy-3.7___windows-2022___x64.txt 0:20:03.201926 4 16___3.9___macos-latest___x64.txt 0:20:15.415178 4 17___3.10___macos-latest___x64.txt 0:20:20.263216 4 20___pypy-3.8___macos-latest___x64.txt 0:20:31.998226 3 1___3___windows-latest___mingw64.txt 0:20:40.812286 4 18___3.11___macos-latest___x64.txt 0:22:47.714749 4 19___pypy-3.7___macos-latest___x64.txt 0:23:04.435859 3 2___3___windows-latest___mingw32.txt 0:25:48.719597 3 14___pypy-3.9___windows-2022___x64.txt 0:26:01.211688 4 21___pypy-3.9___macos-latest___x64.txt 0:28:19.971015 1 1___3___CentOS7__PGI_22.9___x64.txt ``` * Update skipif for Python 3.12a7 (the WIP needs to be handled in a separate PR). --- .github/workflows/upstream.yml | 1 - docs/classes.rst | 10 ++++++++ include/pybind11/detail/internals.h | 3 ++- tests/CMakeLists.txt | 2 ++ tests/test_exceptions.py | 2 +- tests/test_unnamed_namespace_a.cpp | 38 +++++++++++++++++++++++++++++ tests/test_unnamed_namespace_a.py | 34 ++++++++++++++++++++++++++ tests/test_unnamed_namespace_b.cpp | 13 ++++++++++ tests/test_unnamed_namespace_b.py | 5 ++++ 9 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/test_unnamed_namespace_a.cpp create mode 100644 tests/test_unnamed_namespace_a.py create mode 100644 tests/test_unnamed_namespace_b.cpp create mode 100644 tests/test_unnamed_namespace_b.py diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be3cd4050..be643ddfd 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -95,7 +95,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - name: Build (unstable ABI) run: cmake --build build17max -j 2 diff --git a/docs/classes.rst b/docs/classes.rst index c0c53135b..52cd52da3 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -58,6 +58,16 @@ interactive Python session demonstrating this example is shown below: Static member functions can be bound in the same way using :func:`class_::def_static`. +.. note:: + + Binding C++ types in unnamed namespaces (also known as anonymous namespaces) + works reliably on many platforms, but not all. The `XFAIL_CONDITION` in + tests/test_unnamed_namespace_a.py encodes the currently known conditions. + For background see `#4319 `_. + If portability is a concern, it is therefore not recommended to bind C++ + types in unnamed namespaces. It will be safest to manually pick unique + namespace names. + Keyword and default arguments ============================= It is possible to specify keyword and default arguments using the syntax diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 8364ebd1e..aaa7f8686 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -123,7 +123,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d0eb27c0..11ac2215b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,8 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_union + test_unnamed_namespace_a + test_unnamed_namespace_b test_vector_unique_ptr_member test_virtual_functions) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 1b4c89add..a92ab59a3 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -319,7 +319,7 @@ def test_error_already_set_what_with_happy_exceptions( @pytest.mark.skipif( # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 6)", + "sys.version_info == (3, 12, 0, 'alpha', 7)", reason="WIP: https://github.com/python/cpython/issues/102594", ) @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") diff --git a/tests/test_unnamed_namespace_a.cpp b/tests/test_unnamed_namespace_a.cpp new file mode 100644 index 000000000..2152e64bd --- /dev/null +++ b/tests/test_unnamed_namespace_a.cpp @@ -0,0 +1,38 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_a, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_a_any_struct"); + } else { + m.attr("unnamed_namespace_a_any_struct") = py::none(); + } + m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; + m.attr("defined_WIN32_or__WIN32") = +#if defined(WIN32) || defined(_WIN32) + true; +#else + false; +#endif + m.attr("defined___clang__") = +#if defined(__clang__) + true; +#else + false; +#endif + m.attr("defined__LIBCPP_VERSION") = +#if defined(_LIBCPP_VERSION) + true; +#else + false; +#endif + m.attr("defined___GLIBCXX__") = +#if defined(__GLIBCXX__) + true; +#else + false; +#endif +} diff --git a/tests/test_unnamed_namespace_a.py b/tests/test_unnamed_namespace_a.py new file mode 100644 index 000000000..9d9856c5a --- /dev/null +++ b/tests/test_unnamed_namespace_a.py @@ -0,0 +1,34 @@ +import pytest + +from pybind11_tests import unnamed_namespace_a as m +from pybind11_tests import unnamed_namespace_b as mb + +XFAIL_CONDITION = ( + "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" + " or " + "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" + " and " + "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" +) +XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False) +@pytest.mark.parametrize( + "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct] +) +def test_have_class_any_struct(any_struct): + assert any_struct is not None + + +def test_have_at_least_one_class_any_struct(): + assert ( + m.unnamed_namespace_a_any_struct is not None + or mb.unnamed_namespace_b_any_struct is not None + ) + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True) +def test_have_both_class_any_struct(): + assert m.unnamed_namespace_a_any_struct is not None + assert mb.unnamed_namespace_b_any_struct is not None diff --git a/tests/test_unnamed_namespace_b.cpp b/tests/test_unnamed_namespace_b.cpp new file mode 100644 index 000000000..f97757a7d --- /dev/null +++ b/tests/test_unnamed_namespace_b.cpp @@ -0,0 +1,13 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_b, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_b_any_struct"); + } else { + m.attr("unnamed_namespace_b_any_struct") = py::none(); + } +} diff --git a/tests/test_unnamed_namespace_b.py b/tests/test_unnamed_namespace_b.py new file mode 100644 index 000000000..4bcaa7a6c --- /dev/null +++ b/tests/test_unnamed_namespace_b.py @@ -0,0 +1,5 @@ +from pybind11_tests import unnamed_namespace_b as m + + +def test_have_attr_any_struct(): + assert hasattr(m, "unnamed_namespace_b_any_struct") From 3f366ff888c846a3bc39af36615f99dc9e424e51 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 27 Apr 2023 07:24:48 -0700 Subject: [PATCH 50/83] Remove stray comment. (Oversight in PR #4631. Noticed by chance.) (#4641) --- tests/test_vector_unique_ptr_member.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp index 657743e4d..680cf9a6d 100644 --- a/tests/test_vector_unique_ptr_member.cpp +++ b/tests/test_vector_unique_ptr_member.cpp @@ -45,8 +45,6 @@ using namespace pybind11_tests::vector_unique_ptr_member; py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } -// PYBIND11_SMART_HOLDER_TYPE_CASTERS(VectorOwner) - TEST_SUBMODULE(vector_unique_ptr_member, m) { py::class_(m, "VectorOwner") .def_static("Create", &VectorOwner::Create) From 5e946c2fa5b3c3170d702f57311492a8f838444e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 14:32:32 -0400 Subject: [PATCH 51/83] chore(deps): update pre-commit hooks (#4605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/psf/black: 23.1.0 → 23.3.0](https://github.com/psf/black/compare/23.1.0...23.3.0) - [github.com/Lucas-C/pre-commit-hooks: v1.4.2 → v1.5.1](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.4.2...v1.5.1) - [github.com/charliermarsh/ruff-pre-commit: v0.0.254 → v0.0.260](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.254...v0.0.260) - [github.com/PyCQA/pylint: v2.16.4 → v3.0.0a6](https://github.com/PyCQA/pylint/compare/v2.16.4...v3.0.0a6) - [github.com/pre-commit/mirrors-mypy: v1.0.1 → v1.1.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.0.1...v1.1.1) - [github.com/codespell-project/codespell: v2.2.2 → v2.2.4](https://github.com/codespell-project/codespell/compare/v2.2.2...v2.2.4) - [github.com/pre-commit/mirrors-clang-format: v15.0.7 → v16.0.0](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.7...v16.0.0) * style: pre-commit fixes * style: fix issues * Update tests/test_call_policies.py * Update tests/test_call_policies.py * fix: ignore code in file * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++---- include/pybind11/detail/type_caster_base.h | 6 +-- pyproject.toml | 1 + tests/test_iostream.py | 58 +++++++++++----------- tests/test_stl.py | 2 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c4316192..51b6f0a41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "23.1.0" # Keep in sync with blacken-docs + rev: "23.3.0" # Keep in sync with blacken-docs hooks: - id: black @@ -51,11 +51,11 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.1.0 # keep in sync with black hook + - black==23.3.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.4.2" + rev: "v1.5.1" hooks: - id: remove-tabs @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.254 + rev: v0.0.263 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.4" + rev: "v3.0.0a6" hooks: - id: pylint files: ^pybind11 @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.0.1" + rev: "v1.1.1" hooks: - id: mypy args: [] @@ -120,11 +120,11 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.2" + rev: "v2.2.4" hooks: - id: codespell exclude: ".supp$" - args: ["-x", ".codespell-ignore-lines"] + args: ["-x.codespell-ignore-lines", "-Lccompiler"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.7" + rev: "v16.0.0" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 34dcd26ce..8f3d0f379 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -258,9 +258,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; diff --git a/pyproject.toml b/pyproject.toml index 96cf84265..e3655aca7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,3 +93,4 @@ isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests [tool.ruff.per-file-ignores] "tests/**" = ["EM", "N"] +"tests/test_call_policies.py" = ["PLC1901"] diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 79b2a2b8c..d283eb152 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -9,16 +9,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg @@ -30,7 +30,7 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset0(capsys): @@ -40,7 +40,7 @@ def test_captured_utf8_2byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset1(capsys): @@ -50,7 +50,7 @@ def test_captured_utf8_2byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset0(capsys): @@ -60,7 +60,7 @@ def test_captured_utf8_3byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset1(capsys): @@ -70,7 +70,7 @@ def test_captured_utf8_3byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset2(capsys): @@ -80,7 +80,7 @@ def test_captured_utf8_3byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset0(capsys): @@ -90,7 +90,7 @@ def test_captured_utf8_4byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset1(capsys): @@ -100,7 +100,7 @@ def test_captured_utf8_4byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset2(capsys): @@ -110,7 +110,7 @@ def test_captured_utf8_4byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset3(capsys): @@ -120,7 +120,7 @@ def test_captured_utf8_4byte_offset3(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_guard_capture(capsys): @@ -128,7 +128,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_series_captured(capture): @@ -145,7 +145,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -164,15 +164,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" - assert stream.getvalue() == "" + assert not stderr + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -182,16 +182,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -221,13 +221,13 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream), m.ostream_redirect(): m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stream.getvalue() == msg stream = StringIO() @@ -235,7 +235,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() def test_redirect_err(capfd): @@ -248,7 +248,7 @@ def test_redirect_err(capfd): m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr assert stream.getvalue() == msg2 @@ -262,8 +262,8 @@ def test_redirect_both(capfd): m.raw_output(msg) m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg assert stream2.getvalue() == msg2 diff --git a/tests/test_stl.py b/tests/test_stl.py index b7a0d1c0d..2d9dcc89f 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -367,7 +367,7 @@ def test_issue_1561(): """check fix for issue #1561""" bar = m.Issue1561Outer() bar.list = [m.Issue1561Inner("bar")] - bar.list + assert bar.list assert bar.list[0].data == "bar" From 956390a87f6e2ee6ea20438f3faa05222415018f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Apr 2023 10:15:45 -0400 Subject: [PATCH 52/83] fix(cmake): only define lto if CMAKE's IPO setting is unset (#4643) Signed-off-by: Henry Schreiner --- tools/pybind11Common.cmake | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 0c985bc8e..e4ff1e408 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -5,8 +5,8 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +20,7 @@ Adds the following functions:: # CMake 3.10 has an include_guard command, but we can't use that yet # include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) +if(TARGET pybind11::pybind11) return() endif() @@ -372,11 +372,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- From da91926295638b2b1c0c85a568a2d66ca3646565 Mon Sep 17 00:00:00 2001 From: biergaizi Date: Mon, 1 May 2023 14:14:52 +0000 Subject: [PATCH 53/83] fix: remove -stdlib=libc++ from setup helpers, not needed on modern Pythons (#4639) * Inject -stdlib=libc++ on macOS only when it's supported, close #4637. On macOS, by default, pybind11 currently unconditionally set the compiler flag "-stdlib=libc++" in Pybind11Extension.__init__(), regardless of which compiler is used. This flag is required for clang, but is invalid for GCC. If GCC is used, it causes compilation failures in all Python projects that use pybind11, with the error message: arm64-apple-darwin22-gcc: error: unrecognized command-line option -stdlib=libc++. This commit uses has_flag() to detect whether "-stdlib=libc++" on macOS, and injects this flag from build_ext.build_extensions(), rather than setting it unconditionally. Signed-off-by: Yifeng Li * revert: just remove flags --------- Signed-off-by: Yifeng Li Co-authored-by: Henry Schreiner --- pybind11/setup_helpers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 0fb4679e4..cb279f27e 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -144,7 +144,6 @@ class Pybind11Extension(_Extension): # type: ignore[misc] self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -154,11 +153,7 @@ class Pybind11Extension(_Extension): # type: ignore[misc] c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: From dff75a62bd33ff35312aedf696c17f0300301b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:34:51 -0400 Subject: [PATCH 54/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.5 to 1.8.6 (#4650) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.5 to 1.8.6. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.5...v1.8.6) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7f686b13f..6d9be3b1d 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From b3e88ecf894269b6b3fe822a7551c95247a3c47e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:35:21 -0400 Subject: [PATCH 55/83] chore(deps): update pre-commit hooks (#4648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.1.1 → v1.2.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.1.1...v1.2.0) - [github.com/pre-commit/mirrors-clang-format: v16.0.0 → v16.0.2](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.0...v16.0.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51b6f0a41..aca9b3ca2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.1.1" + rev: "v1.2.0" hooks: - id: mypy args: [] @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.0" + rev: "v16.0.2" hooks: - id: clang-format types_or: [c++, c, cuda] From f70165463328c218d118204efc13aac93783d17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 5 May 2023 07:39:05 +0200 Subject: [PATCH 56/83] Introduce recursive_container_traits (#4623) * Testing * Similar fix for std::vector * Fix infinite recursion check: 1) Apply to is_copy_assignable additionally 2) Check infinite recursion for map-like types * style: pre-commit fixes * Optional commit that demonstrates the limitations of this PR * Fix positioning of container bindings The bindings were previously in a block that was only activated if numpy was available. * Suggestions from code review: API side * Suggestions from code review: Test side * Suggestions from code review 1) Renaming: is_recursive_container and MutuallyRecursiveContainerPair(MV|VM) 2) Avoid ambiguous specializations of is_recursive_container * Some little fixes * Reordering of structs * Add recursive checks for is_move_constructible * Static testing for pybind11 type traits * More precise checking of recursive types Instead of a trait `is_recursive_container`, use a trait `recursive_container_traits` with dependent type `recursive_container_traits::type_to_check_recursively`. So, instead of just checking if a type is recursive and then trying to somehow deal with it, recursively-defined traits such as is_move_constructible can now directly ask this trait where the recursion should proceed. * Review suggestions 1. Use std::conditional 2. Fix typo * Remove leftover include from test --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/type_caster_base.h | 205 +++++++++++++++--- include/pybind11/stl_bind.h | 8 +- tests/test_copy_move.cpp | 238 +++++++++++++++++++++ tests/test_stl_binders.cpp | 44 ++++ tests/test_stl_binders.py | 18 ++ 5 files changed, 485 insertions(+), 28 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8f3d0f379..16387506c 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -822,26 +822,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; + +template +struct impl_recursive_container_traits< + Container, + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + // std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when // T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; -template -struct is_move_constructible : std::is_move_constructible {}; - -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. -template -struct is_copy_constructible< - Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -852,14 +1005,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 7cfd996dd..49f1b7782 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -61,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp index 28c244564..f54733550 100644 --- a/tests/test_copy_move.cpp +++ b/tests/test_copy_move.cpp @@ -13,6 +13,8 @@ #include "constructor_stats.h" #include "pybind11_tests.h" +#include + template struct empty { static const derived &get_one() { return instance_; } @@ -293,3 +295,239 @@ TEST_SUBMODULE(copy_move_policies, m) { // Make sure that cast from pytype rvalue to other pytype works m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast(); }); } + +/* + * Rest of the file: + * static_assert based tests for pybind11 adaptations of + * std::is_move_constructible, std::is_copy_constructible and + * std::is_copy_assignable (no adaptation of std::is_move_assignable). + * Difference between pybind11 and std traits: pybind11 traits will also check + * the contained value_types. + */ + +struct NotMovable { + NotMovable() = default; + NotMovable(NotMovable const &) = default; + NotMovable(NotMovable &&) = delete; + NotMovable &operator=(NotMovable const &) = default; + NotMovable &operator=(NotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyable { + NotCopyable() = default; + NotCopyable(NotCopyable const &) = delete; + NotCopyable(NotCopyable &&) = default; + NotCopyable &operator=(NotCopyable const &) = delete; + NotCopyable &operator=(NotCopyable &&) = default; +}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovable { + NotCopyableNotMovable() = default; + NotCopyableNotMovable(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable(NotCopyableNotMovable &&) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct RecursiveVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct RecursiveMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index ca9630bd1..1681760aa 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -70,6 +70,44 @@ NestMap *times_hundred(int n) { return m; } +/* + * Recursive data structures as test for issue #4623 + */ +struct RecursiveVector : std::vector { + using Parent = std::vector; + using Parent::Parent; +}; + +struct RecursiveMap : std::map { + using Parent = std::map; + using Parent::Parent; +}; + +/* + * Pybind11 does not catch more complicated recursion schemes, such as mutual + * recursion. + * In that case custom recursive_container_traits specializations need to be added, + * thus manually telling pybind11 about the recursion. + */ +struct MutuallyRecursiveContainerPairMV; +struct MutuallyRecursiveContainerPairVM; + +struct MutuallyRecursiveContainerPairMV : std::map {}; +struct MutuallyRecursiveContainerPairVM : std::vector {}; + +namespace pybind11 { +namespace detail { +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(stl_binders, m) { // test_vector_int py::bind_vector>(m, "VectorInt", py::buffer_protocol()); @@ -129,6 +167,12 @@ TEST_SUBMODULE(stl_binders, m) { m, "VectorUndeclStruct", py::buffer_protocol()); }); + // Bind recursive container types + py::bind_vector(m, "RecursiveVector"); + py::bind_map(m, "RecursiveMap"); + py::bind_map(m, "MutuallyRecursiveContainerPairMV"); + py::bind_vector(m, "MutuallyRecursiveContainerPairVM"); + // The rest depends on numpy: try { py::module_::import("numpy"); diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 7dca742a9..e002f5b67 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -335,3 +335,21 @@ def test_map_view_types(): assert type(unordered_map_string_double.items()) is items_type assert type(map_string_double_const.items()) is items_type assert type(unordered_map_string_double_const.items()) is items_type + + +def test_recursive_vector(): + recursive_vector = m.RecursiveVector() + recursive_vector.append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + # Can't use len() since test_stl_binders.cpp does not include stl.h, + # so the necessary conversion is missing + assert recursive_vector[0].count(m.RecursiveVector()) == 2 + + +def test_recursive_map(): + recursive_map = m.RecursiveMap() + recursive_map[100] = m.RecursiveMap() + recursive_map[100][101] = m.RecursiveMap() + recursive_map[100][102] = m.RecursiveMap() + assert list(recursive_map[100].keys()) == [101, 102] From 90312a6ee8de69f582e165b25338ead4f1a6ccc2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 7 May 2023 10:15:53 -0700 Subject: [PATCH 57/83] Add `type_caster` (#4601) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `type_caster` (tests are still incomplete). * Fix oversight (`const PyObject *`). * Ensure `type_caster` only works for `PyObject *` * Move `is_same_ignoring_cvref` into `detail` namespace. * Add test_cast_nullptr * Change is_same_ignoring_cvref from variable template to using. ``` test_type_caster_pyobject_ptr.cpp:8:23: error: variable templates only available with ‘-std=c++14’ or ‘-std=gnu++14’ [-Werror] 8 | static constexpr bool is_same_ignoring_cvref = std::is_same, U>::value; | ^~~~~~~~~~~~~~~~~~~~~~ ``` * Remove `return_value_policy::reference_internal` `keep_alive` feature (because of doubts about it actually being useful). * Add missing test, fix bug (missing `throw error_already_set();`), various cosmetic changes. * Move `type_caster` from test to new include (pybind11/type_caster_pyobject_ptr.h) * Add new header file to CMakeLists.txt and tests/extra_python_package/test_files.py * Backport changes from https://github.com/google/pywrapcc/pull/30021 to https://github.com/pybind/pybind11/pull/4601 * Fix oversight in test (to resolve a valgrind leak detection error) and add a related comment in cast.h. No production code changes. Make tests more sensitive by using `ValueHolder` instead of empty tuples and dicts. Manual leak checks with `while True:` & top command repeated for all tests. * Add tests for interop with stl.h `list_caster` (No production code changes.) * Bug fix in test. Minor comment enhancements. * Change `type_caster::name` to `object`, as suggested by @Skylion007 * Expand comment for the new `T cast(const handle &handle)` [`T` = `PyObject *`] * Add `T cast(object &&obj)` overload as suggested by @Skylion007 The original suggestion leads to `error: call to 'cast' is ambiguous` (full error message below), therefore SFINAE guarding is needed. ``` clang++ -o pybind11/tests/test_type_caster_pyobject_ptr.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:1: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:13: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/detail/class.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/attr.h:14: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1165:12: error: call to 'cast' is ambiguous return pybind11::cast(std::move(*this)); ^~~~~~~~~~~~~~~~~ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:109:70: note: in instantiation of function template specialization 'pybind11::object::cast<_object *>' requested here return hfunc.f(std::forward(args)...).template cast(); ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:103:16: note: in instantiation of member function 'pybind11::detail::type_caster>::load(pybind11::handle, bool)::func_wrapper::operator()' requested here struct func_wrapper { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1456:47: note: in instantiation of member function 'pybind11::detail::type_caster>::load' requested here if ((... || !std::get(argcasters).load(call.args[Is], call.args_convert[Is]))) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1434:50: note: in instantiation of function template specialization 'pybind11::detail::argument_loader &, int>::load_impl_sequence<0UL, 1UL>' requested here bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:227:33: note: in instantiation of member function 'pybind11::detail::argument_loader &, int>::load_args' requested here if (!args_converter.load_args(call)) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:101:9: note: in instantiation of function template specialization 'pybind11::cpp_function::initialize<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), _object *, const std::function<_object *(int)> &, int, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy>' requested here initialize( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1163:22: note: in instantiation of function template specialization 'pybind11::cpp_function::cpp_function<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy, void>' requested here cpp_function func(std::forward(f), ^ /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:48:7: note: in instantiation of function template specialization 'pybind11::module_::def<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::return_value_policy>' requested here m.def( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1077:3: note: candidate function [with T = _object *, $1 = 0] T cast(object &&obj) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1149:1: note: candidate function [with T = _object *] cast(object &&object) { ^ 1 error generated. ``` --- CMakeLists.txt | 3 +- include/pybind11/cast.h | 34 ++++- include/pybind11/detail/common.h | 4 + include/pybind11/type_caster_pyobject_ptr.h | 61 +++++++++ tests/CMakeLists.txt | 1 + tests/extra_python_package/test_files.py | 1 + tests/test_type_caster_pyobject_ptr.cpp | 130 ++++++++++++++++++++ tests/test_type_caster_pyobject_ptr.py | 104 ++++++++++++++++ 8 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 include/pybind11/type_caster_pyobject_ptr.h create mode 100644 tests/test_type_caster_pyobject_ptr.cpp create mode 100644 tests/test_type_caster_pyobject_ptr.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d9320388..25a7d1498 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,8 @@ set(PYBIND11_HEADERS include/pybind11/pytypes.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9d1ce15d4..db3934118 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1041,7 +1041,11 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference::value, @@ -1055,6 +1059,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 129039252..1ff0df09f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -661,6 +661,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 000000000..aa914f9e1 --- /dev/null +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 11ac2215b..5574663b6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -154,6 +154,7 @@ set(PYBIND11_TEST_FILES test_stl_binders test_tagbased_polymorphic test_thread + test_type_caster_pyobject_ptr test_union test_unnamed_namespace_a test_unnamed_namespace_b diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index dd6393bf0..e5982f962 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -43,6 +43,7 @@ main_headers = { "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/type_caster_pyobject_ptr.h", } detail_headers = { diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp new file mode 100644 index 000000000..1667ea126 --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -0,0 +1,130 @@ +#include +#include +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace { + +std::vector make_vector_pyobject_ptr(const py::object &ValueHolder) { + std::vector vec_obj; + for (int i = 1; i < 3; i++) { + vec_obj.push_back(ValueHolder(i * 93).release().ptr()); + } + // This vector now owns the refcounts. + return vec_obj; +} + +} // namespace + +TEST_SUBMODULE(type_caster_pyobject_ptr, m) { + m.def("cast_from_pyobject_ptr", []() { + PyObject *ptr = PyLong_FromLongLong(6758L); + return py::cast(ptr, py::return_value_policy::take_ownership); + }); + m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) { + auto rc1 = obj.ref_count(); + auto *ptr = py::cast(obj); + auto rc2 = obj.ref_count(); + if (rc2 != rc1 + 1) { + return -1; + } + return 100 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_object_to_pyobject_ptr", [](py::object obj) { + py::handle hdl = obj; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(obj)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 300 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_list_to_pyobject_ptr", [](py::list lst) { + // This is to cover types implicitly convertible to object. + py::handle hdl = lst; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(lst)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 400 - static_cast(py::len(py::reinterpret_steal(ptr))); + }); + + m.def( + "return_pyobject_ptr", + []() { return PyLong_FromLongLong(2314L); }, + py::return_value_policy::take_ownership); + m.def("pass_pyobject_ptr", [](PyObject *ptr) { + return 200 - py::reinterpret_borrow(ptr).attr("value").cast(); + }); + + m.def("call_callback_with_object_return", + [](const std::function &cb, int value) { return cb(value); }); + m.def( + "call_callback_with_pyobject_ptr_return", + [](const std::function &cb, int value) { return cb(value); }, + py::return_value_policy::take_ownership); + m.def( + "call_callback_with_pyobject_ptr_arg", + [](const std::function &cb, py::handle obj) { return cb(obj.ptr()); }, + py::arg("cb"), // This triggers return_value_policy::automatic_reference + py::arg("obj")); + + m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { + if (set_error) { + PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + } + PyObject *ptr = nullptr; + py::cast(ptr); + }); + + m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { + PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::cast(Py_None); + }); + + m.def("pass_list_pyobject_ptr", [](const std::vector &vec_obj) { + int acc = 0; + for (const auto &ptr : vec_obj) { + acc = acc * 1000 + py::reinterpret_borrow(ptr).attr("value").cast(); + } + return acc; + }); + + m.def("return_list_pyobject_ptr_take_ownership", + make_vector_pyobject_ptr, + // Ownership is transferred one-by-one when the vector is converted to a Python list. + py::return_value_policy::take_ownership); + + m.def("return_list_pyobject_ptr_reference", + make_vector_pyobject_ptr, + // Ownership is not transferred. + py::return_value_policy::reference); + + m.def("dec_ref_each_pyobject_ptr", [](const std::vector &vec_obj) { + std::size_t i = 0; + for (; i < vec_obj.size(); i++) { + py::handle h(vec_obj[i]); + if (static_cast(h.ref_count()) < 2) { + break; // Something is badly wrong. + } + h.dec_ref(); + } + return i; + }); + + m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {}); + +#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing. + { + PyObject *ptr = nullptr; + (void) py::cast(*ptr); + } +#endif +} diff --git a/tests/test_type_caster_pyobject_ptr.py b/tests/test_type_caster_pyobject_ptr.py new file mode 100644 index 000000000..1f1ece2ba --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.py @@ -0,0 +1,104 @@ +import pytest + +from pybind11_tests import type_caster_pyobject_ptr as m + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below. +class ValueHolder: + def __init__(self, value): + self.value = value + + +def test_cast_from_pyobject_ptr(): + assert m.cast_from_pyobject_ptr() == 6758 + + +def test_cast_handle_to_pyobject_ptr(): + assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76 + + +def test_cast_object_to_pyobject_ptr(): + assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257 + + +def test_cast_list_to_pyobject_ptr(): + assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395 + + +def test_return_pyobject_ptr(): + assert m.return_pyobject_ptr() == 2314 + + +def test_pass_pyobject_ptr(): + assert m.pass_pyobject_ptr(ValueHolder(82)) == 118 + + +@pytest.mark.parametrize( + "call_callback", + [ + m.call_callback_with_object_return, + m.call_callback_with_pyobject_ptr_return, + ], +) +def test_call_callback_with_object_return(call_callback): + def cb(value): + if value < 0: + raise ValueError("Raised from cb") + return ValueHolder(1000 - value) + + assert call_callback(cb, 287).value == 713 + + with pytest.raises(ValueError, match="^Raised from cb$"): + call_callback(cb, -1) + + +def test_call_callback_with_pyobject_ptr_arg(): + def cb(obj): + return 300 - obj.value + + assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261 + + +@pytest.mark.parametrize("set_error", [True, False]) +def test_cast_to_python_nullptr(set_error): + expected = { + True: r"^Reflective of healthy error handling\.$", + False: ( + r"^Internal error: pybind11::error_already_set called " + r"while Python error indicator not set\.$" + ), + }[set_error] + with pytest.raises(RuntimeError, match=expected): + m.cast_to_pyobject_ptr_nullptr(set_error) + + +def test_cast_to_python_non_nullptr_with_error_set(): + with pytest.raises(SystemError) as excinfo: + m.cast_to_pyobject_ptr_non_nullptr_with_error_set() + assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()" + assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling." + + +def test_pass_list_pyobject_ptr(): + acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)]) + assert acc == 842452 + + +def test_return_list_pyobject_ptr_take_ownership(): + vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + + +def test_return_list_pyobject_ptr_reference(): + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + +def test_type_caster_name_via_incompatible_function_arguments_type_error(): + with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"): + m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202)) From e9b961d9b913575c07ba28c038c3706731768da6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 8 May 2023 10:13:54 -0700 Subject: [PATCH 58/83] Elide to-python conversion of setter return values (#4621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reproducer for property setter with return type that is not wrapped. * Use `py::class_()` to work around the return value conversion issue. * WIP drop_return_value * Remove struct drop_return_value * Introduce `return_value_policy::return_none` for use by setters. * Add `is_setter` to attr.h and use from `.def_property()` * Merge the new test into test_methods_and_attributes * Remove return_none return_value_policy again. * Fix oversight (NOLINTNEXTLINE placement). * Simplification (for the better) found while searching for a way to resolve GCC build failures. Example of failure resolved by this change: g++ (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 ``` cd /build/tests && /usr/bin/c++ -DPYBIND11_TEST_EIGEN -Dpybind11_tests_EXPORTS -I/mounted_pybind11/include -isystem /usr/include/python3.8 -isystem /build/_deps/eigen-src -g -std=c++17 -fPIC -fvisibility=hidden -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -MD -MT tests/CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -MF CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o.d -o CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -c /mounted_pybind11/tests/test_buffers.cpp In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h: In instantiation of ‘pybind11::class_& pybind11::class_::def_property(const char*, const Getter&, const Setter&, const Extra& ...) [with Getter = pybind11::cpp_function; Setter = std::nullptr_t; Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’: /mounted_pybind11/include/pybind11/pybind11.h:1716:58: required from ‘pybind11::class_& pybind11::class_::def_property_readonly(const char*, const pybind11::cpp_function&, const Extra& ...) [with Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/include/pybind11/pybind11.h:1684:9: required from ‘pybind11::class_& pybind11::class_::def_readonly(const char*, const D C::*, const Extra& ...) [with C = pybind11::buffer_info; D = long int; Extra = {}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/tests/test_buffers.cpp:209:61: required from here /mounted_pybind11/include/pybind11/pybind11.h:1740:25: error: call of overloaded ‘cpp_function(std::nullptr_t&, pybind11::is_setter)’ is ambiguous 1740 | name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /mounted_pybind11/include/pybind11/pybind11.h:101:5: note: candidate: ‘pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = std::nullptr_t&; Extra = {pybind11::is_setter}; = void]’ 101 | cpp_function(Func &&f, const Extra &...extra) { | ^~~~~~~~~~~~ In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h:87:5: note: candidate: ‘pybind11::cpp_function::cpp_function(std::nullptr_t, const Extra& ...) [with Extra = {pybind11::is_setter}; std::nullptr_t = std::nullptr_t]’ 87 | cpp_function(std::nullptr_t, const Extra &...) {} | ^~~~~~~~~~~~ ``` * Bug fix: obvious in hindsight. I thought the original version was incrementing the reference count for None, but no. Discovered via many failing tests in the wild (10s of thousands). It is very tricky to construct a meaningful unit test for this bug specifically. It's unlikely to come back, because 10s of thousands of tests will fail again. --- include/pybind11/attr.h | 16 +++++++++++-- include/pybind11/pybind11.h | 18 ++++++++++---- tests/test_methods_and_attributes.cpp | 34 +++++++++++++++++++++++++++ tests/test_methods_and_attributes.py | 9 +++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index b5e3b7b22..1044db94d 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -26,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -426,6 +432,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 6205effd6..28ebc2229 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -84,6 +84,7 @@ public: cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -244,10 +245,16 @@ protected: using Guard = extract_guard_t; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -1729,7 +1736,8 @@ public: template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 815dd5e98..31d46eb7e 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -177,6 +177,38 @@ struct RValueRefParam { std::size_t func4(std::string &&s) const & { return s.size(); } }; +namespace pybind11_tests { +namespace exercise_is_setter { + +struct FieldBase { + int int_value() const { return int_value_; } + + FieldBase &SetIntValue(int int_value) { + int_value_ = int_value; + return *this; + } + +private: + int int_value_ = -99; +}; + +struct Field : FieldBase {}; + +void add_bindings(py::module &m) { + py::module sm = m.def_submodule("exercise_is_setter"); + // NOTE: FieldBase is not wrapped, therefore ... + py::class_(sm, "Field") + .def(py::init<>()) + .def_property( + "int_value", + &Field::int_value, + &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted. + ); +} + +} // namespace exercise_is_setter +} // namespace pybind11_tests + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_ emna(m, "ExampleMandA"); @@ -456,4 +488,6 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("func2", &RValueRefParam::func2) .def("func3", &RValueRefParam::func3) .def("func4", &RValueRefParam::func4); + + pybind11_tests::exercise_is_setter::add_bindings(m); } diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index a85468575..955a85f67 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -522,3 +522,12 @@ def test_rvalue_ref_param(): assert r.func2("1234") == 4 assert r.func3("12345") == 5 assert r.func4("123456") == 6 + + +def test_is_setter(): + fld = m.exercise_is_setter.Field() + assert fld.int_value == -99 + setter_return = fld.int_value = 100 + assert isinstance(setter_return, int) + assert setter_return == 100 + assert fld.int_value == 100 From cca4c51ca463ea02fa504331ff21bc313c80c7f3 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Tue, 9 May 2023 08:04:20 -0600 Subject: [PATCH 59/83] Update errors in string "Explicit conversions" docs (#4658) `PyUnicode_DecodeLatin1` requires you to pass in the `error` parameter. The code as it is in the docs didn't compile. There is a reference leak in the example code. `PyUnicode_DecodeLatin1` returns a new reference. Calling `py::str(PyObject*)` calls `PyObject_Str`, which also returns a new reference. That reference is managed by the `py::str` constructor (which correctly steals the reference, using the `stolen_t` constructor), but the original reference returned by `PyUnicode_DecodeLatin1` is never decref'd: it never makes it into an `object`, and it's never manually decremented. This fixes both of those issues. The code compiles, and I viewed the sphinx docs locally. --- docs/advanced/cast/strings.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/advanced/cast/strings.rst b/docs/advanced/cast/strings.rst index e246c5219..271716b4b 100644 --- a/docs/advanced/cast/strings.rst +++ b/docs/advanced/cast/strings.rst @@ -101,8 +101,11 @@ conversion has the same overhead as implicit conversion. m.def("str_output", []() { std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 - py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); - return py_s; + py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr); + if (!py_s) { + throw py::error_already_set(); + } + return py::reinterpret_steal(py_s); } ); @@ -113,7 +116,8 @@ conversion has the same overhead as implicit conversion. The `Python C API `_ provides -several built-in codecs. +several built-in codecs. Note that these all return *new* references, so +use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`. One could also use a third party encoding library such as libiconv to transcode From d72ffb448c58b4ffb08b5ad629bc788646e2d59e Mon Sep 17 00:00:00 2001 From: Joyce Date: Mon, 15 May 2023 14:02:25 -0300 Subject: [PATCH 60/83] ci: set minimal permissions to github workflows (#4665) * set ci.yml minimal permissions Signed-off-by: Joyce * set configure.yml minimal permissions Signed-off-by: Joyce * set format.yml minimal permissions Signed-off-by: Joyce * set pip.yml minimal permissions Signed-off-by: Joyce * set upstream.yml minimal permissions Signed-off-by: Joyce * set labeler.yml minimal permissions Signed-off-by: Joyce * Update ci.yml to read all Signed-off-by: Joyce * test labeler.yml Signed-off-by: Joyce * restore the if at labeler.yml Signed-off-by: Joyce --------- Signed-off-by: Joyce --- .github/workflows/ci.yml | 2 ++ .github/workflows/configure.yml | 3 +++ .github/workflows/format.yml | 3 +++ .github/workflows/labeler.yml | 5 +++++ .github/workflows/pip.yml | 3 +++ .github/workflows/upstream.yml | 3 +++ 6 files changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c88f0797f..8c2aba343 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,8 @@ on: - stable - v* +permissions: read-all + concurrency: group: test-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index b469a69d1..4ae22281c 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,6 +9,9 @@ on: - stable - v* +permissions: + contents: read + env: # For cmake: VERBOSE: 1 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 46489feb3..b8242ee52 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,6 +12,9 @@ on: - stable - "v*" +permissions: + contents: read + env: FORCE_COLOR: 3 # For cmake: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 165a2fd87..858a4a0e2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,10 +3,15 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: label: name: Labeler runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/labeler@main diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 6d9be3b1d..c1feb6fe1 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -12,6 +12,9 @@ on: types: - published +permissions: + contents: read + env: PIP_ONLY_BINARY: numpy diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be643ddfd..4acfbfce7 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -5,6 +5,9 @@ on: workflow_dispatch: pull_request: +permissions: + contents: read + concurrency: group: upstream-${{ github.ref }} cancel-in-progress: true From 19816f0db7b89468104f5ed284c0ec9d7e312c44 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 22 May 2023 13:44:03 -0700 Subject: [PATCH 61/83] chore: update changelog, with target date for v2.11.0 release (#4677) * Remove .dev1 from version number. * [skip ci] Update changelog.rst * [ci skip] Fix pre-commit: rst ``code`` is two backticks * Apply fix suggested by @henryiii * [ci skip] Set target date for the release to June 2, 2023 * [ci skip] Apply more fixes suggested by @henryiii (I missed those before). * [ci skip] Revert "Remove .dev1 from version number." This reverts commit afc80134cb1035be2913fd8153a16a864181c52c. --- docs/changelog.rst | 54 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58cc40983..32f6606aa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,10 +10,25 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -IN DEVELOPMENT --------------- +Version 2.11.0 (June 2, 2023) +----------------------------- -Changes will be summarized here periodically. +New features: + +* ``pybind11::detail::is_move_constructible`` can now be specialized for cases + in which ``std::is_move_constructible`` does not work as needed. This is + very similar to the long-established + ``pybind11::detail::is_copy_constructible``. + `#4631 `_ + +* Introduce ``recursive_container_traits``. + `#4623 `_ + +* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic + wrapping of APIs that make use of ``PyObject *``. This header needs to + included explicitly (i.e. it is not included implicitly + with ``pybind/pybind11.h``). + `#4601 `_ Changes: @@ -31,6 +46,28 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* Setter return values (which are inaccessible for all practical purposes) are + no longer converted to Python (only to be discarded). + `#4621 `_ + +* Allow lambda specified to function definition to be ``noexcept(true)`` + in C++17. + `#4593 `_ + +* Get rid of recursive template instantiations for concatenating type + signatures on C++17 and higher. + `#4587 `_ + +* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 + ABI version for Python 3.12 is version 5. (The default ABI version + for Python versions up to and including 3.11 is still version 4.). + `#4570 `_ + +* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds + use ``std::hash`` and ``std::equal_to`` + instead of string-based type comparisons. This resolves issues when binding + types defined in the unnamed namespace. + `#4319 `_ Build system improvements: @@ -40,8 +77,17 @@ Build system improvements: * Moved the linting framework over to Ruff. `#4483 `_ +* Skip lto checks and target generation when + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. + `#4643 `_ + +* No longer inject ``-stdlib=libc++``, not needed for modern Pythons + (macOS 10.9+). + `#4639 `_ + + Version 2.10.4 (Mar 16, 2023) ----------------------------- +----------------------------- Changes: From ce9bbc0a213bfb21b77ada8de5346be5e350e3c4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:03:33 -0700 Subject: [PATCH 62/83] Python 3.11+: Add `__notes__` to `error_already_set::what()` output. (#4678) * First version adding `__notes__` to `error_already_set::what()` output. * Fix trivial oversight (missing adjustment in existing test). * Minor enhancements of new code. * Re-enable `cmake --target cpptest -j 2` * Revert "Re-enable `cmake --target cpptest -j 2`" This reverts commit 60816285e9e7b95b7d33a60805888b80b2bec641. * Add general comment explaining why the `error_fetch_and_normalize` code is so unusual. --- include/pybind11/pytypes.h | 70 +++++++++++++++++++++++++++++++++----- tests/test_exceptions.py | 36 +++++++++++++++----- 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f11ed5da7..f5d3f34f3 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -471,13 +471,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -492,6 +503,14 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -506,12 +525,12 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 // This behavior runs the risk of masking errors in the error handling, but avoids a // conflict with PyPy, which relies on the normalization here to change OSError to // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). m_lazy_error_string = exc_type_name_norm; -#else +# else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -523,6 +542,7 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +# endif #endif } @@ -558,6 +578,40 @@ struct error_fetch_and_normalize { } } } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif } else { result = ""; } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index a92ab59a3..ccac4536d 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -317,13 +317,7 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what -@pytest.mark.skipif( - # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 7)", - reason="WIP: https://github.com/python/cpython/issues/102594", -) -@pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") -def test_flaky_exception_failure_point_init(): +def _test_flaky_exception_failure_point_init_before_py_3_12(): with pytest.raises(RuntimeError) as excinfo: m.error_already_set_what(FlakyException, ("failure_point_init",)) lines = str(excinfo.value).splitlines() @@ -337,7 +331,33 @@ def test_flaky_exception_failure_point_init(): # Checking the first two lines of the traceback as formatted in error_string(): assert "test_exceptions.py(" in lines[3] assert lines[3].endswith("): __init__") - assert lines[4].endswith("): test_flaky_exception_failure_point_init") + assert lines[4].endswith( + "): _test_flaky_exception_failure_point_init_before_py_3_12" + ) + + +def _test_flaky_exception_failure_point_init_py_3_12(): + # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594 + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_init",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init") + assert lines[1] == "__notes__ (len=1):" + assert "Normalization failed:" in lines[2] + assert "FlakyException" in lines[2] + + +@pytest.mark.skipif( + "env.PYPY and sys.version_info[:2] < (3, 12)", + reason="PyErr_NormalizeException Segmentation fault", +) +def test_flaky_exception_failure_point_init(): + if sys.version_info[:2] < (3, 12): + _test_flaky_exception_failure_point_init_before_py_3_12() + else: + _test_flaky_exception_failure_point_init_py_3_12() def test_flaky_exception_failure_point_str(): From 6e6bcca5b28461cd16b2af70bba837a4b24482c0 Mon Sep 17 00:00:00 2001 From: Joyce Date: Tue, 23 May 2023 14:05:25 -0300 Subject: [PATCH 63/83] Create s Security Policy (#4671) * Create SECURITY.md * Update test_files.py to include SECURITY.md file * Update MANIFEST.in to include SECURITY.md file --- MANIFEST.in | 2 +- SECURITY.md | 13 +++++++++++++ tests/extra_python_package/test_files.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/MANIFEST.in b/MANIFEST.in index 31632acc3..7ce83c552 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..3d74611f2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e5982f962..5b9e14569 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -111,6 +111,7 @@ sdist_files = { "MANIFEST.in", "README.rst", "PKG-INFO", + "SECURITY.md", } local_sdist_files = { From 8e1f9d5c40f74632799c8a44287d32d18a6c8630 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:49:32 -0700 Subject: [PATCH 64/83] Add `format_descriptor<>` & `npy_format_descriptor<>` `PyObject *` specializations. (#4674) * Add `npy_format_descriptor` to enable `py::array_t` to/from-python conversions. * resolve clang-tidy warning * Use existing constructor instead of adding a static method. Thanks @Skylion007 for pointing out. * Add `format_descriptor` Trivial addition, but still in search for a meaningful test. * Add test_format_descriptor_format * Ensure the Eigen `type_caster`s do not segfault when loading arrays with dtype=object * Use `static_assert()` `!std::is_pointer<>` to replace runtime guards. * Add comments to explain how to check for ref-count bugs. (NO code changes.) * Make the "Pointer types ... are not supported" message Eigen-specific, as suggested by @Lalaland. Move to new pybind11/eigen/common.h header. * Change "format_descriptor_format" implementation as suggested by @Lalaland. Additional tests meant to ensure consistency between py::format_descriptor<>, np.array, np.format_parser turn out to be useful only to highlight long-standing inconsistencies. * resolve clang-tidy warning * Account for np.float128, np.complex256 not being available on Windows, in a future-proof way. * Fully address i|q|l ambiguity (hopefully). * Remove the new `np.format_parser()`-based test, it's much more distracting than useful. * Use bi.itemsize to disambiguate "l" or "L" * Use `py::detail::compare_buffer_info::compare()` to validate the `format_descriptor::format()` strings. * Add `buffer_info::compare` to make `detail::compare_buffer_info::compare` more visible & accessible. * silence clang-tidy warning * pytest-compatible access to np.float128, np.complex256 * Revert "pytest-compatible access to np.float128, np.complex256" This reverts commit e9a289c50fc07199806d14ded644215ab6f03afa. * Use `sizeof(long double) == sizeof(double)` instead of `std::is_same<>` * Report skipped `long double` tests. * Change the name of the new `buffer_info` member function to `item_type_is_equivalent_to`. Add comment defining "equivalent" by example. * Change `item_type_is_equivalent_to<>()` from `static` function to member function, as suggested by @Lalaland --- CMakeLists.txt | 1 + include/pybind11/buffer_info.h | 17 +++++- include/pybind11/detail/common.h | 9 +++ include/pybind11/eigen/common.h | 9 +++ include/pybind11/eigen/matrix.h | 13 +++++ include/pybind11/eigen/tensor.h | 5 ++ include/pybind11/numpy.h | 18 ++++-- tests/extra_python_package/test_files.py | 1 + tests/test_buffers.cpp | 35 ++++++++++++ tests/test_buffers.py | 57 +++++++++++++++++++ tests/test_numpy_array.cpp | 26 +++++++++ tests/test_numpy_array.py | 71 ++++++++++++++++++++++++ 12 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 include/pybind11/eigen/common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 25a7d1498..0ad74db2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,7 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h include/pybind11/eigen/matrix.h include/pybind11/eigen/tensor.h include/pybind11/embed.h diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 06120d556..b99ee8bef 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ private: PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1ff0df09f..c2e6f9ec8 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1025,6 +1025,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; diff --git a/include/pybind11/eigen/common.h b/include/pybind11/eigen/common.h new file mode 100644 index 000000000..24f56d158 --- /dev/null +++ b/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index bd533bed3..8d4342f81 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -10,6 +10,7 @@ #pragma once #include "../numpy.h" +#include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: @@ -287,6 +288,8 @@ handle eigen_encapsulate(Type *src) { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using props = EigenProps; bool load(handle src, bool convert) { @@ -405,6 +408,9 @@ private: // Base class for casting reference/map/block/etc. objects back to python. template struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + private: using props = EigenProps; @@ -457,6 +463,8 @@ private: using Type = Eigen::Ref; using props = EigenProps; using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::Map; using Array = array_t struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + protected: using Matrix = Eigen::Matrix; @@ -632,6 +643,8 @@ public: template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using StorageIndex = remove_reference_t().outerIndexPtr())>; using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index de7dcba89..25d12baca 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -8,6 +8,7 @@ #pragma once #include "../numpy.h" +#include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); @@ -164,6 +165,8 @@ PYBIND11_WARNING_POP template struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using Helper = eigen_tensor_helper; static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); @@ -359,6 +362,8 @@ struct get_storage_pointer_type struct type_caster, typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 854d6e87f..36077ec04 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -564,6 +564,8 @@ public: m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -1283,12 +1285,16 @@ private: public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 5b9e14569..57387dd8b 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -57,6 +57,7 @@ detail_headers = { } eigen_headers = { + "include/pybind11/eigen/common.h", "include/pybind11/eigen/matrix.h", "include/pybind11/eigen/tensor.h", } diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 6b6e8cba7..b5b8c355b 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -7,12 +7,47 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" #include "pybind11_tests.h" TEST_SUBMODULE(buffers, m) { + m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double)); + + m.def("format_descriptor_format_buffer_info_equiv", + [](const std::string &cpp_name, const py::buffer &buffer) { + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *format_table = new std::map; + static auto *equiv_table + = new std::map; + if (format_table->empty()) { +#define PYBIND11_ASSIGN_HELPER(...) \ + (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format(); \ + (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>; + PYBIND11_ASSIGN_HELPER(PyObject *) + PYBIND11_ASSIGN_HELPER(bool) + PYBIND11_ASSIGN_HELPER(std::int8_t) + PYBIND11_ASSIGN_HELPER(std::uint8_t) + PYBIND11_ASSIGN_HELPER(std::int16_t) + PYBIND11_ASSIGN_HELPER(std::uint16_t) + PYBIND11_ASSIGN_HELPER(std::int32_t) + PYBIND11_ASSIGN_HELPER(std::uint32_t) + PYBIND11_ASSIGN_HELPER(std::int64_t) + PYBIND11_ASSIGN_HELPER(std::uint64_t) + PYBIND11_ASSIGN_HELPER(float) + PYBIND11_ASSIGN_HELPER(double) + PYBIND11_ASSIGN_HELPER(long double) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) +#undef PYBIND11_ASSIGN_HELPER + } + return std::pair( + (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))()); + }); + // test_from_python / test_to_python: class Matrix { public: diff --git a/tests/test_buffers.py b/tests/test_buffers.py index eb58c4675..63d9d869f 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -10,6 +10,63 @@ from pybind11_tests import buffers as m np = pytest.importorskip("numpy") +if m.long_double_and_double_have_same_size: + # Determined by the compiler used to build the pybind11 tests + # (e.g. MSVC gets here, but MinGW might not). + np_float128 = None + np_complex256 = None +else: + # Determined by the compiler used to build numpy (e.g. MinGW). + np_float128 = getattr(np, *["float128"] * 2) + np_complex256 = getattr(np, *["complex256"] * 2) + +CPP_NAME_FORMAT_NP_DTYPE_TABLE = [ + ("PyObject *", "O", object), + ("bool", "?", np.bool_), + ("std::int8_t", "b", np.int8), + ("std::uint8_t", "B", np.uint8), + ("std::int16_t", "h", np.int16), + ("std::uint16_t", "H", np.uint16), + ("std::int32_t", "i", np.int32), + ("std::uint32_t", "I", np.uint32), + ("std::int64_t", "q", np.int64), + ("std::uint64_t", "Q", np.uint64), + ("float", "f", np.float32), + ("double", "d", np.float64), + ("long double", "g", np_float128), + ("std::complex", "Zf", np.complex64), + ("std::complex", "Zd", np.complex128), + ("std::complex", "Zg", np_complex256), +] +CPP_NAME_FORMAT_TABLE = [ + (cpp_name, format) + for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE + if np_dtype is not None +] +CPP_NAME_NP_DTYPE_TABLE = [ + (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE +] + + +@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE) +def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype): + if np_dtype is None: + pytest.skip( + f"cpp_name=`{cpp_name}`: `long double` and `double` have same size." + ) + if isinstance(np_dtype, str): + pytest.skip(f"np.{np_dtype} does not exist.") + np_array = np.array([], dtype=np_dtype) + for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE: + format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv( + other_cpp_name, np_array + ) + assert format == expected_format + if other_cpp_name == cpp_name: + assert np_array_is_matching + else: + assert not np_array_is_matching + def test_from_python(): with pytest.raises(RuntimeError) as excinfo: diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index b118e2c6c..8c122a865 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -523,4 +523,30 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); sm.def("round_trip_float", [](double d) { return d; }); + + sm.def("pass_array_pyobject_ptr_return_sum_str_values", + [](const py::array_t &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; + }); + + sm.def("pass_array_pyobject_ptr_return_as_list", + [](const py::array_t &objs) -> py::list { return objs; }); + + sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t arr_from_list(static_cast(arr_size)); + PyObject **data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(data[i] == nullptr); + data[i] = py::cast(objs[i].attr("value")); + } + return arr_from_list; + }); + + sm.def("return_array_pyobject_ptr_from_list", + [](const py::list &objs) -> py::array_t { return objs; }); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 070813d3a..12e7d17d1 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -595,3 +595,74 @@ def test_round_trip_float(): arr = np.zeros((), np.float64) arr[()] = 37.2 assert m.round_trip_float(arr) == 37.2 + + +# HINT: An easy and robust way (although only manual unfortunately) to check for +# ref-count leaks in the test_.*pyobject_ptr.* functions below is to +# * temporarily insert `while True:` (one-by-one), +# * run this test, and +# * run the Linux `top` command in another shell to visually monitor +# `RES` for a minute or two. +# If there is a leak, it is usually evident in seconds because the `RES` +# value increases without bounds. (Don't forget to Ctrl-C the test!) + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below: +# * Ref-count leaks will be immediately evident. +# * Sanitizers are much more likely to detect heap-use-after-free due to +# other ref-count bugs. +class PyValueHolder: + def __init__(self, value): + self.value = value + + +def WrapWithPyValueHolder(*values): + return [PyValueHolder(v) for v in values] + + +def UnwrapPyValueHolder(vhs): + return [vh.value for vh in vhs] + + +def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) + ) + == "-3four5.0" + ) + + +def test_pass_array_pyobject_ptr_return_sum_str_values_list(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + WrapWithPyValueHolder(2, "three", -4.0) + ) + == "2three-4.0" + ) + + +def test_pass_array_pyobject_ptr_return_as_list(): + # Intentionally all temporaries, do not change. + assert UnwrapPyValueHolder( + m.pass_array_pyobject_ptr_return_as_list( + np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) + ) + ) == [-1, "two", 3.0] + + +@pytest.mark.parametrize( + ("return_array_pyobject_ptr", "unwrap"), + [ + (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + ], +) +def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): + # Intentionally all temporaries, do not change. + arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + assert isinstance(arr_from_list, np.ndarray) + assert arr_from_list.dtype == np.dtype("O") + assert unwrap(arr_from_list) == [6, "seven", -8.0] From d0232b119f5f0cf2bf6cecacaf099c301c811ca8 Mon Sep 17 00:00:00 2001 From: "T.Yamada" Date: Thu, 25 May 2023 13:39:36 +0900 Subject: [PATCH 65/83] Use annotated for array (#4679) * use Annotated for std::array docstring * add tests * fix test * style: pre-commit fixes * fix valarray annotation * style: pre-commit fixes * refix test * add FixedSize * style: pre-commit fixes * Update test_stl.py * style: pre-commit fixes --------- Co-authored-by: Taiju Yamada Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/stl.h | 8 ++++---- tests/test_stl.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 84c517108..f39f44f7c 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -273,11 +273,11 @@ public: } PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name + const_name(const_name(""), const_name("Annotated[")) + + const_name("List[") + value_conv::name + const_name("]") + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + const_name(", FixedSize(") + + const_name() + const_name(")]"))); }; template diff --git a/tests/test_stl.py b/tests/test_stl.py index 2d9dcc89f..8a614f8b8 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -39,8 +39,11 @@ def test_array(doc): assert m.load_array(lst) assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> List[int[2]]" - assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool" + assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert ( + doc(m.load_array) + == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + ) def test_valarray(doc): From 29487dee8f86c984605570c298b21c79ea3bada3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 7 Jun 2023 12:00:37 -0700 Subject: [PATCH 66/83] =?UTF-8?q?Disable=20=F0=9F=90=8D=203=20=E2=80=A2=20?= =?UTF-8?q?CentOS7=20/=20PGI=2022.9=20=E2=80=A2=20x64=20(#4691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2aba343..91144241f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,6 +400,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: + if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 runs-on: ubuntu-latest name: "🐍 3 • CentOS7 / PGI 22.9 • x64" container: centos:7 From 3617f3554aa5096e2148281c14be99afe9c138e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 12:57:38 -0700 Subject: [PATCH 67/83] chore(deps): update pre-commit hooks (#4689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.263 → v0.0.270](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.263...v0.0.270) - [github.com/pre-commit/mirrors-mypy: v1.2.0 → v1.3.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.2.0...v1.3.0) - [github.com/shellcheck-py/shellcheck-py: v0.9.0.2 → v0.9.0.5](https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.2...v0.9.0.5) - [github.com/pre-commit/mirrors-clang-format: v16.0.2 → v16.0.4](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.2...v16.0.4) * Resolve new ruff error: ``` tests/test_pytypes.py:478:20: PLW0130 Duplicate value `3` in set ``` The reduction from `{3, 3}` to `{3}` never happened in pybind11 code, therefore this change is essentially a no-op for the purpose of unit-testing pybind11. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 8 ++++---- tests/test_pytypes.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aca9b3ca2..bcc3f2723 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.263 + rev: v0.0.270 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.2.0" + rev: "v1.3.0" hooks: - id: mypy args: [] @@ -128,7 +128,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.2" + rev: "v0.9.0.5" hooks: - id: shellcheck @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.2" + rev: "v16.0.4" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 946b94140..afb7a1ce8 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -475,7 +475,7 @@ def test_pybind11_str_raw_str(): assert cvt({}) == "{}" assert cvt({3: 4}) == "{3: 4}" assert cvt(set()) == "set()" - assert cvt({3, 3}) == "{3}" + assert cvt({3}) == "{3}" valid_orig = "DZ" valid_utf8 = valid_orig.encode("utf-8") From c679a9209557ef0324a19f4618c556b60e251009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:49:14 -0700 Subject: [PATCH 68/83] chore(deps): bump deadsnakes/action from 3.0.0 to 3.0.1 (#4687) Bumps [deadsnakes/action](https://github.com/deadsnakes/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/deadsnakes/action/releases) - [Commits](https://github.com/deadsnakes/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: deadsnakes/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91144241f..b844d8133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,7 +203,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v3.0.0 + uses: deadsnakes/action@v3.0.1 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} From 0e43fcc75e6b7429e3511dfb44343ec05a0ab843 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 10 Jun 2023 10:14:08 -0700 Subject: [PATCH 69/83] Python 3.12b2 testing (#4695) * Uncomment `Interface test` sections (test_embed) * fix: setuptools has been removed from default installs in 3.12 --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 6 +++--- .github/workflows/upstream.yml | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b844d8133..45a6738f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,9 @@ jobs: # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" @@ -242,8 +244,6 @@ jobs: python -m pip install -r tests/requirements.txt - name: Configure - env: - SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 4acfbfce7..d4220be22 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -1,4 +1,3 @@ - name: Upstream on: @@ -65,8 +64,8 @@ jobs: - name: Python tests C++11 run: cmake --build build11 --target pytest -j 2 - # - name: C++11 tests - # run: cmake --build build11 --target cpptest -j 2 + - name: C++11 tests + run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 run: cmake --build build11 --target test_cmake_build @@ -80,14 +79,14 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - - name: Build + - name: Build C++17 run: cmake --build build17 -j 2 - - name: Python tests + - name: Python tests C++17 run: cmake --build build17 --target pytest - # - name: C++ tests - # run: cmake --build build17 --target cpptest + - name: C++17 tests + run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) @@ -105,10 +104,12 @@ jobs: - name: Python tests (unstable ABI) run: cmake --build build17max --target pytest - - name: Interface test + - name: Interface test (unstable ABI) run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools From 86f60a0c072e5bd5cab79233f712c321fe8deca7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 16 Jun 2023 14:09:40 -0700 Subject: [PATCH 70/83] pre-commit markdown-it-py<3 (for Python 3.7 compatibility) (#4704) * markdown-it-py dropped Python 3.7 support, but we are type checking for Python 3.7 from a newer version of Python. Keep using markdown-it-py<3 as long as we support Python 3.7. * Shuffle order of pre-commit checks for better agility. --- .pre-commit-config.yaml | 91 +++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcc3f2723..2d508276e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,48 @@ ci: exclude: ^tools/JoinPaths.cmake$ repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v16.0.4" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.3.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.270 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.3.0" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v4.4.0" @@ -39,12 +81,6 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: "23.3.0" # Keep in sync with blacken-docs - hooks: - - id: black - # Also code format the docs - repo: https://github.com/asottile/blacken-docs rev: "1.13.0" @@ -66,13 +102,6 @@ repos: - id: fix-ligatures - id: fix-smartquotes -# Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 - hooks: - - id: ruff - args: ["--fix", "--show-fixes"] - # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks rev: "v1.10.0" @@ -81,32 +110,6 @@ repos: - id: rst-directive-colons - id: rst-inline-touching-normal - -# PyLint has native support - not always usable, but works for us -- repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a6" - hooks: - - id: pylint - files: ^pybind11 - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: "v0.6.13" - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ - -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" - hooks: - - id: mypy - args: [] - exclude: ^(tests|docs)/ - additional_dependencies: [nox, rich] - # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest rev: "0.49" @@ -141,9 +144,9 @@ repos: entry: PyBind|Numpy|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ -# Clang format the codebase automatically -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.0a6" hooks: - - id: clang-format - types_or: [c++, c, cuda] + - id: pylint + files: ^pybind11 From 849322806cd4b3697ad1d35eedd6d0352c5f267a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 17 Jun 2023 07:02:23 -0700 Subject: [PATCH 71/83] Systematically add `PIP_BREAK_SYSTEM_PACKAGES` to all .yml files from which pip is called. (#4705) * Systematically add PIP_BREAK_SYSTEM_PACKAGES to all .yml files from which pip is called. * Try gcc:10-bullseye (because gcc:10 is broken: https://github.com/docker-library/gcc/issues/95) * bug fix (matrix did not work as hoped) --- .github/workflows/ci.yml | 17 +++++++++-------- .github/workflows/configure.yml | 1 + .github/workflows/pip.yml | 1 + .github/workflows/upstream.yml | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45a6738f7..1d5d502b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy FORCE_COLOR: 3 PYTEST_TIMEOUT: 300 @@ -455,16 +456,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11 } - - { gcc: 7, std: 17 } - - { gcc: 8, std: 14 } - - { gcc: 8, std: 17 } - - { gcc: 10, std: 17 } - - { gcc: 11, std: 20 } - - { gcc: 12, std: 20 } + - { gcc: 7, std: 11, container_suffix: "" } + - { gcc: 7, std: 17, container_suffix: "" } + - { gcc: 8, std: 14, container_suffix: "" } + - { gcc: 8, std: 17, container_suffix: "" } + - { gcc: 10, std: 17, container_suffix: "-bullseye" } + - { gcc: 11, std: 20, container_suffix: "" } + - { gcc: 12, std: 20, container_suffix: "" } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" - container: "gcc:${{ matrix.gcc }}" + container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 4ae22281c..82a493900 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -13,6 +13,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 # For cmake: VERBOSE: 1 diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c1feb6fe1..04d95a28c 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -16,6 +16,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy jobs: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index d4220be22..dd8a1c960 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -12,6 +12,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: ":all:" # For cmake: VERBOSE: 1 From bc1bcf7c05b6097c2f0c993a776a975d7332f279 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 21 Jun 2023 13:25:10 -0400 Subject: [PATCH 72/83] chore: 3.12 + cleanup (#4713) Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 2 ++ .pre-commit-config.yaml | 1 + pybind11/setup_helpers.py | 8 ++++---- pyproject.toml | 18 ++++++++++-------- setup.cfg | 1 + setup.py | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d5d502b0..1d61ad6a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: - '3.9' - '3.10' - '3.11' + - '3.12' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -74,6 +75,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Setup Boost (Linux) # Can't use boost + define _ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d508276e..9a829d72b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,6 +54,7 @@ repos: - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. - nox - rich + - types-setuptools # CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index cb279f27e..aeeee9dcf 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -66,8 +66,8 @@ try: from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import build_ext as _build_ext # type: ignore[assignment] + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors @@ -84,7 +84,7 @@ STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -266,7 +266,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search diff --git a/pyproject.toml b/pyproject.toml index e3655aca7..59c15ea63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + [tool.check-manifest] ignore = [ "tests/**", @@ -15,6 +16,7 @@ ignore = [ "noxfile.py", ] + [tool.mypy] files = ["pybind11"] python_version = "3.6" @@ -24,7 +26,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*"] ignore_missing_imports = true @@ -55,10 +57,11 @@ messages_control.disable = [ "unused-argument", # covered by Ruff ARG ] + [tool.ruff] select = [ "E", "F", "W", # flake8 - "B", "B904", # flake8-bugbear + "B", # flake8-bugbear "I", # isort "N", # pep8-naming "ARG", # flake8-unused-arguments @@ -77,14 +80,13 @@ select = [ "YTT", # flake8-2020 ] ignore = [ - "PLR", # Design related pylint - "E501", # Line too long (Black is enough) - "PT011", # Too broad with raises in pytest - "PT004", # Fixture that doesn't return needs underscore (no, it is fine) - "SIM118",# iter(x) is not always the same as iter(x.keys()) + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) ] target-version = "py37" -typing-modules = ["scikit_build_core._compat.typing"] src = ["src"] unfixable = ["T20"] exclude = [] diff --git a/setup.cfg b/setup.cfg index c35dd07ae..92e6c953a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython diff --git a/setup.py b/setup.py index 68573519c..9fea7d35c 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def get_and_replace( # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] +class SDist(setuptools.command.sdist.sdist): def make_release_tree(self, base_dir: str, files: List[str]) -> None: super().make_release_tree(base_dir, files) From e10da79b6ee2554be364ef14df1c988f94df02ea Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 24 Jun 2023 12:12:35 -0700 Subject: [PATCH 73/83] Undo ci.yml gcc10 workaround after docker-library/gcc#95 was resolved. (#4717) The gcc10 workaround was introduced with PR #4705. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d61ad6a9..0d6401412 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,16 +458,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11, container_suffix: "" } - - { gcc: 7, std: 17, container_suffix: "" } - - { gcc: 8, std: 14, container_suffix: "" } - - { gcc: 8, std: 17, container_suffix: "" } - - { gcc: 10, std: 17, container_suffix: "-bullseye" } - - { gcc: 11, std: 20, container_suffix: "" } - - { gcc: 12, std: 20, container_suffix: "" } + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" - container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" + container: "gcc:${{ matrix.gcc }}" steps: - uses: actions/checkout@v3 From 2fb3d7cbde264a0b3f921e802f287195387e8263 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 27 Jun 2023 15:08:32 -0700 Subject: [PATCH 74/83] Trivial refactoring to make the capsule API more user friendly. (#4720) * Trivial refactoring to make the capsule API more user friendly. * Use new API in production code. Thanks @Lalaland for pointing this out. --- include/pybind11/pybind11.h | 2 +- include/pybind11/pytypes.h | 51 ++++++++++++++++++++++--------------- tests/test_pytypes.cpp | 9 +++++++ tests/test_pytypes.py | 13 ++++++++++ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 28ebc2229..3bce1a01b 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -508,8 +508,8 @@ protected: rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); - rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f5d3f34f3..c93e3d3b9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1925,28 +1925,13 @@ public: } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr && PyErr_Occurred()) { - throw error_already_set(); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (destructor != nullptr) { - destructor(ptr); - } - }); - - if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -2014,6 +1999,32 @@ private: return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 1028bb58e..b4ee64289 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -260,6 +260,15 @@ TEST_SUBMODULE(pytypes, m) { }); }); + m.def("return_capsule_with_destructor_3", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + py::print("original name: {}"_s.format(cap.name())); + return cap; + }); + m.def("return_renamed_capsule_with_destructor_2", []() { py::print("creating capsule"); auto cap = py::capsule((void *) 1234, [](void *ptr) { diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index afb7a1ce8..eda7a20a9 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -319,6 +319,19 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_destructor_3() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + destructing capsule: 1233 + original name: oname + """ + ) + with capture: a = m.return_renamed_capsule_with_destructor_2() del a From 5ccb9e412d8974e8eeac1b061d9077ac0bd365e1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 4 Jul 2023 08:38:14 -0400 Subject: [PATCH 75/83] chore: ruff moved to astral-sh (#4726) * chore: ruff moved to astral-sh Committed via https://github.com/asottile/all-repos * ci: bump CUDA version --- .github/workflows/ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d6401412..39b4c237e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,8 +337,8 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04" - container: nvidia/cuda:11.7.0-devel-ubuntu22.04 + name: "🐍 3.10 • CUDA 12.2 • Ubuntu 22.04" + container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a829d72b..d406624f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: - id: black # Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit +- repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.270 hooks: - id: ruff From 782b6281a2aec358fb54dc1badc4b8ae1988bd41 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 9 Jul 2023 09:40:47 -0700 Subject: [PATCH 76/83] Drop PyPy 3.7 from GitHub Actions (ci.yml) (#4728) * Remove `pypy-3.7` from ci.yml * Update .github/workflows/ci.yml --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39b4c237e..48f7c5e93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,9 @@ jobs: - '3.10' - '3.11' - '3.12' - - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the From d462dd91a58116a043974b198462864a527e84ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:43:20 -0700 Subject: [PATCH 77/83] chore(deps): bump scipy from 1.8.0 to 1.10.0 in /tests (#4731) Bumps [scipy](https://github.com/scipy/scipy) from 1.8.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.8.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 04aafa8cf..4ba101119 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,4 +6,4 @@ numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" pytest==7.0.0 pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy==1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" +scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" From 47dc0c4bd16b7a97edb2072e93e6960647c3872f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:31:43 -0700 Subject: [PATCH 78/83] chore(deps): update pre-commit hooks (#4727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v16.0.4 → v16.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.4...v16.0.6) - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.270 → v0.0.276](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.270...v0.0.276) - [github.com/pre-commit/mirrors-mypy: v1.3.0 → v1.4.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.4.1) - [github.com/asottile/blacken-docs: 1.13.0 → 1.14.0](https://github.com/asottile/blacken-docs/compare/1.13.0...1.14.0) - [github.com/codespell-project/codespell: v2.2.4 → v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d406624f3..86ac965d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" + rev: "v16.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] @@ -38,14 +38,14 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.276 hooks: - id: ruff args: ["--fix", "--show-fixes"] # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" + rev: "v1.4.1" hooks: - id: mypy args: [] @@ -84,7 +84,7 @@ repos: # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "1.13.0" + rev: "1.14.0" hooks: - id: blacken-docs additional_dependencies: @@ -124,7 +124,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.4" + rev: "v2.2.5" hooks: - id: codespell exclude: ".supp$" From b2732c6e1a96eb97e79359c049871c41e2cd34c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 08:55:18 -0700 Subject: [PATCH 79/83] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 (#4718) * chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.6 to 1.8.7. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.6...v1.8.7) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Apply suggestions from code review --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 04d95a28c..d6687b441 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -102,13 +102,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 6d22dba82f1789f11a8eb2c2debbcbd4d2d8a969 Mon Sep 17 00:00:00 2001 From: Luc de Jonckheere Date: Wed, 12 Jul 2023 17:56:40 +0200 Subject: [PATCH 80/83] Warning on comparing wrapper enums with is (#4732) * Warning on comparing wrapper enums with is * backticks for quoting and link to related issue --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- docs/classes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/classes.rst b/docs/classes.rst index 52cd52da3..4f2167dac 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -549,3 +549,7 @@ The ``name`` property returns the name of the enum value as a unicode string. ... By default, these are omitted to conserve space. + +.. warning:: + + Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 `_ for background). From 2e5f5c4cf8efdd62acbeb4632e8988b2148f6caf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 12 Jul 2023 13:10:24 -0500 Subject: [PATCH 81/83] fix: support CMake 3.27, drop 3.4 (#4719) * fix: support CMake 3.27, drop 3.4 Signed-off-by: Henry Schreiner * Update upgrade.rst * Update upgrade.rst * Update upgrade.rst --------- Signed-off-by: Henry Schreiner --- .github/workflows/configure.yml | 12 ++++++++---- CMakeLists.txt | 8 ++++---- docs/advanced/embedding.rst | 2 +- docs/compiling.rst | 9 ++++++--- docs/faq.rst | 3 ++- docs/upgrade.rst | 14 ++++++++++++++ tests/CMakeLists.txt | 11 ++++------- tests/test_cmake_build/CMakeLists.txt | 3 --- .../installed_embed/CMakeLists.txt | 8 ++++---- .../installed_function/CMakeLists.txt | 8 ++++---- .../installed_target/CMakeLists.txt | 8 ++++---- .../subdirectory_embed/CMakeLists.txt | 8 ++++---- .../subdirectory_function/CMakeLists.txt | 8 ++++---- .../subdirectory_target/CMakeLists.txt | 8 ++++---- tools/pybind11Common.cmake | 10 +++++++++- tools/pybind11Config.cmake.in | 4 +++- 16 files changed, 75 insertions(+), 49 deletions(-) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 82a493900..ec7cd612d 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -26,20 +26,24 @@ jobs: matrix: runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] - cmake: ["3.23"] + cmake: ["3.26"] include: - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.4 + cmake: "3.5" + + - runs-on: ubuntu-20.04 + arch: x64 + cmake: "3.27" - runs-on: macos-latest arch: x64 - cmake: 3.7 + cmake: "3.7" - runs-on: windows-2019 arch: x64 # x86 compilers seem to be missing on 2019 image - cmake: 3.18 + cmake: "3.18" name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ad74db2b..87ec10346 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,15 +5,15 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.22) + cmake_policy(VERSION 3.26) endif() # Avoid infinite recursion if tests include this as a subdirectory diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dd980d483..e6a1686f8 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/docs/compiling.rst b/docs/compiling.rst index 2b543be0b..1fd098bec 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -241,7 +241,7 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) add_subdirectory(pybind11) @@ -261,6 +261,9 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. versionchanged:: 2.6 CMake 3.4+ is required. +.. versionchanged:: 2.11 + CMake 3.5+ is required. + Further information can be found at :doc:`cmake/index`. pybind11_add_module @@ -495,7 +498,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -553,7 +556,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) diff --git a/docs/faq.rst b/docs/faq.rst index 28498e7df..1eb00efad 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -284,7 +284,8 @@ There are three possible solutions: COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead of the old, deprecated search tools, and these modules are much better at - finding the correct Python. + finding the correct Python. If FindPythonLibs/Interp are not available + (CMake 3.27+), then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6a9db2d08..b13d21f5e 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,20 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.11: + +v2.11 +===== + +* The minimum version of CMake is now 3.5. A future version will likely move to + requiring something like CMake 3.15. Note that CMake 3.27 is removing the + long-deprecated support for ``FindPythonInterp`` if you set 3.27 as the + minimum or maximum supported version. To prepare for that future, CMake 3.15+ + using ``FindPython`` or setting ``PYBIND11_FINDPYTHON`` is highly recommended, + otherwise pybind11 will automatically switch to using ``FindPython`` if + ``FindPythonInterp`` is not available. + + .. _upgrade-guide-2.9: v2.9 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5574663b6..80ee9c1f1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,20 +5,17 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.21) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.21) + cmake_policy(VERSION 3.26) endif() -# Only needed for CMake < 3.5 support -include(CMakeParseArguments) - # Filter out items; print an optional message if any items filtered. This ignores extensions. # # Usage: diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 8bfaa386a..e5aa975cf 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -1,6 +1,3 @@ -# Built-in in CMake 3.5+ -include(CMakeParseArguments) - add_custom_target(test_cmake_build) function(pybind11_add_build_test name) diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index f7d693998..d9dcb45e4 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_embed CXX) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index d7ca4db55..2f4f64275 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) project(test_installed_module CXX) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_function CXX) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index bc5e101f1..a981e236f 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_target CXX) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index 58cdd7cfd..f286746b9 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_embed CXX) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 01557c439..275a75c0b 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_function CXX) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index ba82fdee2..37bb2c56e 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_target CXX) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index e4ff1e408..308d1b70d 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -163,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) diff --git a/tools/pybind11Config.cmake.in b/tools/pybind11Config.cmake.in index 9383e8c67..5734f437b 100644 --- a/tools/pybind11Config.cmake.in +++ b/tools/pybind11Config.cmake.in @@ -63,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ From b33d06f61543878a81a433ed2d472b61d3fca0ca Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 12 Jul 2023 16:20:08 -0400 Subject: [PATCH 82/83] bugfix: fixes a test suite bug in the __new__ example (#4698) * bugfix: fixes a test suite bug in the __new__ example * See https://github.com/pybind/pybind11/pull/4698#discussion_r1227107682 --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/test_class.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_class.cpp b/tests/test_class.cpp index ca925917e..7241bc881 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -85,7 +85,7 @@ TEST_SUBMODULE(class_, m) { .def_static("new_instance", &NoConstructor::new_instance, "Return an instance"); py::class_(m, "NoConstructorNew") - .def(py::init([](const NoConstructorNew &self) { return self; })) // Need a NOOP __init__ + .def(py::init([]() { return nullptr; })) // Need a NOOP __init__ .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); From 1a917f1852eb7819b671fc3fa862840f4c491a07 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 14 Jul 2023 08:20:38 -0700 Subject: [PATCH 83/83] docs: preparation for v2.11.0 release (#4744) * Remove .dev1 from version number. * Update Changelog (starting from `nox -s make_changelog` output) * Miscellaneous minor fixes from proofreading in GitHub web view. * docs: minor changelog updates Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner --- docs/changelog.rst | 47 +++++++++++++++++++++++++++----- docs/release.rst | 10 ++++--- include/pybind11/detail/common.h | 4 +-- pybind11/_version.py | 2 +- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 32f6606aa..7bb115b90 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,15 +10,15 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -Version 2.11.0 (June 2, 2023) +Version 2.11.0 (July 14, 2023) ----------------------------- New features: -* ``pybind11::detail::is_move_constructible`` can now be specialized for cases - in which ``std::is_move_constructible`` does not work as needed. This is - very similar to the long-established - ``pybind11::detail::is_copy_constructible``. +* The newly added ``pybind11::detail::is_move_constructible`` trait can be + specialized for cases in which ``std::is_move_constructible`` does not work + as needed. This is very similar to the long-established + ``pybind11::detail::is_copy_constructible``. `#4631 `_ * Introduce ``recursive_container_traits``. @@ -30,6 +30,19 @@ New features: with ``pybind/pybind11.h``). `#4601 `_ +* ``format_descriptor<>`` & ``npy_format_descriptor<>`` ``PyObject *`` + specializations were added. The latter enables ``py::array_t`` + to/from-python conversions. + `#4674 `_ + +* ``buffer_info`` gained an ``item_type_is_equivalent_to()`` member + function. + `#4674 `_ + +* The ``capsule`` API gained a user-friendly constructor + (``py::capsule(ptr, "name", dtor)``). + `#4720 `_ + Changes: * ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & @@ -46,6 +59,12 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* The docstring generation for the ``std::array``-list caster was fixed. + Previously, signatures included the size of the list in a non-standard, + non-spec compliant way. The new format conforms to PEP 593. + **Tooling for processing the docstrings may need to be updated accordingly.** + `#4679 `_ + * Setter return values (which are inaccessible for all practical purposes) are no longer converted to Python (only to be discarded). `#4621 `_ @@ -58,7 +77,7 @@ Changes: signatures on C++17 and higher. `#4587 `_ -* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 +* Compatibility with Python 3.12 (beta). Note that the minimum pybind11 ABI version for Python 3.12 is version 5. (The default ABI version for Python versions up to and including 3.11 is still version 4.). `#4570 `_ @@ -69,15 +88,23 @@ Changes: types defined in the unnamed namespace. `#4319 `_ +* Python exception ``__notes__`` (introduced with Python 3.11) are now added to + the ``error_already_set::what()`` output. + `#4678 `_ + Build system improvements: +* CMake 3.27 support was added, CMake 3.4 support was dropped. + FindPython will be used if ``FindPythonInterp`` is not present. + `#4719 `_ + * Update clang-tidy to 15 in CI. `#4387 `_ * Moved the linting framework over to Ruff. `#4483 `_ -* Skip lto checks and target generation when +* Skip ``lto`` checks and target generation when ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. `#4643 `_ @@ -85,6 +112,12 @@ Build system improvements: (macOS 10.9+). `#4639 `_ +* PyPy 3.10 support was added, PyPy 3.7 support was dropped. + `#4728 `_ + +* Testing with Python 3.12 beta releases was added. + `#4713 `_ + Version 2.10.4 (Mar 16, 2023) ----------------------------- diff --git a/docs/release.rst b/docs/release.rst index e761cdf7a..4950c3b88 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -33,10 +33,12 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Ensure that all the information in ``setup.cfg`` is up-to-date, like supported Python versions. - - Add release date in ``docs/changelog.rst``. - - Check to make sure - `needs-changelog `_ - issues are entered in the changelog (clear the label when done). + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + - Note that the ``make_changelog`` command inspects + `needs changelog `_. + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it fails due to a known flake issue, either ignore or restart CI.) - Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index c2e6f9ec8..27c426db5 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_PATCH 0 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020B0000 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5c..1c310e0c9 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.11.0" version_info = tuple(_to_int(s) for s in __version__.split("."))