Merge branch 'master' of https://github.com/pybind/pybind11 into skylion007/classmethod

.codespell-ignore-lines

@@ -0,0 +1,24 @@
+template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t>
+    template <typename ThisT>
+        auto &this_ = static_cast<ThisT &>(*this);
+                if (load_impl<ThisT>(temp, false)) {
+        ssize_t nd = 0;
+        auto trivial = broadcast(buffers, nd, shape);
+        auto ndim = (size_t) nd;
+    int nd;
+    ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; }
+        using op = op_impl<id, ot, Base, L_type, R_type>;
+template <op_id id, op_type ot, typename L, typename R>
+    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])
+struct IntStruct {
+    explicit IntStruct(int v) : value(v){};
+    ~IntStruct() { value = -value; }
+    IntStruct(const IntStruct &) = default;
+    IntStruct &operator=(const IntStruct &) = default;
+    py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); }));
+    py::implicitly_convertible<int, IntStruct>();
+    m.def("test", [](int expected, const IntStruct &in) {
+        [](int expected, const IntStruct &in) {

.github/workflows/ci.yml

@@ -80,7 +80,7 @@ jobs:
       run: brew install boost
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Cache wheels
       if: runner.os == 'macOS'
@@ -202,7 +202,7 @@ jobs:
         debug: ${{ matrix.python-debug }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Valgrind cache
       if: matrix.valgrind
@@ -324,8 +324,8 @@ jobs:
   # Testing NVCC; forces sources to behave like .cu files
   cuda:
     runs-on: ubuntu-latest
-    name: "🐍 3.8 • CUDA 11.2 • Ubuntu 20.04"
+    name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04"
-    container: nvidia/cuda:11.2.2-devel-ubuntu20.04
+    container: nvidia/cuda:11.7.0-devel-ubuntu22.04
 
     steps:
     - uses: actions/checkout@v3
@@ -465,7 +465,7 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Configure
       shell: bash
@@ -737,6 +737,9 @@ jobs:
             args: -DCMAKE_CXX_STANDARD=20
           - python: 3.8
             args: -DCMAKE_CXX_STANDARD=17
+          - python: 3.7
+            args: -DCMAKE_CXX_STANDARD=14
+
 
     name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}"
     runs-on: windows-2019
@@ -751,10 +754,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.10.0
+      uses: ilammy/msvc-dev-cmd@v1.11.0
       with:
         arch: x86
 
@@ -804,10 +807,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.10.0
+      uses: ilammy/msvc-dev-cmd@v1.11.0
       with:
         arch: x86
 
@@ -855,7 +858,7 @@ jobs:
         python3 -m pip install -r tests/requirements.txt
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Configure C++20
       run: >

.github/workflows/configure.yml

@@ -51,7 +51,7 @@ jobs:
     # An action for adding a specific version of CMake:
     #   https://github.com/jwlawson/actions-setup-cmake
     - name: Setup CMake ${{ matrix.cmake }}
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
       with:
         cmake-version: ${{ matrix.cmake }}
 

.github/workflows/upstream.yml

@@ -31,7 +31,7 @@ jobs:
       run: sudo apt-get install libboost-dev
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare env
       run: |

.pre-commit-config.yaml

@@ -12,6 +12,12 @@
 #
 # See https://github.com/pre-commit/pre-commit
 
+
+ci:
+  autoupdate_commit_msg: "chore(deps): update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+  autoupdate_schedule: monthly
+
 # third-party content
 exclude: ^tools/JoinPaths.cmake$
 
@@ -35,7 +41,7 @@ repos:
 
 # Upgrade old Python syntax
 - repo: https://github.com/asottile/pyupgrade
-  rev: "v2.37.3"
+  rev: "v2.38.2"
   hooks:
   - id: pyupgrade
     args: [--py36-plus]
@@ -48,7 +54,7 @@ repos:
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black
-  rev: "22.6.0" # Keep in sync with blacken-docs
+  rev: "22.8.0" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
@@ -58,16 +64,16 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==22.6.0 # keep in sync with black hook
+    - black==22.8.0 # keep in sync with black hook
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: "v1.3.0"
+  rev: "v1.3.1"
   hooks:
   - id: remove-tabs
 
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.3.1"
+  rev: "0.4.0"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
@@ -110,7 +116,7 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v2.14.5"
+  rev: "v2.15.3"
   hooks:
   - id: pylint
     files: ^pybind11
@@ -126,7 +132,7 @@ repos:
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v0.971"
+  rev: "v0.981"
   hooks:
   - id: mypy
     args: []
@@ -143,12 +149,14 @@ repos:
     additional_dependencies: [cmake, ninja]
 
 # Check for spelling
+# Use tools/codespell_ignore_lines_from_errors.py
+# to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.1.0"
+  rev: "v2.2.1"
   hooks:
   - id: codespell
     exclude: ".supp$"
-    args: ["-L", "nd,ot,thist"]
+    args: ["-x", ".codespell-ignore-lines"]
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py

docs/advanced/misc.rst

@@ -39,15 +39,42 @@ The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds.
 Global Interpreter Lock (GIL)
 =============================
 
-When calling a C++ function from Python, the GIL is always held.
+The Python C API dictates that the Global Interpreter Lock (GIL) must always
+be held by the current thread to safely access Python objects. As a result,
+when Python calls into C++ via pybind11 the GIL must be held, and pybind11
+will never implicitly release the GIL.
+
+.. code-block:: cpp
+
+    void my_function() {
+        /* GIL is held when this function is called from Python */
+    }
+
+    PYBIND11_MODULE(example, m) {
+        m.def("my_function", &my_function);
+    }
+
+pybind11 will ensure that the GIL is held when it knows that it is calling
+Python code. For example, if a Python callback is passed to C++ code via
+``std::function``, when C++ code calls the function the built-in wrapper
+will acquire the GIL before calling the Python callback. Similarly, the
+``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling
+back into Python.
+
+When writing C++ code that is called from other C++ code, if that code accesses
+Python state, it must explicitly acquire and release the GIL.
+
 The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be
 used to acquire and release the global interpreter lock in the body of a C++
 function call. In this way, long-running C++ code can be parallelized using
-multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this
+multiple Python threads, **but great care must be taken** when any
+:class:`gil_scoped_release` appear: if there is any way that the C++ code
+can access Python objects, :class:`gil_scoped_acquire` should be used to
+reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this
 could be realized as follows (important changes highlighted):
 
 .. code-block:: cpp
-    :emphasize-lines: 8,9,31,32
+    :emphasize-lines: 8,30,31
 
     class PyAnimal : public Animal {
     public:
@@ -56,9 +83,7 @@ could be realized as follows (important changes highlighted):
 
         /* Trampoline (need one for each virtual function) */
         std::string go(int n_times) {
-            /* Acquire GIL before calling Python code */
+            /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */
-            py::gil_scoped_acquire acquire;
-
             PYBIND11_OVERRIDE_PURE(
                 std::string, /* Return type */
                 Animal,      /* Parent class */
@@ -78,7 +103,8 @@ could be realized as follows (important changes highlighted):
             .def(py::init<>());
 
         m.def("call_go", [](Animal *animal) -> std::string {
-            /* Release GIL before calling into (potentially long-running) C++ code */
+            // GIL is held when called from Python code. Release GIL before
+            // calling into (potentially long-running) C++ code
             py::gil_scoped_release release;
             return call_go(animal);
         });

include/pybind11/cast.h

@@ -1545,7 +1545,7 @@ private:
             throw cast_error_unable_to_convert_call_arg(a.name, a.type);
 #endif
         }
-        m_kwargs[a.name] = a.value;
+        m_kwargs[a.name] = std::move(a.value);
     }
 
     void process(list & /*args_list*/, detail::kwargs_proxy kp) {

include/pybind11/detail/class.h

@@ -55,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec
     return PyProperty_Type.tp_descr_set(self, cls, value);
 }
 
+// Forward declaration to use in `make_static_property_type()`
+inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type);
+
 /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()`
     methods are modified to always use the object type instead of a concrete instance.
     Return value: New reference. */
@@ -87,6 +90,13 @@ inline PyTypeObject *make_static_property_type() {
         pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");
     }
 
+#    if PY_VERSION_HEX >= 0x030C0000
+    // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE.
+    // Since Python-3.12 property-derived types are required to
+    // have dynamic attributes (to set `__doc__`)
+    enable_dynamic_attributes(heap_type);
+#    endif
+
     setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
     PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
 

include/pybind11/detail/common.h

@@ -1033,12 +1033,7 @@ PYBIND11_NAMESPACE_END(detail)
 ///  - regular: static_cast<Return (Class::*)(Arg0, Arg1, Arg2)>(&Class::func)
 ///  - sweet:   overload_cast<Arg0, Arg1, Arg2>(&Class::func)
 template <typename... Args>
-#    if (defined(_MSC_VER) && _MSC_VER < 1920) /* MSVC 2017 */                                    \
+static constexpr detail::overload_cast_impl<Args...> overload_cast{};
-        || (defined(__clang__) && __clang_major__ == 5)
-static constexpr detail::overload_cast_impl<Args...> overload_cast = {};
-#    else
-static constexpr detail::overload_cast_impl<Args...> overload_cast;
-#    endif
 #endif
 
 /// Const member function selector for overload_cast

include/pybind11/detail/internals.h

@@ -412,6 +412,8 @@ PYBIND11_NOINLINE internals &get_internals() {
     // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals.
     struct gil_scoped_acquire_local {
         gil_scoped_acquire_local() : state(PyGILState_Ensure()) {}
+        gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete;
+        gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete;
         ~gil_scoped_acquire_local() { PyGILState_Release(state); }
         const PyGILState_STATE state;
     } gil;
@@ -512,8 +514,13 @@ struct local_internals {
 
 /// Works like `get_internals`, but for things which are locally registered.
 inline local_internals &get_local_internals() {
-    static local_internals locals;
+    // Current static can be created in the interpreter finalization routine. If the later will be
-    return locals;
+    // destroyed in another static variable destructor, creation of this static there will cause
+    // static deinitialization fiasco. In order to avoid it we avoid destruction of the
+    // local_internals static. One can read more about the problem and current solution here:
+    // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables
+    static auto *locals = new local_internals();
+    return *locals;
 }
 
 /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its

include/pybind11/gil.h

@@ -80,6 +80,9 @@ public:
         inc_ref();
     }
 
+    gil_scoped_acquire(const gil_scoped_acquire &) = delete;
+    gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete;
+
     void inc_ref() { ++tstate->gilstate_counter; }
 
     PYBIND11_NOINLINE void dec_ref() {
@@ -144,6 +147,9 @@ public:
         }
     }
 
+    gil_scoped_release(const gil_scoped_acquire &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
+
     /// This method will disable the PyThreadState_DeleteCurrent call and the
     /// GIL won't be acquired. This method should be used if the interpreter
     /// could be shutting down when this is called, as thread deletion is not
@@ -178,6 +184,8 @@ class gil_scoped_acquire {
 
 public:
     gil_scoped_acquire() { state = PyGILState_Ensure(); }
+    gil_scoped_acquire(const gil_scoped_acquire &) = delete;
+    gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete;
     ~gil_scoped_acquire() { PyGILState_Release(state); }
     void disarm() {}
 };
@@ -187,6 +195,8 @@ class gil_scoped_release {
 
 public:
     gil_scoped_release() { state = PyEval_SaveThread(); }
+    gil_scoped_release(const gil_scoped_release &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
     ~gil_scoped_release() { PyEval_RestoreThread(state); }
     void disarm() {}
 };

include/pybind11/numpy.h

@@ -1401,7 +1401,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor>
     oss << '}';
     auto format_str = oss.str();
 
-    // Sanity check: verify that NumPy properly parses our buffer format string
+    // Smoke test: verify that NumPy properly parses our buffer format string
     auto &api = npy_api::get();
     auto arr = array(buffer_info(nullptr, itemsize, format_str, 1));
     if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) {

include/pybind11/operators.h

@@ -84,6 +84,7 @@ struct op_impl {};
 /// Operator implementation generator
 template <op_id id, op_type ot, typename L, typename R>
 struct op_ {
+    static constexpr bool op_enable_if_hook = true;
     template <typename Class, typename... Extra>
     void execute(Class &cl, const Extra &...extra) const {
         using Base = typename Class::type;

include/pybind11/pybind11.h

@@ -1590,14 +1590,14 @@ public:
         return *this;
     }
 
-    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0>
-    class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def(const T &op, const Extra &...extra) {
         op.execute(*this, extra...);
         return *this;
     }
 
-    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0>
-    class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def_cast(const T &op, const Extra &...extra) {
         op.execute_cast(*this, extra...);
         return *this;
     }

include/pybind11/pytypes.h

@@ -473,7 +473,7 @@ struct error_fetch_and_normalize {
                           + " failed to obtain the name "
                             "of the normalized active exception type.");
         }
-#if defined(PYPY_VERSION)
+#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00
         // This behavior runs the risk of masking errors in the error handling, but avoids a
         // conflict with PyPy, which relies on the normalization here to change OSError to
         // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075).
@@ -1830,18 +1830,18 @@ public:
             // guard if destructor called while err indicator is set
             error_scope error_guard;
             auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o));
-            if (destructor == nullptr) {
             if (PyErr_Occurred()) {
                 throw error_already_set();
             }
-                pybind11_fail("Unable to get capsule context");
-            }
             const char *name = get_name_in_error_scope(o);
             void *ptr = PyCapsule_GetPointer(o, name);
             if (ptr == nullptr) {
                 throw error_already_set();
             }
+
+            if (destructor != nullptr) {
                 destructor(ptr);
+            }
         });
 
         if (!m_ptr || PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) {
@@ -2023,14 +2023,20 @@ public:
     detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; }
     template <typename T>
     void append(T &&val) /* py-non-const */ {
-        PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr());
+        if (PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) != 0) {
+            throw error_already_set();
+        }
     }
     template <typename IdxType,
               typename ValType,
               detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0>
     void insert(const IdxType &index, ValType &&val) /* py-non-const */ {
-        PyList_Insert(
+        if (PyList_Insert(m_ptr,
-            m_ptr, ssize_t_cast(index), detail::object_or_cast(std::forward<ValType>(val)).ptr());
+                          ssize_t_cast(index),
+                          detail::object_or_cast(std::forward<ValType>(val)).ptr())
+            != 0) {
+            throw error_already_set();
+        }
     }
 };
 

include/pybind11/stl.h

@@ -49,17 +49,31 @@ constexpr forwarded_type<T, U> forward_like(U &&u) {
     return std::forward<detail::forwarded_type<T, U>>(std::forward<U>(u));
 }
 
+// Checks if a container has a STL style reserve method.
+// This will only return true for a `reserve()` with a `void` return.
+template <typename C>
+using has_reserve_method = std::is_same<decltype(std::declval<C>().reserve(0)), void>;
+
 template <typename Type, typename Key>
 struct set_caster {
     using type = Type;
     using key_conv = make_caster<Key>;
 
+private:
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
+    void reserve_maybe(const anyset &s, Type *) {
+        value.reserve(s.size());
+    }
+    void reserve_maybe(const anyset &, void *) {}
+
+public:
     bool load(handle src, bool convert) {
         if (!isinstance<anyset>(src)) {
             return false;
         }
         auto s = reinterpret_borrow<anyset>(src);
         value.clear();
+        reserve_maybe(s, &value);
         for (auto entry : s) {
             key_conv conv;
             if (!conv.load(entry, convert)) {
@@ -94,12 +108,21 @@ struct map_caster {
     using key_conv = make_caster<Key>;
     using value_conv = make_caster<Value>;
 
+private:
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
+    void reserve_maybe(const dict &d, Type *) {
+        value.reserve(d.size());
+    }
+    void reserve_maybe(const dict &, void *) {}
+
+public:
     bool load(handle src, bool convert) {
         if (!isinstance<dict>(src)) {
             return false;
         }
         auto d = reinterpret_borrow<dict>(src);
         value.clear();
+        reserve_maybe(d, &value);
         for (auto it : d) {
             key_conv kconv;
             value_conv vconv;
@@ -160,9 +183,7 @@ struct list_caster {
     }
 
 private:
-    template <
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
-        typename T = Type,
-        enable_if_t<std::is_same<decltype(std::declval<T>().reserve(0)), void>::value, int> = 0>
     void reserve_maybe(const sequence &s, Type *) {
         value.reserve(s.size());
     }

tests/constructor_stats.h

@@ -115,7 +115,7 @@ public:
 #if defined(PYPY_VERSION)
         PyObject *globals = PyEval_GetGlobals();
         PyObject *result = PyRun_String("import gc\n"
-                                        "for i in range(2):"
+                                        "for i in range(2):\n"
                                         "    gc.collect()\n",
                                         Py_file_input,
                                         globals,

tests/test_class.cpp

@@ -36,6 +36,26 @@ struct NoBraceInitialization {
     std::vector<int> vec;
 };
 
+namespace test_class {
+namespace pr4220_tripped_over_this { // PR #4227
+
+template <int>
+struct SoEmpty {};
+
+template <typename T>
+std::string get_msg(const T &) {
+    return "This is really only meant to exercise successful compilation.";
+}
+
+using Empty0 = SoEmpty<0x0>;
+
+void bind_empty0(py::module_ &m) {
+    py::class_<Empty0>(m, "Empty0").def(py::init<>()).def("get_msg", get_msg<Empty0>);
+}
+
+} // namespace pr4220_tripped_over_this
+} // namespace test_class
+
 TEST_SUBMODULE(class_, m) {
     // test_instance
     struct NoConstructor {
@@ -525,6 +545,8 @@ TEST_SUBMODULE(class_, m) {
         py::class_<OtherDuplicateNested>(gt, "OtherDuplicateNested");
         py::class_<OtherDuplicateNested>(gt, "YetAnotherDuplicateNested");
     });
+
+    test_class::pr4220_tripped_over_this::bind_empty0(m);
 }
 
 template <int N>

tests/test_class.py

@@ -476,3 +476,10 @@ def test_register_duplicate_class():
         m.register_duplicate_nested_class_type(ClassScope)
     expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!'
     assert str(exc_info.value) == expected
+
+
+def test_pr4220_tripped_over_this():
+    assert (
+        m.Empty0().get_msg()
+        == "This is really only meant to exercise successful compilation."
+    )

tests/test_eigen.cpp

@@ -197,11 +197,40 @@ TEST_SUBMODULE(eigen, m) {
 
     // Return a block of a matrix (gives non-standard strides)
     m.def("block",
-          [](const Eigen::Ref<const Eigen::MatrixXd> &x,
+          [m](const py::object &x_obj,
               int start_row,
               int start_col,
               int block_rows,
-             int block_cols) { return x.block(start_row, start_col, block_rows, block_cols); });
+              int block_cols) {
+              return m.attr("_block")(x_obj, x_obj, start_row, start_col, block_rows, block_cols);
+          });
+
+    m.def(
+        "_block",
+        [](const py::object &x_obj,
+           const Eigen::Ref<const Eigen::MatrixXd> &x,
+           int start_row,
+           int start_col,
+           int block_rows,
+           int block_cols) {
+            // See PR #4217 for background. This test is a bit over the top, but might be useful
+            // as a concrete example to point to when explaining the dangling reference trap.
+            auto i0 = py::make_tuple(0, 0);
+            auto x0_orig = x_obj[*i0].cast<double>();
+            if (x(0, 0) != x0_orig) {
+                throw std::runtime_error(
+                    "Something in the type_caster for Eigen::Ref is terribly wrong.");
+            }
+            double x0_mod = x0_orig + 1;
+            x_obj[*i0] = x0_mod;
+            auto copy_detected = (x(0, 0) != x0_mod);
+            x_obj[*i0] = x0_orig;
+            if (copy_detected) {
+                throw std::runtime_error("type_caster for Eigen::Ref made a copy.");
+            }
+            return x.block(start_row, start_col, block_rows, block_cols);
+        },
+        py::keep_alive<0, 1>());
 
     // test_eigen_return_references, test_eigen_keepalive
     // return value referencing/copying tests:

tests/test_eigen.py

@@ -217,15 +217,24 @@ def test_negative_stride_from_python(msg):
     )
 
 
+def test_block_runtime_error_type_caster_eigen_ref_made_a_copy():
+    with pytest.raises(RuntimeError) as excinfo:
+        m.block(ref, 0, 0, 0, 0)
+    assert str(excinfo.value) == "type_caster for Eigen::Ref made a copy."
+
+
 def test_nonunit_stride_to_python():
     assert np.all(m.diagonal(ref) == ref.diagonal())
     assert np.all(m.diagonal_1(ref) == ref.diagonal(1))
     for i in range(-5, 7):
         assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), f"m.diagonal_n({i})"
 
-    assert np.all(m.block(ref, 2, 1, 3, 3) == ref[2:5, 1:4])
+    # Must be order="F", otherwise the type_caster will make a copy and
-    assert np.all(m.block(ref, 1, 4, 4, 2) == ref[1:, 4:])
+    # m.block() will return a dangling reference (heap-use-after-free).
-    assert np.all(m.block(ref, 1, 4, 3, 2) == ref[1:4, 4:])
+    rof = np.asarray(ref, order="F")
+    assert np.all(m.block(rof, 2, 1, 3, 3) == rof[2:5, 1:4])
+    assert np.all(m.block(rof, 1, 4, 4, 2) == rof[1:, 4:])
+    assert np.all(m.block(rof, 1, 4, 3, 2) == rof[1:4, 4:])
 
 
 def test_eigen_ref_to_python():
@@ -251,14 +260,14 @@ def array_copy_but_one(a, r, c, v):
 def test_eigen_return_references():
     """Tests various ways of returning references and non-referencing copies"""
 
-    master = np.ones((10, 10))
+    primary = np.ones((10, 10))
     a = m.ReturnTester()
     a_get1 = a.get()
     assert not a_get1.flags.owndata and a_get1.flags.writeable
-    assign_both(a_get1, master, 3, 3, 5)
+    assign_both(a_get1, primary, 3, 3, 5)
     a_get2 = a.get_ptr()
     assert not a_get2.flags.owndata and a_get2.flags.writeable
-    assign_both(a_get1, master, 2, 3, 6)
+    assign_both(a_get1, primary, 2, 3, 6)
 
     a_view1 = a.view()
     assert not a_view1.flags.owndata and not a_view1.flags.writeable
@@ -271,25 +280,25 @@ def test_eigen_return_references():
 
     a_copy1 = a.copy_get()
     assert a_copy1.flags.owndata and a_copy1.flags.writeable
-    np.testing.assert_array_equal(a_copy1, master)
+    np.testing.assert_array_equal(a_copy1, primary)
     a_copy1[7, 7] = -44  # Shouldn't affect anything else
-    c1want = array_copy_but_one(master, 7, 7, -44)
+    c1want = array_copy_but_one(primary, 7, 7, -44)
     a_copy2 = a.copy_view()
     assert a_copy2.flags.owndata and a_copy2.flags.writeable
-    np.testing.assert_array_equal(a_copy2, master)
+    np.testing.assert_array_equal(a_copy2, primary)
     a_copy2[4, 4] = -22  # Shouldn't affect anything else
-    c2want = array_copy_but_one(master, 4, 4, -22)
+    c2want = array_copy_but_one(primary, 4, 4, -22)
 
     a_ref1 = a.ref()
     assert not a_ref1.flags.owndata and a_ref1.flags.writeable
-    assign_both(a_ref1, master, 1, 1, 15)
+    assign_both(a_ref1, primary, 1, 1, 15)
     a_ref2 = a.ref_const()
     assert not a_ref2.flags.owndata and not a_ref2.flags.writeable
     with pytest.raises(ValueError):
         a_ref2[5, 5] = 33
     a_ref3 = a.ref_safe()
     assert not a_ref3.flags.owndata and a_ref3.flags.writeable
-    assign_both(a_ref3, master, 0, 7, 99)
+    assign_both(a_ref3, primary, 0, 7, 99)
     a_ref4 = a.ref_const_safe()
     assert not a_ref4.flags.owndata and not a_ref4.flags.writeable
     with pytest.raises(ValueError):
@@ -297,23 +306,23 @@ def test_eigen_return_references():
 
     a_copy3 = a.copy_ref()
     assert a_copy3.flags.owndata and a_copy3.flags.writeable
-    np.testing.assert_array_equal(a_copy3, master)
+    np.testing.assert_array_equal(a_copy3, primary)
     a_copy3[8, 1] = 11
-    c3want = array_copy_but_one(master, 8, 1, 11)
+    c3want = array_copy_but_one(primary, 8, 1, 11)
     a_copy4 = a.copy_ref_const()
     assert a_copy4.flags.owndata and a_copy4.flags.writeable
-    np.testing.assert_array_equal(a_copy4, master)
+    np.testing.assert_array_equal(a_copy4, primary)
     a_copy4[8, 4] = 88
-    c4want = array_copy_but_one(master, 8, 4, 88)
+    c4want = array_copy_but_one(primary, 8, 4, 88)
 
     a_block1 = a.block(3, 3, 2, 2)
     assert not a_block1.flags.owndata and a_block1.flags.writeable
     a_block1[0, 0] = 55
-    master[3, 3] = 55
+    primary[3, 3] = 55
     a_block2 = a.block_safe(2, 2, 3, 2)
     assert not a_block2.flags.owndata and a_block2.flags.writeable
     a_block2[2, 1] = -123
-    master[4, 3] = -123
+    primary[4, 3] = -123
     a_block3 = a.block_const(6, 7, 4, 3)
     assert not a_block3.flags.owndata and not a_block3.flags.writeable
     with pytest.raises(ValueError):
@@ -321,18 +330,18 @@ def test_eigen_return_references():
 
     a_copy5 = a.copy_block(2, 2, 2, 3)
     assert a_copy5.flags.owndata and a_copy5.flags.writeable
-    np.testing.assert_array_equal(a_copy5, master[2:4, 2:5])
+    np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5])
     a_copy5[1, 1] = 777
-    c5want = array_copy_but_one(master[2:4, 2:5], 1, 1, 777)
+    c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777)
 
     a_corn1 = a.corners()
     assert not a_corn1.flags.owndata and a_corn1.flags.writeable
     a_corn1 *= 50
     a_corn1[1, 1] = 999
-    master[0, 0] = 50
+    primary[0, 0] = 50
-    master[0, 9] = 50
+    primary[0, 9] = 50
-    master[9, 0] = 50
+    primary[9, 0] = 50
-    master[9, 9] = 999
+    primary[9, 9] = 999
     a_corn2 = a.corners_const()
     assert not a_corn2.flags.owndata and not a_corn2.flags.writeable
     with pytest.raises(ValueError):
@@ -340,22 +349,22 @@ def test_eigen_return_references():
 
     # All of the changes made all the way along should be visible everywhere
     # now (except for the copies, of course)
-    np.testing.assert_array_equal(a_get1, master)
+    np.testing.assert_array_equal(a_get1, primary)
-    np.testing.assert_array_equal(a_get2, master)
+    np.testing.assert_array_equal(a_get2, primary)
-    np.testing.assert_array_equal(a_view1, master)
+    np.testing.assert_array_equal(a_view1, primary)
-    np.testing.assert_array_equal(a_view2, master)
+    np.testing.assert_array_equal(a_view2, primary)
-    np.testing.assert_array_equal(a_ref1, master)
+    np.testing.assert_array_equal(a_ref1, primary)
-    np.testing.assert_array_equal(a_ref2, master)
+    np.testing.assert_array_equal(a_ref2, primary)
-    np.testing.assert_array_equal(a_ref3, master)
+    np.testing.assert_array_equal(a_ref3, primary)
-    np.testing.assert_array_equal(a_ref4, master)
+    np.testing.assert_array_equal(a_ref4, primary)
-    np.testing.assert_array_equal(a_block1, master[3:5, 3:5])
+    np.testing.assert_array_equal(a_block1, primary[3:5, 3:5])
-    np.testing.assert_array_equal(a_block2, master[2:5, 2:4])
+    np.testing.assert_array_equal(a_block2, primary[2:5, 2:4])
-    np.testing.assert_array_equal(a_block3, master[6:10, 7:10])
+    np.testing.assert_array_equal(a_block3, primary[6:10, 7:10])
     np.testing.assert_array_equal(
-        a_corn1, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1]
+        a_corn1, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1]
     )
     np.testing.assert_array_equal(
-        a_corn2, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1]
+        a_corn2, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1]
     )
 
     np.testing.assert_array_equal(a_copy1, c1want)

tests/test_pytypes.cpp

@@ -289,6 +289,12 @@ TEST_SUBMODULE(pytypes, m) {
         return capsule;
     });
 
+    m.def("return_capsule_with_explicit_nullptr_dtor", []() {
+        py::print("creating capsule with explicit nullptr dtor");
+        return py::capsule(reinterpret_cast<void *>(1234),
+                           static_cast<void (*)(void *)>(nullptr)); // PR #4221
+    });
+
     // test_accessors
     m.def("accessor_api", [](const py::object &o) {
         auto d = py::dict();

tests/test_pytypes.py

@@ -299,6 +299,17 @@ def test_capsule(capture):
     """
     )
 
+    with capture:
+        a = m.return_capsule_with_explicit_nullptr_dtor()
+        del a
+        pytest.gc_collect()
+    assert (
+        capture.unordered
+        == """
+        creating capsule with explicit nullptr dtor
+    """
+    )
+
 
 def test_accessors():
     class SubTestObject:

tools/codespell_ignore_lines_from_errors.py

@@ -0,0 +1,35 @@
+"""Simple script for rebuilding .codespell-ignore-lines
+
+Usage:
+
+cat < /dev/null > .codespell-ignore-lines
+pre-commit run --all-files codespell >& /tmp/codespell_errors.txt
+python3 tools/codespell_ignore_lines_from_errors.py /tmp/codespell_errors.txt > .codespell-ignore-lines
+
+git diff to review changes, then commit, push.
+"""
+
+import sys
+from typing import List
+
+
+def run(args: List[str]) -> None:
+    assert len(args) == 1, "codespell_errors.txt"
+    cache = {}
+    done = set()
+    for line in sorted(open(args[0]).read().splitlines()):
+        i = line.find(" ==> ")
+        if i > 0:
+            flds = line[:i].split(":")
+            if len(flds) >= 2:
+                filename, line_num = flds[:2]
+                if filename not in cache:
+                    cache[filename] = open(filename).read().splitlines()
+                supp = cache[filename][int(line_num) - 1]
+                if supp not in done:
+                    print(supp)
+                    done.add(supp)
+
+
+if __name__ == "__main__":
+    run(args=sys.argv[1:])