Merge branch 'master' into sh_merge_master

docs/advanced/classes.rst

@@ -1261,3 +1261,37 @@ object, just like ``type(ob)`` in Python.
     Other types, like ``py::type::of<int>()``, do not work, see :ref:`type-conversions`.
 
 .. versionadded:: 2.6
+
+Custom type setup
+=================
+
+For advanced use cases, such as enabling garbage collection support, you may
+wish to directly manipulate the `PyHeapTypeObject` corresponding to a
+``py::class_`` definition.
+
+You can do that using ``py::custom_type_setup``:
+
+.. code-block:: cpp
+
+   struct OwnsPythonObjects {
+       py::object value = py::none();
+   };
+   py::class_<OwnsPythonObjects> cls(
+       m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) {
+           auto *type = &heap_type->ht_type;
+           type->tp_flags |= Py_TPFLAGS_HAVE_GC;
+           type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
+               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+               Py_VISIT(self.value.ptr());
+               return 0;
+           };
+           type->tp_clear = [](PyObject *self_base) {
+               auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
+               self.value = py::none();
+               return 0;
+           };
+       }));
+   cls.def(py::init<>());
+   cls.def_readwrite("value", &OwnsPythonObjects::value);
+
+.. versionadded:: 2.8

docs/reference.rst

@@ -63,6 +63,9 @@ Convenience functions converting to Python types
 .. doxygenfunction:: make_key_iterator(Iterator, Sentinel, Extra &&...)
 .. doxygenfunction:: make_key_iterator(Type &, Extra&&...)
 
+.. doxygenfunction:: make_value_iterator(Iterator, Sentinel, Extra &&...)
+.. doxygenfunction:: make_value_iterator(Type &, Extra&&...)
+
 .. _extras:
 
 Passing extra arguments to ``def`` or ``class_``

include/pybind11/attr.h

@@ -12,6 +12,8 @@
 
 #include "cast.h"
 
+#include <functional>
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
 /// \addtogroup annotations
@@ -79,6 +81,23 @@ struct metaclass {
     explicit metaclass(handle value) : value(value) { }
 };
 
+/// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that
+/// may be used to customize the Python type.
+///
+/// The callback is invoked immediately before `PyType_Ready`.
+///
+/// Note: This is an advanced interface, and uses of it may require changes to
+/// work with later versions of pybind11.  You may wish to consult the
+/// implementation of `make_new_python_type` in `detail/classes.h` to understand
+/// the context in which the callback will be run.
+struct custom_type_setup {
+    using callback = std::function<void(PyHeapTypeObject *heap_type)>;
+
+    explicit custom_type_setup(callback value) : value(std::move(value)) {}
+
+    callback value;
+};
+
 /// Annotation that marks a class as local to the module:
 struct module_local { const bool value;
     constexpr explicit module_local(bool v = true) : value(v) {}
@@ -272,6 +291,9 @@ struct type_record {
     /// Custom metaclass (optional)
     handle metaclass;
 
+    /// Custom type setup.
+    custom_type_setup::callback custom_type_setup_callback;
+
     /// Multiple inheritance marker
     bool multiple_inheritance : 1;
 
@@ -476,6 +498,13 @@ struct process_attribute<dynamic_attr> : process_attribute_default<dynamic_attr>
     static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; }
 };
 
+template <>
+struct process_attribute<custom_type_setup> {
+    static void init(const custom_type_setup &value, type_record *r) {
+        r->custom_type_setup_callback = value.value;
+    }
+};
+
 template <>
 struct process_attribute<is_final> : process_attribute_default<is_final> {
     static void init(const is_final &, type_record *r) { r->is_final = true; }

include/pybind11/detail/class.h

@@ -685,11 +685,13 @@ inline PyObject* make_new_python_type(const type_record &rec) {
     if (rec.buffer_protocol)
         enable_buffer_protocol(heap_type);
 
+    if (rec.custom_type_setup_callback)
+        rec.custom_type_setup_callback(heap_type);
+
     if (PyType_Ready(type) < 0)
         pybind11_fail(std::string(rec.name) + ": PyType_Ready failed (" + error_string() + ")!");
 
-    assert(rec.dynamic_attr ? PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)
-                            : !PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));
+    assert(!rec.dynamic_attr || PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));
 
     /* Register type with the parent scope */
     if (rec.scope)

include/pybind11/detail/internals.h

@@ -85,12 +85,13 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // On CPython < 3.4 and on PyPy, `PyThread_set_key_value` strangely does not set
 // the value if it has already been set.  Instead, it must first be deleted and
 // then set again.
+inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) {
+    PyThread_delete_key_value(key);
+    PyThread_set_key_value(key, value);
+}
 #        define PYBIND11_TLS_DELETE_VALUE(key) PyThread_delete_key_value(key)
 #        define PYBIND11_TLS_REPLACE_VALUE(key, value)                                            \
-            do {                                                                                  \
-                PyThread_delete_key_value((key));                                                 \
-                PyThread_set_key_value((key), (value));                                           \
-            } while (false)
+            ::pybind11::detail::tls_replace_value((key), (value))
 #    else
 #        define PYBIND11_TLS_DELETE_VALUE(key) PyThread_set_key_value((key), nullptr)
 #        define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_set_key_value((key), (value))

include/pybind11/pybind11.h

@@ -2069,25 +2069,54 @@ inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_t
     return res;
 }
 
-template <typename Iterator, typename Sentinel, bool KeyIterator, return_value_policy Policy>
+/* There are a large number of apparently unused template arguments because
+ * each combination requires a separate py::class_ registration.
+ */
+template <typename Access, return_value_policy Policy, typename Iterator, typename Sentinel, typename ValueType, typename... Extra>
 struct iterator_state {
     Iterator it;
     Sentinel end;
     bool first_or_done;
 };
 
-PYBIND11_NAMESPACE_END(detail)
+// Note: these helpers take the iterator by non-const reference because some
+// iterators in the wild can't be dereferenced when const. C++ needs the extra parens in decltype
+// to enforce an lvalue. The & after Iterator is required for MSVC < 16.9. SFINAE cannot be
+// reused for result_type due to bugs in ICC, NVCC, and PGI compilers. See PR #3293.
+template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()))>
+struct iterator_access {
+    using result_type = decltype((*std::declval<Iterator &>()));
+    // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
+    result_type operator()(Iterator &it) const {
+        return *it;
+    }
+};
 
-/// Makes a python iterator from a first and past-the-end C++ InputIterator.
-template <return_value_policy Policy = return_value_policy::reference_internal,
+template <typename Iterator, typename SFINAE = decltype(((*std::declval<Iterator &>()).first)) >
+struct iterator_key_access {
+    using result_type = decltype(((*std::declval<Iterator &>()).first));
+    result_type operator()(Iterator &it) const {
+        return (*it).first;
+    }
+};
+
+template <typename Iterator, typename SFINAE = decltype(((*std::declval<Iterator &>()).second))>
+struct iterator_value_access {
+    using result_type = decltype(((*std::declval<Iterator &>()).second));
+    result_type operator()(Iterator &it) const {
+        return (*it).second;
+    }
+};
+
+template <typename Access,
+          return_value_policy Policy,
           typename Iterator,
           typename Sentinel,
-#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
-          typename ValueType = decltype(*std::declval<Iterator>()),
-#endif
+          typename ValueType,
           typename... Extra>
-iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
-    using state = detail::iterator_state<Iterator, Sentinel, false, Policy>;
+iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&... extra) {
+    using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>;
+    // TODO: state captures only the types of Extra, not the values
 
     if (!detail::get_type_info(typeid(state), false)) {
         class_<state>(handle(), "iterator", pybind11::module_local())
@@ -2101,7 +2130,7 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
                     s.first_or_done = true;
                     throw stop_iteration();
                 }
-                return *s.it;
+                return Access()(s.it);
             // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
             }, std::forward<Extra>(extra)..., Policy);
     }
@@ -2109,35 +2138,55 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
     return cast(state{first, last, true});
 }
 
-/// Makes an python iterator over the keys (`.first`) of a iterator over pairs from a
+PYBIND11_NAMESPACE_END(detail)
+
+/// Makes a python iterator from a first and past-the-end C++ InputIterator.
+template <return_value_policy Policy = return_value_policy::reference_internal,
+          typename Iterator,
+          typename Sentinel,
+          typename ValueType = typename detail::iterator_access<Iterator>::result_type,
+          typename... Extra>
+iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
+    return detail::make_iterator_impl<
+        detail::iterator_access<Iterator>,
+        Policy,
+        Iterator,
+        Sentinel,
+        ValueType,
+        Extra...>(first, last, std::forward<Extra>(extra)...);
+}
+
+/// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a
 /// first and past-the-end InputIterator.
 template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Iterator,
           typename Sentinel,
-#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
-          typename KeyType = decltype((*std::declval<Iterator>()).first),
-#endif
+          typename KeyType = typename detail::iterator_key_access<Iterator>::result_type,
           typename... Extra>
 iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
-    using state = detail::iterator_state<Iterator, Sentinel, true, Policy>;
+    return detail::make_iterator_impl<
+        detail::iterator_key_access<Iterator>,
+        Policy,
+        Iterator,
+        Sentinel,
+        KeyType,
+        Extra...>(first, last, std::forward<Extra>(extra)...);
+}
 
-    if (!detail::get_type_info(typeid(state), false)) {
-        class_<state>(handle(), "iterator", pybind11::module_local())
-            .def("__iter__", [](state &s) -> state& { return s; })
-            .def("__next__", [](state &s) -> detail::remove_cv_t<KeyType> {
-                if (!s.first_or_done)
-                    ++s.it;
-                else
-                    s.first_or_done = false;
-                if (s.it == s.end) {
-                    s.first_or_done = true;
-                    throw stop_iteration();
-                }
-                return (*s.it).first;
-            }, std::forward<Extra>(extra)..., Policy);
-    }
-
-    return cast(state{first, last, true});
+/// Makes a python iterator over the values (`.second`) of a iterator over pairs from a
+/// first and past-the-end InputIterator.
+template <return_value_policy Policy = return_value_policy::reference_internal,
+          typename Iterator,
+          typename Sentinel,
+          typename ValueType = typename detail::iterator_value_access<Iterator>::result_type,
+          typename... Extra>
+iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
+    return detail::make_iterator_impl<
+        detail::iterator_value_access<Iterator>,
+        Policy, Iterator,
+        Sentinel,
+        ValueType,
+        Extra...>(first, last, std::forward<Extra>(extra)...);
 }
 
 /// Makes an iterator over values of an stl container or other container supporting
@@ -2154,6 +2203,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
     return make_key_iterator<Policy>(std::begin(value), std::end(value), extra...);
 }
 
+/// Makes an iterator over the values (`.second`) of a stl map-like container supporting
+/// `std::begin()`/`std::end()`
+template <return_value_policy Policy = return_value_policy::reference_internal,
+          typename Type, typename... Extra> iterator make_value_iterator(Type &value, Extra&&... extra) {
+    return make_value_iterator<Policy>(std::begin(value), std::end(value), extra...);
+}
+
 template <typename InputType, typename OutputType> void implicitly_convertible() {
     struct set_flag {
         bool &flag;

include/pybind11/pytypes.h

@@ -259,8 +259,11 @@ public:
 
     object& operator=(const object &other) {
         other.inc_ref();
-        dec_ref();
+        // Use temporary variable to ensure `*this` remains valid while
+        // `Py_XDECREF` executes, in case `*this` is accessible from Python.
+        handle temp(m_ptr);
         m_ptr = other.m_ptr;
+        temp.dec_ref();
         return *this;
     }
 

tests/CMakeLists.txt

@@ -118,6 +118,7 @@ set(PYBIND11_TEST_FILES
     test_constants_and_functions.cpp
     test_copy_move.cpp
     test_custom_type_casters.cpp
+    test_custom_type_setup.cpp
     test_docstring_options.cpp
     test_eigen.cpp
     test_enum.cpp

tests/test_custom_type_setup.cpp

@@ -0,0 +1,41 @@
+/*
+    tests/test_custom_type_setup.cpp -- Tests `pybind11::custom_type_setup`
+
+    Copyright (c) Google LLC
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include <pybind11/pybind11.h>
+
+#include "pybind11_tests.h"
+
+namespace py = pybind11;
+
+namespace {
+
+struct OwnsPythonObjects {
+    py::object value = py::none();
+};
+} // namespace
+
+TEST_SUBMODULE(custom_type_setup, m) {
+    py::class_<OwnsPythonObjects> cls(
+        m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) {
+            auto *type = &heap_type->ht_type;
+            type->tp_flags |= Py_TPFLAGS_HAVE_GC;
+            type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
+                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                Py_VISIT(self.value.ptr());
+                return 0;
+            };
+            type->tp_clear = [](PyObject *self_base) {
+                auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
+                self.value = py::none();
+                return 0;
+            };
+        }));
+    cls.def(py::init<>());
+    cls.def_readwrite("value", &OwnsPythonObjects::value);
+}

tests/test_custom_type_setup.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+import gc
+import weakref
+
+import pytest
+
+import env  # noqa: F401
+from pybind11_tests import custom_type_setup as m
+
+
+@pytest.fixture
+def gc_tester():
+    """Tests that an object is garbage collected.
+
+    Assumes that any unreferenced objects are fully collected after calling
+    `gc.collect()`.  That is true on CPython, but does not appear to reliably
+    hold on PyPy.
+    """
+
+    weak_refs = []
+
+    def add_ref(obj):
+        # PyPy does not support `gc.is_tracked`.
+        if hasattr(gc, "is_tracked"):
+            assert gc.is_tracked(obj)
+        weak_refs.append(weakref.ref(obj))
+
+    yield add_ref
+
+    gc.collect()
+    for ref in weak_refs:
+        assert ref() is None
+
+
+# PyPy does not seem to reliably garbage collect.
+@pytest.mark.skipif("env.PYPY")
+def test_self_cycle(gc_tester):
+    obj = m.OwnsPythonObjects()
+    obj.value = obj
+    gc_tester(obj)
+
+
+# PyPy does not seem to reliably garbage collect.
+@pytest.mark.skipif("env.PYPY")
+def test_indirect_cycle(gc_tester):
+    obj = m.OwnsPythonObjects()
+    obj_list = [obj]
+    obj.value = obj_list
+    gc_tester(obj)

tests/test_sequences_and_iterators.cpp

@@ -15,6 +15,7 @@
 
 #include <algorithm>
 #include <utility>
+#include <vector>
 
 #ifdef PYBIND11_HAS_OPTIONAL
 #include <optional>
@@ -37,6 +38,29 @@ bool operator==(const NonZeroIterator<std::pair<A, B>>& it, const NonZeroSentine
     return !(*it).first || !(*it).second;
 }
 
+class NonCopyableInt {
+public:
+    explicit NonCopyableInt(int value) : value_(value) {}
+    NonCopyableInt(const NonCopyableInt &) = delete;
+    NonCopyableInt(NonCopyableInt &&other) noexcept : value_(other.value_) {
+        other.value_ = -1;  // detect when an unwanted move occurs
+    }
+    NonCopyableInt &operator=(const NonCopyableInt &) = delete;
+    NonCopyableInt &operator=(NonCopyableInt &&other) noexcept {
+        value_ = other.value_;
+        other.value_ = -1;  // detect when an unwanted move occurs
+        return *this;
+    }
+    int get() const { return value_; }
+    void set(int value) { value_ = value; }
+    ~NonCopyableInt() = default;
+private:
+    int value_;
+};
+using NonCopyableIntPair = std::pair<NonCopyableInt, NonCopyableInt>;
+PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableInt>);
+PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableIntPair>);
+
 template <typename PythonType>
 py::list test_random_access_iterator(PythonType x) {
     if (x.size() < 5)
@@ -288,6 +312,10 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
         .def(
             "items",
             [](const StringMap &map) { return py::make_iterator(map.begin(), map.end()); },
+            py::keep_alive<0, 1>())
+        .def(
+            "values",
+            [](const StringMap &map) { return py::make_value_iterator(map.begin(), map.end()); },
             py::keep_alive<0, 1>());
 
     // test_generalized_iterators
@@ -308,19 +336,62 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
         .def("nonzero_keys", [](const IntPairs& s) {
             return py::make_key_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
         }, py::keep_alive<0, 1>())
+        .def("nonzero_values", [](const IntPairs& s) {
+            return py::make_value_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
+        }, py::keep_alive<0, 1>())
+
+        // test single-argument make_iterator
         .def("simple_iterator", [](IntPairs& self) {
             return py::make_iterator(self);
         }, py::keep_alive<0, 1>())
         .def("simple_keys", [](IntPairs& self) {
             return py::make_key_iterator(self);
         }, py::keep_alive<0, 1>())
+        .def("simple_values", [](IntPairs& self) {
+            return py::make_value_iterator(self);
+        }, py::keep_alive<0, 1>())
 
-        // test iterator with keep_alive (doesn't work so not used at runtime, but tests compile)
-        .def("make_iterator_keep_alive", [](IntPairs& self) {
-            return py::make_iterator(self, py::keep_alive<0, 1>());
+        // Test iterator with an Extra (doesn't do anything useful, so not used
+        // at runtime, but tests need to be able to compile with the correct
+        // overload. See PR #3293.
+        .def("_make_iterator_extras", [](IntPairs& self) {
+            return py::make_iterator(self, py::call_guard<int>());
+        }, py::keep_alive<0, 1>())
+        .def("_make_key_extras", [](IntPairs& self) {
+            return py::make_key_iterator(self, py::call_guard<int>());
+        }, py::keep_alive<0, 1>())
+        .def("_make_value_extras", [](IntPairs& self) {
+            return py::make_value_iterator(self, py::call_guard<int>());
         }, py::keep_alive<0, 1>())
         ;
 
+    // test_iterater_referencing
+    py::class_<NonCopyableInt>(m, "NonCopyableInt")
+        .def(py::init<int>())
+        .def("set", &NonCopyableInt::set)
+        .def("__int__", &NonCopyableInt::get)
+        ;
+    py::class_<std::vector<NonCopyableInt>>(m, "VectorNonCopyableInt")
+        .def(py::init<>())
+        .def("append", [](std::vector<NonCopyableInt> &vec, int value) {
+            vec.emplace_back(value);
+        })
+        .def("__iter__", [](std::vector<NonCopyableInt> &vec) {
+            return py::make_iterator(vec.begin(), vec.end());
+        })
+        ;
+    py::class_<std::vector<NonCopyableIntPair>>(m, "VectorNonCopyableIntPair")
+        .def(py::init<>())
+        .def("append", [](std::vector<NonCopyableIntPair> &vec, const std::pair<int, int> &value) {
+            vec.emplace_back(NonCopyableInt(value.first), NonCopyableInt(value.second));
+        })
+        .def("keys", [](std::vector<NonCopyableIntPair> &vec) {
+            return py::make_key_iterator(vec.begin(), vec.end());
+        })
+        .def("values", [](std::vector<NonCopyableIntPair> &vec) {
+            return py::make_value_iterator(vec.begin(), vec.end());
+        })
+        ;
 
 #if 0
     // Obsolete: special data structure for exposing custom iterator types to python

tests/test_sequences_and_iterators.py

@@ -36,6 +36,10 @@ def test_generalized_iterators():
     assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_keys()) == [1]
     assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_keys()) == []
 
+    assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).nonzero_values()) == [2, 4]
+    assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_values()) == [2]
+    assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_values()) == []
+
     # __next__ must continue to raise StopIteration
     it = m.IntPairs([(0, 0)]).nonzero()
     for _ in range(3):
@@ -55,6 +59,31 @@ def test_generalized_iterators_simple():
         (0, 5),
     ]
     assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_keys()) == [1, 3, 0]
+    assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_values()) == [2, 4, 5]
+
+
+def test_iterator_referencing():
+    """Test that iterators reference rather than copy their referents."""
+    vec = m.VectorNonCopyableInt()
+    vec.append(3)
+    vec.append(5)
+    assert [int(x) for x in vec] == [3, 5]
+    # Increment everything to make sure the referents can be mutated
+    for x in vec:
+        x.set(int(x) + 1)
+    assert [int(x) for x in vec] == [4, 6]
+
+    vec = m.VectorNonCopyableIntPair()
+    vec.append([3, 4])
+    vec.append([5, 7])
+    assert [int(x) for x in vec.keys()] == [3, 5]
+    assert [int(x) for x in vec.values()] == [4, 7]
+    for x in vec.keys():
+        x.set(int(x) + 1)
+    for x in vec.values():
+        x.set(int(x) + 10)
+    assert [int(x) for x in vec.keys()] == [4, 6]
+    assert [int(x) for x in vec.values()] == [14, 17]
 
 
 def test_sliceable():
@@ -160,6 +189,7 @@ def test_map_iterator():
         assert sm[k] == expected[k]
     for k, v in sm.items():
         assert v == expected[k]
+    assert list(sm.values()) == [expected[k] for k in sm]
 
     it = iter(m.StringMap({}))
     for _ in range(3):  # __next__ must continue to raise StopIteration

tools/pybind11NewTools.cmake

@@ -82,6 +82,15 @@ if(NOT DEFINED ${_Python}_EXECUTABLE)
 
 endif()
 
+if(NOT ${_Python}_EXECUTABLE STREQUAL PYTHON_EXECUTABLE_LAST)
+  # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
+  unset(PYTHON_IS_DEBUG CACHE)
+  unset(PYTHON_MODULE_EXTENSION CACHE)
+  set(PYTHON_EXECUTABLE_LAST
+      "${${_Python}_EXECUTABLE}"
+      CACHE INTERNAL "Python executable during the last CMake run")
+endif()
+
 if(NOT DEFINED PYTHON_IS_DEBUG)
   # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
   execute_process(

tools/pybind11Tools.cmake

@@ -45,31 +45,25 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
 find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} MODULE REQUIRED ${_pybind11_quiet})
 list(REMOVE_AT CMAKE_MODULE_PATH -1)
 
+# Makes a normal variable a cached variable
+macro(_PYBIND11_PROMOTE_TO_CACHE NAME)
+  set(_tmp_ptc "${${NAME}}")
+  # CMake 3.21 complains if a cached variable is shadowed by a normal one
+  unset(${NAME})
+  set(${NAME}
+      "${_tmp_ptc}"
+      CACHE INTERNAL "")
+endmacro()
+
 # Cache variables so pybind11_add_module can be used in parent projects
-set(PYTHON_INCLUDE_DIRS
-    ${PYTHON_INCLUDE_DIRS}
-    CACHE INTERNAL "")
-set(PYTHON_LIBRARIES
-    ${PYTHON_LIBRARIES}
-    CACHE INTERNAL "")
-set(PYTHON_MODULE_PREFIX
-    ${PYTHON_MODULE_PREFIX}
-    CACHE INTERNAL "")
-set(PYTHON_MODULE_EXTENSION
-    ${PYTHON_MODULE_EXTENSION}
-    CACHE INTERNAL "")
-set(PYTHON_VERSION_MAJOR
-    ${PYTHON_VERSION_MAJOR}
-    CACHE INTERNAL "")
-set(PYTHON_VERSION_MINOR
-    ${PYTHON_VERSION_MINOR}
-    CACHE INTERNAL "")
-set(PYTHON_VERSION
-    ${PYTHON_VERSION}
-    CACHE INTERNAL "")
-set(PYTHON_IS_DEBUG
-    "${PYTHON_IS_DEBUG}"
-    CACHE INTERNAL "")
+_pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS)
+_pybind11_promote_to_cache(PYTHON_LIBRARIES)
+_pybind11_promote_to_cache(PYTHON_MODULE_PREFIX)
+_pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION)
+_pybind11_promote_to_cache(PYTHON_VERSION_MAJOR)
+_pybind11_promote_to_cache(PYTHON_VERSION_MINOR)
+_pybind11_promote_to_cache(PYTHON_VERSION)
+_pybind11_promote_to_cache(PYTHON_IS_DEBUG)
 
 if(PYBIND11_MASTER_PROJECT)
   if(PYTHON_MODULE_EXTENSION MATCHES "pypy")