Merge branch 'master' into python_multiple_inheritance_test

.github/CONTRIBUTING.md

@@ -135,7 +135,7 @@ The valid options are:
 * Use `-G` and the name of a generator to use something different. `cmake
   --help` lists the generators available.
       - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give
-        you automatic mulithreading on all your CMake projects!
+        you automatic multithreading on all your CMake projects!
 * Open the `CMakeLists.txt` with QtCreator to generate for that IDE.
 * You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file
   that some tools expect.

.github/workflows/ci.yml

@@ -195,9 +195,10 @@ jobs:
       matrix:
         include:
         # TODO: Fails on 3.10, investigate
-        - python-version: "3.9"
+        # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4889
-          python-debug: true
+        # - python-version: "3.9"
-          valgrind: true
+        #   python-debug: true
+        #   valgrind: true
         - python-version: "3.11"
           python-debug: false
 
@@ -1052,7 +1053,7 @@ jobs:
         uses: jwlawson/actions-setup-cmake@v1.14
 
       - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v3
+        uses: seanmiddleditch/gha-setup-ninja@v4
 
       - name: Run pip installs
         run: |

.pre-commit-config.yaml

@@ -25,27 +25,27 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v16.0.6"
+  rev: "v17.0.3"
   hooks:
   - id: clang-format
     types_or: [c++, c, cuda]
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: "23.7.0" # Keep in sync with blacken-docs
+  rev: "23.10.1" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
 # Ruff, the Python auto-correcting linter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.287
+  rev: v0.1.2
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v1.5.1"
+  rev: "v1.6.1"
   hooks:
   - id: mypy
     args: []
@@ -67,7 +67,7 @@ repos:
 
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: "v4.4.0"
+  rev: "v4.5.0"
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -98,7 +98,7 @@ repos:
 
 # Avoid directional quotes
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.5.0"
+  rev: "0.6.2"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
@@ -124,7 +124,7 @@ repos:
 # Use tools/codespell_ignore_lines_from_errors.py
 # to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.2.5"
+  rev: "v2.2.6"
   hooks:
   - id: codespell
     exclude: ".supp$"
@@ -132,7 +132,7 @@ repos:
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: "v0.9.0.5"
+  rev: "v0.9.0.6"
   hooks:
   - id: shellcheck
 
@@ -142,12 +142,12 @@ repos:
   - id: disallow-caps
     name: Disallow improper capitalization
     language: pygrep
-    entry: PyBind|Numpy|Cmake|CCache|PyTest
+    entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest
     exclude: ^\.pre-commit-config.yaml$
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.0.0a7"
+  rev: "v3.0.1"
   hooks:
   - id: pylint
     files: ^pybind11

CMakeLists.txt

@@ -5,6 +5,11 @@
 # All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+# Propagate this policy (FindPythonInterp removal) so it can be detected later
+if(NOT CMAKE_VERSION VERSION_LESS "3.27")
+  cmake_policy(GET CMP0148 _pybind11_cmp0148)
+endif()
+
 cmake_minimum_required(VERSION 3.5)
 
 # The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with
@@ -16,6 +21,11 @@ else()
   cmake_policy(VERSION 3.26)
 endif()
 
+if(_pybind11_cmp0148)
+  cmake_policy(SET CMP0148 ${_pybind11_cmp0148})
+  unset(_pybind11_cmp0148)
+endif()
+
 # Avoid infinite recursion if tests include this as a subdirectory
 if(DEFINED PYBIND11_MASTER_PROJECT)
   return()
@@ -132,6 +142,7 @@ set(PYBIND11_HEADERS
     include/pybind11/embed.h
     include/pybind11/eval.h
     include/pybind11/gil.h
+    include/pybind11/gil_safe_call_once.h
     include/pybind11/iostream.h
     include/pybind11/functional.h
     include/pybind11/numpy.h

docs/advanced/exceptions.rst

@@ -141,15 +141,14 @@ standard python RuntimeError:
 
 .. code-block:: cpp
 
-    // This is a static object, so we must leak the Python reference:
+    PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> exc_storage;
-    // It is undefined when the destructor will run, possibly only after the
+    exc_storage.call_once_and_store_result(
-    // Python interpreter is finalized already.
+        [&]() { return py::exception<MyCustomException>(m, "MyCustomError"); });
-    static py::handle exc = py::exception<MyCustomException>(m, "MyCustomError").release();
     py::register_exception_translator([](std::exception_ptr p) {
         try {
             if (p) std::rethrow_exception(p);
         } catch (const MyCustomException &e) {
-            py::set_error(exc, e.what());
+            py::set_error(exc_storage.get_stored(), e.what());
         } catch (const OtherException &e) {
             py::set_error(PyExc_RuntimeError, e.what());
         }

include/pybind11/cast.h

@@ -42,13 +42,15 @@ using make_caster = type_caster<intrinsic_t<type>>;
 // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T
 template <typename T>
 typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) {
-    return caster.operator typename make_caster<T>::template cast_op_type<T>();
+    using result_t = typename make_caster<T>::template cast_op_type<T>; // See PR #4893
+    return caster.operator result_t();
 }
 template <typename T>
 typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>
 cast_op(make_caster<T> &&caster) {
-    return std::move(caster).operator typename make_caster<T>::
+    using result_t = typename make_caster<T>::template cast_op_type<
-        template cast_op_type<typename std::add_rvalue_reference<T>::type>();
+        typename std::add_rvalue_reference<T>::type>; // See PR #4893
+    return std::move(caster).operator result_t();
 }
 
 template <typename type>

include/pybind11/detail/common.h

@@ -118,6 +118,14 @@
 #    endif
 #endif
 
+#if defined(PYBIND11_CPP20)
+#    define PYBIND11_CONSTINIT constinit
+#    define PYBIND11_DTOR_CONSTEXPR constexpr
+#else
+#    define PYBIND11_CONSTINIT
+#    define PYBIND11_DTOR_CONSTEXPR
+#endif
+
 // Compiler version assertions
 #if defined(__INTEL_COMPILER)
 #    if __INTEL_COMPILER < 1800

include/pybind11/detail/init.h

@@ -65,7 +65,7 @@ constexpr bool is_alias(void *) {
 }
 
 // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall
-// back to brace aggregate initiailization so that for aggregate initialization can be used with
+// back to brace aggregate initialization so that for aggregate initialization can be used with
 // py::init, e.g.  `py::init<int, int>` to initialize a `struct T { int a; int b; }`.  For
 // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually
 // works, but will not do the expected thing when `T` has an `initializer_list<T>` constructor).

include/pybind11/detail/internals.h

@@ -451,6 +451,7 @@ inline object get_python_state_dict() {
 #endif
     if (!state_dict) {
         raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED");
+        throw error_already_set();
     }
     return state_dict;
 }
@@ -463,6 +464,7 @@ inline internals **get_internals_pp_from_capsule(handle obj) {
     void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr);
     if (raw_ptr == nullptr) {
         raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED");
+        throw error_already_set();
     }
     return static_cast<internals **>(raw_ptr);
 }

include/pybind11/detail/type_caster_base.h

@@ -521,8 +521,10 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i
 inline PyThreadState *get_thread_state_unchecked() {
 #if defined(PYPY_VERSION)
     return PyThreadState_GET();
-#else
+#elif PY_VERSION_HEX < 0x030D0000
     return _PyThreadState_UncheckedGet();
+#else
+    return PyThreadState_GetUnchecked();
 #endif
 }
 

include/pybind11/gil.h

@@ -11,6 +11,8 @@
 
 #include "detail/common.h"
 
+#include <cassert>
+
 #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 #    include "detail/internals.h"
 #endif
@@ -137,7 +139,9 @@ private:
 
 class gil_scoped_release {
 public:
+    // PRECONDITION: The GIL must be held when this constructor is called.
     explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) {
+        assert(PyGILState_Check());
         // `get_internals()` must be called here unconditionally in order to initialize
         // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an
         // initialization race could occur as multiple threads try `gil_scoped_acquire`.
@@ -201,7 +205,11 @@ class gil_scoped_release {
     PyThreadState *state;
 
 public:
-    gil_scoped_release() : state{PyEval_SaveThread()} {}
+    // PRECONDITION: The GIL must be held when this constructor is called.
+    gil_scoped_release() {
+        assert(PyGILState_Check());
+        state = PyEval_SaveThread();
+    }
     gil_scoped_release(const gil_scoped_release &) = delete;
     gil_scoped_release &operator=(const gil_scoped_release &) = delete;
     ~gil_scoped_release() { PyEval_RestoreThread(state); }

include/pybind11/gil_safe_call_once.h

@@ -0,0 +1,91 @@
+// Copyright (c) 2023 The pybind Community.
+
+#pragma once
+
+#include "detail/common.h"
+#include "gil.h"
+
+#include <cassert>
+#include <mutex>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+// Use the `gil_safe_call_once_and_store` class below instead of the naive
+//
+//   static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE!
+//
+// which has two serious issues:
+//
+//     1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and
+//     2. deadlocks in multi-threaded processes (because of missing lock ordering).
+//
+// The following alternative avoids both problems:
+//
+//   PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> storage;
+//   auto &imported_obj = storage // Do NOT make this `static`!
+//       .call_once_and_store_result([]() {
+//           return py::module_::import("module_name");
+//       })
+//       .get_stored();
+//
+// The parameter of `call_once_and_store_result()` must be callable. It can make
+// CPython API calls, and in particular, it can temporarily release the GIL.
+//
+// `T` can be any C++ type, it does not have to involve CPython API types.
+//
+// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`),
+// is not ideal. If the main thread is the one to actually run the `Callable`,
+// then a `KeyboardInterrupt` will interrupt it if it is running normal Python
+// code. The situation is different if a non-main thread runs the
+// `Callable`, and then the main thread starts waiting for it to complete:
+// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will
+// get processed only when it is the main thread's turn again and it is running
+// normal Python code. However, this will be unnoticeable for quick call-once
+// functions, which is usually the case.
+template <typename T>
+class gil_safe_call_once_and_store {
+public:
+    // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called.
+    template <typename Callable>
+    gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) {
+        if (!is_initialized_) { // This read is guarded by the GIL.
+            // Multiple threads may enter here, because the GIL is released in the next line and
+            // CPython API calls in the `fn()` call below may release and reacquire the GIL.
+            gil_scoped_release gil_rel; // Needed to establish lock ordering.
+            std::call_once(once_flag_, [&] {
+                // Only one thread will ever enter here.
+                gil_scoped_acquire gil_acq;
+                ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL.
+                is_initialized_ = true;   // This write is guarded by the GIL.
+            });
+            // All threads will observe `is_initialized_` as true here.
+        }
+        // Intentionally not returning `T &` to ensure the calling code is self-documenting.
+        return *this;
+    }
+
+    // This must only be called after `call_once_and_store_result()` was called.
+    T &get_stored() {
+        assert(is_initialized_);
+        PYBIND11_WARNING_PUSH
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5
+        // Needed for gcc 4.8.5
+        PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing")
+#endif
+        return *reinterpret_cast<T *>(storage_);
+        PYBIND11_WARNING_POP
+    }
+
+    constexpr gil_safe_call_once_and_store() = default;
+    PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default;
+
+private:
+    alignas(T) char storage_[sizeof(T)] = {};
+    std::once_flag once_flag_ = {};
+    bool is_initialized_ = false;
+    // The `is_initialized_`-`storage_` pair is very similar to `std::optional`,
+    // but the latter does not have the triviality properties of former,
+    // therefore `std::optional` is not a viable alternative here.
+};
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/numpy.h

@@ -10,7 +10,10 @@
 #pragma once
 
 #include "pybind11.h"
+#include "detail/common.h"
 #include "complex.h"
+#include "gil_safe_call_once.h"
+#include "pytypes.h"
 
 #include <algorithm>
 #include <array>
@@ -120,6 +123,20 @@ inline numpy_internals &get_numpy_internals() {
     return *ptr;
 }
 
+PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) {
+    module_ numpy = module_::import("numpy");
+    str version_string = numpy.attr("__version__");
+
+    module_ numpy_lib = module_::import("numpy.lib");
+    object numpy_version = numpy_lib.attr("NumpyVersion")(version_string);
+    int major_version = numpy_version.attr("major").cast<int>();
+
+    /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially
+        became a private module. */
+    std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core";
+    return module_::import((numpy_core_path + "." + submodule_name).c_str());
+}
+
 template <typename T>
 struct same_size {
     template <typename U>
@@ -192,8 +209,8 @@ struct npy_api {
     };
 
     static npy_api &get() {
-        static npy_api api = lookup();
+        PYBIND11_CONSTINIT static gil_safe_call_once_and_store<npy_api> storage;
-        return api;
+        return storage.call_once_and_store_result(lookup).get_stored();
     }
 
     bool PyArray_Check_(PyObject *obj) const {
@@ -263,9 +280,13 @@ private:
     };
 
     static npy_api lookup() {
-        module_ m = module_::import("numpy.core.multiarray");
+        module_ m = detail::import_numpy_core_submodule("multiarray");
         auto c = m.attr("_ARRAY_API");
         void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr);
+        if (api_ptr == nullptr) {
+            raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer.");
+            throw error_already_set();
+        }
         npy_api api;
 #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func];
         DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion);
@@ -625,13 +646,14 @@ public:
     char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; }
 
 private:
-    static object _dtype_from_pep3118() {
+    static object &_dtype_from_pep3118() {
-        static PyObject *obj = module_::import("numpy.core._internal")
+        PYBIND11_CONSTINIT static gil_safe_call_once_and_store<object> storage;
-                                   .attr("_dtype_from_pep3118")
+        return storage
-                                   .cast<object>()
+            .call_once_and_store_result([]() {
-                                   .release()
+                return detail::import_numpy_core_submodule("_internal")
-                                   .ptr();
+                    .attr("_dtype_from_pep3118");
-        return reinterpret_borrow<object>(obj);
+            })
+            .get_stored();
     }
 
     dtype strip_padding(ssize_t itemsize) {

include/pybind11/pybind11.h

@@ -14,7 +14,9 @@
 #include "detail/init.h"
 #include "attr.h"
 #include "gil.h"
+#include "gil_safe_call_once.h"
 #include "options.h"
+#include "typing.h"
 
 #include <cstdlib>
 #include <cstring>
@@ -57,11 +59,13 @@ inline std::string replace_newlines_and_squash(const char *text) {
     std::string result(text);
     bool previous_is_whitespace = false;
 
-    // Do not modify string representations
+    if (result.size() >= 2) {
-    char first_char = result[0];
+        // Do not modify string representations
-    char last_char = result[result.size() - 1];
+        char first_char = result[0];
-    if (first_char == last_char && first_char == '\'') {
+        char last_char = result[result.size() - 1];
-        return result;
+        if (first_char == last_char && first_char == '\'') {
+            return result;
+        }
     }
     result.clear();
 
@@ -1140,7 +1144,7 @@ protected:
                     }
                     msg += "kwargs: ";
                     bool first = true;
-                    for (auto kwarg : kwargs) {
+                    for (const auto &kwarg : kwargs) {
                         if (first) {
                             first = false;
                         } else {
@@ -2445,7 +2449,7 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_access<Iterator>::result_type,
           typename... Extra>
-iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) {
+typing::Iterator<ValueType> make_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_access<Iterator>,
                                       Policy,
                                       Iterator,
@@ -2463,7 +2467,7 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename KeyType = typename detail::iterator_key_access<Iterator>::result_type,
           typename... Extra>
-iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
+typing::Iterator<KeyType> make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_key_access<Iterator>,
                                       Policy,
                                       Iterator,
@@ -2481,7 +2485,7 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_value_access<Iterator>::result_type,
           typename... Extra>
-iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
+typing::Iterator<ValueType> make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_value_access<Iterator>,
                                       Policy,
                                       Iterator,
@@ -2496,8 +2500,10 @@ iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
 /// `std::begin()`/`std::end()`
 template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Type,
+          typename ValueType = typename detail::iterator_access<
+              decltype(std::begin(std::declval<Type &>()))>::result_type,
           typename... Extra>
-iterator make_iterator(Type &value, Extra &&...extra) {
+typing::Iterator<ValueType> make_iterator(Type &value, Extra &&...extra) {
     return make_iterator<Policy>(
         std::begin(value), std::end(value), std::forward<Extra>(extra)...);
 }
@@ -2506,8 +2512,10 @@ iterator make_iterator(Type &value, Extra &&...extra) {
 /// `std::begin()`/`std::end()`
 template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Type,
+          typename KeyType = typename detail::iterator_key_access<
+              decltype(std::begin(std::declval<Type &>()))>::result_type,
           typename... Extra>
-iterator make_key_iterator(Type &value, Extra &&...extra) {
+typing::Iterator<KeyType> make_key_iterator(Type &value, Extra &&...extra) {
     return make_key_iterator<Policy>(
         std::begin(value), std::end(value), std::forward<Extra>(extra)...);
 }
@@ -2516,8 +2524,10 @@ iterator make_key_iterator(Type &value, Extra &&...extra) {
 /// `std::begin()`/`std::end()`
 template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Type,
+          typename ValueType = typename detail::iterator_value_access<
+              decltype(std::begin(std::declval<Type &>()))>::result_type,
           typename... Extra>
-iterator make_value_iterator(Type &value, Extra &&...extra) {
+typing::Iterator<ValueType> make_value_iterator(Type &value, Extra &&...extra) {
     return make_value_iterator<Policy>(
         std::begin(value), std::end(value), std::forward<Extra>(extra)...);
 }
@@ -2600,23 +2610,14 @@ public:
 };
 
 PYBIND11_NAMESPACE_BEGIN(detail)
-// Returns a reference to a function-local static exception object used in the simple
-// register_exception approach below.  (It would be simpler to have the static local variable
-// directly in register_exception, but that makes clang <3.5 segfault - issue #1349).
-template <typename CppException>
-exception<CppException> &get_exception_object() {
-    static exception<CppException> ex;
-    return ex;
-}
 
 // Helper function for register_exception and register_local_exception
 template <typename CppException>
 exception<CppException> &
 register_exception_impl(handle scope, const char *name, handle base, bool isLocal) {
-    auto &ex = detail::get_exception_object<CppException>();
+    PYBIND11_CONSTINIT static gil_safe_call_once_and_store<exception<CppException>> exc_storage;
-    if (!ex) {
+    exc_storage.call_once_and_store_result(
-        ex = exception<CppException>(scope, name, base);
+        [&]() { return exception<CppException>(scope, name, base); });
-    }
 
     auto register_func
         = isLocal ? &register_local_exception_translator : &register_exception_translator;
@@ -2628,10 +2629,10 @@ register_exception_impl(handle scope, const char *name, handle base, bool isLoca
         try {
             std::rethrow_exception(p);
         } catch (const CppException &e) {
-            set_error(detail::get_exception_object<CppException>(), e.what());
+            set_error(exc_storage.get_stored(), e.what());
         }
     });
-    return ex;
+    return exc_storage.get_stored();
 }
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/pytypes.h

@@ -305,19 +305,19 @@ private:
             "https://pybind11.readthedocs.io/en/stable/advanced/"
             "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n"
             "If you are convinced there is no bug in your code, you can #define "
-            "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF"
+            "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF "
             "to disable this check. In that case you have to ensure this #define is consistently "
             "used for all translation units linked into a given pybind11 extension, otherwise "
             "there will be ODR violations.",
             function_name.c_str());
-        fflush(stderr);
         if (Py_TYPE(m_ptr)->tp_name != nullptr) {
             fprintf(stderr,
-                    "The failing %s call was triggered on a %s object.\n",
+                    " The failing %s call was triggered on a %s object.",
                     function_name.c_str(),
                     Py_TYPE(m_ptr)->tp_name);
-            fflush(stderr);
         }
+        fprintf(stderr, "\n");
+        fflush(stderr);
         throw std::runtime_error(function_name + " PyGILState_Check() failure.");
     }
 #endif

pyproject.toml

@@ -19,7 +19,7 @@ ignore = [
 
 [tool.mypy]
 files = ["pybind11"]
-python_version = "3.6"
+python_version = "3.7"
 strict = true
 show_error_codes = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -57,10 +57,13 @@ messages_control.disable = [
   "unused-argument",  # covered by Ruff ARG
 ]
 
-
 [tool.ruff]
-select = [
+target-version = "py37"
-  "E", "F", "W", # flake8
+src = ["src"]
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = [
   "B",           # flake8-bugbear
   "I",           # isort
   "N",           # pep8-naming
@@ -86,13 +89,10 @@ ignore = [
   "PT004",  # Fixture that doesn't return needs underscore (no, it is fine)
   "SIM118", # iter(x) is not always the same as iter(x.keys())
 ]
-target-version = "py37"
-src = ["src"]
 unfixable = ["T20"]
 exclude = []
-line-length = 120
 isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/**" = ["EM", "N", "E721"]
 "tests/test_call_policies.py" = ["PLC1901"]

tests/extra_python_package/test_files.py

@@ -35,6 +35,7 @@ main_headers = {
     "include/pybind11/eval.h",
     "include/pybind11/functional.h",
     "include/pybind11/gil.h",
+    "include/pybind11/gil_safe_call_once.h",
     "include/pybind11/iostream.h",
     "include/pybind11/numpy.h",
     "include/pybind11/operators.h",

tests/requirements.txt

@@ -3,7 +3,8 @@ numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux"
 numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
 numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10"
 numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" and python_version<"3.11"
-pytest==7.0.0
+pytest==7.0.0; platform_python_implementation!="PyPy" and python_version=="3.6"
+pytest==7.2.0; platform_python_implementation!="PyPy" and python_version>="3.7"
 pytest-timeout
 scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10"
 scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10"

tests/test_exceptions.cpp

@@ -6,6 +6,8 @@
     All rights reserved. Use of this source code is governed by a
     BSD-style license that can be found in the LICENSE file.
 */
+#include <pybind11/gil_safe_call_once.h>
+
 #include "test_exceptions.h"
 
 #include "local_bindings.h"
@@ -114,7 +116,9 @@ TEST_SUBMODULE(exceptions, m) {
           []() { throw std::runtime_error("This exception was intentionally thrown."); });
 
     // PLEASE KEEP IN SYNC with docs/advanced/exceptions.rst
-    static py::handle ex = py::exception<MyException>(m, "MyException").release();
+    PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> ex_storage;
+    ex_storage.call_once_and_store_result(
+        [&]() { return py::exception<MyException>(m, "MyException"); });
     py::register_exception_translator([](std::exception_ptr p) {
         try {
             if (p) {
@@ -122,7 +126,7 @@ TEST_SUBMODULE(exceptions, m) {
             }
         } catch (const MyException &e) {
             // Set MyException as the active python error
-            py::set_error(ex, e.what());
+            py::set_error(ex_storage.get_stored(), e.what());
         }
     });
 

tests/test_kwargs_and_defaults.cpp

@@ -85,6 +85,8 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         "kw_lb_func7",
         [](const std::string &) {},
         py::arg("str_arg") = "First line.\n  Second line.");
+    m.def(
+        "kw_lb_func8", [](const CustomRepr &) {}, py::arg("custom") = CustomRepr(""));
 
     // test_args_and_kwargs
     m.def("args_function", [](py::args args) -> py::tuple {

tests/test_kwargs_and_defaults.py

@@ -55,6 +55,10 @@ def test_function_signatures(doc):
         doc(m.kw_lb_func7)
         == "kw_lb_func7(str_arg: str = 'First line.\\n  Second line.') -> None"
     )
+    assert (
+        doc(m.kw_lb_func8)
+        == "kw_lb_func8(custom: m.kwargs_and_defaults.CustomRepr = ) -> None"
+    )
 
 
 def test_named_arguments():

tests/test_numpy_array.py

@@ -298,7 +298,7 @@ def test_constructors():
     results = m.converting_constructors([1, 2, 3])
     for a in results.values():
         np.testing.assert_array_equal(a, [1, 2, 3])
-    assert results["array"].dtype == np.int_
+    assert results["array"].dtype == np.dtype(int)
     assert results["array_t<int32>"].dtype == np.int32
     assert results["array_t<double>"].dtype == np.float64
 

tests/test_numpy_dtypes.cpp

@@ -157,7 +157,7 @@ py::array mkarray_via_buffer(size_t n) {
     do {                                                                                          \
         (s).bool_ = (i) % 2 != 0;                                                                 \
         (s).uint_ = (uint32_t) (i);                                                               \
-        (s).float_ = (float) (i) *1.5f;                                                           \
+        (s).float_ = (float) (i) * 1.5f;                                                          \
         (s).ldbl_ = (long double) (i) * -2.5L;                                                    \
     } while (0)
 

tests/test_pytypes.cpp

@@ -662,8 +662,8 @@ TEST_SUBMODULE(pytypes, m) {
 // This is "most correct" and enforced on these platforms.
 #    define PYBIND11_AUTO_IT auto it
 #else
-// This works on many platforms and is (unfortunately) reflective of existing user code.
+    // This works on many platforms and is (unfortunately) reflective of existing user code.
-// NOLINTNEXTLINE(bugprone-macro-parentheses)
+    // NOLINTNEXTLINE(bugprone-macro-parentheses)
 #    define PYBIND11_AUTO_IT auto &it
 #endif
 

tests/test_sequences_and_iterators.py

@@ -58,6 +58,15 @@ def test_generalized_iterators_simple():
     assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_values()) == [2, 4, 5]
 
 
+def test_iterator_doc_annotations():
+    assert m.IntPairs.nonref.__doc__.endswith("-> Iterator[tuple[int, int]]\n")
+    assert m.IntPairs.nonref_keys.__doc__.endswith("-> Iterator[int]\n")
+    assert m.IntPairs.nonref_values.__doc__.endswith("-> Iterator[int]\n")
+    assert m.IntPairs.simple_iterator.__doc__.endswith("-> Iterator[tuple[int, int]]\n")
+    assert m.IntPairs.simple_keys.__doc__.endswith("-> Iterator[int]\n")
+    assert m.IntPairs.simple_values.__doc__.endswith("-> Iterator[int]\n")
+
+
 def test_iterator_referencing():
     """Test that iterators reference rather than copy their referents."""
     vec = m.VectorNonCopyableInt()

tools/FindPythonLibsNew.cmake

@@ -95,6 +95,22 @@ if(NOT PythonLibsNew_FIND_VERSION)
   set(PythonLibsNew_FIND_VERSION "3.6")
 endif()
 
+if(NOT CMAKE_VERSION VERSION_LESS "3.27")
+  cmake_policy(GET CMP0148 _pybind11_cmp0148)
+  if(NOT _pybind11_cmp0148)
+    message(
+      AUTHOR_WARNING
+        "Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs "
+        "modules are removed.  Run \"cmake --help-policy CMP0148\" for policy "
+        "details.  Use the cmake_policy command to set the policy and suppress "
+        "this warning, or preferably upgrade to using FindPython, either by "
+        "calling it explicitly before pybind11, or by setting "
+        "PYBIND11_FINDPYTHON ON before pybind11.")
+  endif()
+  cmake_policy(SET CMP0148 OLD)
+  unset(_pybind11_cmp0148)
+endif()
+
 find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required}
              ${_pythonlibs_quiet})
 
@@ -172,13 +188,20 @@ _pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST)
 _pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX)
 _pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR)
 _pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES)
-_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION)
 _pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG)
 _pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P)
 _pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX)
 _pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR)
 _pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH)
 
+list(GET _PYTHON_VALUES 4 _PYTHON_MODULE_EXT_SUFFIX)
+if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+  get_filename_component(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+endif()
+if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION)
+  get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+endif()
+
 # Make sure the Python has the same pointer-size as the chosen compiler
 # Skip if CMAKE_SIZEOF_VOID_P is not defined
 # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)

tools/pybind11NewTools.cmake

@@ -95,25 +95,36 @@ endif()
 
 # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
 # required for PyPy3 (as of 7.3.1)
-if(NOT DEFINED PYTHON_MODULE_EXTENSION)
+if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
   execute_process(
     COMMAND
       "${${_Python}_EXECUTABLE}" "-c"
       "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
-    OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION
+    OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
-    ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR
+    ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
     OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-  if(_PYTHON_MODULE_EXTENSION STREQUAL "")
+  if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
     message(
       FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
-                  "package is not installed. Full error message:\n${_PYTHON_MODULE_EXTENSION_ERR}")
+                  "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
+    )
   endif()
 
   # This needs to be available for the pybind11_extension function
-  set(PYTHON_MODULE_EXTENSION
+  if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
-      "${_PYTHON_MODULE_EXTENSION}"
+    get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
-      CACHE INTERNAL "")
+    set(PYTHON_MODULE_DEBUG_POSTFIX
+        "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+        CACHE INTERNAL "")
+  endif()
+
+  if(NOT DEFINED PYTHON_MODULE_EXTENSION)
+    get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+    set(PYTHON_MODULE_EXTENSION
+        "${_PYTHON_MODULE_EXTENSION}"
+        CACHE INTERNAL "")
+  endif()
 endif()
 
 # Python debug libraries expose slightly different objects before 3.8
@@ -253,6 +264,9 @@ endfunction()
 
 function(pybind11_extension name)
   # The extension is precomputed
-  set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${PYTHON_MODULE_EXTENSION}")
+  set_target_properties(
-
+    ${name}
+    PROPERTIES PREFIX ""
+               DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}"
+               SUFFIX "${PYTHON_MODULE_EXTENSION}")
 endfunction()

tools/pybind11Tools.cmake

@@ -65,6 +65,7 @@ _pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS)
 _pybind11_promote_to_cache(PYTHON_LIBRARIES)
 _pybind11_promote_to_cache(PYTHON_MODULE_PREFIX)
 _pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION)
+_pybind11_promote_to_cache(PYTHON_MODULE_DEBUG_POSTFIX)
 _pybind11_promote_to_cache(PYTHON_VERSION_MAJOR)
 _pybind11_promote_to_cache(PYTHON_VERSION_MINOR)
 _pybind11_promote_to_cache(PYTHON_VERSION)
@@ -148,8 +149,11 @@ endif()
 
 function(pybind11_extension name)
   # The prefix and extension are provided by FindPythonLibsNew.cmake
-  set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}"
+  set_target_properties(
-                                           SUFFIX "${PYTHON_MODULE_EXTENSION}")
+    ${name}
+    PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}"
+               DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}"
+               SUFFIX "${PYTHON_MODULE_EXTENSION}")
 endfunction()
 
 # Build a Python extension module: