Merge 2b08a25e9c into b7c33009ac

* Start pybind11v3

* Remove all code for PYBIND11_INTERNALS_VERSION 4 and 5

* Fix oversight: pybind11/_version.py needs to be updated as well.

* Add @pytest.mark.skipif("env.GRAALPY", reason="TODO debug segfault")

.github/workflows/ci.yml

@@ -1124,7 +1124,7 @@ jobs:
         uses: jwlawson/actions-setup-cmake@v2.0
 
       - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v5
+        uses: seanmiddleditch/gha-setup-ninja@v6
 
       - name: Run pip installs
         run: |

.github/workflows/pip.yml

@@ -103,7 +103,7 @@ jobs:
     - uses: actions/download-artifact@v4
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@7668571508540a607bdfd90a87a560489fe372eb # v2.1.0
+      uses: actions/attest-build-provenance@520d128f165991a6c774bcb264f323e3d70747f4 # v2.2.0
       with:
         subject-path: "*/pybind11*"
 

.pre-commit-config.yaml

@@ -25,14 +25,14 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v19.1.6"
+  rev: "v19.1.7"
   hooks:
   - id: clang-format
     types_or: [c++, c, cuda]
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.6
+  rev: v0.9.4
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -119,7 +119,7 @@ repos:
 # Use tools/codespell_ignore_lines_from_errors.py
 # to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.3.0"
+  rev: "v2.4.1"
   hooks:
   - id: codespell
     exclude: ".supp$"
@@ -142,14 +142,14 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.3.3"
+  rev: "v3.3.4"
   hooks:
   - id: pylint
     files: ^pybind11
 
 # Check schemas on some of our YAML files
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.30.0
+  rev: 0.31.1
   hooks:
   - id: check-readthedocs
   - id: check-github-workflows

docs/advanced/cast/custom.rst

@@ -61,19 +61,16 @@ type is explicitly allowed.
 
     template <>
     struct type_caster<user_space::Point2D> {
-        // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple`
-        PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple"));
-        // `arg_name` and `return_name` may optionally be used to specify type hints separately for
-        // arguments and return values.
+        // This macro inserts a lot of boilerplate code and sets the type hint.
+        // `io_name` is used to specify different type hints for arguments and return values.
         // The signature of our negate function would then look like:
         // `negate(Sequence[float]) -> tuple[float, float]`
-        static constexpr auto arg_name = const_name("Sequence[float]");
-        static constexpr auto return_name = const_name("tuple[float, float]");
+        PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]"));
 
         // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments
         // are used to indicate the return value policy and parent object (for
         // return_value_policy::reference_internal) and are often ignored by custom casters.
-        // The return value should reflect the type hint specified by `return_name`.
+        // The return value should reflect the type hint specified by the second argument of `io_name`.
         static handle
         cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) {
             return py::make_tuple(number.x, number.y).release();
@@ -81,7 +78,8 @@ type is explicitly allowed.
 
         // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The
         // second argument indicates whether implicit conversions should be allowed.
-        // The accepted types should reflect the type hint specified by `arg_name`.
+        // The accepted types should reflect the type hint specified by the first argument of
+        // `io_name`.
         bool load(handle src, bool /*convert*/) {
             // Check if handle is a Sequence
             if (!py::isinstance<py::sequence>(src)) {

docs/basics.rst

@@ -142,7 +142,7 @@ On Linux, the above example can be compiled using the following command:
 
 .. code-block:: bash
 
-    $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix)
+    $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3 -m pybind11 --extension-suffix)
 
 .. note::
 

include/pybind11/attr.h

@@ -81,6 +81,10 @@ struct dynamic_attr {};
 /// Annotation which enables the buffer protocol for a type
 struct buffer_protocol {};
 
+/// Annotation which enables releasing the GIL before calling the C++ destructor of wrapped
+/// instances (pybind/pybind11#1446).
+struct release_gil_before_calling_cpp_dtor {};
+
 /// Annotation which requests that a special metaclass is created for a type
 struct metaclass {
     handle value;
@@ -272,7 +276,8 @@ struct function_record {
 struct type_record {
     PYBIND11_NOINLINE type_record()
         : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false),
-          default_holder(true), module_local(false), is_final(false) {}
+          default_holder(true), module_local(false), is_final(false),
+          release_gil_before_calling_cpp_dtor(false) {}
 
     /// Handle to the parent scope
     handle scope;
@@ -331,6 +336,9 @@ struct type_record {
     /// Is the class inheritable from python classes?
     bool is_final : 1;
 
+    /// Solves pybind/pybind11#1446
+    bool release_gil_before_calling_cpp_dtor : 1;
+
     PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) {
         auto *base_info = detail::get_type_info(base, false);
         if (!base_info) {
@@ -603,6 +611,14 @@ struct process_attribute<module_local> : process_attribute_default<module_local>
     static void init(const module_local &l, type_record *r) { r->module_local = l.value; }
 };
 
+template <>
+struct process_attribute<release_gil_before_calling_cpp_dtor>
+    : process_attribute_default<release_gil_before_calling_cpp_dtor> {
+    static void init(const release_gil_before_calling_cpp_dtor &, type_record *r) {
+        r->release_gil_before_calling_cpp_dtor = true;
+    }
+};
+
 /// Process a 'prepend' attribute, putting this at the beginning of the overload chain
 template <>
 struct process_attribute<prepend> : process_attribute_default<prepend> {

include/pybind11/cast.h

@@ -34,39 +34,6 @@ PYBIND11_WARNING_DISABLE_MSVC(4127)
 
 PYBIND11_NAMESPACE_BEGIN(detail)
 
-// Type trait checker for `descr`
-template <typename>
-struct is_descr : std::false_type {};
-
-template <size_t N, typename... Ts>
-struct is_descr<descr<N, Ts...>> : std::true_type {};
-
-template <size_t N, typename... Ts>
-struct is_descr<const descr<N, Ts...>> : std::true_type {};
-
-// Use arg_name instead of name when available
-template <typename T, typename SFINAE = void>
-struct as_arg_type {
-    static constexpr auto name = T::name;
-};
-
-template <typename T>
-struct as_arg_type<T, typename std::enable_if<is_descr<decltype(T::arg_name)>::value>::type> {
-    static constexpr auto name = T::arg_name;
-};
-
-// Use return_name instead of name when available
-template <typename T, typename SFINAE = void>
-struct as_return_type {
-    static constexpr auto name = T::name;
-};
-
-template <typename T>
-struct as_return_type<T,
-                      typename std::enable_if<is_descr<decltype(T::return_name)>::value>::type> {
-    static constexpr auto name = T::return_name;
-};
-
 template <typename type, typename SFINAE = void>
 class type_caster : public type_caster_base<type> {};
 template <typename type>
@@ -1113,8 +1080,6 @@ struct pyobject_caster {
         return src.inc_ref();
     }
     PYBIND11_TYPE_CASTER(type, handle_type_name<type>::name);
-    static constexpr auto arg_name = as_arg_type<handle_type_name<type>>::name;
-    static constexpr auto return_name = as_return_type<handle_type_name<type>>::name;
 };
 
 template <typename T>
@@ -1668,7 +1633,7 @@ public:
                   "py::args cannot be specified more than once");
 
     static constexpr auto arg_names
-        = ::pybind11::detail::concat(type_descr(as_arg_type<make_caster<Args>>::name)...);
+        = ::pybind11::detail::concat(type_descr(make_caster<Args>::name)...);
 
     bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); }
 

include/pybind11/detail/class.h

@@ -574,9 +574,9 @@ extern "C" inline int pybind11_clear(PyObject *self) {
 inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) {
     auto *type = &heap_type->ht_type;
     type->tp_flags |= Py_TPFLAGS_HAVE_GC;
-#if PY_VERSION_HEX < 0x030B0000
-    type->tp_dictoffset = type->tp_basicsize;           // place dict at the end
-    type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it
+#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) // For PyPy see PR #5508
+    type->tp_dictoffset = type->tp_basicsize;            // place dict at the end
+    type->tp_basicsize += (ssize_t) sizeof(PyObject *);  // and allocate enough space for it
 #else
     type->tp_flags |= Py_TPFLAGS_MANAGED_DICT;
 #endif

include/pybind11/detail/common.h

@@ -14,13 +14,13 @@
 #    error "PYTHON < 3.8 IS UNSUPPORTED. pybind11 v2.13 was the last to support Python 3.7."
 #endif
 
-#define PYBIND11_VERSION_MAJOR 2
-#define PYBIND11_VERSION_MINOR 14
+#define PYBIND11_VERSION_MAJOR 3
+#define PYBIND11_VERSION_MINOR 0
 #define PYBIND11_VERSION_PATCH 0.dev1
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x020E00D1
+#define PYBIND11_VERSION_HEX 0x030000D1
 
 // Define some generic pybind11 helper macros for warning management.
 //

include/pybind11/detail/descr.h

@@ -99,6 +99,13 @@ constexpr descr<1, Type> const_name() {
     return {'%'};
 }
 
+// Use a different name based on whether the parameter is used as input or output
+template <size_t N1, size_t N2>
+constexpr descr<N1 + N2 + 1> io_name(char const (&text1)[N1], char const (&text2)[N2]) {
+    return const_name("@") + const_name(text1) + const_name("@") + const_name(text2)
+           + const_name("@");
+}
+
 // If "_" is defined as a macro, py::detail::_ cannot be provided.
 // It is therefore best to use py::detail::const_name universally.
 // This block is for backward compatibility only.
@@ -167,5 +174,15 @@ constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) {
     return const_name("{") + descr + const_name("}");
 }
 
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> arg_descr(const descr<N, Ts...> &descr) {
+    return const_name("@^") + descr + const_name("@!");
+}
+
+template <size_t N, typename... Ts>
+constexpr descr<N + 4, Ts...> return_descr(const descr<N, Ts...> &descr) {
+    return const_name("@$") + descr + const_name("@!");
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/internals.h

@@ -37,22 +37,12 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER)
-// Version bump for Python 3.12+, before first 3.12 beta release.
-// Version bump for MSVC piggy-backed on PR #4779. See comments there.
-#        ifdef Py_GIL_DISABLED
-#            define PYBIND11_INTERNALS_VERSION 6
-#        else
-#            define PYBIND11_INTERNALS_VERSION 5
-#        endif
-#    else
-#        define PYBIND11_INTERNALS_VERSION 4
-#    endif
+#    define PYBIND11_INTERNALS_VERSION 6
 #endif
 
-// This requirement is mainly to reduce the support burden (see PR #4570).
-static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5,
-              "pybind11 ABI version 5 is the minimum for Python 3.12+");
+#if PYBIND11_INTERNALS_VERSION < 6
+#    error "PYBIND11_INTERNALS_VERSION 6 is the minimum for all platforms for pybind11v3."
+#endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -71,40 +61,29 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // Thread Specific Storage (TSS) API.
 // Avoid unnecessary allocation of `Py_tss_t`, since we cannot use
 // `Py_LIMITED_API` anyway.
-#if PYBIND11_INTERNALS_VERSION > 4
-#    define PYBIND11_TLS_KEY_REF Py_tss_t &
-#    if defined(__clang__)
-#        define PYBIND11_TLS_KEY_INIT(var)                                                        \
-            _Pragma("clang diagnostic push")                                         /**/         \
-                _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/         \
-                Py_tss_t var                                                                      \
-                = Py_tss_NEEDS_INIT;                                                              \
-            _Pragma("clang diagnostic pop")
-#    elif defined(__GNUC__) && !defined(__INTEL_COMPILER)
-#        define PYBIND11_TLS_KEY_INIT(var)                                                        \
-            _Pragma("GCC diagnostic push")                                         /**/           \
-                _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/           \
-                Py_tss_t var                                                                      \
-                = Py_tss_NEEDS_INIT;                                                              \
-            _Pragma("GCC diagnostic pop")
-#    else
-#        define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT;
-#    endif
-#    define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0)
-#    define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key))
-#    define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value))
-#    define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr)
-#    define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key))
+#define PYBIND11_TLS_KEY_REF Py_tss_t &
+#if defined(__clang__)
+#    define PYBIND11_TLS_KEY_INIT(var)                                                            \
+        _Pragma("clang diagnostic push")                                         /**/             \
+            _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/             \
+            Py_tss_t var                                                                          \
+            = Py_tss_NEEDS_INIT;                                                                  \
+        _Pragma("clang diagnostic pop")
+#elif defined(__GNUC__) && !defined(__INTEL_COMPILER)
+#    define PYBIND11_TLS_KEY_INIT(var)                                                            \
+        _Pragma("GCC diagnostic push")                                         /**/               \
+            _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/               \
+            Py_tss_t var                                                                          \
+            = Py_tss_NEEDS_INIT;                                                                  \
+        _Pragma("GCC diagnostic pop")
 #else
-#    define PYBIND11_TLS_KEY_REF Py_tss_t *
-#    define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr;
-#    define PYBIND11_TLS_KEY_CREATE(var)                                                          \
-        (((var) = PyThread_tss_alloc()) != nullptr && (PyThread_tss_create((var)) == 0))
-#    define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key))
-#    define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value))
-#    define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr)
-#    define PYBIND11_TLS_FREE(key) PyThread_tss_free(key)
+#    define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT;
 #endif
+#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0)
+#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key))
+#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value))
+#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr)
+#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key))
 
 // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly
 // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module
@@ -112,8 +91,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name,
 // which works.  If not under a known-good stl, provide our own name-based hash and equality
 // functions that use the type name.
-#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__))                                     \
-    || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION))
+#if !defined(_LIBCPP_VERSION)
 inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; }
 using type_hash = std::hash<std::type_index>;
 using type_equal_to = std::equal_to<std::type_index>;
@@ -201,35 +179,26 @@ struct internals {
     std::forward_list<ExceptionTranslator> registered_exception_translators;
     std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across
                                                          // extensions
-#if PYBIND11_INTERNALS_VERSION == 4
-    std::vector<PyObject *> unused_loader_patient_stack_remove_at_v5;
-#endif
-    std::forward_list<std::string> static_strings; // Stores the std::strings backing
-                                                   // detail::c_str()
+    std::forward_list<std::string> static_strings;       // Stores the std::strings backing
+                                                         // detail::c_str()
     PyTypeObject *static_property_type;
     PyTypeObject *default_metaclass;
     PyObject *instance_base;
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PYBIND11_TLS_KEY_INIT(tstate)
-#if PYBIND11_INTERNALS_VERSION > 4
     PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-#endif // PYBIND11_INTERNALS_VERSION > 4
     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PyInterpreterState *istate = nullptr;
 
-#if PYBIND11_INTERNALS_VERSION > 4
     // Note that we have to use a std::string to allocate memory to ensure a unique address
     // We want unique addresses since we use pointer equality to compare function records
     std::string function_record_capsule_name = internals_function_record_capsule_name;
-#endif
 
     internals() = default;
     internals(const internals &other) = delete;
     internals &operator=(const internals &other) = delete;
     ~internals() {
-#if PYBIND11_INTERNALS_VERSION > 4
         PYBIND11_TLS_FREE(loader_life_support_tls_key);
-#endif // PYBIND11_INTERNALS_VERSION > 4
 
         // This destructor is called *after* Py_Finalize() in finalize_interpreter().
         // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is
@@ -394,7 +363,7 @@ inline void translate_local_exception(std::exception_ptr p) {
 
 inline object get_python_state_dict() {
     object state_dict;
-#if PYBIND11_INTERNALS_VERSION <= 4 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
+#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)
     state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
 #else
 #    if PY_VERSION_HEX < 0x03090000
@@ -492,13 +461,12 @@ PYBIND11_NOINLINE internals &get_internals() {
         }
         PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate);
 
-#if PYBIND11_INTERNALS_VERSION > 4
         // NOLINTNEXTLINE(bugprone-assignment-in-if-condition)
         if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) {
             pybind11_fail("get_internals: could not successfully initialize the "
                           "loader_life_support TSS key!");
         }
-#endif
+
         internals_ptr->istate = tstate->interp;
         state_dict[PYBIND11_INTERNALS_ID] = capsule(reinterpret_cast<void *>(internals_pp));
         internals_ptr->registered_exception_translators.push_front(&translate_exception);
@@ -528,40 +496,6 @@ PYBIND11_NOINLINE internals &get_internals() {
 struct local_internals {
     type_map<type_info *> registered_types_cpp;
     std::forward_list<ExceptionTranslator> registered_exception_translators;
-#if PYBIND11_INTERNALS_VERSION == 4
-
-    // For ABI compatibility, we can't store the loader_life_support TLS key in
-    // the `internals` struct directly.  Instead, we store it in `shared_data` and
-    // cache a copy in `local_internals`.  If we allocated a separate TLS key for
-    // each instance of `local_internals`, we could end up allocating hundreds of
-    // TLS keys if hundreds of different pybind11 modules are loaded (which is a
-    // plausible number).
-    PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-
-    // Holds the shared TLS key for the loader_life_support stack.
-    struct shared_loader_life_support_data {
-        PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
-        shared_loader_life_support_data() {
-            // NOLINTNEXTLINE(bugprone-assignment-in-if-condition)
-            if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) {
-                pybind11_fail("local_internals: could not successfully initialize the "
-                              "loader_life_support TLS key!");
-            }
-        }
-        // We can't help but leak the TLS key, because Python never unloads extension modules.
-    };
-
-    local_internals() {
-        auto &internals = get_internals();
-        // Get or create the `loader_life_support_stack_key`.
-        auto &ptr = internals.shared_data["_life_support"];
-        if (!ptr) {
-            ptr = new shared_loader_life_support_data;
-        }
-        loader_life_support_tls_key
-            = static_cast<shared_loader_life_support_data *>(ptr)->loader_life_support_tls_key;
-    }
-#endif //  PYBIND11_INTERNALS_VERSION == 4
 };
 
 /// Works like `get_internals`, but for things which are locally registered.
@@ -668,7 +602,7 @@ const char *c_str(Args &&...args) {
 
 inline const char *get_function_record_capsule_name() {
     // On GraalPy, pointer equality of the names is currently not guaranteed
-#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON)
+#if !defined(GRAALVM_PYTHON)
     return get_internals().function_record_capsule_name.c_str();
 #else
     return nullptr;

include/pybind11/detail/type_caster_base.h

@@ -43,11 +43,7 @@ private:
 
     // Store stack pointer in thread-local storage.
     static PYBIND11_TLS_KEY_REF get_stack_tls_key() {
-#if PYBIND11_INTERNALS_VERSION == 4
-        return get_local_internals().loader_life_support_tls_key;
-#else
         return get_internals().loader_life_support_tls_key;
-#endif
     }
     static loader_life_support *get_stack_top() {
         return static_cast<loader_life_support *>(PYBIND11_TLS_GET_VALUE(get_stack_tls_key()));

include/pybind11/eigen/matrix.h

@@ -225,19 +225,22 @@ struct EigenProps {
         = !show_c_contiguous && show_order && requires_col_major;
 
     static constexpr auto descriptor
-        = const_name("numpy.ndarray[") + npy_format_descriptor<Scalar>::name + const_name("[")
+        = const_name("typing.Annotated[")
+          + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<Scalar>::name + io_name("", "]") + const_name(", \"[")
           + const_name<fixed_rows>(const_name<(size_t) rows>(), const_name("m")) + const_name(", ")
-          + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n")) + const_name("]")
-          +
+          + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n"))
+          + const_name("]\"")
           // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to
           // be satisfied: writeable=True (for a mutable reference), and, depending on the map's
           // stride options, possibly f_contiguous or c_contiguous.  We include them in the
           // descriptor output to provide some hint as to why a TypeError is occurring (otherwise
-          // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and
-          // an error message that you *gave* a numpy.ndarray of the right type and dimensions.
-          const_name<show_writeable>(", flags.writeable", "")
-          + const_name<show_c_contiguous>(", flags.c_contiguous", "")
-          + const_name<show_f_contiguous>(", flags.f_contiguous", "") + const_name("]");
+          // it can be confusing to see that a function accepts a
+          // 'typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3,2]"]' and an error message
+          // that you *gave* a numpy.ndarray of the right type and dimensions.
+          + const_name<show_writeable>(", \"flags.writeable\"", "")
+          + const_name<show_c_contiguous>(", \"flags.c_contiguous\"", "")
+          + const_name<show_f_contiguous>(", \"flags.f_contiguous\"", "") + const_name("]");
 };
 
 // Casts an Eigen type to numpy array.  If given a base, the numpy array references the src data,
@@ -316,8 +319,11 @@ struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> {
             return false;
         }
 
+        PYBIND11_WARNING_PUSH
+        PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") // See PR #5516
         // Allocate the new type, then build a numpy reference into it
         value = Type(fits.rows, fits.cols);
+        PYBIND11_WARNING_POP
         auto ref = reinterpret_steal<array>(eigen_ref_array<props>(value));
         if (dims == 1) {
             ref = ref.squeeze();
@@ -438,7 +444,9 @@ public:
         }
     }
 
-    static constexpr auto name = props::descriptor;
+    // return_descr forces the use of NDArray instead of ArrayLike in args
+    // since Ref<...> args can only accept arrays.
+    static constexpr auto name = return_descr(props::descriptor);
 
     // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return
     // types but not bound arguments).  We still provide them (with an explicitly delete) so that

include/pybind11/eigen/tensor.h

@@ -124,13 +124,16 @@ struct eigen_tensor_helper<
 template <typename Type, bool ShowDetails, bool NeedsWriteable = false>
 struct get_tensor_descriptor {
     static constexpr auto details
-        = const_name<NeedsWriteable>(", flags.writeable", "") + const_name
+        = const_name<NeedsWriteable>(", \"flags.writeable\"", "") + const_name
               < static_cast<int>(Type::Layout)
-          == static_cast<int>(Eigen::RowMajor) > (", flags.c_contiguous", ", flags.f_contiguous");
+          == static_cast<int>(Eigen::RowMajor)
+                 > (", \"flags.c_contiguous\"", ", \"flags.f_contiguous\"");
     static constexpr auto value
-        = const_name("numpy.ndarray[") + npy_format_descriptor<typename Type::Scalar>::name
-          + const_name("[") + eigen_tensor_helper<remove_cv_t<Type>>::dimensions_descriptor
-          + const_name("]") + const_name<ShowDetails>(details, const_name("")) + const_name("]");
+        = const_name("typing.Annotated[")
+          + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<typename Type::Scalar>::name + io_name("", "]")
+          + const_name(", \"[") + eigen_tensor_helper<remove_cv_t<Type>>::dimensions_descriptor
+          + const_name("]\"") + const_name<ShowDetails>(details, const_name("")) + const_name("]");
 };
 
 // When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes<T, 0> does not have the begin() member
@@ -502,7 +505,10 @@ protected:
     std::unique_ptr<MapType> value;
 
 public:
-    static constexpr auto name = get_tensor_descriptor<Type, true, needs_writeable>::value;
+    // return_descr forces the use of NDArray instead of ArrayLike since refs can only reference
+    // arrays
+    static constexpr auto name
+        = return_descr(get_tensor_descriptor<Type, true, needs_writeable>::value);
     explicit operator MapType *() { return value.get(); }
     explicit operator MapType &() { return *value; }
     explicit operator MapType &&() && { return std::move(*value); }

include/pybind11/numpy.h

@@ -175,7 +175,6 @@ inline numpy_internals &get_numpy_internals() {
 PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) {
     module_ numpy = module_::import("numpy");
     str version_string = numpy.attr("__version__");
-
     module_ numpy_lib = module_::import("numpy.lib");
     object numpy_version = numpy_lib.attr("NumpyVersion")(version_string);
     int major_version = numpy_version.attr("major").cast<int>();
@@ -2183,7 +2182,8 @@ vectorize_helper<Func, Return, Args...> vectorize_extractor(const Func &f, Retur
 template <typename T, int Flags>
 struct handle_type_name<array_t<T, Flags>> {
     static constexpr auto name
-        = const_name("numpy.ndarray[") + npy_format_descriptor<T>::name + const_name("]");
+        = io_name("typing.Annotated[numpy.typing.ArrayLike, ", "numpy.typing.NDArray[")
+          + npy_format_descriptor<T>::name + const_name("]");
 };
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/pybind11.h

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <memory>
 #include <new>
+#include <stack>
 #include <string>
 #include <utility>
 #include <vector>
@@ -336,8 +337,8 @@ protected:
 
         /* Generate a readable signature describing the function's arguments and return
            value types */
-        static constexpr auto signature = const_name("(") + cast_in::arg_names
-                                          + const_name(") -> ") + as_return_type<cast_out>::name;
+        static constexpr auto signature
+            = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name;
         PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types();
 
         /* Register the function with Python from generic (non-templated) code */
@@ -440,6 +441,13 @@ protected:
         std::string signature;
         size_t type_index = 0, arg_index = 0;
         bool is_starred = false;
+        // `is_return_value.top()` is true if we are currently inside the return type of the
+        // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops
+        // back to the previous state.
+        std::stack<bool> is_return_value({false});
+        // The following characters have special meaning in the signature parsing. Literals
+        // containing these are escaped with `!`.
+        std::string special_chars("!@%{}-");
         for (const auto *pc = text; *pc != '\0'; ++pc) {
             const auto c = *pc;
 
@@ -493,7 +501,57 @@ protected:
                 } else {
                     signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name()));
                 }
+            } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) {
+                // typing::Literal escapes special characters with !
+                signature += *++pc;
+            } else if (c == '@') {
+                // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see
+                // typing::Callable/detail::arg_descr/detail::return_descr)
+                if (*(pc + 1) == '^') {
+                    is_return_value.emplace(false);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '$') {
+                    is_return_value.emplace(true);
+                    ++pc;
+                    continue;
+                }
+                if (*(pc + 1) == '!') {
+                    is_return_value.pop();
+                    ++pc;
+                    continue;
+                }
+                // Handle types that differ depending on whether they appear
+                // in an argument or a return value position (see io_name<text1, text2>).
+                // For named arguments (py::arg()) with noconvert set, return value type is used.
+                ++pc;
+                if (!is_return_value.top()
+                    && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) {
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                } else {
+                    while (*pc != '\0' && *pc != '@') {
+                        ++pc;
+                    }
+                    if (*pc == '@') {
+                        ++pc;
+                    }
+                    while (*pc != '\0' && *pc != '@') {
+                        signature += *pc++;
+                    }
+                }
             } else {
+                if (c == '-' && *(pc + 1) == '>') {
+                    is_return_value.emplace(true);
+                }
                 signature += c;
             }
         }
@@ -1607,7 +1665,6 @@ public:
         record.type_align = alignof(conditional_t<has_alias, type_alias, type> &);
         record.holder_size = sizeof(holder_type);
         record.init_instance = init_instance;
-        record.dealloc = dealloc;
         record.default_holder = detail::is_instantiation<std::unique_ptr, holder_type>::value;
 
         set_operator_new<type>(&record);
@@ -1618,6 +1675,12 @@ public:
         /* Process optional arguments, if any */
         process_attributes<Extra...>::init(extra..., &record);
 
+        if (record.release_gil_before_calling_cpp_dtor) {
+            record.dealloc = dealloc_release_gil_before_calling_cpp_dtor;
+        } else {
+            record.dealloc = dealloc_without_manipulating_gil;
+        }
+
         generic_type::initialize(record);
 
         if (has_alias) {
@@ -1944,15 +2007,14 @@ private:
         init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr<type>());
     }
 
-    /// Deallocates an instance; via holder, if constructed; otherwise via operator delete.
-    static void dealloc(detail::value_and_holder &v_h) {
-        // We could be deallocating because we are cleaning up after a Python exception.
-        // If so, the Python error indicator will be set. We need to clear that before
-        // running the destructor, in case the destructor code calls more Python.
-        // If we don't, the Python API will exit with an exception, and pybind11 will
-        // throw error_already_set from the C++ destructor which is forbidden and triggers
-        // std::terminate().
-        error_scope scope;
+    // Deallocates an instance; via holder, if constructed; otherwise via operator delete.
+    // NOTE: The Python error indicator needs to cleared BEFORE this function is called.
+    // This is because we could be deallocating while cleaning up after a Python exception.
+    // If the error indicator is not cleared but the C++ destructor code makes Python C API
+    // calls, those calls are likely to generate a new exception, and pybind11 will then
+    // throw `error_already_set` from the C++ destructor. This is forbidden and will
+    // trigger std::terminate().
+    static void dealloc_impl(detail::value_and_holder &v_h) {
         if (v_h.holder_constructed()) {
             v_h.holder<holder_type>().~holder_type();
             v_h.set_holder_constructed(false);
@@ -1963,6 +2025,32 @@ private:
         v_h.value_ptr() = nullptr;
     }
 
+    static void dealloc_without_manipulating_gil(detail::value_and_holder &v_h) {
+        error_scope scope;
+        dealloc_impl(v_h);
+    }
+
+    static void dealloc_release_gil_before_calling_cpp_dtor(detail::value_and_holder &v_h) {
+        error_scope scope;
+        // Intentionally not using `gil_scoped_release` because the non-simple
+        // version unconditionally calls `get_internals()`.
+        // `Py_BEGIN_ALLOW_THREADS`, `Py_END_ALLOW_THREADS` cannot be used
+        // because those macros include `{` and `}`.
+        PyThreadState *py_ts = PyEval_SaveThread();
+        try {
+            dealloc_impl(v_h);
+        } catch (...) {
+            // This code path is expected to be unreachable unless there is a
+            // bug in pybind11 itself.
+            // An alternative would be to mark this function, or
+            // `dealloc_impl()`, with `nothrow`, but that would be a subtle
+            // behavior change and could make debugging more difficult.
+            PyEval_RestoreThread(py_ts);
+            throw;
+        }
+        PyEval_RestoreThread(py_ts);
+    }
+
     static detail::function_record *get_function_record(handle h) {
         h = detail::get_function(h);
         if (!h) {

include/pybind11/stl/filesystem.h

@@ -106,9 +106,7 @@ public:
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, const_name("os.PathLike"));
-    static constexpr auto arg_name = const_name("Union[os.PathLike, str, bytes]");
-    static constexpr auto return_name = const_name("Path");
+    PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "pathlib.Path"));
 };
 
 #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)

include/pybind11/typing.h

@@ -16,6 +16,13 @@
 
 #include <algorithm>
 
+#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L
+#    define PYBIND11_TYPING_H_HAS_STRING_LITERAL
+#    include <numeric>
+#    include <ranges>
+#    include <string_view>
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(typing)
 
@@ -112,8 +119,7 @@ class Never : public none {
     using none::none;
 };
 
-#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L
-#    define PYBIND11_TYPING_H_HAS_STRING_LITERAL
+#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)
 template <size_t N>
 struct StringLiteral {
     constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, name); }
@@ -143,13 +149,6 @@ struct handle_type_name<typing::Tuple<Types...>> {
     static constexpr auto name = const_name("tuple[")
                                  + ::pybind11::detail::concat(make_caster<Types>::name...)
                                  + const_name("]");
-    static constexpr auto arg_name
-        = const_name("tuple[")
-          + ::pybind11::detail::concat(as_arg_type<make_caster<Types>>::name...) + const_name("]");
-    static constexpr auto return_name
-        = const_name("tuple[")
-          + ::pybind11::detail::concat(as_return_type<make_caster<Types>>::name...)
-          + const_name("]");
 };
 
 template <>
@@ -163,58 +162,32 @@ struct handle_type_name<typing::Tuple<T, ellipsis>> {
     // PEP 484 specifies this syntax for a variable-length tuple
     static constexpr auto name
         = const_name("tuple[") + make_caster<T>::name + const_name(", ...]");
-    static constexpr auto arg_name
-        = const_name("tuple[") + as_arg_type<make_caster<T>>::name + const_name(", ...]");
-    static constexpr auto return_name
-        = const_name("tuple[") + as_return_type<make_caster<T>>::name + const_name(", ...]");
 };
 
 template <typename K, typename V>
 struct handle_type_name<typing::Dict<K, V>> {
     static constexpr auto name = const_name("dict[") + make_caster<K>::name + const_name(", ")
                                  + make_caster<V>::name + const_name("]");
-    static constexpr auto arg_name = const_name("dict[") + as_arg_type<make_caster<K>>::name
-                                     + const_name(", ") + as_arg_type<make_caster<V>>::name
-                                     + const_name("]");
-    static constexpr auto return_name = const_name("dict[") + as_return_type<make_caster<K>>::name
-                                        + const_name(", ") + as_return_type<make_caster<V>>::name
-                                        + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::List<T>> {
     static constexpr auto name = const_name("list[") + make_caster<T>::name + const_name("]");
-    static constexpr auto arg_name
-        = const_name("list[") + as_arg_type<make_caster<T>>::name + const_name("]");
-    static constexpr auto return_name
-        = const_name("list[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::Set<T>> {
     static constexpr auto name = const_name("set[") + make_caster<T>::name + const_name("]");
-    static constexpr auto arg_name
-        = const_name("set[") + as_arg_type<make_caster<T>>::name + const_name("]");
-    static constexpr auto return_name
-        = const_name("set[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::Iterable<T>> {
     static constexpr auto name = const_name("Iterable[") + make_caster<T>::name + const_name("]");
-    static constexpr auto arg_name
-        = const_name("Iterable[") + as_arg_type<make_caster<T>>::name + const_name("]");
-    static constexpr auto return_name
-        = const_name("Iterable[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::Iterator<T>> {
     static constexpr auto name = const_name("Iterator[") + make_caster<T>::name + const_name("]");
-    static constexpr auto arg_name
-        = const_name("Iterator[") + as_arg_type<make_caster<T>>::name + const_name("]");
-    static constexpr auto return_name
-        = const_name("Iterator[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
 template <typename Return, typename... Args>
@@ -222,8 +195,9 @@ struct handle_type_name<typing::Callable<Return(Args...)>> {
     using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
     static constexpr auto name
         = const_name("Callable[[")
-          + ::pybind11::detail::concat(as_arg_type<make_caster<Args>>::name...) + const_name("], ")
-          + as_return_type<make_caster<retval_type>>::name + const_name("]");
+          + ::pybind11::detail::concat(::pybind11::detail::arg_descr(make_caster<Args>::name)...)
+          + const_name("], ") + ::pybind11::detail::return_descr(make_caster<retval_type>::name)
+          + const_name("]");
 };
 
 template <typename Return>
@@ -231,7 +205,7 @@ struct handle_type_name<typing::Callable<Return(ellipsis)>> {
     // PEP 484 specifies this syntax for defining only return types of callables
     using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
     static constexpr auto name = const_name("Callable[..., ")
-                                 + as_return_type<make_caster<retval_type>>::name
+                                 + ::pybind11::detail::return_descr(make_caster<retval_type>::name)
                                  + const_name("]");
 };
 
@@ -245,22 +219,11 @@ struct handle_type_name<typing::Union<Types...>> {
     static constexpr auto name = const_name("Union[")
                                  + ::pybind11::detail::concat(make_caster<Types>::name...)
                                  + const_name("]");
-    static constexpr auto arg_name
-        = const_name("Union[")
-          + ::pybind11::detail::concat(as_arg_type<make_caster<Types>>::name...) + const_name("]");
-    static constexpr auto return_name
-        = const_name("Union[")
-          + ::pybind11::detail::concat(as_return_type<make_caster<Types>>::name...)
-          + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::Optional<T>> {
     static constexpr auto name = const_name("Optional[") + make_caster<T>::name + const_name("]");
-    static constexpr auto arg_name
-        = const_name("Optional[") + as_arg_type<make_caster<T>>::name + const_name("]");
-    static constexpr auto return_name
-        = const_name("Optional[") + as_return_type<make_caster<T>>::name + const_name("]");
 };
 
 template <typename T>
@@ -273,19 +236,14 @@ struct handle_type_name<typing::ClassVar<T>> {
     static constexpr auto name = const_name("ClassVar[") + make_caster<T>::name + const_name("]");
 };
 
-// TypeGuard and TypeIs use as_return_type to use the return type if available, which is usually
-// the narrower type.
-
 template <typename T>
 struct handle_type_name<typing::TypeGuard<T>> {
-    static constexpr auto name
-        = const_name("TypeGuard[") + as_return_type<make_caster<T>>::name + const_name("]");
+    static constexpr auto name = const_name("TypeGuard[") + make_caster<T>::name + const_name("]");
 };
 
 template <typename T>
 struct handle_type_name<typing::TypeIs<T>> {
-    static constexpr auto name
-        = const_name("TypeIs[") + as_return_type<make_caster<T>>::name + const_name("]");
+    static constexpr auto name = const_name("TypeIs[") + make_caster<T>::name + const_name("]");
 };
 
 template <>
@@ -299,15 +257,35 @@ struct handle_type_name<typing::Never> {
 };
 
 #if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)
+template <typing::StringLiteral StrLit>
+consteval auto sanitize_string_literal() {
+    constexpr std::string_view v(StrLit.name);
+    constexpr std::string_view special_chars("!@%{}-");
+    constexpr auto num_special_chars = std::accumulate(
+        special_chars.begin(), special_chars.end(), (size_t) 0, [&v](auto acc, const char &c) {
+            return std::move(acc) + std::ranges::count(v, c);
+        });
+    char result[v.size() + num_special_chars + 1];
+    size_t i = 0;
+    for (auto c : StrLit.name) {
+        if (special_chars.find(c) != std::string_view::npos) {
+            result[i++] = '!';
+        }
+        result[i++] = c;
+    }
+    return typing::StringLiteral(result);
+}
+
 template <typing::StringLiteral... Literals>
 struct handle_type_name<typing::Literal<Literals...>> {
-    static constexpr auto name = const_name("Literal[")
-                                 + pybind11::detail::concat(const_name(Literals.name)...)
-                                 + const_name("]");
+    static constexpr auto name
+        = const_name("Literal[")
+          + pybind11::detail::concat(const_name(sanitize_string_literal<Literals>().name)...)
+          + const_name("]");
 };
 template <typing::StringLiteral StrLit>
 struct handle_type_name<typing::TypeVar<StrLit>> {
-    static constexpr auto name = const_name(StrLit.name);
+    static constexpr auto name = const_name(sanitize_string_literal<StrLit>().name);
 };
 #endif
 

pybind11/__main__.py

@@ -71,6 +71,11 @@ def main() -> None:
         action="store_true",
         help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
     )
+    parser.add_argument(
+        "--extension-suffix",
+        action="store_true",
+        help="Print the extension for a Python module",
+    )
     args = parser.parse_args()
     if not sys.argv[1:]:
         parser.print_help()
@@ -80,6 +85,8 @@ def main() -> None:
         print(quote(get_cmake_dir()))
     if args.pkgconfigdir:
         print(quote(get_pkgconfig_dir()))
+    if args.extension_suffix:
+        print(sysconfig.get_config_var("EXT_SUFFIX"))
 
 
 if __name__ == "__main__":

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s: str) -> int | str:
         return s
 
 
-__version__ = "2.14.0.dev1"
+__version__ = "3.0.0.dev1"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

tests/CMakeLists.txt

@@ -115,6 +115,7 @@ set(PYBIND11_TEST_FILES
     test_callbacks
     test_chrono
     test_class
+    test_class_release_gil_before_calling_cpp_dtor
     test_const_name
     test_constants_and_functions
     test_copy_move

tests/conftest.py

@@ -212,9 +212,9 @@ def pytest_configure():
 
 def pytest_report_header(config):
     del config  # Unused.
-    assert (
-        pybind11_tests.compiler_info is not None
-    ), "Please update pybind11_tests.cpp if this assert fails."
+    assert pybind11_tests.compiler_info is not None, (
+        "Please update pybind11_tests.cpp if this assert fails."
+    )
     return (
         "C++ Info:"
         f" {pybind11_tests.compiler_info}"

tests/test_callbacks.cpp

@@ -269,12 +269,7 @@ TEST_SUBMODULE(callbacks, m) {
     rec_capsule.set_name(rec_capsule_name);
     m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr()));
 
-    // This test requires a new ABI version to pass
-#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON)
     // rec_capsule with nullptr name
     py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); });
     m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr()));
-#else
-    m.add_object("custom_function2", py::none());
-#endif
 }

tests/test_callbacks.py

@@ -217,9 +217,7 @@ def test_custom_func():
     assert m.roundtrip(m.custom_function)(4) == 36
 
 
-@pytest.mark.skipif(
-    m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low"
-)
+@pytest.mark.skipif("env.GRAALPY", reason="TODO debug segfault")
 def test_custom_func2():
     assert m.custom_function2(3) == 27
     assert m.roundtrip(m.custom_function2)(3) == 27

tests/test_class_release_gil_before_calling_cpp_dtor.cpp

@@ -0,0 +1,53 @@
+#include <pybind11/pybind11.h>
+
+#include "pybind11_tests.h"
+
+#include <string>
+#include <unordered_map>
+
+namespace pybind11_tests {
+namespace class_release_gil_before_calling_cpp_dtor {
+
+using RegistryType = std::unordered_map<std::string, int>;
+
+static RegistryType &PyGILState_Check_Results() {
+    static RegistryType singleton; // Local static variables have thread-safe initialization.
+    return singleton;
+}
+
+template <int> // Using int as a trick to easily generate a series of types.
+struct ProbeType {
+private:
+    std::string unique_key;
+
+public:
+    explicit ProbeType(const std::string &unique_key) : unique_key{unique_key} {}
+
+    ~ProbeType() {
+        RegistryType &reg = PyGILState_Check_Results();
+        assert(reg.count(unique_key) == 0);
+        reg[unique_key] = PyGILState_Check();
+    }
+};
+
+} // namespace class_release_gil_before_calling_cpp_dtor
+} // namespace pybind11_tests
+
+TEST_SUBMODULE(class_release_gil_before_calling_cpp_dtor, m) {
+    using namespace pybind11_tests::class_release_gil_before_calling_cpp_dtor;
+
+    py::class_<ProbeType<0>>(m, "ProbeType0").def(py::init<std::string>());
+
+    py::class_<ProbeType<1>>(m, "ProbeType1", py::release_gil_before_calling_cpp_dtor())
+        .def(py::init<std::string>());
+
+    m.def("PopPyGILState_Check_Result", [](const std::string &unique_key) -> std::string {
+        RegistryType &reg = PyGILState_Check_Results();
+        if (reg.count(unique_key) == 0) {
+            return "MISSING";
+        }
+        int res = reg[unique_key];
+        reg.erase(unique_key);
+        return std::to_string(res);
+    });
+}

tests/test_class_release_gil_before_calling_cpp_dtor.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import gc
+
+import pytest
+
+from pybind11_tests import class_release_gil_before_calling_cpp_dtor as m
+
+
+@pytest.mark.parametrize(
+    ("probe_type", "unique_key", "expected_result"),
+    [
+        (m.ProbeType0, "without_manipulating_gil", "1"),
+        (m.ProbeType1, "release_gil_before_calling_cpp_dtor", "0"),
+    ],
+)
+def test_gil_state_check_results(probe_type, unique_key, expected_result):
+    probe_type(unique_key)
+    gc.collect()
+    result = m.PopPyGILState_Check_Result(unique_key)
+    assert result == expected_result

tests/test_docs_advanced_cast_custom.cpp

@@ -20,19 +20,16 @@ namespace detail {
 
 template <>
 struct type_caster<user_space::Point2D> {
-    // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple`
-    PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple"));
-    // `arg_name` and `return_name` may optionally be used to specify type hints separately for
-    // arguments and return values.
+    // This macro inserts a lot of boilerplate code and sets the type hint.
+    // `io_name` is used to specify different type hints for arguments and return values.
     // The signature of our negate function would then look like:
     // `negate(Sequence[float]) -> tuple[float, float]`
-    static constexpr auto arg_name = const_name("Sequence[float]");
-    static constexpr auto return_name = const_name("tuple[float, float]");
+    PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]"));
 
     // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments
     // are used to indicate the return value policy and parent object (for
     // return_value_policy::reference_internal) and are often ignored by custom casters.
-    // The return value should reflect the type hint specified by `return_name`.
+    // The return value should reflect the type hint specified by the second argument of `io_name`.
     static handle
     cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) {
         return py::make_tuple(number.x, number.y).release();
@@ -40,7 +37,8 @@ struct type_caster<user_space::Point2D> {
 
     // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The
     // second argument indicates whether implicit conversions should be allowed.
-    // The accepted types should reflect the type hint specified by `arg_name`.
+    // The accepted types should reflect the type hint specified by the first argument of
+    // `io_name`.
     bool load(handle src, bool /*convert*/) {
         // Check if handle is a Sequence
         if (!py::isinstance<py::sequence>(src)) {

tests/test_eigen_matrix.cpp

@@ -440,4 +440,8 @@ TEST_SUBMODULE(eigen_matrix, m) {
         py::module_::import("numpy").attr("ones")(10);
         return v[0](5);
     });
+    m.def("round_trip_vector", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return x; });
+    m.def("round_trip_dense", [](const DenseMatrixR &m) -> DenseMatrixR { return m; });
+    m.def("round_trip_dense_ref",
+          [](const Eigen::Ref<DenseMatrixR> &m) -> Eigen::Ref<DenseMatrixR> { return m; });
 }

tests/test_eigen_matrix.py

@@ -95,19 +95,20 @@ def test_mutator_descriptors():
     with pytest.raises(TypeError) as excinfo:
         m.fixed_mutator_r(zc)
     assert (
-        "(arg0: numpy.ndarray[numpy.float32[5, 6],"
-        " flags.writeable, flags.c_contiguous]) -> None" in str(excinfo.value)
+        '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]",'
+        ' "flags.writeable", "flags.c_contiguous"]) -> None' in str(excinfo.value)
     )
     with pytest.raises(TypeError) as excinfo:
         m.fixed_mutator_c(zr)
     assert (
-        "(arg0: numpy.ndarray[numpy.float32[5, 6],"
-        " flags.writeable, flags.f_contiguous]) -> None" in str(excinfo.value)
+        '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]",'
+        ' "flags.writeable", "flags.f_contiguous"]) -> None' in str(excinfo.value)
     )
     with pytest.raises(TypeError) as excinfo:
         m.fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype="float32"))
-    assert "(arg0: numpy.ndarray[numpy.float32[5, 6], flags.writeable]) -> None" in str(
-        excinfo.value
+    assert (
+        '(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[5, 6]", "flags.writeable"]) -> None'
+        in str(excinfo.value)
     )
     zr.flags.writeable = False
     with pytest.raises(TypeError):
@@ -201,7 +202,7 @@ def test_negative_stride_from_python(msg):
         msg(excinfo.value)
         == """
         double_threer(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: numpy.ndarray[numpy.float32[1, 3], flags.writeable]) -> None
+            1. (arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[1, 3]", "flags.writeable"]) -> None
 
         Invoked with: """
         + repr(np.array([5.0, 4.0, 3.0], dtype="float32"))
@@ -213,7 +214,7 @@ def test_negative_stride_from_python(msg):
         msg(excinfo.value)
         == """
         double_threec(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: numpy.ndarray[numpy.float32[3, 1], flags.writeable]) -> None
+            1. (arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[3, 1]", "flags.writeable"]) -> None
 
         Invoked with: """
         + repr(np.array([7.0, 4.0, 1.0], dtype="float32"))
@@ -244,9 +245,9 @@ def test_eigen_ref_to_python():
     chols = [m.cholesky1, m.cholesky2, m.cholesky3, m.cholesky4]
     for i, chol in enumerate(chols, start=1):
         mymat = chol(np.array([[1.0, 2, 4], [2, 13, 23], [4, 23, 77]]))
-        assert np.all(
-            mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]])
-        ), f"cholesky{i}"
+        assert np.all(mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]])), (
+            f"cholesky{i}"
+        )
 
 
 def assign_both(a1, a2, r, c, v):
@@ -634,16 +635,16 @@ def test_nocopy_wrapper():
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(int_matrix_colmajor)
     assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
-    assert ", flags.f_contiguous" in str(excinfo.value)
+    assert ', "flags.f_contiguous"' in str(excinfo.value)
     assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(int_matrix_rowmajor)
     assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
-    assert ", flags.f_contiguous" in str(excinfo.value)
+    assert ', "flags.f_contiguous"' in str(excinfo.value)
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(dbl_matrix_rowmajor)
     assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
-    assert ", flags.f_contiguous" in str(excinfo.value)
+    assert ', "flags.f_contiguous"' in str(excinfo.value)
 
     # For the row-major test, we take a long matrix in row-major, so only the third is allowed:
     with pytest.raises(TypeError) as excinfo:
@@ -651,20 +652,20 @@ def test_nocopy_wrapper():
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
     )
-    assert ", flags.c_contiguous" in str(excinfo.value)
+    assert ', "flags.c_contiguous"' in str(excinfo.value)
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_rm_nocopy(dbl_matrix_colmajor)
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
     )
-    assert ", flags.c_contiguous" in str(excinfo.value)
+    assert ', "flags.c_contiguous"' in str(excinfo.value)
     assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_rm_nocopy(dbl_matrix_rowmajor)
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
     )
-    assert ", flags.c_contiguous" in str(excinfo.value)
+    assert ', "flags.c_contiguous"' in str(excinfo.value)
 
 
 def test_eigen_ref_life_support():
@@ -700,25 +701,25 @@ def test_dense_signature(doc):
     assert (
         doc(m.double_col)
         == """
-        double_col(arg0: numpy.ndarray[numpy.float32[m, 1]]) -> numpy.ndarray[numpy.float32[m, 1]]
+        double_col(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, 1]"]) -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, 1]"]
     """
     )
     assert (
         doc(m.double_row)
         == """
-        double_row(arg0: numpy.ndarray[numpy.float32[1, n]]) -> numpy.ndarray[numpy.float32[1, n]]
+        double_row(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[1, n]"]) -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[1, n]"]
     """
     )
     assert doc(m.double_complex) == (
         """
-        double_complex(arg0: numpy.ndarray[numpy.complex64[m, 1]])"""
-        """ -> numpy.ndarray[numpy.complex64[m, 1]]
+        double_complex(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex64, "[m, 1]"])"""
+        """ -> typing.Annotated[numpy.typing.NDArray[numpy.complex64], "[m, 1]"]
     """
     )
     assert doc(m.double_mat_rm) == (
         """
-        double_mat_rm(arg0: numpy.ndarray[numpy.float32[m, n]])"""
-        """ -> numpy.ndarray[numpy.float32[m, n]]
+        double_mat_rm(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, n]"])"""
+        """ -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]"]
     """
     )
 
@@ -817,3 +818,22 @@ def test_custom_operator_new():
     o = m.CustomOperatorNew()
     np.testing.assert_allclose(o.a, 0.0)
     np.testing.assert_allclose(o.b.diagonal(), 1.0)
+
+
+def test_arraylike_signature(doc):
+    assert doc(m.round_trip_vector) == (
+        'round_trip_vector(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, 1]"])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, 1]"]'
+    )
+    assert doc(m.round_trip_dense) == (
+        'round_trip_dense(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32, "[m, n]"])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]"]'
+    )
+    assert doc(m.round_trip_dense_ref) == (
+        'round_trip_dense_ref(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]", "flags.writeable", "flags.c_contiguous"])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float32], "[m, n]", "flags.writeable", "flags.c_contiguous"]'
+    )
+    m.round_trip_vector([1.0, 2.0])
+    m.round_trip_dense([[1.0, 2.0], [3.0, 4.0]])
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.round_trip_dense_ref([[1.0, 2.0], [3.0, 4.0]])

tests/test_eigen_tensor.py

@@ -271,23 +271,46 @@ def test_round_trip_references_actually_refer(m):
 @pytest.mark.parametrize("m", submodules)
 def test_doc_string(m, doc):
     assert (
-        doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]"
+        doc(m.copy_tensor)
+        == 'copy_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]'
     )
     assert (
         doc(m.copy_fixed_tensor)
-        == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2]]"
+        == 'copy_fixed_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3, 5, 2]"]'
     )
     assert (
         doc(m.reference_const_tensor)
-        == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]"
+        == 'reference_const_tensor() -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]'
     )
 
-    order_flag = f"flags.{m.needed_options.lower()}_contiguous"
+    order_flag = f'"flags.{m.needed_options.lower()}_contiguous"'
     assert doc(m.round_trip_view_tensor) == (
-        f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])"
-        f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]"
+        f'round_trip_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}])'
+        f' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}]'
     )
     assert doc(m.round_trip_const_view_tensor) == (
-        f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])"
-        " -> numpy.ndarray[numpy.float64[?, ?, ?]]"
+        f'round_trip_const_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", {order_flag}])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]'
     )
+
+
+@pytest.mark.parametrize("m", submodules)
+def test_arraylike_signature(m, doc):
+    order_flag = f'"flags.{m.needed_options.lower()}_contiguous"'
+    assert doc(m.round_trip_tensor) == (
+        'round_trip_tensor(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float64, "[?, ?, ?]"])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]'
+    )
+    assert doc(m.round_trip_tensor_noconvert) == (
+        'round_trip_tensor_noconvert(tensor: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"])'
+        ' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]"]'
+    )
+    assert doc(m.round_trip_view_tensor) == (
+        f'round_trip_view_tensor(arg0: typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}])'
+        f' -> typing.Annotated[numpy.typing.NDArray[numpy.float64], "[?, ?, ?]", "flags.writeable", {order_flag}]'
+    )
+    m.round_trip_tensor(tensor_ref.tolist())
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.round_trip_tensor_noconvert(tensor_ref.tolist())
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.round_trip_view_tensor(tensor_ref.tolist())

tests/test_gil_scoped.py

@@ -108,6 +108,10 @@ def test_nested_acquire():
 
 
 @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads")
+@pytest.mark.skipif(
+    env.GRAALPY and sys.platform == "darwin",
+    reason="Transiently crashes on GraalPy on OS X",
+)
 def test_multi_acquire_release_cross_module():
     for bits in range(16 * 8):
         internals_ids = m.test_multi_acquire_release_cross_module(bits)

tests/test_numpy_array.cpp

@@ -586,4 +586,13 @@ TEST_SUBMODULE(numpy_array, sm) {
     sm.def("return_array_pyobject_ptr_from_list", return_array_from_list<PyObject *>);
     sm.def("return_array_handle_from_list", return_array_from_list<py::handle>);
     sm.def("return_array_object_from_list", return_array_from_list<py::object>);
+
+    sm.def(
+        "round_trip_array_t",
+        [](const py::array_t<float> &x) -> py::array_t<float> { return x; },
+        py::arg("x"));
+    sm.def(
+        "round_trip_array_t_noconvert",
+        [](const py::array_t<float> &x) -> py::array_t<float> { return x; },
+        py::arg("x").noconvert());
 }

tests/test_numpy_array.py

@@ -321,13 +321,13 @@ def test_overload_resolution(msg):
         msg(excinfo.value)
         == """
         overloaded(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: numpy.ndarray[numpy.float64]) -> str
-            2. (arg0: numpy.ndarray[numpy.float32]) -> str
-            3. (arg0: numpy.ndarray[numpy.int32]) -> str
-            4. (arg0: numpy.ndarray[numpy.uint16]) -> str
-            5. (arg0: numpy.ndarray[numpy.int64]) -> str
-            6. (arg0: numpy.ndarray[numpy.complex128]) -> str
-            7. (arg0: numpy.ndarray[numpy.complex64]) -> str
+            1. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]) -> str
+            2. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float32]) -> str
+            3. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int32]) -> str
+            4. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.uint16]) -> str
+            5. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int64]) -> str
+            6. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex128]) -> str
+            7. (arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.complex64]) -> str
 
         Invoked with: 'not an array'
     """
@@ -343,8 +343,8 @@ def test_overload_resolution(msg):
     assert m.overloaded3(np.array([1], dtype="intc")) == "int"
     expected_exc = """
         overloaded3(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: numpy.ndarray[numpy.int32]) -> str
-            2. (arg0: numpy.ndarray[numpy.float64]) -> str
+            1. (arg0: numpy.typing.NDArray[numpy.int32]) -> str
+            2. (arg0: numpy.typing.NDArray[numpy.float64]) -> str
 
         Invoked with: """
 
@@ -528,7 +528,7 @@ def test_index_using_ellipsis():
     ],
 )
 def test_format_descriptors_for_floating_point_types(test_func):
-    assert "numpy.ndarray[numpy.float" in test_func.__doc__
+    assert "numpy.typing.ArrayLike, numpy.float" in test_func.__doc__
 
 
 @pytest.mark.parametrize("forcecast", [False, True])
@@ -687,3 +687,17 @@ def test_return_array_object_cpp_loop(return_array, unwrap):
     assert isinstance(arr_from_list, np.ndarray)
     assert arr_from_list.dtype == np.dtype("O")
     assert unwrap(arr_from_list) == [6, "seven", -8.0]
+
+
+def test_arraylike_signature(doc):
+    assert (
+        doc(m.round_trip_array_t)
+        == "round_trip_array_t(x: typing.Annotated[numpy.typing.ArrayLike, numpy.float32]) -> numpy.typing.NDArray[numpy.float32]"
+    )
+    assert (
+        doc(m.round_trip_array_t_noconvert)
+        == "round_trip_array_t_noconvert(x: numpy.typing.NDArray[numpy.float32]) -> numpy.typing.NDArray[numpy.float32]"
+    )
+    m.round_trip_array_t([1, 2, 3])
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.round_trip_array_t_noconvert([1, 2, 3])

tests/test_numpy_dtypes.py

@@ -373,7 +373,7 @@ def test_complex_array():
 def test_signature(doc):
     assert (
         doc(m.create_rec_nested)
-        == "create_rec_nested(arg0: int) -> numpy.ndarray[NestedStruct]"
+        == "create_rec_nested(arg0: int) -> numpy.typing.NDArray[NestedStruct]"
     )
 
 

tests/test_numpy_vectorize.py

@@ -150,7 +150,7 @@ def test_docs(doc):
     assert (
         doc(m.vectorized_func)
         == """
-        vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object
+        vectorized_func(arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.int32], arg1: typing.Annotated[numpy.typing.ArrayLike, numpy.float32], arg2: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]) -> object
     """
     )
 
@@ -212,12 +212,12 @@ def test_passthrough_arguments(doc):
         + ", ".join(
             [
                 "arg0: float",
-                "arg1: numpy.ndarray[numpy.float64]",
-                "arg2: numpy.ndarray[numpy.float64]",
-                "arg3: numpy.ndarray[numpy.int32]",
+                "arg1: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]",
+                "arg2: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]",
+                "arg3: typing.Annotated[numpy.typing.ArrayLike, numpy.int32]",
                 "arg4: int",
                 "arg5: m.numpy_vectorize.NonPODClass",
-                "arg6: numpy.ndarray[numpy.float64]",
+                "arg6: typing.Annotated[numpy.typing.ArrayLike, numpy.float64]",
             ]
         )
         + ") -> object"

tests/test_pytypes.cpp

@@ -142,7 +142,6 @@ typedef py::typing::TypeVar<"V"> TypeVarV;
 // RealNumber:
 // * in arguments -> float | int
 // * in return -> float
-// * fallback -> complex
 // The choice of types is not really useful, but just made different for testing purposes.
 // According to `PEP 484 – Type Hints` annotating with `float` also allows `int`,
 // so using `float | int` could be replaced by just `float`.
@@ -156,15 +155,17 @@ namespace detail {
 
 template <>
 struct type_caster<RealNumber> {
-    PYBIND11_TYPE_CASTER(RealNumber, const_name("complex"));
-    static constexpr auto arg_name = const_name("Union[float, int]");
-    static constexpr auto return_name = const_name("float");
+    PYBIND11_TYPE_CASTER(RealNumber, io_name("Union[float, int]", "float"));
 
     static handle cast(const RealNumber &number, return_value_policy, handle) {
         return py::float_(number.value).release();
     }
 
-    bool load(handle src, bool) {
+    bool load(handle src, bool convert) {
+        // If we're in no-convert mode, only load if given a float
+        if (!convert && !py::isinstance<py::float_>(src)) {
+            return false;
+        }
         if (!py::isinstance<py::float_>(src) && !py::isinstance<py::int_>(src)) {
             return false;
         }
@@ -970,6 +971,19 @@ TEST_SUBMODULE(pytypes, m) {
         .value("BLUE", literals::Color::BLUE);
 
     m.def("annotate_literal", [](literals::LiteralFoo &o) -> py::object { return o; });
+    // Literal with `@`, `%`, `{`, `}`, and `->`
+    m.def("identity_literal_exclamation", [](const py::typing::Literal<"\"!\""> &x) { return x; });
+    m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; });
+    m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; });
+    m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; });
+    m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; });
+    m.def("identity_literal_arrow_with_io_name",
+          [](const py::typing::Literal<"\"->\""> &x, const RealNumber &) { return x; });
+    m.def("identity_literal_arrow_with_callable",
+          [](const py::typing::Callable<RealNumber(const py::typing::Literal<"\"->\""> &,
+                                                   const RealNumber &)> &x) { return x; });
+    m.def("identity_literal_all_special_chars",
+          [](const py::typing::Literal<"\"!@!!->{%}\""> &x) { return x; });
     m.def("annotate_generic_containers",
           [](const py::typing::List<typevar::TypeVarT> &l) -> py::typing::List<typevar::TypeVarV> {
               return l;
@@ -1070,6 +1084,14 @@ TEST_SUBMODULE(pytypes, m) {
     m.attr("defined___cpp_inline_variables") = false;
 #endif
     m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; });
+    m.def(
+        "half_of_number_convert",
+        [](const RealNumber &x) { return RealNumber{x.value / 2}; },
+        py::arg("x"));
+    m.def(
+        "half_of_number_noconvert",
+        [](const RealNumber &x) { return RealNumber{x.value / 2}; },
+        py::arg("x").noconvert());
     // std::vector<T>
     m.def("half_of_number_vector", [](const std::vector<RealNumber> &x) {
         std::vector<RealNumber> result;
@@ -1130,6 +1152,16 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("identity_iterable", [](const py::typing::Iterable<RealNumber> &x) { return x; });
     // Iterator<T>
     m.def("identity_iterator", [](const py::typing::Iterator<RealNumber> &x) { return x; });
+    // Callable<R(A)> identity
+    m.def("identity_callable",
+          [](const py::typing::Callable<RealNumber(const RealNumber &)> &x) { return x; });
+    // Callable<R(...)> identity
+    m.def("identity_callable_ellipsis",
+          [](const py::typing::Callable<RealNumber(py::ellipsis)> &x) { return x; });
+    // Nested Callable<R(A)> identity
+    m.def("identity_nested_callable",
+          [](const py::typing::Callable<py::typing::Callable<RealNumber(const RealNumber &)>(
+                 py::typing::Callable<RealNumber(const RealNumber &)>)> &x) { return x; });
     // Callable<R(A)>
     m.def("apply_callable",
           [](const RealNumber &x, const py::typing::Callable<RealNumber(const RealNumber &)> &f) {

tests/test_pytypes.py

@@ -52,10 +52,10 @@ def test_from_iterable(pytype, from_iter_func):
 
 def test_iterable(doc):
     assert doc(m.get_iterable) == "get_iterable() -> Iterable"
-    lins = [1, 2, 3]
-    i = m.get_first_item_from_iterable(lins)
+    lst = [1, 2, 3]
+    i = m.get_first_item_from_iterable(lst)
     assert i == 1
-    i = m.get_second_item_from_iterable(lins)
+    i = m.get_second_item_from_iterable(lst)
     assert i == 2
 
 
@@ -67,13 +67,13 @@ def test_list(capture, doc):
     assert m.list_no_args() == []
     assert m.list_ssize_t() == []
     assert m.list_size_t() == []
-    lins = [1, 2]
-    m.list_insert_ssize_t(lins)
-    assert lins == [1, 83, 2]
-    m.list_insert_size_t(lins)
-    assert lins == [1, 83, 2, 57]
-    m.list_clear(lins)
-    assert lins == []
+    lst = [1, 2]
+    m.list_insert_ssize_t(lst)
+    assert lst == [1, 83, 2]
+    m.list_insert_size_t(lst)
+    assert lst == [1, 83, 2, 57]
+    m.list_clear(lst)
+    assert lst == []
 
     with capture:
         lst = m.get_list()
@@ -1044,6 +1044,39 @@ def test_literal(doc):
         doc(m.annotate_literal)
         == 'annotate_literal(arg0: Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]) -> object'
     )
+    # The characters !, @, %, {, } and -> are used in the signature parser as special characters, but Literal should escape those for the parser to work.
+    assert (
+        doc(m.identity_literal_exclamation)
+        == 'identity_literal_exclamation(arg0: Literal["!"]) -> Literal["!"]'
+    )
+    assert (
+        doc(m.identity_literal_at)
+        == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]'
+    )
+    assert (
+        doc(m.identity_literal_percent)
+        == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]'
+    )
+    assert (
+        doc(m.identity_literal_curly_open)
+        == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]'
+    )
+    assert (
+        doc(m.identity_literal_curly_close)
+        == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]'
+    )
+    assert (
+        doc(m.identity_literal_arrow_with_io_name)
+        == 'identity_literal_arrow_with_io_name(arg0: Literal["->"], arg1: Union[float, int]) -> Literal["->"]'
+    )
+    assert (
+        doc(m.identity_literal_arrow_with_callable)
+        == 'identity_literal_arrow_with_callable(arg0: Callable[[Literal["->"], Union[float, int]], float]) -> Callable[[Literal["->"], Union[float, int]], float]'
+    )
+    assert (
+        doc(m.identity_literal_all_special_chars)
+        == 'identity_literal_all_special_chars(arg0: Literal["!@!!->{%}"]) -> Literal["!@!!->{%}"]'
+    )
 
 
 @pytest.mark.skipif(
@@ -1195,15 +1228,22 @@ def test_final_annotation() -> None:
 
 def test_arg_return_type_hints(doc):
     assert doc(m.half_of_number) == "half_of_number(arg0: Union[float, int]) -> float"
+    assert (
+        doc(m.half_of_number_convert)
+        == "half_of_number_convert(x: Union[float, int]) -> float"
+    )
+    assert (
+        doc(m.half_of_number_noconvert) == "half_of_number_noconvert(x: float) -> float"
+    )
     assert m.half_of_number(2.0) == 1.0
     assert m.half_of_number(2) == 1.0
     assert m.half_of_number(0) == 0
     assert isinstance(m.half_of_number(0), float)
     assert not isinstance(m.half_of_number(0), int)
-    # std::vector<T> should use fallback type (complex is not really useful but just used for testing)
+    # std::vector<T>
     assert (
         doc(m.half_of_number_vector)
-        == "half_of_number_vector(arg0: list[complex]) -> list[complex]"
+        == "half_of_number_vector(arg0: list[Union[float, int]]) -> list[float]"
     )
     # Tuple<T, T>
     assert (
@@ -1245,6 +1285,21 @@ def test_arg_return_type_hints(doc):
         doc(m.identity_iterator)
         == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]"
     )
+    # Callable<R(A)> identity
+    assert (
+        doc(m.identity_callable)
+        == "identity_callable(arg0: Callable[[Union[float, int]], float]) -> Callable[[Union[float, int]], float]"
+    )
+    # Callable<R(...)> identity
+    assert (
+        doc(m.identity_callable_ellipsis)
+        == "identity_callable_ellipsis(arg0: Callable[..., float]) -> Callable[..., float]"
+    )
+    # Nested Callable<R(A)> identity
+    assert (
+        doc(m.identity_nested_callable)
+        == "identity_nested_callable(arg0: Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]) -> Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]"
+    )
     # Callable<R(A)>
     assert (
         doc(m.apply_callable)

tests/test_smart_ptr.py

@@ -313,9 +313,8 @@ def test_smart_ptr_from_default():
     instance = m.HeldByDefaultHolder()
     with pytest.raises(RuntimeError) as excinfo:
         m.HeldByDefaultHolder.load_shared_ptr(instance)
-    assert (
-        "Unable to load a custom holder type from a "
-        "default-holder instance" in str(excinfo.value)
+    assert "Unable to load a custom holder type from a default-holder instance" in str(
+        excinfo.value
     )
 
 

tests/test_stl.py

@@ -265,19 +265,19 @@ def test_fs_path(doc):
     assert m.parent_path(PseudoBytesPath()) == Path("foo")
     assert (
         doc(m.parent_path)
-        == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> Path"
+        == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> pathlib.Path"
     )
-    # std::vector should use name (for arg_name/return_name typing classes must be used)
+    # std::vector
     assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")]
     assert (
         doc(m.parent_paths)
-        == "parent_paths(arg0: list[os.PathLike]) -> list[os.PathLike]"
+        == "parent_paths(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]"
     )
     # py::typing::List
     assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")]
     assert (
         doc(m.parent_paths_list)
-        == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]"
+        == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]"
     )
     # Nested py::typing::List
     assert m.parent_paths_nested_list([["foo/bar"], ["foo/baz", "foo/buzz"]]) == [
@@ -286,13 +286,13 @@ def test_fs_path(doc):
     ]
     assert (
         doc(m.parent_paths_nested_list)
-        == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[Path]]"
+        == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[pathlib.Path]]"
     )
     # py::typing::Tuple
     assert m.parent_paths_tuple(("foo/bar", "foo/baz")) == (Path("foo"), Path("foo"))
     assert (
         doc(m.parent_paths_tuple)
-        == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[Path, Path]"
+        == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[pathlib.Path, pathlib.Path]"
     )
     # py::typing::Dict
     assert m.parent_paths_dict(
@@ -308,7 +308,7 @@ def test_fs_path(doc):
     }
     assert (
         doc(m.parent_paths_dict)
-        == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, Path]"
+        == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, pathlib.Path]"
     )
 
 

tests/test_unnamed_namespace_a.cpp

@@ -10,7 +10,6 @@ TEST_SUBMODULE(unnamed_namespace_a, m) {
     } else {
         m.attr("unnamed_namespace_a_any_struct") = py::none();
     }
-    m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION;
     m.attr("defined_WIN32_or__WIN32") =
 #if defined(WIN32) || defined(_WIN32)
         true;

tests/test_unnamed_namespace_a.py

@@ -5,13 +5,7 @@ import pytest
 from pybind11_tests import unnamed_namespace_a as m
 from pybind11_tests import unnamed_namespace_b as mb
 
-XFAIL_CONDITION = (
-    "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))"
-    " or "
-    "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32"
-    " and "
-    "(m.defined___clang__ or m.defined__LIBCPP_VERSION))"
-)
+XFAIL_CONDITION = "not m.defined_WIN32_or__WIN32 and (m.defined___clang__ or m.defined__LIBCPP_VERSION)"
 XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319"
 
 

tools/FindPythonLibsNew.cmake

@@ -200,6 +200,16 @@ if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
 endif()
 if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION)
   get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+  if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "") AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}"
+                                                          STREQUAL "${PYTHON_MODULE_EXTENSION}"))
+    message(
+      AUTHOR_WARNING,
+      "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", "
+      "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". "
+      "This may cause problems when importing the Python extensions. "
+      "If you are using cross-compiling Python, you may need to "
+      "set PYTHON_MODULE_EXTENSION manually.")
+  endif()
 endif()
 
 # Make sure the Python has the same pointer-size as the chosen compiler

tools/pybind11NewTools.cmake

@@ -171,6 +171,16 @@ if(NOT _PYBIND11_CROSSCOMPILING)
       set(PYTHON_MODULE_EXTENSION
           "${_PYTHON_MODULE_EXTENSION}"
           CACHE INTERNAL "")
+      if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "")
+         AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "${PYTHON_MODULE_EXTENSION}"))
+        message(
+          AUTHOR_WARNING,
+          "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", "
+          "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". "
+          "This may cause problems when importing the Python extensions. "
+          "If you are using cross-compiling Python, you may need to "
+          "set PYTHON_MODULE_EXTENSION manually.")
+      endif()
     endif()
   endif()
 else()
@@ -307,7 +317,7 @@ function(pybind11_add_module target_name)
   if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454
     # Use case-insensitive comparison to match the result of $<CONFIG:cfgs>
     string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
-    if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO)
+    if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE)
       # Strip unnecessary sections of the binary on Linux/macOS
       pybind11_strip(${target_name})
     endif()

tools/pybind11Tools.cmake

@@ -196,7 +196,7 @@ function(pybind11_add_module target_name)
   if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454
     # Use case-insensitive comparison to match the result of $<CONFIG:cfgs>
     string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
-    if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO)
+    if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE)
       pybind11_strip(${target_name})
     endif()
   endif()