Merge branch 'master' into annotated_any

.pre-commit-config.yaml

@@ -25,27 +25,22 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v16.0.6"
+  rev: "v17.0.4"
   hooks:
   - id: clang-format
     types_or: [c++, c, cuda]
 
-# Black, the code formatter, natively supports pre-commit
-- repo: https://github.com/psf/black-pre-commit-mirror
-  rev: "23.9.1" # Keep in sync with blacken-docs
-  hooks:
-  - id: black
-
-# Ruff, the Python auto-correcting linter written in Rust
+# Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.292
+  rev: v0.1.4
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
+  - id: ruff-format
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v1.5.1"
+  rev: "v1.6.1"
   hooks:
   - id: mypy
     args: []
@@ -67,7 +62,7 @@ repos:
 
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: "v4.4.0"
+  rev: "v4.5.0"
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -88,7 +83,7 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==23.3.0 # keep in sync with black hook
+    - black==23.*
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
@@ -98,7 +93,7 @@ repos:
 
 # Avoid directional quotes
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.5.0"
+  rev: "0.6.2"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
@@ -147,7 +142,7 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v3.0.0"
+  rev: "v3.0.1"
   hooks:
   - id: pylint
     files: ^pybind11

include/pybind11/detail/class.h

@@ -189,12 +189,10 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P
         return nullptr;
     }
 
-    // This must be a pybind11 instance
-    auto *instance = reinterpret_cast<detail::instance *>(self);
-
     // Ensure that the base __init__ function(s) were called
-    for (const auto &vh : values_and_holders(instance)) {
-        if (!vh.holder_constructed()) {
+    values_and_holders vhs(self);
+    for (const auto &vh : vhs) {
+        if (!vh.holder_constructed() && !vhs.is_redundant_value_and_holder(vh)) {
             PyErr_Format(PyExc_TypeError,
                          "%.200s.__init__() must be called when overriding __init__",
                          get_fully_qualified_tp_name(vh.type->type).c_str());

include/pybind11/detail/type_caster_base.h

@@ -102,8 +102,22 @@ public:
 inline std::pair<decltype(internals::registered_types_py)::iterator, bool>
 all_type_info_get_cache(PyTypeObject *type);
 
+// Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762.
+inline void all_type_info_add_base_most_derived_first(std::vector<type_info *> &bases,
+                                                      type_info *addl_base) {
+    for (auto it = bases.begin(); it != bases.end(); it++) {
+        type_info *existing_base = *it;
+        if (PyType_IsSubtype(addl_base->type, existing_base->type) != 0) {
+            bases.insert(it, addl_base);
+            return;
+        }
+    }
+    bases.push_back(addl_base);
+}
+
 // Populates a just-created cache entry.
 PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_info *> &bases) {
+    assert(bases.empty());
     std::vector<PyTypeObject *> check;
     for (handle parent : reinterpret_borrow<tuple>(t->tp_bases)) {
         check.push_back((PyTypeObject *) parent.ptr());
@@ -136,7 +150,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_
                     }
                 }
                 if (!found) {
-                    bases.push_back(tinfo);
+                    all_type_info_add_base_most_derived_first(bases, tinfo);
                 }
             }
         } else if (type->tp_bases) {
@@ -322,18 +336,29 @@ public:
     explicit values_and_holders(instance *inst)
         : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {}
 
+    explicit values_and_holders(PyObject *obj)
+        : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) {
+        if (!tinfo.empty()) {
+            inst = reinterpret_cast<instance *>(obj);
+        }
+    }
+
     struct iterator {
     private:
         instance *inst = nullptr;
         const type_vec *types = nullptr;
         value_and_holder curr;
         friend struct values_and_holders;
-        iterator(instance *inst, const type_vec *tinfo)
-            : inst{inst}, types{tinfo},
-              curr(inst /* instance */,
-                   types->empty() ? nullptr : (*types)[0] /* type info */,
+        iterator(instance *inst, const type_vec *tinfo) : inst{inst}, types{tinfo} {
+            if (inst != nullptr) {
+                assert(!types->empty());
+                curr = value_and_holder(
+                    inst /* instance */,
+                    (*types)[0] /* type info */,
                     0, /* vpos: (non-simple types only): the first vptr comes first */
-                   0 /* index */) {}
+                    0 /* index */);
+            }
+        }
         // Past-the-end iterator:
         explicit iterator(size_t end) : curr(end) {}
 
@@ -364,6 +389,16 @@ public:
     }
 
     size_t size() { return tinfo.size(); }
+
+    // Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762.
+    bool is_redundant_value_and_holder(const value_and_holder &vh) {
+        for (size_t i = 0; i < vh.index; i++) {
+            if (PyType_IsSubtype(tinfo[i]->type, tinfo[vh.index]->type) != 0) {
+                return true;
+            }
+        }
+        return false;
+    }
 };
 
 /**
@@ -486,8 +521,10 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i
 inline PyThreadState *get_thread_state_unchecked() {
 #if defined(PYPY_VERSION)
     return PyThreadState_GET();
-#else
+#elif PY_VERSION_HEX < 0x030D0000
     return _PyThreadState_UncheckedGet();
+#else
+    return PyThreadState_GetUnchecked();
 #endif
 }
 

include/pybind11/pybind11.h

@@ -2760,7 +2760,11 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
         if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
             PyObject *locals = PyEval_GetLocals();
             if (locals != nullptr) {
+#        if PY_VERSION_HEX >= 0x030b0000
+                PyObject *co_varnames = PyCode_GetVarnames(f_code);
+#        else
                 PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames");
+#        endif
                 PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0);
                 Py_DECREF(co_varnames);
                 PyObject *self_caller = dict_getitem(locals, self_arg);

include/pybind11/pytypes.h

@@ -310,14 +310,14 @@ private:
             "used for all translation units linked into a given pybind11 extension, otherwise "
             "there will be ODR violations.",
             function_name.c_str());
-        fflush(stderr);
         if (Py_TYPE(m_ptr)->tp_name != nullptr) {
             fprintf(stderr,
-                    "The failing %s call was triggered on a %s object.\n",
+                    " The failing %s call was triggered on a %s object.",
                     function_name.c_str(),
                     Py_TYPE(m_ptr)->tp_name);
-            fflush(stderr);
         }
+        fprintf(stderr, "\n");
+        fflush(stderr);
         throw std::runtime_error(function_name + " PyGILState_Check() failure.");
     }
 #endif

pybind11/setup_helpers.py

@@ -66,7 +66,9 @@ try:
     from setuptools import Extension as _Extension
     from setuptools.command.build_ext import build_ext as _build_ext
 except ImportError:
-    from distutils.command.build_ext import build_ext as _build_ext  # type: ignore[assignment]
+    from distutils.command.build_ext import (  # type: ignore[assignment]
+        build_ext as _build_ext,
+    )
     from distutils.extension import Extension as _Extension  # type: ignore[assignment]
 
 import distutils.ccompiler

pyproject.toml

@@ -19,7 +19,7 @@ ignore = [
 
 [tool.mypy]
 files = ["pybind11"]
-python_version = "3.6"
+python_version = "3.7"
 strict = true
 show_error_codes = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -57,10 +57,12 @@ messages_control.disable = [
   "unused-argument",  # covered by Ruff ARG
 ]
 
-
 [tool.ruff]
-select = [
-  "E", "F", "W", # flake8
+target-version = "py37"
+src = ["src"]
+
+[tool.ruff.lint]
+extend-select = [
   "B",           # flake8-bugbear
   "I",           # isort
   "N",           # pep8-naming
@@ -68,7 +70,6 @@ select = [
   "C4",          # flake8-comprehensions
   "EM",          # flake8-errmsg
   "ICN",         # flake8-import-conventions
-  "ISC",         # flake8-implicit-str-concat
   "PGH",         # pygrep-hooks
   "PIE",         # flake8-pie
   "PL",          # pylint
@@ -86,13 +87,9 @@ ignore = [
   "PT004",  # Fixture that doesn't return needs underscore (no, it is fine)
   "SIM118", # iter(x) is not always the same as iter(x.keys())
 ]
-target-version = "py37"
-src = ["src"]
 unfixable = ["T20"]
-exclude = []
-line-length = 120
 isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "tests/**" = ["EM", "N", "E721"]
 "tests/test_call_policies.py" = ["PLC1901"]

tests/CMakeLists.txt

@@ -144,6 +144,7 @@ set(PYBIND11_TEST_FILES
     test_opaque_types
     test_operator_overloading
     test_pickling
+    test_python_multiple_inheritance
     test_pytypes
     test_sequences_and_iterators
     test_smart_ptr

tests/test_class.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 import pytest
 
 import env
@@ -203,6 +205,18 @@ def test_inheritance_init(msg):
     assert msg(exc_info.value) == expected
 
 
+@pytest.mark.parametrize(
+    "mock_return_value", [None, (1, 2, 3), m.Pet("Polly", "parrot"), m.Dog("Molly")]
+)
+def test_mock_new(mock_return_value):
+    with mock.patch.object(
+        m.Pet, "__new__", return_value=mock_return_value
+    ) as mock_new:
+        obj = m.Pet("Noname", "Nospecies")
+    assert obj is mock_return_value
+    mock_new.assert_called_once_with(m.Pet, "Noname", "Nospecies")
+
+
 def test_automatic_upcasting():
     assert type(m.return_class_1()).__name__ == "DerivedClass1"
     assert type(m.return_class_2()).__name__ == "DerivedClass2"

tests/test_enum.py

@@ -60,9 +60,7 @@ Members:
 
   ETwo : Docstring for ETwo
 
-  EThree : Docstring for EThree""".split(
-        "\n"
-    ):
+  EThree : Docstring for EThree""".split("\n"):
         assert docstring_line in m.UnscopedEnum.__doc__
 
     # Unscoped enums will accept ==/!= int comparisons

tests/test_methods_and_attributes.py

@@ -232,26 +232,30 @@ def test_no_mixed_overloads():
 
     with pytest.raises(RuntimeError) as excinfo:
         m.ExampleMandA.add_mixed_overloads1()
-    assert str(
-        excinfo.value
-    ) == "overloading a method with both static and instance methods is not supported; " + (
+    assert (
+        str(excinfo.value)
+        == "overloading a method with both static and instance methods is not supported; "
+        + (
             "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more details"
             if not detailed_error_messages_enabled
             else "error while attempting to bind static method ExampleMandA.overload_mixed1"
             "(arg0: float) -> str"
         )
+    )
 
     with pytest.raises(RuntimeError) as excinfo:
         m.ExampleMandA.add_mixed_overloads2()
-    assert str(
-        excinfo.value
-    ) == "overloading a method with both static and instance methods is not supported; " + (
+    assert (
+        str(excinfo.value)
+        == "overloading a method with both static and instance methods is not supported; "
+        + (
             "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more details"
             if not detailed_error_messages_enabled
             else "error while attempting to bind instance method ExampleMandA.overload_mixed2"
             "(self: pybind11_tests.methods_and_attributes.ExampleMandA, arg0: int, arg1: int)"
             " -> str"
         )
+    )
 
 
 @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])

tests/test_python_multiple_inheritance.cpp

@@ -0,0 +1,45 @@
+#include "pybind11_tests.h"
+
+namespace test_python_multiple_inheritance {
+
+// Copied from:
+// https://github.com/google/clif/blob/5718e4d0807fd3b6a8187dde140069120b81ecef/clif/testing/python_multiple_inheritance.h
+
+struct CppBase {
+    explicit CppBase(int value) : base_value(value) {}
+    int get_base_value() const { return base_value; }
+    void reset_base_value(int new_value) { base_value = new_value; }
+
+private:
+    int base_value;
+};
+
+struct CppDrvd : CppBase {
+    explicit CppDrvd(int value) : CppBase(value), drvd_value(value * 3) {}
+    int get_drvd_value() const { return drvd_value; }
+    void reset_drvd_value(int new_value) { drvd_value = new_value; }
+
+    int get_base_value_from_drvd() const { return get_base_value(); }
+    void reset_base_value_from_drvd(int new_value) { reset_base_value(new_value); }
+
+private:
+    int drvd_value;
+};
+
+} // namespace test_python_multiple_inheritance
+
+TEST_SUBMODULE(python_multiple_inheritance, m) {
+    using namespace test_python_multiple_inheritance;
+
+    py::class_<CppBase>(m, "CppBase")
+        .def(py::init<int>())
+        .def("get_base_value", &CppBase::get_base_value)
+        .def("reset_base_value", &CppBase::reset_base_value);
+
+    py::class_<CppDrvd, CppBase>(m, "CppDrvd")
+        .def(py::init<int>())
+        .def("get_drvd_value", &CppDrvd::get_drvd_value)
+        .def("reset_drvd_value", &CppDrvd::reset_drvd_value)
+        .def("get_base_value_from_drvd", &CppDrvd::get_base_value_from_drvd)
+        .def("reset_base_value_from_drvd", &CppDrvd::reset_base_value_from_drvd);
+}

tests/test_python_multiple_inheritance.py

@@ -0,0 +1,35 @@
+# Adapted from:
+# https://github.com/google/clif/blob/5718e4d0807fd3b6a8187dde140069120b81ecef/clif/testing/python/python_multiple_inheritance_test.py
+
+from pybind11_tests import python_multiple_inheritance as m
+
+
+class PC(m.CppBase):
+    pass
+
+
+class PPCC(PC, m.CppDrvd):
+    pass
+
+
+def test_PC():
+    d = PC(11)
+    assert d.get_base_value() == 11
+    d.reset_base_value(13)
+    assert d.get_base_value() == 13
+
+
+def test_PPCC():
+    d = PPCC(11)
+    assert d.get_drvd_value() == 33
+    d.reset_drvd_value(55)
+    assert d.get_drvd_value() == 55
+
+    assert d.get_base_value() == 11
+    assert d.get_base_value_from_drvd() == 11
+    d.reset_base_value(20)
+    assert d.get_base_value() == 20
+    assert d.get_base_value_from_drvd() == 20
+    d.reset_base_value_from_drvd(30)
+    assert d.get_base_value() == 30
+    assert d.get_base_value_from_drvd() == 30

tools/pybind11Config.cmake.in

@@ -149,7 +149,7 @@ default is ``MODULE``. There are several options:
 ``OPT_SIZE``
   Optimize for size, even if the ``CMAKE_BUILD_TYPE`` is not ``MinSizeRel``.
 ``THIN_LTO``
-  Use thin TLO instead of regular if there's a choice (pybind11's selection
+  Use thin LTO instead of regular if there's a choice (pybind11's selection
   is disabled if ``CMAKE_INTERPROCEDURAL_OPTIMIZATIONS`` is set).
 ``WITHOUT_SOABI``
   Disable the SOABI component (``PYBIND11_NEWPYTHON`` mode only).

tools/pybind11NewTools.cmake

@@ -32,12 +32,22 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     set(Python_ROOT_DIR "$ENV{pythonLocation}")
   endif()
 
-  find_package(Python 3.6 REQUIRED COMPONENTS Interpreter Development ${_pybind11_quiet})
+  # Development.Module support (required for manylinux) started in 3.18
+  if(CMAKE_VERSION VERSION_LESS 3.18)
+    set(_pybind11_dev_component Development)
+  else()
+    set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed)
+  endif()
+
+  find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
+                                              ${_pybind11_quiet})
 
   # If we are in submodule mode, export the Python targets to global targets.
   # If this behavior is not desired, FindPython _before_ pybind11.
   if(NOT is_config)
+    if(TARGET Python::Python)
       set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
+    endif()
     set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
     if(TARGET Python::Module)
       set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)

tools/pybind11Tools.cmake

@@ -43,7 +43,7 @@ endif()
 
 # A user can set versions manually too
 set(Python_ADDITIONAL_VERSIONS
-    "3.11;3.10;3.9;3.8;3.7;3.6"
+    "3.12;3.11;3.10;3.9;3.8;3.7;3.6"
     CACHE INTERNAL "")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")