Merge branch 'v2.10' into stable

.github/workflows/ci.yml

@@ -283,8 +283,6 @@ jobs:
         include:
           - clang: 5
             std: 14
-          - clang: 10
-            std: 20
           - clang: 10
             std: 17
           - clang: 11

.pre-commit-config.yaml

@@ -41,20 +41,20 @@ repos:
 
 # Upgrade old Python syntax
 - repo: https://github.com/asottile/pyupgrade
-  rev: "v3.3.0"
+  rev: "v3.3.1"
   hooks:
   - id: pyupgrade
     args: [--py36-plus]
 
 # Nicely sort includes
 - repo: https://github.com/PyCQA/isort
-  rev: "5.10.1"
+  rev: "5.11.4"
   hooks:
   - id: isort
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black
-  rev: "22.10.0" # Keep in sync with blacken-docs
+  rev: "22.12.0" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
@@ -116,7 +116,7 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v2.15.8"
+  rev: "v2.15.9"
   hooks:
   - id: pylint
     files: ^pybind11
@@ -160,7 +160,7 @@ repos:
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: "v0.8.0.4"
+  rev: "v0.9.0.2"
   hooks:
   - id: shellcheck
 
@@ -175,7 +175,7 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v15.0.4"
+  rev: "v15.0.6"
   hooks:
   - id: clang-format
     types_or: [c++, c, cuda]

docs/advanced/misc.rst

@@ -118,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy
     m.def("call_go", &call_go, py::call_guard<py::gil_scoped_release>());
 
 
+Common Sources Of Global Interpreter Lock Errors
+==================================================================
+
+Failing to properly hold the Global Interpreter Lock (GIL) is one of the
+more common sources of bugs within code that uses pybind11. If you are
+running into GIL related errors, we highly recommend you consult the
+following checklist.
+
+- Do you have any global variables that are pybind11 objects or invoke
+  pybind11 functions in either their constructor or destructor? You are generally
+  not allowed to invoke any Python function in a global static context. We recommend
+  using lazy initialization and then intentionally leaking at the end of the program.
+
+- Do you have any pybind11 objects that are members of other C++ structures? One
+  commonly overlooked requirement is that pybind11 objects have to increase their reference count
+  whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke
+  the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very
+  tricky to track for complicated programs Think carefully when you make a pybind11 object
+  a member in another struct.
+
+- C++ destructors that invoke Python functions can be particularly troublesome as
+  destructors can sometimes get invoked in weird and unexpected circumstances as a result
+  of exceptions.
+
+- You should try running your code in a debug build. That will enable additional assertions
+  within pybind11 that will throw exceptions on certain GIL handling errors
+  (reference counting operations).
+
 Binding sequence data types, iterators, the slicing protocol, etc.
 ==================================================================
 

docs/changelog.rst

@@ -15,6 +15,39 @@ IN DEVELOPMENT
 
 Changes will be summarized here periodically.
 
+Changes:
+
+* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` &
+  ``dec_ref()`` are now enabled by default again.
+  `#4246 <https://github.com/pybind/pybind11/pull/4246>`_
+
+Build system improvements:
+
+* Update clang-tidy to 15 in CI.
+  `#4387 <https://github.com/pybind/pybind11/pull/4387>`_
+
+
+Version 2.10.3 (Jan 3, 2023)
+----------------------------
+
+Changes:
+
+* Temporarily made our GIL status assertions (added in 2.10.2) disabled by
+  default (re-enable manually by defining
+  ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11).
+  `#4432 <https://github.com/pybind/pybind11/pull/4432>`_
+
+* Improved error messages when ``inc_ref``/``dec_ref`` are called with an
+  invalid GIL state.
+  `#4427 <https://github.com/pybind/pybind11/pull/4427>`_
+  `#4436 <https://github.com/pybind/pybind11/pull/4436>`_
+
+Bug Fixes:
+
+* Some minor touchups found by static analyzers.
+  `#4440 <https://github.com/pybind/pybind11/pull/4440>`_
+
+
 Version 2.10.2 (Dec 20, 2022)
 -----------------------------
 
@@ -30,7 +63,7 @@ Changes:
 * ``PyGILState_Check()``'s  were integrated to ``pybind11::handle``
   ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by
   ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if
-  ``NDEBUG`` is not defined.
+  ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11)
   `#4246 <https://github.com/pybind/pybind11/pull/4246>`_
 
 * Add option for enable/disable enum members in docstring.

include/pybind11/attr.h

@@ -399,7 +399,7 @@ struct process_attribute<doc> : process_attribute_default<doc> {
 template <>
 struct process_attribute<const char *> : process_attribute_default<const char *> {
     static void init(const char *d, function_record *r) { r->doc = const_cast<char *>(d); }
-    static void init(const char *d, type_record *r) { r->doc = const_cast<char *>(d); }
+    static void init(const char *d, type_record *r) { r->doc = d; }
 };
 template <>
 struct process_attribute<char *> : process_attribute<const char *> {};

include/pybind11/detail/common.h

@@ -11,11 +11,11 @@
 
 #define PYBIND11_VERSION_MAJOR 2
 #define PYBIND11_VERSION_MINOR 10
-#define PYBIND11_VERSION_PATCH 2
+#define PYBIND11_VERSION_PATCH 3
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x020A0200
+#define PYBIND11_VERSION_HEX 0x020A0300
 
 // Define some generic pybind11 helper macros for warning management.
 //
@@ -329,7 +329,8 @@ PYBIND11_WARNING_POP
          && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the       \
                                   process (see PR #4268) */                                       \
     && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
-#    define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
+// The following define will be enabled by default in the 2.11 release
+// define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
 #endif
 
 // #define PYBIND11_STR_LEGACY_PERMISSIVE

include/pybind11/eigen/tensor.h

@@ -176,7 +176,7 @@ struct type_caster<Type, typename eigen_tensor_helper<Type>::ValidType> {
                 return false;
             }
 
-            if (!convert && !temp.dtype().is(dtype::of<typename Type::Scalar>())) {
+            if (!temp.dtype().is(dtype::of<typename Type::Scalar>())) {
                 return false;
             }
         }

include/pybind11/pytypes.h

@@ -250,9 +250,9 @@ public:
 #ifdef PYBIND11_HANDLE_REF_DEBUG
         inc_ref_counter(1);
 #endif
-#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
+#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
         if (m_ptr != nullptr && !PyGILState_Check()) {
-            throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure.");
+            throw_gilstate_error("pybind11::handle::inc_ref()");
         }
 #endif
         Py_XINCREF(m_ptr);
@@ -265,9 +265,9 @@ public:
         this function automatically. Returns a reference to itself.
     \endrst */
     const handle &dec_ref() const & {
-#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
+#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
         if (m_ptr != nullptr && !PyGILState_Check()) {
-            throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure.");
+            throw_gilstate_error("pybind11::handle::dec_ref()");
         }
 #endif
         Py_XDECREF(m_ptr);
@@ -296,8 +296,28 @@ public:
 protected:
     PyObject *m_ptr = nullptr;
 
-#ifdef PYBIND11_HANDLE_REF_DEBUG
 private:
+#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
+    void throw_gilstate_error(const std::string &function_name) const {
+        fprintf(
+            stderr,
+            "%s is being called while the GIL is either not held or invalid. Please see "
+            "https://pybind11.readthedocs.io/en/stable/advanced/"
+            "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n",
+            function_name.c_str());
+        fflush(stderr);
+        if (Py_TYPE(m_ptr)->tp_name != nullptr) {
+            fprintf(stderr,
+                    "The failing %s call was triggered on a %s object.\n",
+                    function_name.c_str(),
+                    Py_TYPE(m_ptr)->tp_name);
+            fflush(stderr);
+        }
+        throw std::runtime_error(function_name + " PyGILState_Check() failure.");
+    }
+#endif
+
+#ifdef PYBIND11_HANDLE_REF_DEBUG
     static std::size_t inc_ref_counter(std::size_t add) {
         thread_local std::size_t counter = 0;
         counter += add;
@@ -443,7 +463,7 @@ PYBIND11_NAMESPACE_BEGIN(detail)
 
 // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class).
 inline const char *obj_class_name(PyObject *obj) {
-    if (Py_TYPE(obj) == &PyType_Type) {
+    if (PyType_Check(obj)) {
         return reinterpret_cast<PyTypeObject *>(obj)->tp_name;
     }
     return Py_TYPE(obj)->tp_name;
@@ -481,7 +501,7 @@ struct error_fetch_and_normalize {
                             "active exception.");
         }
         const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr());
-        if (exc_type_name_orig == nullptr) {
+        if (exc_type_name_norm == nullptr) {
             pybind11_fail("Internal error: " + std::string(called)
                           + " failed to obtain the name "
                             "of the normalized active exception type.");

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]:
         return s
 
 
-__version__ = "2.10.2"
+__version__ = "2.10.3"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

tests/test_class.cpp

@@ -55,6 +55,8 @@ void bind_empty0(py::module_ &m) {
 } // namespace test_class
 
 TEST_SUBMODULE(class_, m) {
+    m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); });
+
     // test_instance
     struct NoConstructor {
         NoConstructor() = default;

tests/test_class.py

@@ -1,10 +1,19 @@
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import ConstructorStats, UserType
 from pybind11_tests import class_ as m
 
 
+def test_obj_class_name():
+    if env.PYPY:
+        expected_name = "UserType"
+    else:
+        expected_name = "pybind11_tests.UserType"
+    assert m.obj_class_name(UserType(1)) == expected_name
+    assert m.obj_class_name(UserType) == expected_name
+
+
 def test_repr():
     assert "pybind11_type" in repr(type(UserType))
     assert "UserType" in repr(UserType)

tests/test_pytypes.cpp

@@ -99,6 +99,8 @@ void m_defs(py::module_ &m) {
 } // namespace handle_from_move_only_type_with_operator_PyObject
 
 TEST_SUBMODULE(pytypes, m) {
+    m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); });
+
     handle_from_move_only_type_with_operator_PyObject::m_defs(m);
 
     // test_bool

tests/test_pytypes.py

@@ -9,6 +9,12 @@ from pybind11_tests import detailed_error_messages_enabled
 from pybind11_tests import pytypes as m
 
 
+def test_obj_class_name():
+    assert m.obj_class_name(None) == "NoneType"
+    assert m.obj_class_name(list) == "list"
+    assert m.obj_class_name([]) == "list"
+
+
 def test_handle_from_move_only_type_with_operator_PyObject():  # noqa: N802
     assert m.handle_from_move_only_type_with_operator_PyObject_ncnst()
     assert m.handle_from_move_only_type_with_operator_PyObject_const()

tools/make_changelog.py

@@ -31,8 +31,10 @@ issues = (issue for page in issues_pages for issue in page)
 missing = []
 
 for issue in issues:
-    changelog = ENTRY.findall(issue.body)
-    if changelog:
+    changelog = ENTRY.findall(issue.body or "")
+    if not changelog or not changelog[0]:
+        missing.append(issue)
+    else:
         (msg,) = changelog
         if not msg.startswith("* "):
             msg = "* " + msg
@@ -44,9 +46,6 @@ for issue in issues:
         print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True))
         print()
 
-    else:
-        missing.append(issue)
-
 if missing:
     print()
     print("[blue]" + "-" * 30)