Merge branch 'master' into squash_merge_smart_holder_into_master_preview_1

CMakeLists.txt

@@ -165,7 +165,8 @@ set(PYBIND11_HEADERS
     include/pybind11/stl/filesystem.h
     include/pybind11/trampoline_self_life_support.h
     include/pybind11/type_caster_pyobject_ptr.h
-    include/pybind11/typing.h)
+    include/pybind11/typing.h
+    include/pybind11/warnings.h)
 
 # Compare with grep and warn if mismatched
 if(PYBIND11_MASTER_PROJECT)

docs/faq.rst

@@ -247,6 +247,50 @@ been received, you must either explicitly interrupt execution by throwing
         });
     }
 
+What is a highly conclusive and simple way to find memory leaks (e.g. in pybind11 bindings)?
+============================================================================================
+
+Use ``while True`` & ``top`` (Linux, macOS).
+
+For example, locally change tests/test_type_caster_pyobject_ptr.py like this:
+
+.. code-block:: diff
+
+     def test_return_list_pyobject_ptr_reference():
+    +  while True:
+         vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder)
+         assert [e.value for e in vec_obj] == [93, 186]
+         # Commenting out the next `assert` will leak the Python references.
+         # An easy way to see evidence of the leaks:
+         # Insert `while True:` as the first line of this function and monitor the
+         # process RES (Resident Memory Size) with the Unix top command.
+    -    assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+    +    # assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+
+Then run the test as you would normally do, which will go into the infinite loop.
+
+**In another shell, but on the same machine** run:
+
+.. code-block:: bash
+
+    top
+
+This will show:
+
+.. code-block::
+
+        PID USER      PR  NI    VIRT    RES    SHR S  %CPU  %MEM     TIME+ COMMAND
+    1266095 rwgk      20   0 5207496 611372  45696 R 100.0   0.3   0:08.01 test_type_caste
+
+Look for the number under ``RES`` there. You'll see it going up very quickly.
+
+**Don't forget to Ctrl-C the test command** before your machine becomes
+unresponsive due to swapping.
+
+This method only takes a couple minutes of effort and is very conclusive.
+What you want to see is that the ``RES`` number is stable after a couple
+seconds.
+
 CMake doesn't detect the right Python version
 =============================================
 

include/pybind11/warnings.h

@@ -0,0 +1,75 @@
+/*
+    pybind11/warnings.h: Python warnings wrappers.
+
+    Copyright (c) 2024 Jan Iwaszkiewicz <jiwaszkiewicz6@gmail.com>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "pybind11.h"
+#include "detail/common.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+inline bool PyWarning_Check(PyObject *obj) {
+    int result = PyObject_IsSubclass(obj, PyExc_Warning);
+    if (result == 1) {
+        return true;
+    }
+    if (result == -1) {
+        raise_from(PyExc_SystemError,
+                   "pybind11::detail::PyWarning_Check(): PyObject_IsSubclass() call failed.");
+        throw error_already_set();
+    }
+    return false;
+}
+
+PYBIND11_NAMESPACE_END(detail)
+
+PYBIND11_NAMESPACE_BEGIN(warnings)
+
+inline object
+new_warning_type(handle scope, const char *name, handle base = PyExc_RuntimeWarning) {
+    if (!detail::PyWarning_Check(base.ptr())) {
+        pybind11_fail("pybind11::warnings::new_warning_type(): cannot create custom warning, base "
+                      "must be a subclass of "
+                      "PyExc_Warning!");
+    }
+    if (hasattr(scope, name)) {
+        pybind11_fail("pybind11::warnings::new_warning_type(): an attribute with name \""
+                      + std::string(name) + "\" exists already.");
+    }
+    std::string full_name = scope.attr("__name__").cast<std::string>() + std::string(".") + name;
+    handle h(PyErr_NewException(full_name.c_str(), base.ptr(), nullptr));
+    if (!h) {
+        raise_from(PyExc_SystemError,
+                   "pybind11::warnings::new_warning_type(): PyErr_NewException() call failed.");
+        throw error_already_set();
+    }
+    auto obj = reinterpret_steal<object>(h);
+    scope.attr(name) = obj;
+    return obj;
+}
+
+// Similar to Python `warnings.warn()`
+inline void
+warn(const char *message, handle category = PyExc_RuntimeWarning, int stack_level = 2) {
+    if (!detail::PyWarning_Check(category.ptr())) {
+        pybind11_fail(
+            "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of "
+            "PyExc_Warning!");
+    }
+
+    if (PyErr_WarnEx(category.ptr(), message, stack_level) == -1) {
+        throw error_already_set();
+    }
+}
+
+PYBIND11_NAMESPACE_END(warnings)
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/CMakeLists.txt

@@ -171,7 +171,8 @@ set(PYBIND11_TEST_FILES
     test_unnamed_namespace_a
     test_unnamed_namespace_b
     test_vector_unique_ptr_member
-    test_virtual_functions)
+    test_virtual_functions
+    test_warnings)
 
 # Invoking cmake with something like:
 #     cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" ..

tests/extra_python_package/test_files.py

@@ -49,6 +49,7 @@ main_headers = {
     "include/pybind11/trampoline_self_life_support.h",
     "include/pybind11/type_caster_pyobject_ptr.h",
     "include/pybind11/typing.h",
+    "include/pybind11/warnings.h",
 }
 
 detail_headers = {

tests/test_warnings.cpp

@@ -0,0 +1,46 @@
+/*
+    tests/test_warnings.cpp -- usage of warnings::warn() and warnings categories.
+
+    Copyright (c) 2024 Jan Iwaszkiewicz <jiwaszkiewicz6@gmail.com>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include <pybind11/warnings.h>
+
+#include "pybind11_tests.h"
+
+#include <utility>
+
+TEST_SUBMODULE(warnings_, m) {
+
+    // Test warning mechanism base
+    m.def("warn_and_return_value", []() {
+        std::string message = "This is simple warning";
+        py::warnings::warn(message.c_str(), PyExc_Warning);
+        return 21;
+    });
+
+    m.def("warn_with_default_category", []() { py::warnings::warn("This is RuntimeWarning"); });
+
+    m.def("warn_with_different_category",
+          []() { py::warnings::warn("This is FutureWarning", PyExc_FutureWarning); });
+
+    m.def("warn_with_invalid_category",
+          []() { py::warnings::warn("Invalid category", PyExc_Exception); });
+
+    // Test custom warnings
+    PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> ex_storage;
+    ex_storage.call_once_and_store_result([&]() {
+        return py::warnings::new_warning_type(m, "CustomWarning", PyExc_DeprecationWarning);
+    });
+
+    m.def("warn_with_custom_type", []() {
+        py::warnings::warn("This is CustomWarning", ex_storage.get_stored());
+        return 37;
+    });
+
+    m.def("register_duplicate_warning",
+          [m]() { py::warnings::new_warning_type(m, "CustomWarning", PyExc_RuntimeWarning); });
+}

tests/test_warnings.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import warnings
+
+import pytest
+
+import pybind11_tests  # noqa: F401
+from pybind11_tests import warnings_ as m
+
+
+@pytest.mark.parametrize(
+    ("expected_category", "expected_message", "expected_value", "module_function"),
+    [
+        (Warning, "This is simple warning", 21, m.warn_and_return_value),
+        (RuntimeWarning, "This is RuntimeWarning", None, m.warn_with_default_category),
+        (FutureWarning, "This is FutureWarning", None, m.warn_with_different_category),
+    ],
+)
+def test_warning_simple(
+    expected_category, expected_message, expected_value, module_function
+):
+    with pytest.warns(Warning) as excinfo:
+        value = module_function()
+
+    assert issubclass(excinfo[0].category, expected_category)
+    assert str(excinfo[0].message) == expected_message
+    assert value == expected_value
+
+
+def test_warning_wrong_subclass_fail():
+    with pytest.raises(Exception) as excinfo:
+        m.warn_with_invalid_category()
+
+    assert issubclass(excinfo.type, RuntimeError)
+    assert (
+        str(excinfo.value)
+        == "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of PyExc_Warning!"
+    )
+
+
+def test_warning_double_register_fail():
+    with pytest.raises(Exception) as excinfo:
+        m.register_duplicate_warning()
+
+    assert issubclass(excinfo.type, RuntimeError)
+    assert (
+        str(excinfo.value)
+        == 'pybind11::warnings::new_warning_type(): an attribute with name "CustomWarning" exists already.'
+    )
+
+
+def test_warning_register():
+    assert m.CustomWarning is not None
+
+    with pytest.warns(m.CustomWarning) as excinfo:
+        warnings.warn("This is warning from Python!", m.CustomWarning, stacklevel=1)
+
+    assert issubclass(excinfo[0].category, DeprecationWarning)
+    assert str(excinfo[0].message) == "This is warning from Python!"
+
+
+def test_warning_custom():
+    with pytest.warns(m.CustomWarning) as excinfo:
+        value = m.warn_with_custom_type()
+
+    assert issubclass(excinfo[0].category, DeprecationWarning)
+    assert str(excinfo[0].message) == "This is CustomWarning"
+    assert value == 37