Merge branch 'pybind:master' into master

docs/advanced/misc.rst

@@ -324,6 +324,15 @@ The class ``options`` allows you to selectively suppress auto-generated signatur
         m.def("add", [](int a, int b) { return a + b; }, "A function which adds two numbers");
     }
 
+pybind11 also appends all members of an enum to the resulting enum docstring.
+This default behavior can be disabled by using the ``disable_enum_members_docstring()``
+function of the ``options`` class.
+
+With ``disable_user_defined_docstrings()`` all user defined docstrings of
+``module_::def()``, ``class_::def()`` and ``enum_()`` are disabled, but the
+function signatures and enum members are included in the docstring, unless they
+are disabled separately.
+
 Note that changes to the settings affect only function bindings created during the
 lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function,
 the default settings are restored to prevent unwanted side effects.

include/pybind11/detail/common.h

@@ -323,6 +323,15 @@ PYBIND11_WARNING_POP
 #    define PYBIND11_HAS_U8STRING
 #endif
 
+// See description of PR #4246:
+#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF)                                \
+    && !(defined(PYPY_VERSION)                                                                    \
+         && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the       \
+                                  process (see PR #4268) */                                       \
+    && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
+#    define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF
+#endif
+
 // #define PYBIND11_STR_LEGACY_PERMISSIVE
 // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject
 //             (probably surprising and never documented, but this was the

include/pybind11/detail/type_caster_base.h

@@ -1006,5 +1006,14 @@ protected:
     static Constructor make_move_constructor(...) { return nullptr; }
 };
 
+PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) {
+    if (auto *type_data = get_type_info(ti)) {
+        handle th((PyObject *) type_data->type);
+        return th.attr("__module__").cast<std::string>() + '.'
+               + th.attr("__qualname__").cast<std::string>();
+    }
+    return clean_type_id(ti.name());
+}
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/options.h

@@ -47,6 +47,16 @@ public:
         return *this;
     }
 
+    options &disable_enum_members_docstring() & {
+        global_state().show_enum_members_docstring = false;
+        return *this;
+    }
+
+    options &enable_enum_members_docstring() & {
+        global_state().show_enum_members_docstring = true;
+        return *this;
+    }
+
     // Getter methods (return the global state):
 
     static bool show_user_defined_docstrings() {
@@ -55,6 +65,10 @@ public:
 
     static bool show_function_signatures() { return global_state().show_function_signatures; }
 
+    static bool show_enum_members_docstring() {
+        return global_state().show_enum_members_docstring;
+    }
+
     // This type is not meant to be allocated on the heap.
     void *operator new(size_t) = delete;
 
@@ -63,6 +77,8 @@ private:
         bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings.
         bool show_function_signatures = true;     //< Include auto-generated function signatures
                                                   //  in docstrings.
+        bool show_enum_members_docstring = true;  //< Include auto-generated member list in enum
+                                                  //  docstrings.
     };
 
     static state &global_state() {

include/pybind11/pybind11.h

@@ -1972,29 +1972,35 @@ struct enum_base {
             name("name"),
             is_method(m_base));
 
-        m_base.attr("__doc__") = static_property(
+        if (options::show_enum_members_docstring()) {
-            cpp_function(
+            m_base.attr("__doc__") = static_property(
-                [](handle arg) -> std::string {
+                cpp_function(
-                    std::string docstring;
+                    [](handle arg) -> std::string {
-                    dict entries = arg.attr("__entries");
+                        std::string docstring;
-                    if (((PyTypeObject *) arg.ptr())->tp_doc) {
+                        dict entries = arg.attr("__entries");
-                        docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n";
+                        if (((PyTypeObject *) arg.ptr())->tp_doc) {
-                    }
+                            docstring += std::string(
-                    docstring += "Members:";
+                                reinterpret_cast<PyTypeObject *>(arg.ptr())->tp_doc);
-                    for (auto kv : entries) {
+                            docstring += "\n\n";
-                        auto key = std::string(pybind11::str(kv.first));
-                        auto comment = kv.second[int_(1)];
-                        docstring += "\n\n  " + key;
-                        if (!comment.is_none()) {
-                            docstring += " : " + (std::string) pybind11::str(comment);
                         }
-                    }
+                        docstring += "Members:";
-                    return docstring;
+                        for (auto kv : entries) {
-                },
+                            auto key = std::string(pybind11::str(kv.first));
-                name("__doc__")),
+                            auto comment = kv.second[int_(1)];
-            none(),
+                            docstring += "\n\n  ";
-            none(),
+                            docstring += key;
-            "");
+                            if (!comment.is_none()) {
+                                docstring += " : ";
+                                docstring += pybind11::str(comment).cast<std::string>();
+                            }
+                        }
+                        return docstring;
+                    },
+                    name("__doc__")),
+                none(),
+                none(),
+                "");
+        }
 
         m_base.attr("__members__") = static_property(cpp_function(
                                                          [](handle arg) -> dict {

include/pybind11/pytypes.h

@@ -249,6 +249,11 @@ public:
     const handle &inc_ref() const & {
 #ifdef PYBIND11_HANDLE_REF_DEBUG
         inc_ref_counter(1);
+#endif
+#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
+        if (m_ptr != nullptr && !PyGILState_Check()) {
+            throw std::runtime_error("pybind11::handle::inc_ref() PyGILState_Check() failure.");
+        }
 #endif
         Py_XINCREF(m_ptr);
         return *this;
@@ -260,6 +265,11 @@ public:
         this function automatically. Returns a reference to itself.
     \endrst */
     const handle &dec_ref() const & {
+#if defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
+        if (m_ptr != nullptr && !PyGILState_Check()) {
+            throw std::runtime_error("pybind11::handle::dec_ref() PyGILState_Check() failure.");
+        }
+#endif
         Py_XDECREF(m_ptr);
         return *this;
     }

include/pybind11/stl_bind.h

@@ -10,10 +10,13 @@
 #pragma once
 
 #include "detail/common.h"
+#include "detail/type_caster_base.h"
+#include "cast.h"
 #include "operators.h"
 
 #include <algorithm>
 #include <sstream>
+#include <type_traits>
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
@@ -636,18 +639,52 @@ auto map_if_insertion_operator(Class_ &cl, std::string const &name)
         "Return the canonical string representation of this map.");
 }
 
-template <typename Map>
+template <typename KeyType>
 struct keys_view {
-    Map &map;
+    virtual size_t len() = 0;
+    virtual iterator iter() = 0;
+    virtual bool contains(const KeyType &k) = 0;
+    virtual bool contains(const object &k) = 0;
+    virtual ~keys_view() = default;
 };
 
-template <typename Map>
+template <typename MappedType>
 struct values_view {
+    virtual size_t len() = 0;
+    virtual iterator iter() = 0;
+    virtual ~values_view() = default;
+};
+
+template <typename KeyType, typename MappedType>
+struct items_view {
+    virtual size_t len() = 0;
+    virtual iterator iter() = 0;
+    virtual ~items_view() = default;
+};
+
+template <typename Map, typename KeysView>
+struct KeysViewImpl : public KeysView {
+    explicit KeysViewImpl(Map &map) : map(map) {}
+    size_t len() override { return map.size(); }
+    iterator iter() override { return make_key_iterator(map.begin(), map.end()); }
+    bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); }
+    bool contains(const object &) override { return false; }
     Map &map;
 };
 
-template <typename Map>
+template <typename Map, typename ValuesView>
-struct items_view {
+struct ValuesViewImpl : public ValuesView {
+    explicit ValuesViewImpl(Map &map) : map(map) {}
+    size_t len() override { return map.size(); }
+    iterator iter() override { return make_value_iterator(map.begin(), map.end()); }
+    Map &map;
+};
+
+template <typename Map, typename ItemsView>
+struct ItemsViewImpl : public ItemsView {
+    explicit ItemsViewImpl(Map &map) : map(map) {}
+    size_t len() override { return map.size(); }
+    iterator iter() override { return make_iterator(map.begin(), map.end()); }
     Map &map;
 };
 
@@ -657,9 +694,11 @@ template <typename Map, typename holder_type = std::unique_ptr<Map>, typename...
 class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&...args) {
     using KeyType = typename Map::key_type;
     using MappedType = typename Map::mapped_type;
-    using KeysView = detail::keys_view<Map>;
+    using StrippedKeyType = detail::remove_cvref_t<KeyType>;
-    using ValuesView = detail::values_view<Map>;
+    using StrippedMappedType = detail::remove_cvref_t<MappedType>;
-    using ItemsView = detail::items_view<Map>;
+    using KeysView = detail::keys_view<StrippedKeyType>;
+    using ValuesView = detail::values_view<StrippedMappedType>;
+    using ItemsView = detail::items_view<StrippedKeyType, StrippedMappedType>;
     using Class_ = class_<Map, holder_type>;
 
     // If either type is a non-module-local bound type then make the map binding non-local as well;
@@ -673,12 +712,57 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
     }
 
     Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward<Args>(args)...);
-    class_<KeysView> keys_view(
+    static constexpr auto key_type_descr = detail::make_caster<KeyType>::name;
-        scope, ("KeysView[" + name + "]").c_str(), pybind11::module_local(local));
+    static constexpr auto mapped_type_descr = detail::make_caster<MappedType>::name;
-    class_<ValuesView> values_view(
+    std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text);
-        scope, ("ValuesView[" + name + "]").c_str(), pybind11::module_local(local));
+
-    class_<ItemsView> items_view(
+    // If key type isn't properly wrapped, fall back to C++ names
-        scope, ("ItemsView[" + name + "]").c_str(), pybind11::module_local(local));
+    if (key_type_name == "%") {
+        key_type_name = detail::type_info_description(typeid(KeyType));
+    }
+    // Similarly for value type:
+    if (mapped_type_name == "%") {
+        mapped_type_name = detail::type_info_description(typeid(MappedType));
+    }
+
+    // Wrap KeysView[KeyType] if it wasn't already wrapped
+    if (!detail::get_type_info(typeid(KeysView))) {
+        class_<KeysView> keys_view(
+            scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local));
+        keys_view.def("__len__", &KeysView::len);
+        keys_view.def("__iter__",
+                      &KeysView::iter,
+                      keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
+        );
+        keys_view.def("__contains__",
+                      static_cast<bool (KeysView::*)(const KeyType &)>(&KeysView::contains));
+        // Fallback for when the object is not of the key type
+        keys_view.def("__contains__",
+                      static_cast<bool (KeysView::*)(const object &)>(&KeysView::contains));
+    }
+    // Similarly for ValuesView:
+    if (!detail::get_type_info(typeid(ValuesView))) {
+        class_<ValuesView> values_view(scope,
+                                       ("ValuesView[" + mapped_type_name + "]").c_str(),
+                                       pybind11::module_local(local));
+        values_view.def("__len__", &ValuesView::len);
+        values_view.def("__iter__",
+                        &ValuesView::iter,
+                        keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
+        );
+    }
+    // Similarly for ItemsView:
+    if (!detail::get_type_info(typeid(ItemsView))) {
+        class_<ItemsView> items_view(
+            scope,
+            ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(),
+            pybind11::module_local(local));
+        items_view.def("__len__", &ItemsView::len);
+        items_view.def("__iter__",
+                       &ItemsView::iter,
+                       keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
+        );
+    }
 
     cl.def(init<>());
 
@@ -698,19 +782,25 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
 
     cl.def(
         "keys",
-        [](Map &m) { return KeysView{m}; },
+        [](Map &m) {
+            return std::unique_ptr<KeysView>(new detail::KeysViewImpl<Map, KeysView>(m));
+        },
         keep_alive<0, 1>() /* Essential: keep map alive while view exists */
     );
 
     cl.def(
         "values",
-        [](Map &m) { return ValuesView{m}; },
+        [](Map &m) {
+            return std::unique_ptr<ValuesView>(new detail::ValuesViewImpl<Map, ValuesView>(m));
+        },
         keep_alive<0, 1>() /* Essential: keep map alive while view exists */
     );
 
     cl.def(
         "items",
-        [](Map &m) { return ItemsView{m}; },
+        [](Map &m) {
+            return std::unique_ptr<ItemsView>(new detail::ItemsViewImpl<Map, ItemsView>(m));
+        },
         keep_alive<0, 1>() /* Essential: keep map alive while view exists */
     );
 
@@ -749,36 +839,6 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
 
     cl.def("__len__", &Map::size);
 
-    keys_view.def("__len__", [](KeysView &view) { return view.map.size(); });
-    keys_view.def(
-        "__iter__",
-        [](KeysView &view) { return make_key_iterator(view.map.begin(), view.map.end()); },
-        keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
-    );
-    keys_view.def("__contains__", [](KeysView &view, const KeyType &k) -> bool {
-        auto it = view.map.find(k);
-        if (it == view.map.end()) {
-            return false;
-        }
-        return true;
-    });
-    // Fallback for when the object is not of the key type
-    keys_view.def("__contains__", [](KeysView &, const object &) -> bool { return false; });
-
-    values_view.def("__len__", [](ValuesView &view) { return view.map.size(); });
-    values_view.def(
-        "__iter__",
-        [](ValuesView &view) { return make_value_iterator(view.map.begin(), view.map.end()); },
-        keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
-    );
-
-    items_view.def("__len__", [](ItemsView &view) { return view.map.size(); });
-    items_view.def(
-        "__iter__",
-        [](ItemsView &view) { return make_iterator(view.map.begin(), view.map.end()); },
-        keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */
-    );
-
     return cl;
 }
 

setup.cfg

@@ -46,5 +46,5 @@ zip_safe = False
 max-line-length = 120
 show_source = True
 exclude = .git, __pycache__, build, dist, docs, tools, venv
-extend-ignore = E203, E722, B903, B950
+extend-ignore = E203, E722, B903, B905, B950
 extend-select = B9

tests/test_docstring_options.cpp

@@ -85,4 +85,57 @@ TEST_SUBMODULE(docstring_options, m) {
                           &DocstringTestFoo::setValue,
                           "This is a property docstring");
     }
+
+    {
+        enum class DocstringTestEnum1 { Member1, Member2 };
+
+        py::enum_<DocstringTestEnum1>(m, "DocstringTestEnum1", "Enum docstring")
+            .value("Member1", DocstringTestEnum1::Member1)
+            .value("Member2", DocstringTestEnum1::Member2);
+    }
+
+    {
+        py::options options;
+        options.enable_enum_members_docstring();
+
+        enum class DocstringTestEnum2 { Member1, Member2 };
+
+        py::enum_<DocstringTestEnum2>(m, "DocstringTestEnum2", "Enum docstring")
+            .value("Member1", DocstringTestEnum2::Member1)
+            .value("Member2", DocstringTestEnum2::Member2);
+    }
+
+    {
+        py::options options;
+        options.disable_enum_members_docstring();
+
+        enum class DocstringTestEnum3 { Member1, Member2 };
+
+        py::enum_<DocstringTestEnum3>(m, "DocstringTestEnum3", "Enum docstring")
+            .value("Member1", DocstringTestEnum3::Member1)
+            .value("Member2", DocstringTestEnum3::Member2);
+    }
+
+    {
+        py::options options;
+        options.disable_user_defined_docstrings();
+
+        enum class DocstringTestEnum4 { Member1, Member2 };
+
+        py::enum_<DocstringTestEnum4>(m, "DocstringTestEnum4", "Enum docstring")
+            .value("Member1", DocstringTestEnum4::Member1)
+            .value("Member2", DocstringTestEnum4::Member2);
+    }
+
+    {
+        py::options options;
+        options.disable_user_defined_docstrings();
+        options.disable_enum_members_docstring();
+
+        enum class DocstringTestEnum5 { Member1, Member2 };
+
+        py::enum_<DocstringTestEnum5>(m, "DocstringTestEnum5", "Enum docstring")
+            .value("Member1", DocstringTestEnum5::Member1)
+            .value("Member2", DocstringTestEnum5::Member2);
+    }
 }

tests/test_docstring_options.py

@@ -39,3 +39,26 @@ def test_docstring_options():
     # Suppression of user-defined docstrings for non-function objects
     assert not m.DocstringTestFoo.__doc__
     assert not m.DocstringTestFoo.value_prop.__doc__
+
+    # Check existig behaviour of enum docstings
+    assert (
+        m.DocstringTestEnum1.__doc__
+        == "Enum docstring\n\nMembers:\n\n  Member1\n\n  Member2"
+    )
+
+    # options.enable_enum_members_docstring()
+    assert (
+        m.DocstringTestEnum2.__doc__
+        == "Enum docstring\n\nMembers:\n\n  Member1\n\n  Member2"
+    )
+
+    # options.disable_enum_members_docstring()
+    assert m.DocstringTestEnum3.__doc__ == "Enum docstring"
+
+    # options.disable_user_defined_docstrings()
+    assert m.DocstringTestEnum4.__doc__ == "Members:\n\n  Member1\n\n  Member2"
+
+    # options.disable_user_defined_docstrings()
+    # options.disable_enum_members_docstring()
+    # When all options are disabled, no docstring (instead of an empty one) should be generated
+    assert m.DocstringTestEnum5.__doc__ is None

tests/test_stl_binders.py

@@ -309,3 +309,29 @@ def test_map_delitem():
     del um["ua"]
     assert sorted(list(um)) == ["ub"]
     assert sorted(list(um.items())) == [("ub", 2.6)]
+
+
+def test_map_view_types():
+    map_string_double = m.MapStringDouble()
+    unordered_map_string_double = m.UnorderedMapStringDouble()
+    map_string_double_const = m.MapStringDoubleConst()
+    unordered_map_string_double_const = m.UnorderedMapStringDoubleConst()
+
+    assert map_string_double.keys().__class__.__name__ == "KeysView[str]"
+    assert map_string_double.values().__class__.__name__ == "ValuesView[float]"
+    assert map_string_double.items().__class__.__name__ == "ItemsView[str, float]"
+
+    keys_type = type(map_string_double.keys())
+    assert type(unordered_map_string_double.keys()) is keys_type
+    assert type(map_string_double_const.keys()) is keys_type
+    assert type(unordered_map_string_double_const.keys()) is keys_type
+
+    values_type = type(map_string_double.values())
+    assert type(unordered_map_string_double.values()) is values_type
+    assert type(map_string_double_const.values()) is values_type
+    assert type(unordered_map_string_double_const.values()) is values_type
+
+    items_type = type(map_string_double.items())
+    assert type(unordered_map_string_double.items()) is items_type
+    assert type(map_string_double_const.items()) is items_type
+    assert type(unordered_map_string_double_const.items()) is items_type