Merge branch 'pybind:master' into master

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -6,7 +6,8 @@ body:
   - type: markdown
     attributes:
       value: |
-        Maintainers will only make a best effort to triage PRs. Please do your best to make the issue as easy to act on as possible, and only open if clearly a problem with pybind11 (ask first if unsure).
+        Please do your best to make the issue as easy to act on as possible, and only submit here if there is clearly a problem with pybind11 (ask first if unsure). **Note that a reproducer in a PR is much more likely to get immediate attention.**
+
   - type: checkboxes
     id: steps
     attributes:
@@ -20,6 +21,12 @@ body:
         - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new).
           required: false
 
+  - type: Input
+    id: version
+    attributes:
+      label: What version (or hash if on master) of pybind11 are you using?
+      required: true
+
   - type: textarea
     id: description
     attributes:
@@ -40,6 +47,14 @@ body:
         The code should be minimal, have no external dependencies, isolate the
         function(s) that cause breakage. Submit matched and complete C++ and
         Python snippets that can be easily compiled and run to diagnose the
-        issue. If possible, make a PR with a new, failing test to give us a
+        issue. — Note that a reproducer in a PR is much more likely to get
-        starting point to work on!
+        immediate attention: failing tests in the pybind11 CI are the best
+        starting point for working out fixes.
       render: text
+
+  - type: Input
+    id: regression
+    attributes:
+      label: Is this a regression? Put the last known working version here if it is.
+      description: Put the last known working version here if this is a regression.
+      value: Not a regression

docs/changelog.rst

@@ -103,9 +103,6 @@ Performance and style:
 
 Build system improvements:
 
-* Experimental support for ``cmake.modules`` entrypoint.
-  `#4258 <https://github.com/pybind/pybind11/pull/4258>`_
-
 * Include a pkg-config file when installing pybind11, such as in the Python
   package.
   `#4077 <https://github.com/pybind/pybind11/pull/4077>`_

include/pybind11/cast.h

@@ -248,7 +248,7 @@ public:
         return false;
     }
     static handle cast(T, return_value_policy /* policy */, handle /* parent */) {
-        return none().inc_ref();
+        return none().release();
     }
     PYBIND11_TYPE_CASTER(T, const_name("None"));
 };
@@ -291,7 +291,7 @@ public:
         if (ptr) {
             return capsule(ptr).release();
         }
-        return none().inc_ref();
+        return none().release();
     }
 
     template <typename T>
@@ -537,7 +537,7 @@ public:
 
     static handle cast(const CharT *src, return_value_policy policy, handle parent) {
         if (src == nullptr) {
-            return pybind11::none().inc_ref();
+            return pybind11::none().release();
         }
         return StringCaster::cast(StringType(src), policy, parent);
     }
@@ -1179,11 +1179,9 @@ enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&)
     pybind11_fail("Internal error: cast_safe fallback invoked");
 }
 template <typename T>
-enable_if_t<std::is_same<void, intrinsic_t<T>>::value, void> cast_safe(object &&) {}
+enable_if_t<std::is_void<T>::value, void> cast_safe(object &&) {}
 template <typename T>
-enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>,
+enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>, std::is_void<T>>::value, T>
-                            std::is_same<void, intrinsic_t<T>>>::value,
-            T>
 cast_safe(object &&o) {
     return pybind11::cast<T>(std::move(o));
 }

include/pybind11/detail/common.h

@@ -205,10 +205,6 @@
 #    endif
 #endif
 
-#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
-#    define PYBIND11_HAS_U8STRING
-#endif
-
 #include <Python.h>
 #if PY_VERSION_HEX < 0x03060000
 #    error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5."
@@ -259,6 +255,11 @@
 #    endif
 #endif
 
+// Must be after including <version> or one of the other headers specified by the standard
+#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
+#    define PYBIND11_HAS_U8STRING
+#endif
+
 // #define PYBIND11_STR_LEGACY_PERMISSIVE
 // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject
 //             (probably surprising and never documented, but this was the

include/pybind11/functional.h

@@ -110,7 +110,7 @@ public:
     template <typename Func>
     static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) {
         if (!f_) {
-            return none().inc_ref();
+            return none().release();
         }
 
         auto result = f_.template target<function_type>();

include/pybind11/pybind11.h

@@ -2333,7 +2333,7 @@ template <typename Access,
           typename Sentinel,
           typename ValueType,
           typename... Extra>
-iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) {
     using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>;
     // TODO: state captures only the types of Extra, not the values
 
@@ -2359,7 +2359,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra)
                 Policy);
     }
 
-    return cast(state{std::forward<Iterator>(first), std::forward<Sentinel>(last), true});
+    return cast(state{first, last, true});
 }
 
 PYBIND11_NAMESPACE_END(detail)
@@ -2370,15 +2370,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_access<Iterator>::result_type,
           typename... Extra>
-iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       ValueType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a
@@ -2388,15 +2386,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename KeyType = typename detail::iterator_key_access<Iterator>::result_type,
           typename... Extra>
-iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_key_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       KeyType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a
@@ -2406,15 +2402,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_value_access<Iterator>::result_type,
           typename... Extra>
-iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_value_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       ValueType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes an iterator over values of an stl container or other container supporting

include/pybind11/stl.h

@@ -311,7 +311,7 @@ struct optional_caster {
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!src) {
-            return none().inc_ref();
+            return none().release();
         }
         if (!std::is_lvalue_reference<T>::value) {
             policy = return_value_policy_override<Value>::policy(policy);

setup.py

@@ -144,8 +144,6 @@ with remove_output("pybind11/include", "pybind11/share"):
             stdout=sys.stdout,
             stderr=sys.stderr,
         )
-        if not global_sdist:
-            Path("pybind11/share/cmake/pybind11/__init__.py").touch()
 
     txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd)
     code = compile(txt, setup_py, "exec")

tests/extra_python_package/test_files.py

@@ -166,7 +166,6 @@ def test_build_sdist(monkeypatch, tmpdir):
     files |= {f"pybind11{n}" for n in local_sdist_files}
     files.add("pybind11.egg-info/entry_points.txt")
     files.add("pybind11.egg-info/requires.txt")
-    files.add("pybind11/share/cmake/pybind11/__init__.py")
     assert simpler == files
 
     with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f:
@@ -253,7 +252,6 @@ def tests_build_wheel(monkeypatch, tmpdir):
         "dist-info/entry_points.txt",
         "dist-info/top_level.txt",
     }
-    files.add("pybind11/share/cmake/pybind11/__init__.py")
 
     with zipfile.ZipFile(str(wheel)) as z:
         names = z.namelist()

tests/test_class.cpp

@@ -384,6 +384,8 @@ TEST_SUBMODULE(class_, m) {
 
     protected:
         virtual int foo() const { return value; }
+        virtual void *void_foo() { return static_cast<void *>(&value); }
+        virtual void *get_self() { return static_cast<void *>(this); }
 
     private:
         int value = 42;
@@ -392,6 +394,8 @@ TEST_SUBMODULE(class_, m) {
     class TrampolineB : public ProtectedB {
     public:
         int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); }
+        void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); }
+        void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); }
     };
 
     class PublicistB : public ProtectedB {
@@ -401,11 +405,23 @@ TEST_SUBMODULE(class_, m) {
         // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827)
         ~PublicistB() override{}; // NOLINT(modernize-use-equals-default)
         using ProtectedB::foo;
+        using ProtectedB::get_self;
+        using ProtectedB::void_foo;
     };
 
+    m.def("read_foo", [](const void *original) {
+        const int *ptr = reinterpret_cast<const int *>(original);
+        return *ptr;
+    });
+
+    m.def("pointers_equal",
+          [](const void *original, const void *comparison) { return original == comparison; });
+
     py::class_<ProtectedB, TrampolineB>(m, "ProtectedB")
         .def(py::init<>())
-        .def("foo", &PublicistB::foo);
+        .def("foo", &PublicistB::foo)
+        .def("void_foo", &PublicistB::void_foo)
+        .def("get_self", &PublicistB::get_self);
 
     // test_brace_initialization
     struct BraceInitialization {

tests/test_class.py

@@ -313,6 +313,8 @@ def test_bind_protected_functions():
 
     b = m.ProtectedB()
     assert b.foo() == 42
+    assert m.read_foo(b.void_foo()) == 42
+    assert m.pointers_equal(b.get_self(), b)
 
     class C(m.ProtectedB):
         def __init__(self):

tests/test_sequences_and_iterators.cpp

@@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
           []() { return py::make_iterator<py::return_value_policy::copy>(list); });
     m.def("make_iterator_2",
           []() { return py::make_iterator<py::return_value_policy::automatic>(list); });
+
+    // test_iterator on c arrays
+    // #4100: ensure lvalue required as increment operand
+    class CArrayHolder {
+    public:
+        CArrayHolder(double x, double y, double z) {
+            values[0] = x;
+            values[1] = y;
+            values[2] = z;
+        };
+        double values[3];
+    };
+
+    py::class_<CArrayHolder>(m, "CArrayHolder")
+        .def(py::init<double, double, double>())
+        .def(
+            "__iter__",
+            [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); },
+            py::keep_alive<0, 1>());
 }

tests/test_sequences_and_iterators.py

@@ -241,3 +241,11 @@ def test_iterator_rvp():
     assert list(m.make_iterator_1()) == [1, 2, 3]
     assert list(m.make_iterator_2()) == [1, 2, 3]
     assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2()))
+
+
+def test_carray_iterator():
+    """#4100: Check for proper iterator overload with C-Arrays"""
+    args_gt = list(float(i) for i in range(3))
+    arr_h = m.CArrayHolder(*args_gt)
+    args = list(arr_h)
+    assert args_gt == args

tools/FindPythonLibsNew.cmake

@@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0)
   return()
 endif()
 
+option(
+  PYBIND11_PYTHONLIBS_OVERWRITE
+  "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values."
+  ON)
 # Can manually set values when cross-compiling
 macro(_PYBIND11_GET_IF_UNDEF lst index name)
-  if(NOT DEFINED "${name}")
+  if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}")
     list(GET "${lst}" "${index}" "${name}")
   endif()
 endmacro()

tools/setup_main.py.in

@@ -38,9 +38,6 @@ setup(
         ],
         "pipx.run": [
              "pybind11 = pybind11.__main__:main",
-        ],
-        "cmake.modules": [
-            "pybind11 = pybind11.share.cmake.pybind11",
         ]
     },
     cmdclass=cmdclass