Merge branch 'master' into sh_merge_master

.github/workflows/emscripten.yaml

@@ -5,6 +5,8 @@ on:
   pull_request:
     branches:
     - master
+    - stable
+    - v*
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -23,8 +25,6 @@ jobs:
     - uses: pypa/cibuildwheel@v2.20
       env:
         PYODIDE_BUILD_EXPORTS: whole_archive
-        CFLAGS: -fexceptions
-        LDFLAGS: -fexceptions
       with:
         package-dir: tests
         only: cp312-pyodide_wasm32

.github/workflows/pip.yml

@@ -103,7 +103,7 @@ jobs:
     - uses: actions/download-artifact@v4
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@210c1913531870065f03ce1f9440dd87bc0938cd # v1.4.0
+      uses: actions/attest-build-provenance@310b0a4a3b0b78ef57ecda988ee04b132db73ef8 # v1.4.1
       with:
         subject-path: "*/pybind11*"
 

docs/changelog.rst

@@ -15,6 +15,77 @@ IN DEVELOPMENT
 
 Changes will be summarized here periodically.
 
+New Features:
+
+* Support for Python 3.7 was removed. (Official end-of-life: 2023-06-27).
+  `#5191 <https://github.com/pybind/pybind11/pull/5191>`_
+
+Support for CMake older than 3.15 and some older compilers will also be removed.
+
+Version 2.13.3 (August 13, 2024)
+--------------------------------
+
+Bug fixes:
+
+* Quote paths from pybind11-config
+  `#5302 <https://github.com/pybind/pybind11/pull/5302>`_
+
+
+* Fix typo in Emscripten support when in config mode (CMake)
+  `#5301 <https://github.com/pybind/pybind11/pull/5301>`_
+
+
+Version 2.13.2 (August 13, 2024)
+--------------------------------
+
+New Features:
+
+* A ``pybind11::detail::type_caster_std_function_specializations`` feature was added, to support specializations for
+  ``std::function``'s with return types that require custom to-Python conversion behavior (to primary use case is to catch and
+  convert exceptions).
+  `#4597 <https://github.com/pybind/pybind11/pull/4597>`_
+
+
+Changes:
+
+
+* Use ``PyMutex`` instead of ``std::mutex`` for internal locking in the free-threaded build.
+  `#5219 <https://github.com/pybind/pybind11/pull/5219>`_
+
+* Add a special type annotation for C++ empty tuple.
+  `#5214 <https://github.com/pybind/pybind11/pull/5214>`_
+
+* When compiling for WebAssembly, add the required exception flags (CMake 3.13+).
+  `#5298 <https://github.com/pybind/pybind11/pull/5298>`_
+
+Bug fixes:
+
+* Make ``gil_safe_call_once_and_store`` thread-safe in free-threaded CPython.
+  `#5246 <https://github.com/pybind/pybind11/pull/5246>`_
+
+* A missing ``#include <algorithm>`` in pybind11/typing.h was added to fix build errors (in case user code does not already depend
+  on that include).
+  `#5208 <https://github.com/pybind/pybind11/pull/5208>`_
+
+* Fix regression introduced in #5201 for GCC<10.3 in C++20 mode.
+  `#5205 <https://github.com/pybind/pybind11/pull/5205>`_
+
+
+.. fix(cmake)
+
+* Remove extra = when assigning flto value in the case for Clang in CMake.
+  `#5207 <https://github.com/pybind/pybind11/pull/5207>`_
+
+
+Tests:
+
+* Adding WASM testing to our CI (Pyodide / Emscripten via scikit-build-core).
+  `#4745 <https://github.com/pybind/pybind11/pull/4745>`_
+
+* clang-tidy (in GitHub Actions) was updated from clang 15 to clang 18.
+  `#5272 <https://github.com/pybind/pybind11/pull/5272>`_
+
+
 Version 2.13.1 (June 26, 2024)
 ------------------------------
 

include/pybind11/stl.h

@@ -13,6 +13,7 @@
 #include "detail/common.h"
 
 #include <deque>
+#include <initializer_list>
 #include <list>
 #include <map>
 #include <ostream>
@@ -35,6 +36,89 @@
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
+//
+// Begin: Equivalent of
+//        https://github.com/google/clif/blob/ae4eee1de07cdf115c0c9bf9fec9ff28efce6f6c/clif/python/runtime.cc#L388-L438
+/*
+The three `PyObjectTypeIsConvertibleTo*()` functions below are
+the result of converging the behaviors of pybind11 and PyCLIF
+(http://github.com/google/clif).
+
+Originally PyCLIF was extremely far on the permissive side of the spectrum,
+while pybind11 was very far on the strict side. Originally PyCLIF accepted any
+Python iterable as input for a C++ `vector`/`set`/`map` argument, as long as
+the elements were convertible. The obvious (in hindsight) problem was that
+any empty Python iterable could be passed to any of these C++ types, e.g. `{}`
+was accepted for C++ `vector`/`set` arguments, or `[]` for C++ `map` arguments.
+
+The functions below strike a practical permissive-vs-strict compromise,
+informed by tens of thousands of use cases in the wild. A main objective is
+to prevent accidents and improve readability:
+
+- Python literals must match the C++ types.
+
+- For C++ `set`: The potentially reducing conversion from a Python sequence
+  (e.g. Python `list` or `tuple`) to a C++ `set` must be explicit, by going
+  through a Python `set`.
+
+- However, a Python `set` can still be passed to a C++ `vector`. The rationale
+  is that this conversion is not reducing. Implicit conversions of this kind
+  are also fairly commonly used, therefore enforcing explicit conversions
+  would have an unfavorable cost : benefit ratio; more sloppily speaking,
+  such an enforcement would be more annoying than helpful.
+*/
+
+inline bool PyObjectIsInstanceWithOneOfTpNames(PyObject *obj,
+                                               std::initializer_list<const char *> tp_names) {
+    if (PyType_Check(obj)) {
+        return false;
+    }
+    const char *obj_tp_name = Py_TYPE(obj)->tp_name;
+    for (const auto *tp_name : tp_names) {
+        if (std::strcmp(obj_tp_name, tp_name) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+inline bool PyObjectTypeIsConvertibleToStdVector(PyObject *obj) {
+    if (PySequence_Check(obj) != 0) {
+        return !PyUnicode_Check(obj) && !PyBytes_Check(obj);
+    }
+    return (PyGen_Check(obj) != 0) || (PyAnySet_Check(obj) != 0)
+           || PyObjectIsInstanceWithOneOfTpNames(
+               obj, {"dict_keys", "dict_values", "dict_items", "map", "zip"});
+}
+
+inline bool PyObjectTypeIsConvertibleToStdSet(PyObject *obj) {
+    return (PyAnySet_Check(obj) != 0) || PyObjectIsInstanceWithOneOfTpNames(obj, {"dict_keys"});
+}
+
+inline bool PyObjectTypeIsConvertibleToStdMap(PyObject *obj) {
+    if (PyDict_Check(obj)) {
+        return true;
+    }
+    // Implicit requirement in the conditions below:
+    // A type with `.__getitem__()` & `.items()` methods must implement these
+    // to be compatible with https://docs.python.org/3/c-api/mapping.html
+    if (PyMapping_Check(obj) == 0) {
+        return false;
+    }
+    PyObject *items = PyObject_GetAttrString(obj, "items");
+    if (items == nullptr) {
+        PyErr_Clear();
+        return false;
+    }
+    bool is_convertible = (PyCallable_Check(items) != 0);
+    Py_DECREF(items);
+    return is_convertible;
+}
+
+//
+// End: Equivalent of clif/python/runtime.cc
+//
+
 /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for
 /// forwarding a container element).  Typically used indirect via forwarded_type(), below.
 template <typename T, typename U>
@@ -66,17 +150,10 @@ private:
     }
     void reserve_maybe(const anyset &, void *) {}
 
-public:
+    bool convert_iterable(const iterable &itbl, bool convert) {
-    bool load(handle src, bool convert) {
+        for (const auto &it : itbl) {
-        if (!isinstance<anyset>(src)) {
-            return false;
-        }
-        auto s = reinterpret_borrow<anyset>(src);
-        value.clear();
-        reserve_maybe(s, &value);
-        for (auto entry : s) {
             key_conv conv;
-            if (!conv.load(entry, convert)) {
+            if (!conv.load(it, convert)) {
                 return false;
             }
             value.insert(cast_op<Key &&>(std::move(conv)));
@@ -84,6 +161,29 @@ public:
         return true;
     }
 
+    bool convert_anyset(anyset s, bool convert) {
+        value.clear();
+        reserve_maybe(s, &value);
+        return convert_iterable(s, convert);
+    }
+
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdSet(src.ptr())) {
+            return false;
+        }
+        if (isinstance<anyset>(src)) {
+            value.clear();
+            return convert_anyset(reinterpret_borrow<anyset>(src), convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        assert(isinstance<iterable>(src));
+        value.clear();
+        return convert_iterable(reinterpret_borrow<iterable>(src), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!std::is_lvalue_reference<T>::value) {
@@ -115,15 +215,10 @@ private:
     }
     void reserve_maybe(const dict &, void *) {}
 
-public:
+    bool convert_elements(const dict &d, bool convert) {
-    bool load(handle src, bool convert) {
-        if (!isinstance<dict>(src)) {
-            return false;
-        }
-        auto d = reinterpret_borrow<dict>(src);
         value.clear();
         reserve_maybe(d, &value);
-        for (auto it : d) {
+        for (const auto &it : d) {
             key_conv kconv;
             value_conv vconv;
             if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) {
@@ -134,6 +229,25 @@ public:
         return true;
     }
 
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdMap(src.ptr())) {
+            return false;
+        }
+        if (isinstance<dict>(src)) {
+            return convert_elements(reinterpret_borrow<dict>(src), convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        auto items = reinterpret_steal<object>(PyMapping_Items(src.ptr()));
+        if (!items) {
+            throw error_already_set();
+        }
+        assert(isinstance<iterable>(items));
+        return convert_elements(dict(reinterpret_borrow<iterable>(items)), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         dict d;
@@ -166,20 +280,21 @@ struct list_caster {
     using value_conv = make_caster<Value>;
 
     bool load(handle src, bool convert) {
-        if (!isinstance<sequence>(src) || isinstance<bytes>(src) || isinstance<str>(src)) {
+        if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
             return false;
         }
-        auto s = reinterpret_borrow<sequence>(src);
+        if (isinstance<sequence>(src)) {
-        value.clear();
+            return convert_elements(src, convert);
-        reserve_maybe(s, &value);
-        for (const auto &it : s) {
-            value_conv conv;
-            if (!conv.load(it, convert)) {
-                return false;
-            }
-            value.push_back(cast_op<Value &&>(std::move(conv)));
         }
-        return true;
+        if (!convert) {
+            return false;
+        }
+        // Designed to be behavior-equivalent to passing tuple(src) from Python:
+        // The conversion to a tuple will first exhaust the generator object, to ensure that
+        // the generator is not left in an unpredictable (to the caller) partially-consumed
+        // state.
+        assert(isinstance<iterable>(src));
+        return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
     }
 
 private:
@@ -189,6 +304,20 @@ private:
     }
     void reserve_maybe(const sequence &, void *) {}
 
+    bool convert_elements(handle seq, bool convert) {
+        auto s = reinterpret_borrow<sequence>(seq);
+        value.clear();
+        reserve_maybe(s, &value);
+        for (const auto &it : seq) {
+            value_conv conv;
+            if (!conv.load(it, convert)) {
+                return false;
+            }
+            value.push_back(cast_op<Value &&>(std::move(conv)));
+        }
+        return true;
+    }
+
 public:
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
@@ -237,12 +366,8 @@ private:
         return size == Size;
     }
 
-public:
+    bool convert_elements(handle seq, bool convert) {
-    bool load(handle src, bool convert) {
+        auto l = reinterpret_borrow<sequence>(seq);
-        if (!isinstance<sequence>(src)) {
-            return false;
-        }
-        auto l = reinterpret_borrow<sequence>(src);
         if (!require_size(l.size())) {
             return false;
         }
@@ -257,6 +382,25 @@ public:
         return true;
     }
 
+public:
+    bool load(handle src, bool convert) {
+        if (!PyObjectTypeIsConvertibleToStdVector(src.ptr())) {
+            return false;
+        }
+        if (isinstance<sequence>(src)) {
+            return convert_elements(src, convert);
+        }
+        if (!convert) {
+            return false;
+        }
+        // Designed to be behavior-equivalent to passing tuple(src) from Python:
+        // The conversion to a tuple will first exhaust the generator object, to ensure that
+        // the generator is not left in an unpredictable (to the caller) partially-consumed
+        // state.
+        assert(isinstance<iterable>(src));
+        return convert_elements(tuple(reinterpret_borrow<iterable>(src)), convert);
+    }
+
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         list l(src.size());

pybind11/__main__.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import argparse
+import shlex
 import sys
 import sysconfig
 
@@ -22,7 +23,7 @@ def print_includes() -> None:
         if d and d not in unique_dirs:
             unique_dirs.append(d)
 
-    print(" ".join("-I" + d for d in unique_dirs))
+    print(" ".join(shlex.quote(f"-I{d}") for d in unique_dirs))
 
 
 def main() -> None:
@@ -54,9 +55,9 @@ def main() -> None:
     if args.includes:
         print_includes()
     if args.cmakedir:
-        print(get_cmake_dir())
+        print(shlex.quote(get_cmake_dir()))
     if args.pkgconfigdir:
-        print(get_pkgconfig_dir())
+        print(shlex.quote(get_pkgconfig_dir()))
 
 
 if __name__ == "__main__":

tests/pyproject.toml

@@ -10,6 +10,10 @@ name = "pybind11_tests"
 version = "0.0.1"
 dependencies = ["pytest", "pytest-timeout", "numpy", "scipy"]
 
+[tool.scikit-build]
+# Hide a warning while we also support CMake < 3.15
+cmake.version = ">=3.15"
+
 [tool.scikit-build.cmake.define]
 PYBIND11_FINDPYTHON = true
 

tests/test_stl.cpp

@@ -167,6 +167,14 @@ struct type_caster<ReferenceSensitiveOptional<T>>
 } // namespace detail
 } // namespace PYBIND11_NAMESPACE
 
+int pass_std_vector_int(const std::vector<int> &v) {
+    int zum = 100;
+    for (const int i : v) {
+        zum += 2 * i;
+    }
+    return zum;
+}
+
 TEST_SUBMODULE(stl, m) {
     // test_vector
     m.def("cast_vector", []() { return std::vector<int>{1}; });
@@ -546,4 +554,30 @@ TEST_SUBMODULE(stl, m) {
         []() { return new std::vector<bool>(4513); },
         // Without explicitly specifying `take_ownership`, this function leaks.
         py::return_value_policy::take_ownership);
+
+    m.def("pass_std_vector_int", pass_std_vector_int);
+    m.def("pass_std_vector_pair_int", [](const std::vector<std::pair<int, int>> &v) {
+        int zum = 0;
+        for (const auto &ij : v) {
+            zum += ij.first * 100 + ij.second;
+        }
+        return zum;
+    });
+    m.def("pass_std_array_int_2", [](const std::array<int, 2> &a) {
+        return pass_std_vector_int(std::vector<int>(a.begin(), a.end())) + 1;
+    });
+    m.def("pass_std_set_int", [](const std::set<int> &s) {
+        int zum = 200;
+        for (const int i : s) {
+            zum += 3 * i;
+        }
+        return zum;
+    });
+    m.def("pass_std_map_int", [](const std::map<int, int> &m) {
+        int zum = 500;
+        for (const auto &p : m) {
+            zum += p.first * 1000 + p.second;
+        }
+        return zum;
+    });
 }

tests/test_stl.py

@@ -381,3 +381,129 @@ def test_return_vector_bool_raw_ptr():
     v = m.return_vector_bool_raw_ptr()
     assert isinstance(v, list)
     assert len(v) == 4513
+
+
+@pytest.mark.parametrize(
+    ("fn", "offset"), [(m.pass_std_vector_int, 0), (m.pass_std_array_int_2, 1)]
+)
+def test_pass_std_vector_int(fn, offset):
+    assert fn([7, 13]) == 140 + offset
+    assert fn({6, 2}) == 116 + offset
+    assert fn({"x": 8, "y": 11}.values()) == 138 + offset
+    assert fn({3: None, 9: None}.keys()) == 124 + offset
+    assert fn(i for i in [4, 17]) == 142 + offset
+    assert fn(map(lambda i: i * 3, [8, 7])) == 190 + offset  # noqa: C417
+    with pytest.raises(TypeError):
+        fn({"x": 0, "y": 1})
+    with pytest.raises(TypeError):
+        fn({})
+
+
+def test_pass_std_vector_pair_int():
+    fn = m.pass_std_vector_pair_int
+    assert fn({1: 2, 3: 4}.items()) == 406
+    assert fn(zip([5, 17], [13, 9])) == 2222
+
+
+def test_list_caster_fully_consumes_generator_object():
+    def gen_invalid():
+        yield from [1, 2.0, 3]
+
+    gen_obj = gen_invalid()
+    with pytest.raises(TypeError):
+        m.pass_std_vector_int(gen_obj)
+    assert not tuple(gen_obj)
+
+
+def test_pass_std_set_int():
+    fn = m.pass_std_set_int
+    assert fn({3, 15}) == 254
+    assert fn({5: None, 12: None}.keys()) == 251
+    with pytest.raises(TypeError):
+        fn([])
+    with pytest.raises(TypeError):
+        fn({})
+    with pytest.raises(TypeError):
+        fn({}.values())
+    with pytest.raises(TypeError):
+        fn(i for i in [])
+
+
+def test_set_caster_dict_keys_failure():
+    dict_keys = {1: None, 2.0: None, 3: None}.keys()
+    # The asserts does not really exercise anything in pybind11, but if one of
+    # them fails in some future version of Python, the set_caster load
+    # implementation may need to be revisited.
+    assert tuple(dict_keys) == (1, 2.0, 3)
+    assert tuple(dict_keys) == (1, 2.0, 3)
+    with pytest.raises(TypeError):
+        m.pass_std_set_int(dict_keys)
+    assert tuple(dict_keys) == (1, 2.0, 3)
+
+
+class FakePyMappingMissingItems:
+    def __getitem__(self, _):
+        raise RuntimeError("Not expected to be called.")
+
+
+class FakePyMappingWithItems(FakePyMappingMissingItems):
+    def items(self):
+        return ((1, 3), (2, 4))
+
+
+class FakePyMappingBadItems(FakePyMappingMissingItems):
+    def items(self):
+        return ((1, 2), (3, "x"))
+
+
+class FakePyMappingItemsNotCallable(FakePyMappingMissingItems):
+    @property
+    def items(self):
+        return ((1, 2), (3, 4))
+
+
+class FakePyMappingItemsWithArg(FakePyMappingMissingItems):
+    def items(self, _):
+        return ((1, 2), (3, 4))
+
+
+class FakePyMappingGenObj(FakePyMappingMissingItems):
+    def __init__(self, gen_obj):
+        super().__init__()
+        self.gen_obj = gen_obj
+
+    def items(self):
+        yield from self.gen_obj
+
+
+def test_pass_std_map_int():
+    fn = m.pass_std_map_int
+    assert fn({1: 2, 3: 4}) == 4506
+    with pytest.raises(TypeError):
+        fn([])
+    assert fn(FakePyMappingWithItems()) == 3507
+    with pytest.raises(TypeError):
+        fn(FakePyMappingMissingItems())
+    with pytest.raises(TypeError):
+        fn(FakePyMappingBadItems())
+    with pytest.raises(TypeError):
+        fn(FakePyMappingItemsNotCallable())
+    with pytest.raises(TypeError):
+        fn(FakePyMappingItemsWithArg())
+
+
+@pytest.mark.parametrize(
+    ("items", "expected_exception"),
+    [
+        (((1, 2), (3, "x"), (4, 5)), TypeError),
+        (((1, 2), (3, 4, 5), (6, 7)), ValueError),
+    ],
+)
+def test_map_caster_fully_consumes_generator_object(items, expected_exception):
+    def gen_invalid():
+        yield from items
+
+    gen_obj = gen_invalid()
+    with pytest.raises(expected_exception):
+        m.pass_std_map_int(FakePyMappingGenObj(gen_obj))
+    assert not tuple(gen_obj)

tools/pybind11Common.cmake

@@ -2,7 +2,7 @@
 
 Adds the following targets::
 
-    pybind11::pybind11 - link to headers and pybind11
+    pybind11::pybind11 - link to Python headers and pybind11::headers
     pybind11::module - Adds module links
     pybind11::embed - Adds embed links
     pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set)
@@ -75,6 +75,32 @@ set_property(
   APPEND
   PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11)
 
+# -------------- emscripten requires exceptions enabled -------------
+# _pybind11_no_exceptions is a private mechanism to disable this addition.
+# Please open an issue if you need to use it; it will be removed if no one
+# needs it.
+if(CMAKE_SYSTEM_NAME MATCHES Emscripten AND NOT _pybind11_no_exceptions)
+  if(CMAKE_VERSION VERSION_LESS 3.13)
+    message(WARNING "CMake 3.13+ is required to build for Emscripten. Some flags will be missing")
+  else()
+    if(is_config)
+      set(_tmp_config_target pybind11::pybind11_headers)
+    else()
+      set(_tmp_config_target pybind11_headers)
+    endif()
+
+    set_property(
+      TARGET ${_tmp_config_target}
+      APPEND
+      PROPERTY INTERFACE_LINK_OPTIONS -fexceptions)
+    set_property(
+      TARGET ${_tmp_config_target}
+      APPEND
+      PROPERTY INTERFACE_COMPILE_OPTIONS -fexceptions)
+    unset(_tmp_config_target)
+  endif()
+endif()
+
 # --------------------------- link helper ---------------------------
 
 add_library(pybind11::python_link_helper IMPORTED INTERFACE ${optional_global})
@@ -329,7 +355,7 @@ function(_pybind11_generate_lto target prefer_thin_lto)
 
     if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64")
       # Do nothing
-    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES emscripten)
+    elseif(CMAKE_SYSTEM_NAME MATCHES Emscripten)
       # This compile is very costly when cross-compiling, so set this without checking
       set(PYBIND11_LTO_CXX_FLAGS "-flto${thin}${cxx_append}")
       set(PYBIND11_LTO_LINKER_FLAGS "-flto${thin}${linker_append}")

tools/pybind11Config.cmake.in

@@ -84,7 +84,7 @@ you can either use the basic targets, or use the FindPython tools:
 
   # Python method:
   Python_add_library(MyModule2 src2.cpp)
-  target_link_libraries(MyModule2 pybind11::headers)
+  target_link_libraries(MyModule2 PUBLIC pybind11::headers)
   set_target_properties(MyModule2 PROPERTIES
                                   INTERPROCEDURAL_OPTIMIZATION ON
                                   CXX_VISIBILITY_PRESET ON