From 8d82f298871c8222a922d0c8992a200012750659 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Jul 2022 21:40:33 -0500 Subject: [PATCH 01/71] chore: back to work Signed-off-by: Henry Schreiner --- docs/changelog.rst | 7 +++++++ include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b926b27af..9c8ff423f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,13 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. + + +IN DEVELOPMENT +-------------- + +Changes will be summarized here periodically. + Version 2.10.0 (Jul 15, 2022) ----------------------------- diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1da323f31..05370108f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 0 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0000 +#define PYBIND11_VERSION_HEX 0x020B00D1 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/_version.py b/pybind11/_version.py index b78d5963d..1cb51fc5c 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.0" +__version__ = "2.11.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) From 5116a629e9819aeb6d142e7d94c1feb365adae98 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Jul 2022 21:03:51 -0500 Subject: [PATCH 02/71] fix(spelling): PYTHON_VERSIONS --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index fe36d70fe..2639cdd3a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,7 @@ import nox nox.needs_version = ">=2022.1.7" nox.options.sessions = ["lint", "tests", "tests_packaging"] -PYTHON_VERISONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"] +PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"] if os.environ.get("CI", None): nox.options.error_on_missing_interpreters = True @@ -20,7 +20,7 @@ def lint(session: nox.Session) -> None: session.run("pre-commit", "run", "-a") -@nox.session(python=PYTHON_VERISONS) +@nox.session(python=PYTHON_VERSIONS) def tests(session: nox.Session) -> None: """ Run the tests (requires a compiler). From 59f03ee389c283cde65bd800c8f32ea690daf3fd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Jul 2022 21:43:27 -0500 Subject: [PATCH 03/71] tests: include pypy3.9 in nox if present Signed-off-by: Henry Schreiner --- noxfile.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 2639cdd3a..7be1f2544 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,7 +5,17 @@ import nox nox.needs_version = ">=2022.1.7" nox.options.sessions = ["lint", "tests", "tests_packaging"] -PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"] +PYTHON_VERSIONS = [ + "3.6", + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "pypy3.7", + "pypy3.8", + "pypy3.9", +] if os.environ.get("CI", None): nox.options.error_on_missing_interpreters = True From ef7d971e036e4675ca266b2c990f0fd65c99e360 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 19:39:10 -0500 Subject: [PATCH 04/71] [pre-commit.ci] pre-commit autoupdate (#4082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v2.0.1 → v2.0.4](https://github.com/hadialqattan/pycln/compare/v2.0.1...v2.0.4) - [github.com/PyCQA/pylint: v2.14.4 → v2.14.5](https://github.com/PyCQA/pylint/compare/v2.14.4...v2.14.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba9955a31..cc0df85f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.0.1" + rev: "v2.0.4" hooks: - id: pycln stages: [manual] @@ -107,7 +107,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.14.4" + rev: "v2.14.5" hooks: - id: pylint files: ^pybind11 From f47f1edfe8a4842e2a723f024aae19d70ed426aa Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 20 Jul 2022 11:42:24 -0400 Subject: [PATCH 05/71] Fix #3812 and fix const of inplace assignments (#4065) * Fix #3812 and fix const of inplace assignments * Fix missing tests * Revert operator overloading changes * calculate answer first for tests * Simplify tests * Add more tests * Add a couple more tests * Add test_inplace_lshift, test_inplace_rshift for completeness. * Update tests * Shortcircuit on self assigment and address reviewer comment * broaden skip for self assignment * One more reviewer comment * Document opt behavior and make consistent * Revert unnecessary change * Clarify comment Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/pytypes.h | 74 +++++++++++++++++++++++++------------- tests/test_pytypes.cpp | 34 ++++++++++++++++++ tests/test_pytypes.py | 72 +++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 24 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 339b0961e..ad5edfed9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -155,23 +155,23 @@ public: object operator-() const; object operator~() const; object operator+(object_api const &other) const; - object operator+=(object_api const &other) const; + object operator+=(object_api const &other); object operator-(object_api const &other) const; - object operator-=(object_api const &other) const; + object operator-=(object_api const &other); object operator*(object_api const &other) const; - object operator*=(object_api const &other) const; + object operator*=(object_api const &other); object operator/(object_api const &other) const; - object operator/=(object_api const &other) const; + object operator/=(object_api const &other); object operator|(object_api const &other) const; - object operator|=(object_api const &other) const; + object operator|=(object_api const &other); object operator&(object_api const &other) const; - object operator&=(object_api const &other) const; + object operator&=(object_api const &other); object operator^(object_api const &other) const; - object operator^=(object_api const &other) const; + object operator^=(object_api const &other); object operator<<(object_api const &other) const; - object operator<<=(object_api const &other) const; + object operator<<=(object_api const &other); object operator>>(object_api const &other) const; - object operator>>=(object_api const &other) const; + object operator>>=(object_api const &other); PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; @@ -334,12 +334,15 @@ public: } object &operator=(const object &other) { - other.inc_ref(); - // Use temporary variable to ensure `*this` remains valid while - // `Py_XDECREF` executes, in case `*this` is accessible from Python. - handle temp(m_ptr); - m_ptr = other.m_ptr; - temp.dec_ref(); + // Skip inc_ref and dec_ref if both objects are the same + if (!this->is(other)) { + other.inc_ref(); + // Use temporary variable to ensure `*this` remains valid while + // `Py_XDECREF` executes, in case `*this` is accessible from Python. + handle temp(m_ptr); + m_ptr = other.m_ptr; + temp.dec_ref(); + } return *this; } @@ -353,6 +356,20 @@ public: return *this; } +#define PYBIND11_INPLACE_OP(iop) \ + object iop(object_api const &other) { return operator=(handle::iop(other)); } + + PYBIND11_INPLACE_OP(operator+=) + PYBIND11_INPLACE_OP(operator-=) + PYBIND11_INPLACE_OP(operator*=) + PYBIND11_INPLACE_OP(operator/=) + PYBIND11_INPLACE_OP(operator|=) + PYBIND11_INPLACE_OP(operator&=) + PYBIND11_INPLACE_OP(operator^=) + PYBIND11_INPLACE_OP(operator<<=) + PYBIND11_INPLACE_OP(operator>>=) +#undef PYBIND11_INPLACE_OP + // Calling cast() on an object lvalue just copies (via handle::cast) template T cast() const &; @@ -2364,26 +2381,35 @@ bool object_api::rich_compare(object_api const &other, int value) const { return result; \ } +#define PYBIND11_MATH_OPERATOR_BINARY_INPLACE(iop, fn) \ + template \ + object object_api::iop(object_api const &other) { \ + object result = reinterpret_steal(fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + PYBIND11_MATH_OPERATOR_UNARY(operator~, PyNumber_Invert) PYBIND11_MATH_OPERATOR_UNARY(operator-, PyNumber_Negative) PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) -PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator+=, PyNumber_InPlaceAdd) PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) -PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator-=, PyNumber_InPlaceSubtract) PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) -PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator*=, PyNumber_InPlaceMultiply) PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) -PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator/=, PyNumber_InPlaceTrueDivide) PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) -PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator|=, PyNumber_InPlaceOr) PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) -PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator&=, PyNumber_InPlaceAnd) PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) -PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator^=, PyNumber_InPlaceXor) PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) -PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator<<=, PyNumber_InPlaceLshift) PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) -PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_UNARY #undef PYBIND11_MATH_OPERATOR_BINARY diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index f532e2608..81387fd9f 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -756,4 +756,38 @@ TEST_SUBMODULE(pytypes, m) { } return o; }); + + // testing immutable object augmented assignment: #issue 3812 + m.def("inplace_append", [](py::object &a, const py::object &b) { + a += b; + return a; + }); + m.def("inplace_subtract", [](py::object &a, const py::object &b) { + a -= b; + return a; + }); + m.def("inplace_multiply", [](py::object &a, const py::object &b) { + a *= b; + return a; + }); + m.def("inplace_divide", [](py::object &a, const py::object &b) { + a /= b; + return a; + }); + m.def("inplace_or", [](py::object &a, const py::object &b) { + a |= b; + return a; + }); + m.def("inplace_and", [](py::object &a, const py::object &b) { + a &= b; + return a; + }); + m.def("inplace_lshift", [](py::object &a, const py::object &b) { + a <<= b; + return a; + }); + m.def("inplace_rshift", [](py::object &a, const py::object &b) { + a >>= b; + return a; + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 7a0a8b4ab..bde831738 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -739,3 +739,75 @@ def test_populate_obj_str_attrs(): new_attrs = {k: v for k, v in new_o.__dict__.items() if not k.startswith("_")} assert all(isinstance(v, str) for v in new_attrs.values()) assert len(new_attrs) == pop + + +@pytest.mark.parametrize( + "a,b", [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))] +) +def test_inplace_append(a, b): + expected = a + b + assert m.inplace_append(a, b) == expected + + +@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))]) +def test_inplace_subtract(a, b): + expected = a - b + assert m.inplace_subtract(a, b) == expected + + +@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), ([1], 3)]) +def test_inplace_multiply(a, b): + expected = a * b + assert m.inplace_multiply(a, b) == expected + + +@pytest.mark.parametrize("a,b", [(6, 3), (6.0, 3.0)]) +def test_inplace_divide(a, b): + expected = a / b + assert m.inplace_divide(a, b) == expected + + +@pytest.mark.parametrize( + "a,b", + [ + (False, True), + ( + set(), + { + 1, + }, + ), + ], +) +def test_inplace_or(a, b): + expected = a | b + assert m.inplace_or(a, b) == expected + + +@pytest.mark.parametrize( + "a,b", + [ + (True, False), + ( + {1, 2, 3}, + { + 1, + }, + ), + ], +) +def test_inplace_and(a, b): + expected = a & b + assert m.inplace_and(a, b) == expected + + +@pytest.mark.parametrize("a,b", [(8, 1), (-3, 2)]) +def test_inplace_lshift(a, b): + expected = a << b + assert m.inplace_lshift(a, b) == expected + + +@pytest.mark.parametrize("a,b", [(8, 1), (-2, 2)]) +def test_inplace_rshift(a, b): + expected = a >> b + assert m.inplace_rshift(a, b) == expected From 42b54507eaa23b265e8de144aeacb2455a837486 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 20 Jul 2022 12:02:20 -0400 Subject: [PATCH 06/71] chore: use explicit defaulting in pyobject macros (#4017) * Use equals default in pyobject macros * Remove extra semicolon * Update clang-tidy equals-default rule to not ignore macros * Fix formatting * One last formatting change --- .clang-tidy | 2 ++ include/pybind11/numpy.h | 2 +- include/pybind11/pytypes.h | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index d945a4a27..23018386c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -63,6 +63,8 @@ Checks: | -bugprone-unused-raii, CheckOptions: +- key: modernize-use-equals-default.IgnoreMacros + value: false - key: performance-for-range-copy.WarnOnAllAutoCopies value: true - key: performance-inefficient-string-concatenation.StrictMode diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0291b02d0..4f82f1ec3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -537,7 +537,7 @@ PYBIND11_NAMESPACE_END(detail) class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_) explicit dtype(const buffer_info &info) { dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index ad5edfed9..7a731e22a 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1283,7 +1283,7 @@ public: #define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun) \ PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ - Name() : Parent() {} + Name() = default; #define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ ::pybind11::type_error("Object of type '" \ @@ -1306,7 +1306,7 @@ public: #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ PYBIND11_OBJECT(Name, Parent, CheckFun) \ - Name() : Parent() {} + Name() = default; /// \addtogroup pytypes /// @{ From cb35a3c1432a5e0441aab4a62593fed82664abed Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 21 Jul 2022 06:38:00 -0700 Subject: [PATCH 07/71] For PyPy only, re-enable old behavior (runs the risk of masking bugs) (#4079) * For PyPy only, re-enable old behavior (likely to mask bugs), to avoid segfault with unknown root cause. Change prompted by https://github.com/pybind/pybind11/issues/4075 * Undo the change in tests/test_exceptions.py I turns out (I forgot) that PyPy segfaults in `test_flaky_exception_failure_point_init` already before the `MISMATCH` code path is reached: https://github.com/pybind/pybind11/runs/7383663596 ``` RPython traceback: test_exceptions.py .......X.........Error in cpyext, CPython compatibility layer: File "pypy_module_cpyext.c", line 14052, in wrapper_second_level__star_3_1 File "pypy_module_cpyext_1.c", line 35750, in not_supposed_to_fail Fatal Python error: Segmentation fault Stack (most recent call first, approximate line numbers): File "/home/runner/work/pybind11/pybind11/tests/test_exceptions.py", line 306 in test_flaky_exception_failure_point_init The function PyErr_NormalizeException was not supposed to fail File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/python.py", line 185 in pytest_pyfunc_call File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_callers.py", line 9 in _multicall File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_manager.py", line 77 in _hookexec File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_hooks.py", line 244 in __call__ File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/python.py", line 1716 in runtest File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 159 in pytest_runtest_call File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_callers.py", line 9 in _multicall File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_manager.py", line 77 in _hookexec Fatal error in cpyext, CPython compatibility layer, calling PyErr_NormalizeException Either report a bug or consider not using this particular extension File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_hooks.py", line 244 in __call__ File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 261 in File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 317 in from_call File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 246 in call_runtest_hook File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 218 in call_and_report File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 118 in runtestprotocol File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/runner.py", line 110 in pytest_runtest_protocol File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_callers.py", line 9 in _multicall File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_manager.py", line 77 in _hookexec File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_hooks.py", line 244 in __call__ File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/main.py", line 335 in pytest_runtestloop File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_callers.py", line 9 in _multicall File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_manager.py", line 77 in _hookexec File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_hooks.py", line 244 in __call__ File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/main.py", line 318 in _main File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/main.py", line 255 in wrap_session File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/main.py", line 314 in pytest_cmdline_main File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_callers.py", line 9 in _multicall File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_manager.py", line 77 in _hookexec File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pluggy/_hooks.py", line 244 in __call__ File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/config/__init__.py", line 133 in main File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/_pytest/config/__init__.py", line 181 in console_main File "/opt/hostedtoolcache/PyPy/3.7.13/x64/site-packages/pytest/__main__.py", line 1 in File "/opt/hostedtoolcache/PyPy/3.7.13/x64/lib-python/3/runpy.py", line 62 in _run_code File "/opt/hostedtoolcache/PyPy/3.7.13/x64/lib-python/3/runpy.py", line 170 in _run_module_as_main File "/app_main.py", line 109 in run_toplevel File "/app_main.py", line 652 in run_command_line File "/app_main.py", line 996 in entry_point Segmentation fault (core dumped) ``` * Add test_pypy_oserror_normalization * Disable new `PYPY_VERSION` `#if`, to verify that the new test actually fails. * Restore PYPY_VERSION workaround and update comment to reflect what was learned. * [ci skip] Fix trivial oversight in comment. --- include/pybind11/pytypes.h | 7 +++++++ tests/test_exceptions.cpp | 10 ++++++++++ tests/test_exceptions.py | 6 ++++++ 3 files changed, 23 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 7a731e22a..a837fb9fa 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -473,6 +473,12 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } +#if defined(PYPY_VERSION) + // This behavior runs the risk of masking errors in the error handling, but avoids a + // conflict with PyPy, which relies on the normalization here to change OSError to + // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). + m_lazy_error_string = exc_type_name_norm; +#else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -484,6 +490,7 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +#endif } error_fetch_and_normalize(const error_fetch_and_normalize &) = delete; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3ec999d1d..3583f22a5 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -334,4 +334,14 @@ TEST_SUBMODULE(exceptions, m) { e.restore(); } }); + + // https://github.com/pybind/pybind11/issues/4075 + m.def("test_pypy_oserror_normalization", []() { + try { + py::module_::import("io").attr("open")("this_filename_must_not_exist", "r"); + } catch (const py::error_already_set &e) { + return py::str(e.what()); // str must be built before e goes out of scope. + } + return py::str("UNEXPECTED"); + }); } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index a5984a142..5e3beeedb 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -360,3 +360,9 @@ def test_error_already_set_double_restore(): "Internal error: pybind11::detail::error_fetch_and_normalize::restore()" " called a second time. ORIGINAL ERROR: ValueError: Random error." ) + + +def test_pypy_oserror_normalization(): + # https://github.com/pybind/pybind11/issues/4075 + what = m.test_pypy_oserror_normalization() + assert "this_filename_must_not_exist" in what From 9a2963734de309a6d3f4ffb3e43bb56713a7ccae Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 21 Jul 2022 06:40:34 -0700 Subject: [PATCH 08/71] More systematic gcc & clang coverage (#4083) * More systematic gcc coverage, based on https://github.com/pybind/pybind11/pull/4074#issuecomment-1188385580 * Fix complete fail. * Resolve GCC 11 & 12 "redundant move in return statement" warnings. * Also add clang 11, 12, 13 (to gather info for warning suppressions). * Add & use `PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING` --- .github/workflows/ci.yml | 20 +++++++++++++------- include/pybind11/detail/common.h | 18 ++++++++++++++++++ include/pybind11/numpy.h | 11 +++++++++-- tests/test_kwargs_and_defaults.cpp | 11 ++++++++++- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 412282a4f..5a24ad6ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,6 +281,12 @@ jobs: std: 20 - clang: 10 std: 17 + - clang: 11 + std: 20 + - clang: 12 + std: 20 + - clang: 13 + std: 20 - clang: 14 std: 20 @@ -437,14 +443,14 @@ jobs: strategy: fail-fast: false matrix: - gcc: - - 7 - - latest - std: - - 11 include: - - gcc: 10 - std: 20 + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" container: "gcc:${{ matrix.gcc }}" diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 05370108f..9e6947daa 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1158,6 +1158,24 @@ constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } # define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ #endif +#if defined(__clang__) \ + && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ + available. */ \ + || (__clang_major__ >= 7 \ + && __clang_major__ <= 12) /* Clang 3, 5, 13, 14, 15 do not generate the warning. */ \ + ) +# define PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +// Example: +// tests/test_kwargs_and_defaults.cpp:46:68: error: local variable 'args' will be copied despite +// being returned by name [-Werror,-Wreturn-std-move] +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// test_kwargs_and_defaults.cpp:46:68: note: call 'std::move' explicitly to avoid copying +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// std::move(args) +#endif + // Pybind offers detailed error messages by default for all builts that are debug (through the // negation of ndebug). This can also be manually enabled by users, for any builds, through // defining PYBIND11_DETAILED_ERROR_MESSAGES. diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 4f82f1ec3..56e5edc5a 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1865,9 +1865,13 @@ private: } auto result = returned_array::create(trivial, shape); +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wreturn-std-move" +#endif if (size == 0) { - return std::move(result); + return result; } /* Call the function */ @@ -1878,7 +1882,10 @@ private: apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq); } - return std::move(result); + return result; +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +# pragma clang diagnostic pop +#endif } template diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 7418afefb..2f3cabaf0 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -43,7 +43,16 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a = 0); // test_args_and_kwargs - m.def("args_function", [](py::args args) -> py::tuple { return std::move(args); }); + m.def("args_function", [](py::args args) -> py::tuple { +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wreturn-std-move" +#endif + return args; +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +# pragma clang diagnostic pop +#endif + }); m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(args, kwargs); }); From b07975f492c2eed0409a18353fa23c9969e83e42 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 22 Jul 2022 11:52:01 -0400 Subject: [PATCH 09/71] Fix missing undef in pytypes (#4087) --- include/pybind11/pytypes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index a837fb9fa..6ba1f5f20 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2420,6 +2420,7 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_UNARY #undef PYBIND11_MATH_OPERATOR_BINARY +#undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) From d70f54b073d9961a4b9eebd151e721a1172ecada Mon Sep 17 00:00:00 2001 From: Hyunwook Choi Date: Thu, 28 Jul 2022 02:41:57 +0900 Subject: [PATCH 10/71] docs: Missing semicolons (#4094) Add semicolons to Memory view sample code --- docs/advanced/pycpp/numpy.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index b6ef019ed..07c969305 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -433,7 +433,7 @@ following: { 2, 4 }, // shape (rows, cols) { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes ); - }) + }); This approach is meant for providing a ``memoryview`` for a C/C++ buffer not managed by Python. The user is responsible for managing the lifetime of the @@ -449,7 +449,7 @@ We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: buffer, // buffer pointer sizeof(uint8_t) * 8 // buffer size ); - }) + }); .. versionchanged:: 2.6 ``memoryview::from_memory`` added. From 1e3400b6742288429f2069aaf5febf92d0662dae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 21:37:47 -0400 Subject: [PATCH 11/71] chore(deps): bump pypa/gh-action-pypi-publish from 1.5.0 to 1.5.1 (#4091) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 2c16735e1..f03a39701 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@v1.5.1 with: password: ${{ secrets.pypi_password }} packages_dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@v1.5.1 with: password: ${{ secrets.pypi_password_global }} packages_dir: global/ From 3665530264bbfeb9ec7623635d7cc644a70116b3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 1 Aug 2022 06:18:48 -0700 Subject: [PATCH 12/71] Add `-DPYBIND11_WERROR=ON` to mingw cmake commands (#4073) * Add `-DPYBIND11_WERROR=ON` to mingw cmake commands (and `-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON`). * Using no-destructor idiom to side-step overzealous MINGW warning. * Add __MINGW32__ pragma GCC diagnostic ignored in eigen.h * Add another no-destructor workaround. * Temporarily add -k (keep-going) flags to hopefully speed up finding all warnings. * Revert "Temporarily add -k (keep-going) flags to hopefully speed up finding all warnings." This reverts commit f36b0af8f93e9f6f16c969fb5b646e116f5eaf0f. * Very minor shuffle to avoid MSVC warnings. * Remove all `:BOOL` as suggested by @henryiii --- .github/workflows/ci.yml | 6 +++--- include/pybind11/eigen.h | 5 +++++ tests/test_builtin_casters.cpp | 9 +++++++-- tests/test_stl.cpp | 10 ++++++++-- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a24ad6ab..ac6c536ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -911,7 +911,7 @@ jobs: - name: Configure C++11 # LTO leads to many undefined reference like # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DDOWNLOAD_CATCH=ON -S . -B build + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build - name: Build C++11 run: cmake --build build -j 2 @@ -929,7 +929,7 @@ jobs: run: git clean -fdx - name: Configure C++14 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DDOWNLOAD_CATCH=ON -S . -B build2 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 - name: Build C++14 run: cmake --build build2 -j 2 @@ -947,7 +947,7 @@ jobs: run: git clean -fdx - name: Configure C++17 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DDOWNLOAD_CATCH=ON -S . -B build3 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 - name: Build C++17 run: cmake --build build3 -j 2 diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index 4d221b3d7..831625229 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -27,6 +27,9 @@ # pragma warning(disable : 4127) // C4127: conditional expression is constant # pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types +#elif defined(__MINGW32__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #include @@ -34,6 +37,8 @@ #if defined(_MSC_VER) # pragma warning(pop) +#elif defined(__MINGW32__) +# pragma GCC diagnostic pop #endif // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index 6529f47d0..6ff63edfc 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -266,9 +266,14 @@ TEST_SUBMODULE(builtin_casters, m) { }); m.def("lvalue_nested", []() -> const decltype(lvnested) & { return lvnested; }); - static std::pair int_string_pair{2, "items"}; m.def( - "int_string_pair", []() { return &int_string_pair; }, py::return_value_policy::reference); + "int_string_pair", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *int_string_pair = new std::pair{2, "items"}; + return int_string_pair; + }, + py::return_value_policy::reference); // test_builtins_cast_return_none m.def("return_none_string", []() -> std::string * { return nullptr; }); diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 38d32fda9..dcc92c3f0 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -176,9 +176,14 @@ TEST_SUBMODULE(stl, m) { m.def("load_bool_vector", [](const std::vector &v) { return v.at(0) == true && v.at(1) == false; }); // Unnumbered regression (caused by #936): pointers to stl containers aren't castable - static std::vector lvv{2}; m.def( - "cast_ptr_vector", []() { return &lvv; }, py::return_value_policy::reference); + "cast_ptr_vector", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *v = new std::vector{2}; + return v; + }, + py::return_value_policy::reference); // test_deque m.def("cast_deque", []() { return std::deque{1}; }); @@ -237,6 +242,7 @@ TEST_SUBMODULE(stl, m) { lvn["b"].emplace_back(); // add a list lvn["b"].back().emplace_back(); // add an array lvn["b"].back().emplace_back(); // add another array + static std::vector lvv{2}; m.def("cast_lv_vector", []() -> const decltype(lvv) & { return lvv; }); m.def("cast_lv_array", []() -> const decltype(lva) & { return lva; }); m.def("cast_lv_map", []() -> const decltype(lvm) & { return lvm; }); From aa953710c180fb9da5178c47b4dadcad945119f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 12:18:42 -0400 Subject: [PATCH 13/71] [pre-commit.ci] pre-commit autoupdate (#4090) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.37.1 → v2.37.2](https://github.com/asottile/pyupgrade/compare/v2.37.1...v2.37.2) - [github.com/pre-commit/mirrors-mypy: v0.961 → v0.971](https://github.com/pre-commit/mirrors-mypy/compare/v0.961...v0.971) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc0df85f3..c2c098c5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.37.1" + rev: "v2.37.2" hooks: - id: pyupgrade args: [--py36-plus] @@ -123,7 +123,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.961" + rev: "v0.971" hooks: - id: mypy args: [] From f8e8403b858d11f3d2102e66c06178ab204fd3d6 Mon Sep 17 00:00:00 2001 From: Thomas Eding Date: Mon, 1 Aug 2022 11:31:31 -0700 Subject: [PATCH 14/71] Open pybind11 namespace with consistent visility. (#4098) --- docs/advanced/cast/custom.rst | 4 ++-- docs/advanced/cast/stl.rst | 6 +++--- docs/advanced/classes.rst | 4 ++-- docs/advanced/smart_ptrs.rst | 2 +- tests/test_custom_type_casters.cpp | 14 +++++++------- tests/test_smart_ptr.cpp | 4 ++-- tests/test_stl.cpp | 12 ++++++------ tests/test_tagbased_polymorphic.cpp | 4 ++-- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 1df4d3e14..8138cac61 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -38,7 +38,7 @@ type is explicitly allowed. .. code-block:: cpp - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { public: /** @@ -78,7 +78,7 @@ type is explicitly allowed. return PyLong_FromLong(src.long_value); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail .. note:: diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 109763f7a..03d49b295 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -42,7 +42,7 @@ types: .. code-block:: cpp // `boost::optional` as an example -- can be any `std::optional`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; }} @@ -54,7 +54,7 @@ for custom variant types: .. code-block:: cpp // `boost::variant` as an example -- can be any `std::variant`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : variant_caster> {}; @@ -66,7 +66,7 @@ for custom variant types: return boost::apply_visitor(args...); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail The ``visit_helper`` specialization is not required if your ``name::variant`` provides a ``name::visit()`` function. For any other function name, the specialization must be diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 49ddf5c0b..01a490b72 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -1228,7 +1228,7 @@ whether a downcast is safe, you can proceed by specializing the std::string bark() const { return sound; } }; - namespace pybind11 { + namespace PYBIND11_NAMESPACE { template<> struct polymorphic_type_hook { static const void *get(const Pet *src, const std::type_info*& type) { // note that src may be nullptr @@ -1239,7 +1239,7 @@ whether a downcast is safe, you can proceed by specializing the return src; } }; - } // namespace pybind11 + } // namespace PYBIND11_NAMESPACE When pybind11 wants to convert a C++ pointer of type ``Base*`` to a Python object, it calls ``polymorphic_type_hook::get()`` to diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 5a2220109..3c40ce123 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -157,7 +157,7 @@ specialized: PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr); // Only needed if the type's `.get()` goes by another name - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct holder_helper> { // <-- specialization static const T *get(const SmartPtr &p) { return p.getPointer(); } diff --git a/tests/test_custom_type_casters.cpp b/tests/test_custom_type_casters.cpp index 25540e368..b4af02a45 100644 --- a/tests/test_custom_type_casters.cpp +++ b/tests/test_custom_type_casters.cpp @@ -21,7 +21,7 @@ public: }; class ArgAlwaysConverts {}; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { @@ -74,7 +74,7 @@ public: } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // test_custom_caster_destruction class DestructionTester { @@ -92,7 +92,7 @@ public: return *this; } }; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { @@ -104,7 +104,7 @@ struct type_caster { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // Define type caster outside of `pybind11::detail` and then alias it. namespace other_lib { @@ -112,7 +112,7 @@ struct MyType {}; // Corrupt `py` shorthand alias for surrounding context. namespace py {} // Corrupt unqualified relative `pybind11` namespace. -namespace pybind11 {} +namespace PYBIND11_NAMESPACE {} // Correct alias. namespace py_ = ::pybind11; // Define caster. This is effectively no-op, we only ensure it compiles and we @@ -127,12 +127,12 @@ struct my_caster { }; } // namespace other_lib // Effectively "alias" it into correct namespace (via inheritance). -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster : public other_lib::my_caster {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(custom_type_casters, m) { // test_custom_type_casters diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index 2c23a9a19..6d9efcedc 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -266,14 +266,14 @@ struct ElementList { // It is always possible to construct a ref from an Object* pointer without // possible inconsistencies, hence the 'true' argument at the end. // Make pybind11 aware of the non-standard getter member function -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct holder_helper> { static const T *get(const ref &p) { return p.get_ptr(); } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // Make pybind aware of the ref-counted wrapper type (s): PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true); diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index dcc92c3f0..d45465d68 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -23,7 +23,7 @@ #if defined(PYBIND11_TEST_BOOST) # include -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; @@ -31,7 +31,7 @@ struct type_caster> : optional_caster> {}; template <> struct type_caster : void_caster {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE #endif // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 @@ -43,7 +43,7 @@ using std::variant; # define PYBIND11_TEST_VARIANT 1 using boost::variant; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : variant_caster> {}; @@ -56,7 +56,7 @@ struct visit_helper { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE #endif PYBIND11_MAKE_OPAQUE(std::vector>); @@ -159,13 +159,13 @@ private: std::vector storage; }; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(stl, m) { // test_vector diff --git a/tests/test_tagbased_polymorphic.cpp b/tests/test_tagbased_polymorphic.cpp index 2807e86ba..12ba6532f 100644 --- a/tests/test_tagbased_polymorphic.cpp +++ b/tests/test_tagbased_polymorphic.cpp @@ -117,7 +117,7 @@ std::string Animal::name_of_kind(Kind kind) { return raw_name; } -namespace pybind11 { +namespace PYBIND11_NAMESPACE { template struct polymorphic_type_hook::value>> { static const void *get(const itype *src, const std::type_info *&type) { @@ -125,7 +125,7 @@ struct polymorphic_type_hook(m, "Animal").def_readonly("name", &Animal::name); From 88a1bb926085dc2298cd7bbfc330f232362c793d Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 1 Aug 2022 15:48:44 -0400 Subject: [PATCH 15/71] chore: remove unnecessary temporary std::pair (#4103) --- include/pybind11/pybind11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index d61dcd5c7..c889dc416 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2073,7 +2073,7 @@ struct enum_base { + "\" already exists!"); } - entries[name] = std::make_pair(value, doc); + entries[name] = pybind11::make_tuple(value, doc); m_base.attr(std::move(name)) = std::move(value); } From ba5ccd845a2261e538df651e3d528dc1bece094d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Aug 2022 11:38:07 -0400 Subject: [PATCH 16/71] [pre-commit.ci] pre-commit autoupdate (#4104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/asottile/pyupgrade: v2.37.2 → v2.37.3](https://github.com/asottile/pyupgrade/compare/v2.37.2...v2.37.3) - [github.com/hadialqattan/pycln: v2.0.4 → v2.1.1](https://github.com/hadialqattan/pycln/compare/v2.0.4...v2.1.1) - [github.com/PyCQA/flake8: 4.0.1 → 5.0.2](https://github.com/PyCQA/flake8/compare/4.0.1...5.0.2) * fix: minor touchups for flake8 Signed-off-by: Henry Schreiner Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 6 +++--- setup.cfg | 2 +- tests/test_exceptions.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2c098c5a..e41480d05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.37.2" + rev: "v2.37.3" hooks: - id: pyupgrade args: [--py36-plus] @@ -71,7 +71,7 @@ repos: # Autoremoves unused imports - repo: https://github.com/hadialqattan/pycln - rev: "v2.0.4" + rev: "v2.1.1" hooks: - id: pycln stages: [manual] @@ -99,7 +99,7 @@ repos: # Flake8 also supports pre-commit natively (same author) - repo: https://github.com/PyCQA/flake8 - rev: "4.0.1" + rev: "5.0.2" hooks: - id: flake8 exclude: ^(docs/.*|tools/.*)$ diff --git a/setup.cfg b/setup.cfg index 36fbfed4f..9af50ea48 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,5 +46,5 @@ zip_safe = False max-line-length = 120 show_source = True exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722, B950 +extend-ignore = E203, E722, B903, B950 extend-select = B9 diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 5e3beeedb..70b6ffea9 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -185,8 +185,8 @@ def test_custom(msg): with pytest.raises(m.MyException5) as excinfo: try: m.throws5() - except m.MyException5_1: - raise RuntimeError("Exception error: caught child from parent") + except m.MyException5_1 as err: + raise RuntimeError("Exception error: caught child from parent") from err assert msg(excinfo.value) == "this is a helper-defined translated exception" From 29f4940cd4aa4e954a16d2f20897b458e3098588 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Aug 2022 08:05:40 -0700 Subject: [PATCH 17/71] Fix copy-paste oversight (#4118) --- .github/workflows/upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index a40a6c7d7..95ff4cb83 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -104,7 +104,7 @@ jobs: run: cmake --build build3 --target pytest - name: Interface test - run: cmake --build build2 --target test_cmake_build + run: cmake --build build3 --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools From 8a4bca8216f94f64d9582997db5e7eaf2e4660b3 Mon Sep 17 00:00:00 2001 From: Daniel Dinu Date: Mon, 8 Aug 2022 07:01:39 -0700 Subject: [PATCH 18/71] fix(cmake): use case-insensitive CMAKE_BUILD_TYPE comparisons (#4078) --- tools/pybind11NewTools.cmake | 4 +++- tools/pybind11Tools.cmake | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index abba0fe0e..9e13daf1a 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -233,7 +233,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 5535e872f..1b6045b70 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -208,7 +208,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) pybind11_strip(${target_name}) endif() From 6abb7de6cd12d7a3b6d3afbdf98f3795639466a3 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 8 Aug 2022 14:28:33 -0400 Subject: [PATCH 19/71] chore: Use PyObject_GenericGetDict and PyObject_GenericSetDict functions (#4106) * Try to update getset_dict APIs for 3.11 * Update API for all Python versions * Test ifdef for forward explicit forward compat * Fix ifdef --- include/pybind11/detail/class.h | 39 ++++++++++----------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 42720f844..a98e5e541 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -502,31 +502,6 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { return (PyObject *) heap_type; } -/// dynamic_attr: Support for `d = instance.__dict__`. -extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { - PyObject *&dict = *_PyObject_GetDictPtr(self); - if (!dict) { - dict = PyDict_New(); - } - Py_XINCREF(dict); - return dict; -} - -/// dynamic_attr: Support for `instance.__dict__ = dict()`. -extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) { - if (!PyDict_Check(new_dict)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, not a '%.200s'", - get_fully_qualified_tp_name(Py_TYPE(new_dict)).c_str()); - return -1; - } - PyObject *&dict = *_PyObject_GetDictPtr(self); - Py_INCREF(new_dict); - Py_CLEAR(dict); - dict = new_dict; - return 0; -} - /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { PyObject *&dict = *_PyObject_GetDictPtr(self); @@ -558,9 +533,17 @@ inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - static PyGetSetDef getset[] = { - {const_cast("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr}}; + static PyGetSetDef getset[] = {{ +#if PY_VERSION_HEX < 0x03070000 + const_cast("__dict__"), +#else + "__dict__", +#endif + PyObject_GenericGetDict, + PyObject_GenericSetDict, + nullptr, + nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; type->tp_getset = getset; } From 14c84654f8674b78a4c0462f0520b8fab93bc80c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Aug 2022 21:15:43 -0400 Subject: [PATCH 20/71] [pre-commit.ci] pre-commit autoupdate (#4126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/yesqa: v1.3.0 → v1.4.0](https://github.com/asottile/yesqa/compare/v1.3.0...v1.4.0) - [github.com/PyCQA/flake8: 5.0.2 → 5.0.4](https://github.com/PyCQA/flake8/compare/5.0.2...5.0.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e41480d05..94359ab62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,7 @@ repos: # Automatically remove noqa that are not used - repo: https://github.com/asottile/yesqa - rev: "v1.3.0" + rev: "v1.4.0" hooks: - id: yesqa additional_dependencies: &flake8_dependencies @@ -99,7 +99,7 @@ repos: # Flake8 also supports pre-commit natively (same author) - repo: https://github.com/PyCQA/flake8 - rev: "5.0.2" + rev: "5.0.4" hooks: - id: flake8 exclude: ^(docs/.*|tools/.*)$ From 5bdd3d59becf82d77e3da81226c1f4f869bfaf64 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 9 Aug 2022 00:02:45 -0400 Subject: [PATCH 21/71] feat(cmake): add installation support for pkg-config dependency detection (#4077) * add installation support for pkg-config dependency detection pkg-config is a buildsystem-agnostic alternative to `pybind11Config.cmake` that can be used from build systems other than cmake. Fixes #230 * tests: add test for pkg config Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 3 + CMakeLists.txt | 13 +++ noxfile.py | 4 +- pybind11/__init__.py | 3 +- pybind11/__main__.py | 9 +- pybind11/commands.py | 12 +++ setup.py | 1 + tests/extra_python_package/test_files.py | 124 ++++++++++++----------- tools/JoinPaths.cmake | 23 +++++ tools/pybind11.pc.in | 7 ++ tools/setup_global.py.in | 2 + tools/setup_main.py.in | 2 + 12 files changed, 142 insertions(+), 61 deletions(-) create mode 100644 tools/JoinPaths.cmake create mode 100644 tools/pybind11.pc.in diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 94359ab62..a962f8b79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,9 @@ # # See https://github.com/pre-commit/pre-commit +# third-party content +exclude: ^tools/JoinPaths.cmake$ + repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/CMakeLists.txt b/CMakeLists.txt index 3787982cb..ee0975bc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,9 @@ else() endif() include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") +# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files +# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") # Relative directory setting if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) @@ -262,6 +265,16 @@ if(PYBIND11_INSTALL) NAMESPACE "pybind11::" DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + # pkg-config support + if(NOT prefix_for_pc_file) + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + endif() + join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + # Uninstall target if(PYBIND11_MASTER_PROJECT) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" diff --git a/noxfile.py b/noxfile.py index 7be1f2544..021ced245 100644 --- a/noxfile.py +++ b/noxfile.py @@ -27,7 +27,7 @@ def lint(session: nox.Session) -> None: Lint the codebase (except for clang-format/tidy). """ session.install("pre-commit") - session.run("pre-commit", "run", "-a") + session.run("pre-commit", "run", "-a", *session.posargs) @nox.session(python=PYTHON_VERSIONS) @@ -58,7 +58,7 @@ def tests_packaging(session: nox.Session) -> None: """ session.install("-r", "tests/requirements.txt", "--prefer-binary") - session.run("pytest", "tests/extra_python_package") + session.run("pytest", "tests/extra_python_package", *session.posargs) @nox.session(reuse_venv=True) diff --git a/pybind11/__init__.py b/pybind11/__init__.py index a1be96981..4fbb17079 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -6,11 +6,12 @@ if sys.version_info < (3, 6): from ._version import __version__, version_info -from .commands import get_cmake_dir, get_include +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir __all__ = ( "version_info", "__version__", "get_include", "get_cmake_dir", + "get_pkgconfig_dir", ) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 22fc4471e..8c8953384 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -4,7 +4,7 @@ import argparse import sys import sysconfig -from .commands import get_cmake_dir, get_include +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir def print_includes() -> None: @@ -36,6 +36,11 @@ def main() -> None: action="store_true", help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", ) + parser.add_argument( + "--pkgconfigdir", + action="store_true", + help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() @@ -43,6 +48,8 @@ def main() -> None: print_includes() if args.cmakedir: print(get_cmake_dir()) + if args.pkgconfigdir: + print(get_pkgconfig_dir()) if __name__ == "__main__": diff --git a/pybind11/commands.py b/pybind11/commands.py index a29c8ca61..152fa20ce 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -23,3 +23,15 @@ def get_cmake_dir() -> str: msg = "pybind11 not installed, installation required to access the CMake files" raise ImportError(msg) + + +def get_pkgconfig_dir() -> str: + """ + Return the path to the pybind11 pkgconfig directory. + """ + pkgconfig_installed_path = os.path.join(DIR, "share", "pkgconfig") + if os.path.exists(pkgconfig_installed_path): + return pkgconfig_installed_path + + msg = "pybind11 not installed, installation required to access the pkgconfig files" + raise ImportError(msg) diff --git a/setup.py b/setup.py index a149755bf..68573519c 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,7 @@ with remove_output("pybind11/include", "pybind11/share"): "-DCMAKE_INSTALL_PREFIX=pybind11", "-DBUILD_TESTING=OFF", "-DPYBIND11_NOPYTHON=ON", + "-Dprefix_for_pc_file=${pcfiledir}/../../", ] if "CMAKE_ARGS" in os.environ: fcommand = [ diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index ba16b5224..8e1ddd850 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -12,6 +12,16 @@ import zipfile DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) +PKGCONFIG = """\ +prefix=${{pcfiledir}}/../../ +includedir=${{prefix}}/include + +Name: pybind11 +Description: Seamless operability between C++11 and Python +Version: {VERSION} +Cflags: -I${{includedir}} +""" + main_headers = { "include/pybind11/attr.h", @@ -59,6 +69,10 @@ cmake_files = { "share/cmake/pybind11/pybind11Tools.cmake", } +pkgconfig_files = { + "share/pkgconfig/pybind11.pc", +} + py_files = { "__init__.py", "__main__.py", @@ -69,7 +83,7 @@ py_files = { } headers = main_headers | detail_headers | stl_headers -src_files = headers | cmake_files +src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files @@ -82,6 +96,7 @@ sdist_files = { "pybind11/share", "pybind11/share/cmake", "pybind11/share/cmake/pybind11", + "pybind11/share/pkgconfig", "pyproject.toml", "setup.cfg", "setup.py", @@ -101,22 +116,25 @@ local_sdist_files = { } +def read_tz_file(tar: tarfile.TarFile, name: str) -> bytes: + start = tar.getnames()[0] + "/" + inner_file = tar.extractfile(tar.getmember(f"{start}{name}")) + assert inner_file + with contextlib.closing(inner_file) as f: + return f.read() + + +def normalize_line_endings(value: bytes) -> bytes: + return value.replace(os.linesep.encode("utf-8"), b"\n") + + def test_build_sdist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - out = subprocess.check_output( - [ - sys.executable, - "-m", - "build", - "--sdist", - "--outdir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}"], check=True ) - if hasattr(out, "decode"): - out = out.decode() (sdist,) = tmpdir.visit("*.tar.gz") @@ -125,25 +143,17 @@ def test_build_sdist(monkeypatch, tmpdir): version = start[9:-1] simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() - - with contextlib.closing( - tar.extractfile( - tar.getmember( - start + "pybind11/share/cmake/pybind11/pybind11Config.cmake" - ) - ) - ) as f: - contents = f.read().decode("utf8") - assert 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' in contents + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) files = {f"pybind11/{n}" for n in all_files} files |= sdist_files @@ -154,9 +164,9 @@ def test_build_sdist(monkeypatch, tmpdir): with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: contents = ( - string.Template(f.read().decode()) + string.Template(f.read().decode("utf-8")) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) assert setup_py == contents @@ -164,25 +174,19 @@ def test_build_sdist(monkeypatch, tmpdir): contents = f.read() assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected + def test_build_global_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - out = subprocess.check_output( - [ - sys.executable, - "-m", - "build", - "--sdist", - "--outdir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir)], check=True ) - if hasattr(out, "decode"): - out = out.decode() - (sdist,) = tmpdir.visit("*.tar.gz") with tarfile.open(str(sdist), "r:gz") as tar: @@ -190,15 +194,17 @@ def test_build_global_dist(monkeypatch, tmpdir): version = start[16:-1] simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) files = {f"pybind11/{n}" for n in all_files} files |= sdist_files @@ -209,7 +215,7 @@ def test_build_global_dist(monkeypatch, tmpdir): contents = ( string.Template(f.read().decode()) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) assert setup_py == contents @@ -217,12 +223,16 @@ def test_build_global_dist(monkeypatch, tmpdir): contents = f.read() assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected + def tests_build_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") @@ -249,8 +259,8 @@ def tests_build_global_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") diff --git a/tools/JoinPaths.cmake b/tools/JoinPaths.cmake new file mode 100644 index 000000000..c68d91b84 --- /dev/null +++ b/tools/JoinPaths.cmake @@ -0,0 +1,23 @@ +# This module provides function for joining paths +# known from most languages +# +# SPDX-License-Identifier: (MIT OR CC0-1.0) +# Copyright 2020 Jan Tojnar +# https://github.com/jtojnar/cmake-snips +# +# Modelled after Python’s os.path.join +# https://docs.python.org/3.7/library/os.path.html#os.path.join +# Windows not supported +function(join_paths joined_path first_path_segment) + set(temp_path "${first_path_segment}") + foreach(current_segment IN LISTS ARGN) + if(NOT ("${current_segment}" STREQUAL "")) + if(IS_ABSOLUTE "${current_segment}") + set(temp_path "${current_segment}") + else() + set(temp_path "${temp_path}/${current_segment}") + endif() + endif() + endforeach() + set(${joined_path} "${temp_path}" PARENT_SCOPE) +endfunction() diff --git a/tools/pybind11.pc.in b/tools/pybind11.pc.in new file mode 100644 index 000000000..402f0b357 --- /dev/null +++ b/tools/pybind11.pc.in @@ -0,0 +1,7 @@ +prefix=@prefix_for_pc_file@ +includedir=@includedir_for_pc_file@ + +Name: @PROJECT_NAME@ +Description: Seamless operability between C++11 and Python +Version: @PROJECT_VERSION@ +Cflags: -I${includedir} diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index 8aa387178..d91468c10 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -29,6 +29,7 @@ main_headers = glob.glob("pybind11/include/pybind11/*.h") detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") +pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") headers = main_headers + detail_headers + stl_headers cmdclass = {"install_headers": InstallHeadersNested} @@ -51,6 +52,7 @@ setup( headers=headers, data_files=[ (base + "share/cmake/pybind11", cmake_files), + (base + "share/pkgconfig", pkgconfig_files), (base + "include/pybind11", main_headers), (base + "include/pybind11/detail", detail_headers), (base + "include/pybind11/stl", stl_headers), diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 738d73faa..65198bdb6 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -17,6 +17,7 @@ setup( "pybind11.include.pybind11.detail", "pybind11.include.pybind11.stl", "pybind11.share.cmake.pybind11", + "pybind11.share.pkgconfig", ], package_data={ "pybind11": ["py.typed"], @@ -24,6 +25,7 @@ setup( "pybind11.include.pybind11.detail": ["*.h"], "pybind11.include.pybind11.stl": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], + "pybind11.share.pkgconfig": ["*.pc"], }, extras_require={ "global": ["pybind11_global==$version"] From bbb89da152a0e4f63a4fecc0d4db44f0ad855fdf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 9 Aug 2022 00:03:36 -0400 Subject: [PATCH 22/71] fix(cmake): support vcpkg, try 2 (#4123) Signed-off-by: Henry Schreiner --- tools/pybind11Tools.cmake | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 1b6045b70..0dc61d399 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -115,6 +115,7 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() +# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg if(CMAKE_VERSION VERSION_LESS 3.11) set_property( TARGET pybind11::module @@ -130,16 +131,19 @@ if(CMAKE_VERSION VERSION_LESS 3.11) APPEND PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $) else() + # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside + # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are + # target_link_library keywords rather than real libraries. + add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) + target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) target_link_libraries( pybind11::module INTERFACE pybind11::python_link_helper - "$<$,$>:$>" - ) + "$<$,$>:pybind11::_ClassicPythonLibraries>") target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 - $) - + pybind11::_ClassicPythonLibraries) endif() function(pybind11_extension name) From b884b9dc6b99e41f08322dc198e62717770ea916 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 11 Aug 2022 16:14:17 -0400 Subject: [PATCH 23/71] chore: Add pytests for constructing pytypes from iterable (#4138) * Add some additional pytests * Reorder tests * Further reorder tests * remove stray lines * remove unused fixtures --- tests/test_pytypes.cpp | 5 +++++ tests/test_pytypes.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 81387fd9f..da0dc8f6b 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -109,6 +109,11 @@ TEST_SUBMODULE(pytypes, m) { m.def("get_iterator", [] { return py::iterator(); }); // test_iterable m.def("get_iterable", [] { return py::iterable(); }); + m.def("get_frozenset_from_iterable", + [](const py::iterable &iter) { return py::frozenset(iter); }); + m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); }); + m.def("get_set_from_iterable", [](const py::iterable &iter) { return py::set(iter); }); + m.def("get_tuple_from_iterable", [](const py::iterable &iter) { return py::tuple(iter); }); // test_float m.def("get_float", [] { return py::float_(0.0f); }); // test_list diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index bde831738..a34eaa59e 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -26,6 +26,22 @@ def test_iterator(doc): assert doc(m.get_iterator) == "get_iterator() -> Iterator" +@pytest.mark.parametrize( + "pytype, from_iter_func", + [ + (frozenset, m.get_frozenset_from_iterable), + (list, m.get_list_from_iterable), + (set, m.get_set_from_iterable), + (tuple, m.get_tuple_from_iterable), + ], +) +def test_from_iterable(pytype, from_iter_func): + my_iter = iter(range(10)) + s = from_iter_func(my_iter) + assert type(s) == pytype + assert s == pytype(range(10)) + + def test_iterable(doc): assert doc(m.get_iterable) == "get_iterable() -> Iterable" From 2d59b43cbf8793119fb92726ce8eb33441469e3e Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 11 Aug 2022 21:19:39 -0700 Subject: [PATCH 24/71] Qualify detail::forward_like to avoid conflict. (#4136) C++23 feature: P2445R1 forward_like() --- include/pybind11/stl.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index ab30ecac0..6426bff08 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -78,7 +78,7 @@ struct set_caster { pybind11::set s; for (auto &&value : src) { auto value_ = reinterpret_steal( - key_conv::cast(forward_like(value), policy, parent)); + key_conv::cast(detail::forward_like(value), policy, parent)); if (!value_ || !s.add(std::move(value_))) { return handle(); } @@ -122,9 +122,9 @@ struct map_caster { } for (auto &&kv : src) { auto key = reinterpret_steal( - key_conv::cast(forward_like(kv.first), policy_key, parent)); + key_conv::cast(detail::forward_like(kv.first), policy_key, parent)); auto value = reinterpret_steal( - value_conv::cast(forward_like(kv.second), policy_value, parent)); + value_conv::cast(detail::forward_like(kv.second), policy_value, parent)); if (!key || !value) { return handle(); } @@ -178,7 +178,7 @@ public: ssize_t index = 0; for (auto &&value : src) { auto value_ = reinterpret_steal( - value_conv::cast(forward_like(value), policy, parent)); + value_conv::cast(detail::forward_like(value), policy, parent)); if (!value_) { return handle(); } @@ -242,7 +242,7 @@ public: ssize_t index = 0; for (auto &&value : src) { auto value_ = reinterpret_steal( - value_conv::cast(forward_like(value), policy, parent)); + value_conv::cast(detail::forward_like(value), policy, parent)); if (!value_) { return handle(); } From 81f35d29c621ce2ba8a82eebeb6ad97f842983c0 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 20 Aug 2022 17:05:07 -0400 Subject: [PATCH 25/71] chore: Mark detail:forward_like as constexpr (#4147) --- include/pybind11/stl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 6426bff08..48031c2a6 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -45,7 +45,7 @@ using forwarded_type = conditional_t::value, /// Forwards a value U as rvalue or lvalue according to whether T is rvalue or lvalue; typically /// used for forwarding a container's elements. template -forwarded_type forward_like(U &&u) { +constexpr forwarded_type forward_like(U &&u) { return std::forward>(std::forward(u)); } From 68e6fdaa90fc93979e6d5d1e9f788f464593e8f2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 21 Aug 2022 09:44:01 -0700 Subject: [PATCH 26/71] embed.h Python 3.11 `config.use_environment=1` + `PYTHONPATH` test (#4119) * Add debug fprintf to test_interpreter.cpp * Update `sys.path` from `PYTHONPATH` in Python >= 3.11 branch of `initialize_interpreter()` * Use `config.isolated = 0; config.use_environment = 1;` As suggsted by @vstinner here: https://github.com/pybind/pybind11/pull/4119#issuecomment-1219442853 * Add `TEST_CASE("PYTHONPATH is used to update sys.path")` * Fix clang-tidy error. * Use `_putenv_s()` under Windows. * Fix clang-tidy error: argument name ... in comment does not match parameter name * Remove slash from PYTHONPATH addition, to work around Windows slash-vs-backslash issue. * Use `py::str(...)` instead of `.attr("__str__")` as suggested by @skylion007 Co-authored-by: Aaron Gokaslan Co-authored-by: Aaron Gokaslan --- include/pybind11/embed.h | 2 ++ tests/test_embed/catch.cpp | 18 ++++++++++++++++++ tests/test_embed/test_interpreter.cpp | 7 +++++++ 3 files changed, 27 insertions(+) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index d6999cd77..0ac609e0f 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -150,6 +150,8 @@ inline void initialize_interpreter(bool init_signal_handlers = true, #else PyConfig config; PyConfig_InitIsolatedConfig(&config); + config.isolated = 0; + config.use_environment = 1; config.install_signal_handlers = init_signal_handlers ? 1 : 0; PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp index 96d2e3f92..a03a8b37c 100644 --- a/tests/test_embed/catch.cpp +++ b/tests/test_embed/catch.cpp @@ -20,7 +20,25 @@ namespace py = pybind11; int main(int argc, char *argv[]) { + // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number: + std::string updated_pythonpath("pybind11_test_embed_PYTHONPATH_2099743835476552"); + const char *preexisting_pythonpath = getenv("PYTHONPATH"); + if (preexisting_pythonpath != nullptr) { +#if defined(_WIN32) + updated_pythonpath += ';'; +#else + updated_pythonpath += ':'; +#endif + updated_pythonpath += preexisting_pythonpath; + } +#if defined(_WIN32) + _putenv_s("PYTHONPATH", updated_pythonpath.c_str()); +#else + setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1); +#endif + py::scoped_interpreter guard{}; + auto result = Catch::Session().run(argc, argv); return result < 0xff ? result : 0xff; diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 1c45457a0..6299293b9 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -75,6 +75,13 @@ PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) { d["missing"].cast(); } +TEST_CASE("PYTHONPATH is used to update sys.path") { + // The setup for this TEST_CASE is in catch.cpp! + auto sys_path = py::str(py::module_::import("sys").attr("path")).cast(); + REQUIRE_THAT(sys_path, + Catch::Matchers::Contains("pybind11_test_embed_PYTHONPATH_2099743835476552")); +} + TEST_CASE("Pass classes and data between modules defined in C++ and Python") { auto module_ = py::module_::import("test_interpreter"); REQUIRE(py::hasattr(module_, "DerivedWidget")); From a48ec3e8820064511aac5fab71706166bdd590d3 Mon Sep 17 00:00:00 2001 From: Brad Messer Date: Wed, 24 Aug 2022 10:34:31 -0400 Subject: [PATCH 27/71] Words matter updates (#4155) * Remove sanity check from code base. * Use main over master. * Better alternative that doesn't collide with language keywords/frequent usage words. --- include/pybind11/numpy.h | 2 +- tests/test_eigen.py | 68 ++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 56e5edc5a..369e18649 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1401,7 +1401,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container oss << '}'; auto format_str = oss.str(); - // Sanity check: verify that NumPy properly parses our buffer format string + // Smoke test: verify that NumPy properly parses our buffer format string auto &api = npy_api::get(); auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) { diff --git a/tests/test_eigen.py b/tests/test_eigen.py index a1c114aed..9c6485de3 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen.py @@ -251,14 +251,14 @@ def array_copy_but_one(a, r, c, v): def test_eigen_return_references(): """Tests various ways of returning references and non-referencing copies""" - master = np.ones((10, 10)) + primary = np.ones((10, 10)) a = m.ReturnTester() a_get1 = a.get() assert not a_get1.flags.owndata and a_get1.flags.writeable - assign_both(a_get1, master, 3, 3, 5) + assign_both(a_get1, primary, 3, 3, 5) a_get2 = a.get_ptr() assert not a_get2.flags.owndata and a_get2.flags.writeable - assign_both(a_get1, master, 2, 3, 6) + assign_both(a_get1, primary, 2, 3, 6) a_view1 = a.view() assert not a_view1.flags.owndata and not a_view1.flags.writeable @@ -271,25 +271,25 @@ def test_eigen_return_references(): a_copy1 = a.copy_get() assert a_copy1.flags.owndata and a_copy1.flags.writeable - np.testing.assert_array_equal(a_copy1, master) + np.testing.assert_array_equal(a_copy1, primary) a_copy1[7, 7] = -44 # Shouldn't affect anything else - c1want = array_copy_but_one(master, 7, 7, -44) + c1want = array_copy_but_one(primary, 7, 7, -44) a_copy2 = a.copy_view() assert a_copy2.flags.owndata and a_copy2.flags.writeable - np.testing.assert_array_equal(a_copy2, master) + np.testing.assert_array_equal(a_copy2, primary) a_copy2[4, 4] = -22 # Shouldn't affect anything else - c2want = array_copy_but_one(master, 4, 4, -22) + c2want = array_copy_but_one(primary, 4, 4, -22) a_ref1 = a.ref() assert not a_ref1.flags.owndata and a_ref1.flags.writeable - assign_both(a_ref1, master, 1, 1, 15) + assign_both(a_ref1, primary, 1, 1, 15) a_ref2 = a.ref_const() assert not a_ref2.flags.owndata and not a_ref2.flags.writeable with pytest.raises(ValueError): a_ref2[5, 5] = 33 a_ref3 = a.ref_safe() assert not a_ref3.flags.owndata and a_ref3.flags.writeable - assign_both(a_ref3, master, 0, 7, 99) + assign_both(a_ref3, primary, 0, 7, 99) a_ref4 = a.ref_const_safe() assert not a_ref4.flags.owndata and not a_ref4.flags.writeable with pytest.raises(ValueError): @@ -297,23 +297,23 @@ def test_eigen_return_references(): a_copy3 = a.copy_ref() assert a_copy3.flags.owndata and a_copy3.flags.writeable - np.testing.assert_array_equal(a_copy3, master) + np.testing.assert_array_equal(a_copy3, primary) a_copy3[8, 1] = 11 - c3want = array_copy_but_one(master, 8, 1, 11) + c3want = array_copy_but_one(primary, 8, 1, 11) a_copy4 = a.copy_ref_const() assert a_copy4.flags.owndata and a_copy4.flags.writeable - np.testing.assert_array_equal(a_copy4, master) + np.testing.assert_array_equal(a_copy4, primary) a_copy4[8, 4] = 88 - c4want = array_copy_but_one(master, 8, 4, 88) + c4want = array_copy_but_one(primary, 8, 4, 88) a_block1 = a.block(3, 3, 2, 2) assert not a_block1.flags.owndata and a_block1.flags.writeable a_block1[0, 0] = 55 - master[3, 3] = 55 + primary[3, 3] = 55 a_block2 = a.block_safe(2, 2, 3, 2) assert not a_block2.flags.owndata and a_block2.flags.writeable a_block2[2, 1] = -123 - master[4, 3] = -123 + primary[4, 3] = -123 a_block3 = a.block_const(6, 7, 4, 3) assert not a_block3.flags.owndata and not a_block3.flags.writeable with pytest.raises(ValueError): @@ -321,18 +321,18 @@ def test_eigen_return_references(): a_copy5 = a.copy_block(2, 2, 2, 3) assert a_copy5.flags.owndata and a_copy5.flags.writeable - np.testing.assert_array_equal(a_copy5, master[2:4, 2:5]) + np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5]) a_copy5[1, 1] = 777 - c5want = array_copy_but_one(master[2:4, 2:5], 1, 1, 777) + c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777) a_corn1 = a.corners() assert not a_corn1.flags.owndata and a_corn1.flags.writeable a_corn1 *= 50 a_corn1[1, 1] = 999 - master[0, 0] = 50 - master[0, 9] = 50 - master[9, 0] = 50 - master[9, 9] = 999 + primary[0, 0] = 50 + primary[0, 9] = 50 + primary[9, 0] = 50 + primary[9, 9] = 999 a_corn2 = a.corners_const() assert not a_corn2.flags.owndata and not a_corn2.flags.writeable with pytest.raises(ValueError): @@ -340,22 +340,22 @@ def test_eigen_return_references(): # All of the changes made all the way along should be visible everywhere # now (except for the copies, of course) - np.testing.assert_array_equal(a_get1, master) - np.testing.assert_array_equal(a_get2, master) - np.testing.assert_array_equal(a_view1, master) - np.testing.assert_array_equal(a_view2, master) - np.testing.assert_array_equal(a_ref1, master) - np.testing.assert_array_equal(a_ref2, master) - np.testing.assert_array_equal(a_ref3, master) - np.testing.assert_array_equal(a_ref4, master) - np.testing.assert_array_equal(a_block1, master[3:5, 3:5]) - np.testing.assert_array_equal(a_block2, master[2:5, 2:4]) - np.testing.assert_array_equal(a_block3, master[6:10, 7:10]) + np.testing.assert_array_equal(a_get1, primary) + np.testing.assert_array_equal(a_get2, primary) + np.testing.assert_array_equal(a_view1, primary) + np.testing.assert_array_equal(a_view2, primary) + np.testing.assert_array_equal(a_ref1, primary) + np.testing.assert_array_equal(a_ref2, primary) + np.testing.assert_array_equal(a_ref3, primary) + np.testing.assert_array_equal(a_ref4, primary) + np.testing.assert_array_equal(a_block1, primary[3:5, 3:5]) + np.testing.assert_array_equal(a_block2, primary[2:5, 2:4]) + np.testing.assert_array_equal(a_block3, primary[6:10, 7:10]) np.testing.assert_array_equal( - a_corn1, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + a_corn1, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] ) np.testing.assert_array_equal( - a_corn2, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + a_corn2, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] ) np.testing.assert_array_equal(a_copy1, c1want) From fac23b6f65e6d0b9aa8525712936b036e372eae1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 24 Aug 2022 13:08:24 -0700 Subject: [PATCH 28/71] `error_fetch_and_normalize`: PyPy 7.3.10+ does not need the PR #4079 workaround anymore. (#4154) --- include/pybind11/pytypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 6ba1f5f20..f4ba459f2 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -473,7 +473,7 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 // This behavior runs the risk of masking errors in the error handling, but avoids a // conflict with PyPy, which relies on the normalization here to change OSError to // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). From 0b4c1bc2865cf89abc751809e72a184bde671678 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Aug 2022 20:25:01 -0700 Subject: [PATCH 29/71] test: ConstructorStats newline (PyPy) (#4167) This looks like it lacks a newline. --- tests/constructor_stats.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/constructor_stats.h b/tests/constructor_stats.h index a3835c21e..937f6c233 100644 --- a/tests/constructor_stats.h +++ b/tests/constructor_stats.h @@ -115,7 +115,7 @@ public: #if defined(PYPY_VERSION) PyObject *globals = PyEval_GetGlobals(); PyObject *result = PyRun_String("import gc\n" - "for i in range(2):" + "for i in range(2):\n" " gc.collect()\n", Py_file_input, globals, From 283f10dc55636e495bc2114c6b5baf7cd7064a0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:26:53 -0400 Subject: [PATCH 30/71] chore(deps): bump ilammy/msvc-dev-cmd from 1.10.0 to 1.11.0 (#4161) Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases) - [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/v1.10.0...v1.11.0) --- updated-dependencies: - dependency-name: ilammy/msvc-dev-cmd dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac6c536ce..213c69005 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -754,7 +754,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.12 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.10.0 + uses: ilammy/msvc-dev-cmd@v1.11.0 with: arch: x86 @@ -807,7 +807,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.12 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.10.0 + uses: ilammy/msvc-dev-cmd@v1.11.0 with: arch: x86 From 8756f16ed842e40406018df901f3219b231e2105 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 21:59:48 -0700 Subject: [PATCH 31/71] [pre-commit.ci] pre-commit autoupdate (#4151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/Lucas-C/pre-commit-hooks: v1.3.0 → v1.3.1](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.3.0...v1.3.1) - [github.com/sirosen/texthooks: 0.3.1 → 0.4.0](https://github.com/sirosen/texthooks/compare/0.3.1...0.4.0) - [github.com/PyCQA/pylint: v2.14.5 → v2.15.0](https://github.com/PyCQA/pylint/compare/v2.14.5...v2.15.0) - [github.com/codespell-project/codespell: v2.1.0 → v2.2.1](https://github.com/codespell-project/codespell/compare/v2.1.0...v2.2.1) * Introduce .codespell-ignore-lines for safer (line-based instead of word-based) suppressions. * Fix two issues: 1. ensure sort order; 2. remove duplicates Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .codespell-ignore-lines | 24 ++++++++++++++ .pre-commit-config.yaml | 12 ++++--- tools/codespell_ignore_lines_from_errors.py | 35 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 .codespell-ignore-lines create mode 100644 tools/codespell_ignore_lines_from_errors.py diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines new file mode 100644 index 000000000..2a01d63eb --- /dev/null +++ b/.codespell-ignore-lines @@ -0,0 +1,24 @@ +template + template + auto &this_ = static_cast(*this); + if (load_impl(temp, false)) { + ssize_t nd = 0; + auto trivial = broadcast(buffers, nd, shape); + auto ndim = (size_t) nd; + int nd; + ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } + using op = op_impl; +template + template + class_ &def(const detail::op_ &op, const Extra &...extra) { + class_ &def_cast(const detail::op_ &op, const Extra &...extra) { +@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) +struct IntStruct { + explicit IntStruct(int v) : value(v){}; + ~IntStruct() { value = -value; } + IntStruct(const IntStruct &) = default; + IntStruct &operator=(const IntStruct &) = default; + py::class_(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); + py::implicitly_convertible(); + m.def("test", [](int expected, const IntStruct &in) { + [](int expected, const IntStruct &in) { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a962f8b79..bced50caf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,12 +62,12 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.0" + rev: "v1.3.1" hooks: - id: remove-tabs - repo: https://github.com/sirosen/texthooks - rev: "0.3.1" + rev: "0.4.0" hooks: - id: fix-ligatures - id: fix-smartquotes @@ -110,7 +110,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.14.5" + rev: "v2.15.0" hooks: - id: pylint files: ^pybind11 @@ -143,12 +143,14 @@ repos: additional_dependencies: [cmake, ninja] # Check for spelling +# Use tools/codespell_ignore_lines_from_errors.py +# to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.1.0" + rev: "v2.2.1" hooks: - id: codespell exclude: ".supp$" - args: ["-L", "nd,ot,thist"] + args: ["-x", ".codespell-ignore-lines"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py diff --git a/tools/codespell_ignore_lines_from_errors.py b/tools/codespell_ignore_lines_from_errors.py new file mode 100644 index 000000000..5403ec3ad --- /dev/null +++ b/tools/codespell_ignore_lines_from_errors.py @@ -0,0 +1,35 @@ +"""Simple script for rebuilding .codespell-ignore-lines + +Usage: + +cat < /dev/null > .codespell-ignore-lines +pre-commit run --all-files codespell >& /tmp/codespell_errors.txt +python3 tools/codespell_ignore_lines_from_errors.py /tmp/codespell_errors.txt > .codespell-ignore-lines + +git diff to review changes, then commit, push. +""" + +import sys +from typing import List + + +def run(args: List[str]) -> None: + assert len(args) == 1, "codespell_errors.txt" + cache = {} + done = set() + for line in sorted(open(args[0]).read().splitlines()): + i = line.find(" ==> ") + if i > 0: + flds = line[:i].split(":") + if len(flds) >= 2: + filename, line_num = flds[:2] + if filename not in cache: + cache[filename] = open(filename).read().splitlines() + supp = cache[filename][int(line_num) - 1] + if supp not in done: + print(supp) + done.add(supp) + + +if __name__ == "__main__": + run(args=sys.argv[1:]) From aa8f8baa4e402885e5c89f5dc42a579932c33790 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 09:19:02 -0400 Subject: [PATCH 32/71] [pre-commit.ci] pre-commit autoupdate (#4171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/psf/black: 22.6.0 → 22.8.0](https://github.com/psf/black/compare/22.6.0...22.8.0) * Update .pre-commit-config.yaml Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bced50caf..b23030647 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.6.0" # Keep in sync with blacken-docs + rev: "22.8.0" # Keep in sync with blacken-docs hooks: - id: black @@ -58,7 +58,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==22.6.0 # keep in sync with black hook + - black==22.8.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks From 64f7281874c3b439dc0102b6ad6cfc7586f17651 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 20:04:44 -0400 Subject: [PATCH 33/71] [pre-commit.ci] pre-commit autoupdate (#4178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/pylint: v2.15.0 → v2.15.2](https://github.com/PyCQA/pylint/compare/v2.15.0...v2.15.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b23030647..5d794a749 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -110,7 +110,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.0" + rev: "v2.15.2" hooks: - id: pylint files: ^pybind11 From 8524b20c3caf8a5e868dbbf76b39927c45b41f72 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Thu, 15 Sep 2022 05:56:40 +0900 Subject: [PATCH 34/71] fix: Python-3.12 compatibility (#4168) * fix: Python-3.12 compatibility Enable dynamic attributes for `pybind11_static_property` * Add future-notice comment --- include/pybind11/detail/class.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index a98e5e541..528e716f7 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -55,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec return PyProperty_Type.tp_descr_set(self, cls, value); } +// Forward declaration to use in `make_static_property_type()` +inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type); + /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()` methods are modified to always use the object type instead of a concrete instance. Return value: New reference. */ @@ -87,6 +90,13 @@ inline PyTypeObject *make_static_property_type() { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); } +# if PY_VERSION_HEX >= 0x030C0000 + // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. + // Since Python-3.12 property-derived types are required to + // have dynamic attributes (to set `__doc__`) + enable_dynamic_attributes(heap_type); +# endif + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); From 1874f8fa8767179ab8f8c645a6e8c8c4e4a7c486 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Wed, 14 Sep 2022 17:00:27 -0400 Subject: [PATCH 35/71] Clarify GIL documentation (#4057) --- docs/advanced/misc.rst | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index edab15fcb..71960b803 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -39,15 +39,42 @@ The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. Global Interpreter Lock (GIL) ============================= -When calling a C++ function from Python, the GIL is always held. +The Python C API dictates that the Global Interpreter Lock (GIL) must always +be held by the current thread to safely access Python objects. As a result, +when Python calls into C++ via pybind11 the GIL must be held, and pybind11 +will never implicitly release the GIL. + +.. code-block:: cpp + + void my_function() { + /* GIL is held when this function is called from Python */ + } + + PYBIND11_MODULE(example, m) { + m.def("my_function", &my_function); + } + +pybind11 will ensure that the GIL is held when it knows that it is calling +Python code. For example, if a Python callback is passed to C++ code via +``std::function``, when C++ code calls the function the built-in wrapper +will acquire the GIL before calling the Python callback. Similarly, the +``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling +back into Python. + +When writing C++ code that is called from other C++ code, if that code accesses +Python state, it must explicitly acquire and release the GIL. + The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ function call. In this way, long-running C++ code can be parallelized using -multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this +multiple Python threads, **but great care must be taken** when any +:class:`gil_scoped_release` appear: if there is any way that the C++ code +can access Python objects, :class:`gil_scoped_acquire` should be used to +reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this could be realized as follows (important changes highlighted): .. code-block:: cpp - :emphasize-lines: 8,9,31,32 + :emphasize-lines: 8,30,31 class PyAnimal : public Animal { public: @@ -56,9 +83,7 @@ could be realized as follows (important changes highlighted): /* Trampoline (need one for each virtual function) */ std::string go(int n_times) { - /* Acquire GIL before calling Python code */ - py::gil_scoped_acquire acquire; - + /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */ PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ @@ -78,7 +103,8 @@ could be realized as follows (important changes highlighted): .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { - /* Release GIL before calling into (potentially long-running) C++ code */ + // GIL is held when called from Python code. Release GIL before + // calling into (potentially long-running) C++ code py::gil_scoped_release release; return call_go(animal); }); From 9c04c7b0f1210a5f6ca7cc2e975606a1eaf21316 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 19 Sep 2022 12:56:31 -0400 Subject: [PATCH 36/71] chore: Delete copy ctor/assign for GIL RAIIs (#4183) * chore: Delete copy ctor/assign for GIL RAIIs * Fix typo * Delete copy ops for local gil scoped acquire --- include/pybind11/detail/internals.h | 2 ++ include/pybind11/gil.h | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 6ca5e1482..86991955b 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -412,6 +412,8 @@ PYBIND11_NOINLINE internals &get_internals() { // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} + gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; + gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index a0b5de151..1ef5f0a8c 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -80,6 +80,9 @@ public: inc_ref(); } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + void inc_ref() { ++tstate->gilstate_counter; } PYBIND11_NOINLINE void dec_ref() { @@ -144,6 +147,9 @@ public: } } + gil_scoped_release(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter /// could be shutting down when this is called, as thread deletion is not @@ -178,6 +184,8 @@ class gil_scoped_acquire { public: gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } void disarm() {} }; @@ -187,6 +195,8 @@ class gil_scoped_release { public: gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; From d02f219fb9b9012ce7b27f34c5ee9feb55da08ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 21:11:57 -0400 Subject: [PATCH 37/71] [pre-commit.ci] pre-commit autoupdate (#4189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.37.3 → v2.38.0](https://github.com/asottile/pyupgrade/compare/v2.37.3...v2.38.0) - [github.com/PyCQA/pylint: v2.15.2 → v2.15.3](https://github.com/PyCQA/pylint/compare/v2.15.2...v2.15.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d794a749..508b3ea6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.37.3" + rev: "v2.38.0" hooks: - id: pyupgrade args: [--py36-plus] @@ -110,7 +110,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.2" + rev: "v2.15.3" hooks: - id: pylint files: ^pybind11 From 424ac4fe1bde46894a56775e78a702199a03137b Mon Sep 17 00:00:00 2001 From: Jan Iwaszkiewicz Date: Tue, 20 Sep 2022 19:03:57 +0200 Subject: [PATCH 38/71] fix: Windows compiler, missing object initializer (#4188) * Fix for windows compiler, missing object initializer * Removal of if-else macro for MSVC --- include/pybind11/detail/common.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9e6947daa..6da7ef859 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1033,12 +1033,7 @@ PYBIND11_NAMESPACE_END(detail) /// - regular: static_cast(&Class::func) /// - sweet: overload_cast(&Class::func) template -# if (defined(_MSC_VER) && _MSC_VER < 1920) /* MSVC 2017 */ \ - || (defined(__clang__) && __clang_major__ == 5) -static constexpr detail::overload_cast_impl overload_cast = {}; -# else -static constexpr detail::overload_cast_impl overload_cast; -# endif +static constexpr detail::overload_cast_impl overload_cast{}; #endif /// Const member function selector for overload_cast From 95d0e71a652992bcdc0c54844c4c6e5282247c6b Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 21 Sep 2022 11:20:07 -0400 Subject: [PATCH 39/71] test C++14 on MSVC (#4191) --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 213c69005..fe57e7ce9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -737,6 +737,9 @@ jobs: args: -DCMAKE_CXX_STANDARD=20 - python: 3.8 args: -DCMAKE_CXX_STANDARD=17 + - python: 3.7 + args: -DCMAKE_CXX_STANDARD=14 + name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}" runs-on: windows-2019 From f743bdf8e61b2dc4a8995a0485111bedd9c2cbb1 Mon Sep 17 00:00:00 2001 From: bogdan-lab <60753036+bogdan-lab@users.noreply.github.com> Date: Wed, 21 Sep 2022 21:50:31 +0300 Subject: [PATCH 40/71] Avoid local_internals destruction (#4192) * Avoid local_internals destruction It allows to avoid possible static deinitialization fiasco. * Add link to relevant google style guide discussion --- include/pybind11/detail/internals.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 86991955b..d47084e26 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -514,8 +514,13 @@ struct local_internals { /// Works like `get_internals`, but for things which are locally registered. inline local_internals &get_local_internals() { - static local_internals locals; - return locals; + // Current static can be created in the interpreter finalization routine. If the later will be + // destroyed in another static variable destructor, creation of this static there will cause + // static deinitialization fiasco. In order to avoid it we avoid destruction of the + // local_internals static. One can read more about the problem and current solution here: + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *locals = new local_internals(); + return *locals; } /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its From 5aa0fad5def3fc4d48f29998b6f65e74bacafd19 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sun, 25 Sep 2022 16:10:57 -0400 Subject: [PATCH 41/71] perf: call reserve method in set and map casters (#4194) * Call reserve method in set and map casters too * Refactor template logic into has_reserve_method * Adjust comment for reviews * Rearrange reserve_maybe to not be underneath macro --- include/pybind11/stl.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 48031c2a6..8f243502e 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -49,17 +49,31 @@ constexpr forwarded_type forward_like(U &&u) { return std::forward>(std::forward(u)); } +// Checks if a container has a STL style reserve method. +// This will only return true for a `reserve()` with a `void` return. +template +using has_reserve_method = std::is_same().reserve(0)), void>; + template struct set_caster { using type = Type; using key_conv = make_caster; +private: + template ::value, int> = 0> + void reserve_maybe(const anyset &s, Type *) { + value.reserve(s.size()); + } + void reserve_maybe(const anyset &, void *) {} + +public: bool load(handle src, bool convert) { if (!isinstance(src)) { return false; } auto s = reinterpret_borrow(src); value.clear(); + reserve_maybe(s, &value); for (auto entry : s) { key_conv conv; if (!conv.load(entry, convert)) { @@ -94,12 +108,21 @@ struct map_caster { using key_conv = make_caster; using value_conv = make_caster; +private: + template ::value, int> = 0> + void reserve_maybe(const dict &d, Type *) { + value.reserve(d.size()); + } + void reserve_maybe(const dict &, void *) {} + +public: bool load(handle src, bool convert) { if (!isinstance(src)) { return false; } auto d = reinterpret_borrow(src); value.clear(); + reserve_maybe(d, &value); for (auto it : d) { key_conv kconv; value_conv vconv; @@ -160,9 +183,7 @@ struct list_caster { } private: - template < - typename T = Type, - enable_if_t().reserve(0)), void>::value, int> = 0> + template ::value, int> = 0> void reserve_maybe(const sequence &s, Type *) { value.reserve(s.size()); } From da8c730a62a7f9654433f5e2206bd1135a8b57d8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 20:44:19 -0400 Subject: [PATCH 42/71] [pre-commit.ci] pre-commit autoupdate (#4197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.38.0 → v2.38.2](https://github.com/asottile/pyupgrade/compare/v2.38.0...v2.38.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 508b3ea6e..2c317ea7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.38.0" + rev: "v2.38.2" hooks: - id: pyupgrade args: [--py36-plus] From c78dfe6964579c69692eb8b5e57cbed26e409f6e Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 3 Oct 2022 13:44:09 -0400 Subject: [PATCH 43/71] bugfix: Add error checking to list append and insert (#4208) --- include/pybind11/pytypes.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f4ba459f2..565df4375 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2022,14 +2022,20 @@ public: detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } template void append(T &&val) /* py-non-const */ { - PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()); + if (PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) != 0) { + throw error_already_set(); + } } template ::value, int> = 0> void insert(const IdxType &index, ValType &&val) /* py-non-const */ { - PyList_Insert( - m_ptr, ssize_t_cast(index), detail::object_or_cast(std::forward(val)).ptr()); + if (PyList_Insert(m_ptr, + ssize_t_cast(index), + detail::object_or_cast(std::forward(val)).ptr()) + != 0) { + throw error_already_set(); + } } }; From 600d697648f4eda8c88a6884472bd1d2ec0a68ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 21:25:46 -0400 Subject: [PATCH 44/71] [pre-commit.ci] pre-commit autoupdate (#4210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v0.971 → v0.981](https://github.com/pre-commit/mirrors-mypy/compare/v0.971...v0.981) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c317ea7a..ad07577eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -126,7 +126,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.971" + rev: "v0.981" hooks: - id: mypy args: [] From 8275b769129816f97c7665f64c88a0e05d8987df Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 4 Oct 2022 14:01:26 -0400 Subject: [PATCH 45/71] ci: update pre-commit schedule (#4212) --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad07577eb..742e7a30a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,12 @@ # # See https://github.com/pre-commit/pre-commit + +ci: + autoupdate_commit_msg: "chore(deps): update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_schedule: monthly + # third-party content exclude: ^tools/JoinPaths.cmake$ From 864ed1120c704bf27d6ad5ab40645262d0d1e955 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 6 Oct 2022 16:11:34 -0400 Subject: [PATCH 46/71] chore: steal arg_v.value from copied arg in unpacking_collector (#4219) --- include/pybind11/cast.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index a0e32281b..b89e4946a 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1545,7 +1545,7 @@ private: throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = a.value; + m_kwargs[a.name] = std::move(a.value); } void process(list & /*args_list*/, detail::kwargs_proxy kp) { From 6cb214748d6ec3a61fdc35ee27f9a90d170dd242 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 6 Oct 2022 21:02:57 -0700 Subject: [PATCH 47/71] fix: NVCC 11.4.0 - 11.8.0 host bug workaround (#4220) * Work-Around: NVCC 11.4.0 - 11.8.0 Adds a targeted NVCC work around for limited number of CUDA releases. Fixed in NVCC development. * style: pre-commit fixes * CI: Bump CTK Version 11.2 -> 11.7 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- include/pybind11/pybind11.h | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe57e7ce9..99caabf09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,8 +324,8 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.8 • CUDA 11.2 • Ubuntu 20.04" - container: nvidia/cuda:11.2.2-devel-ubuntu20.04 + name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04" + container: nvidia/cuda:11.7.0-devel-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index c889dc416..3f6e27b75 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1578,6 +1578,22 @@ public: return *this; } +// Nvidia's NVCC is broken between 11.4.0 and 11.8.0 +// https://github.com/pybind/pybind11/issues/4193 +#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ >= 4) \ + && (__CUDACC_VER_MINOR__ <= 8) + template + class_ &def(const T &op, const Extra &...extra) { + op.execute(*this, extra...); + return *this; + } + + template + class_ &def_cast(const T &op, const Extra &...extra) { + op.execute_cast(*this, extra...); + return *this; + } +#else template class_ &def(const detail::op_ &op, const Extra &...extra) { op.execute(*this, extra...); @@ -1589,6 +1605,7 @@ public: op.execute_cast(*this, extra...); return *this; } +#endif template class_ &def(const detail::initimpl::constructor &init, const Extra &...extra) { From 4a4215620920ee659145cd34597e14f0553a73f1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 7 Oct 2022 09:20:38 -0700 Subject: [PATCH 48/71] test_eigen.py test_nonunit_stride_to_python bug fix (ASAN failure) (#4217) * Disable test triggering ASAN failure (to pin-point where the problem is). * Fix unsafe "block" implementation in test_eigen.cpp * Undo changes (i.e. revert back to master). * Detect "type_caster for Eigen::Ref made a copy." This is achieved without * reaching into internals, * making test_eigen.cpp depend on pybind11/numpy.h. * Add comment pointing to PR, for easy reference. --- tests/test_eigen.cpp | 39 ++++++++++++++++++++++++++++++++++----- tests/test_eigen.py | 15 ++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/test_eigen.cpp b/tests/test_eigen.cpp index 591dacc62..b0c7bdb48 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen.cpp @@ -197,11 +197,40 @@ TEST_SUBMODULE(eigen, m) { // Return a block of a matrix (gives non-standard strides) m.def("block", - [](const Eigen::Ref &x, - int start_row, - int start_col, - int block_rows, - int block_cols) { return x.block(start_row, start_col, block_rows, block_cols); }); + [m](const py::object &x_obj, + int start_row, + int start_col, + int block_rows, + int block_cols) { + return m.attr("_block")(x_obj, x_obj, start_row, start_col, block_rows, block_cols); + }); + + m.def( + "_block", + [](const py::object &x_obj, + const Eigen::Ref &x, + int start_row, + int start_col, + int block_rows, + int block_cols) { + // See PR #4217 for background. This test is a bit over the top, but might be useful + // as a concrete example to point to when explaining the dangling reference trap. + auto i0 = py::make_tuple(0, 0); + auto x0_orig = x_obj[*i0].cast(); + if (x(0, 0) != x0_orig) { + throw std::runtime_error( + "Something in the type_caster for Eigen::Ref is terribly wrong."); + } + double x0_mod = x0_orig + 1; + x_obj[*i0] = x0_mod; + auto copy_detected = (x(0, 0) != x0_mod); + x_obj[*i0] = x0_orig; + if (copy_detected) { + throw std::runtime_error("type_caster for Eigen::Ref made a copy."); + } + return x.block(start_row, start_col, block_rows, block_cols); + }, + py::keep_alive<0, 1>()); // test_eigen_return_references, test_eigen_keepalive // return value referencing/copying tests: diff --git a/tests/test_eigen.py b/tests/test_eigen.py index 9c6485de3..713432a81 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen.py @@ -217,15 +217,24 @@ def test_negative_stride_from_python(msg): ) +def test_block_runtime_error_type_caster_eigen_ref_made_a_copy(): + with pytest.raises(RuntimeError) as excinfo: + m.block(ref, 0, 0, 0, 0) + assert str(excinfo.value) == "type_caster for Eigen::Ref made a copy." + + def test_nonunit_stride_to_python(): assert np.all(m.diagonal(ref) == ref.diagonal()) assert np.all(m.diagonal_1(ref) == ref.diagonal(1)) for i in range(-5, 7): assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), f"m.diagonal_n({i})" - assert np.all(m.block(ref, 2, 1, 3, 3) == ref[2:5, 1:4]) - assert np.all(m.block(ref, 1, 4, 4, 2) == ref[1:, 4:]) - assert np.all(m.block(ref, 1, 4, 3, 2) == ref[1:4, 4:]) + # Must be order="F", otherwise the type_caster will make a copy and + # m.block() will return a dangling reference (heap-use-after-free). + rof = np.asarray(ref, order="F") + assert np.all(m.block(rof, 2, 1, 3, 3) == rof[2:5, 1:4]) + assert np.all(m.block(rof, 1, 4, 4, 2) == rof[1:, 4:]) + assert np.all(m.block(rof, 1, 4, 3, 2) == rof[1:4, 4:]) def test_eigen_ref_to_python(): From 7c6f2f80a73e330642226138ae3ddaf448a4f672 Mon Sep 17 00:00:00 2001 From: Daniel Galvez Date: Fri, 7 Oct 2022 12:27:54 -0700 Subject: [PATCH 49/71] fix: PyCapsule_GetDestructor is allowed to return a nullptr destructor (#4221) * fix: PyCapsule_GetDestructor is allowed to return a nullptr destructor Previously, this code would error out if the destructor happened to be a nullptr. This is incorrect. nullptrs are allowed for capsule destructors. "It is legal for a capsule to have a NULL destructor. This makes a NULL return code somewhat ambiguous; use PyCapsule_IsValid() or PyErr_Occurred() to disambiguate." See: https://docs.python.org/3/c-api/capsule.html#c.PyCapsule_GetDestructor I noticed this while working on a type caster related to #3858 DLPack happens to allow the destructor not to be defined on a capsule, and I encountered such a case. See: https://github.com/dmlc/dlpack/blob/e2bdd3bee8cb6501558042633fa59144cc8b7f5f/include/dlpack/dlpack.h#L219 * Add test for the fix. * Update tests/test_pytypes.cpp I tried this locally and it works! I never knew that there are cases where `reinterpret_cast` does not work but `static_cast` does. Let's see if all compilers are happy with this. Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 12 ++++++------ tests/test_pytypes.cpp | 6 ++++++ tests/test_pytypes.py | 11 +++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 565df4375..29506b7ea 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1829,18 +1829,18 @@ public: // guard if destructor called while err indicator is set error_scope error_guard; auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr) { - if (PyErr_Occurred()) { - throw error_already_set(); - } - pybind11_fail("Unable to get capsule context"); + if (PyErr_Occurred()) { + throw error_already_set(); } const char *name = get_name_in_error_scope(o); void *ptr = PyCapsule_GetPointer(o, name); if (ptr == nullptr) { throw error_already_set(); } - destructor(ptr); + + if (destructor != nullptr) { + destructor(ptr); + } }); if (!m_ptr || PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index da0dc8f6b..c95ff8230 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -289,6 +289,12 @@ TEST_SUBMODULE(pytypes, m) { return capsule; }); + m.def("return_capsule_with_explicit_nullptr_dtor", []() { + py::print("creating capsule with explicit nullptr dtor"); + return py::capsule(reinterpret_cast(1234), + static_cast(nullptr)); // PR #4221 + }); + // test_accessors m.def("accessor_api", [](const py::object &o) { auto d = py::dict(); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index a34eaa59e..070849fc5 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -299,6 +299,17 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_explicit_nullptr_dtor() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule with explicit nullptr dtor + """ + ) + def test_accessors(): class SubTestObject: From da104a9efd0357b5c144c67eb641ae0b9e23012d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 9 Oct 2022 21:50:35 -0700 Subject: [PATCH 50/71] Reproducer and fix for issue encountered in smart_holder update. (#4228) * Reproducer for issue encountered in smart_holder update. * clang-tidy compatibility (untested). * Add `enable_if_t` to workaround. * Bug fix: Move `PYBIND11_USING_WORKAROUND_FOR_CUDA_11_4_THROUGH_8` determination to detail/common.h So that it actually is defined in pybind11.h * Try using the workaround (which is nicer than the original code) universally. * Reduce reproducer for CUDA 11.7 issue encountered in smart_holder update. This commit tested in isolation on top of current master + first version of reproducer (62311eb431849d135a5db84f6c75ec390f2ede7c). Succeeds with Debian Clang 14.0.6 C++17 (and probably all other compilers). Fails for CUDA 11.7: ``` cd /build/tests && /usr/local/cuda/bin/nvcc -forward-unknown-to-host-compiler -Dpybind11_tests_EXPORTS -I/mounted_pybind11/include -isystem=/usr/include/python3.10 -g --generate-code=arch=compute_52,code=[compute_52,sm_52] -Xcompiler=-fPIC -Xcompiler=-fvisibility=hidden -Werror all-warnings -std=c++17 -MD -MT tests/CMakeFiles/pybind11_tests.dir/test_class.cpp.o -MF CMakeFiles/pybind11_tests.dir/test_class.cpp.o.d -x cu -c /mounted_pybind11/tests/test_class.cpp -o CMakeFiles/pybind11_tests.dir/test_class.cpp.o /mounted_pybind11/tests/test_class.cpp(53): error: more than one instance of overloaded function "pybind11::class_::def [with type_=test_class::pr4220_tripped_over_this::Empty0, options=<>]" matches the argument list: function template "pybind11::class_ &pybind11::class_::def(const char *, Func &&, const Extra &...) [with type_=test_class::pr4220_tripped_over_this::Empty0, options=<>]" /mounted_pybind11/include/pybind11/pybind11.h(1557): here function template "pybind11::class_ &pybind11::class_::def(const T &, const Extra &...) [with type_=test_class::pr4220_tripped_over_this::Empty0, options=<>]" /mounted_pybind11/include/pybind11/pybind11.h(1586): here argument types are: (const char [8], ) object type is: pybind11::class_ 1 error detected in the compilation of "/mounted_pybind11/tests/test_class.cpp". ``` --- include/pybind11/operators.h | 1 + include/pybind11/pybind11.h | 21 ++------------------- tests/test_class.cpp | 22 ++++++++++++++++++++++ tests/test_class.py | 7 +++++++ 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/include/pybind11/operators.h b/include/pybind11/operators.h index a0c3b78d6..16a88ae17 100644 --- a/include/pybind11/operators.h +++ b/include/pybind11/operators.h @@ -84,6 +84,7 @@ struct op_impl {}; /// Operator implementation generator template struct op_ { + static constexpr bool op_enable_if_hook = true; template void execute(Class &cl, const Extra &...extra) const { using Base = typename Class::type; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3f6e27b75..fb4b7578d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1578,34 +1578,17 @@ public: return *this; } -// Nvidia's NVCC is broken between 11.4.0 and 11.8.0 -// https://github.com/pybind/pybind11/issues/4193 -#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ >= 4) \ - && (__CUDACC_VER_MINOR__ <= 8) - template + template = 0> class_ &def(const T &op, const Extra &...extra) { op.execute(*this, extra...); return *this; } - template + template = 0> class_ &def_cast(const T &op, const Extra &...extra) { op.execute_cast(*this, extra...); return *this; } -#else - template - class_ &def(const detail::op_ &op, const Extra &...extra) { - op.execute(*this, extra...); - return *this; - } - - template - class_ &def_cast(const detail::op_ &op, const Extra &...extra) { - op.execute_cast(*this, extra...); - return *this; - } -#endif template class_ &def(const detail::initimpl::constructor &init, const Extra &...extra) { diff --git a/tests/test_class.cpp b/tests/test_class.cpp index c8b8071dc..18c8d358b 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -36,6 +36,26 @@ struct NoBraceInitialization { std::vector vec; }; +namespace test_class { +namespace pr4220_tripped_over_this { // PR #4227 + +template +struct SoEmpty {}; + +template +std::string get_msg(const T &) { + return "This is really only meant to exercise successful compilation."; +} + +using Empty0 = SoEmpty<0x0>; + +void bind_empty0(py::module_ &m) { + py::class_(m, "Empty0").def(py::init<>()).def("get_msg", get_msg); +} + +} // namespace pr4220_tripped_over_this +} // namespace test_class + TEST_SUBMODULE(class_, m) { // test_instance struct NoConstructor { @@ -517,6 +537,8 @@ TEST_SUBMODULE(class_, m) { py::class_(gt, "OtherDuplicateNested"); py::class_(gt, "YetAnotherDuplicateNested"); }); + + test_class::pr4220_tripped_over_this::bind_empty0(m); } template diff --git a/tests/test_class.py b/tests/test_class.py index ff9196f0f..47ba450fc 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -469,3 +469,10 @@ def test_register_duplicate_class(): m.register_duplicate_nested_class_type(ClassScope) expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!' assert str(exc_info.value) == expected + + +def test_pr4220_tripped_over_this(): + assert ( + m.Empty0().get_msg() + == "This is really only meant to exercise successful compilation." + ) From ff7b69714d76c312b0bcc45e4a8b854783f6dbc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 00:50:40 -0400 Subject: [PATCH 51/71] chore(deps): bump jwlawson/actions-setup-cmake from 1.12 to 1.13 (#4233) Bumps [jwlawson/actions-setup-cmake](https://github.com/jwlawson/actions-setup-cmake) from 1.12 to 1.13. - [Release notes](https://github.com/jwlawson/actions-setup-cmake/releases) - [Commits](https://github.com/jwlawson/actions-setup-cmake/compare/v1.12...v1.13) --- updated-dependencies: - dependency-name: jwlawson/actions-setup-cmake dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/configure.yml | 2 +- .github/workflows/upstream.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99caabf09..38694bcfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: run: brew install boost - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Cache wheels if: runner.os == 'macOS' @@ -202,7 +202,7 @@ jobs: debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Valgrind cache if: matrix.valgrind @@ -465,7 +465,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Configure shell: bash @@ -754,7 +754,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.11.0 @@ -807,7 +807,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.11.0 @@ -858,7 +858,7 @@ jobs: python3 -m pip install -r tests/requirements.txt - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Configure C++20 run: > diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index edcad4198..5ec0dd462 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -51,7 +51,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 with: cmake-version: ${{ matrix.cmake }} diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 95ff4cb83..366284acf 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -31,7 +31,7 @@ jobs: run: sudo apt-get install libboost-dev - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare env run: | From 0927c4d19e33687b463346ecb7d194493ee53b4c Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 11 Oct 2022 16:07:42 -0400 Subject: [PATCH 52/71] chore: Improve PyCapsule exception handling (#4232) * Improve pycapsule error handling corner cases * Handle another corner case * Simplify err handling code --- include/pybind11/pytypes.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 29506b7ea..2e6b755ca 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1829,7 +1829,7 @@ public: // guard if destructor called while err indicator is set error_scope error_guard; auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (PyErr_Occurred()) { + if (destructor == nullptr && PyErr_Occurred()) { throw error_already_set(); } const char *name = get_name_in_error_scope(o); @@ -1843,7 +1843,7 @@ public: } }); - if (!m_ptr || PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) { + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { throw error_already_set(); } } From 8781daf6e6c2a80524d787110094584a28619f2a Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 12 Oct 2022 16:46:40 -0400 Subject: [PATCH 53/71] chore: Optimize iterator advance() call (#4237) --- include/pybind11/pytypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2e6b755ca..0dda48145 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1382,7 +1382,7 @@ public: private: void advance() { value = reinterpret_steal(PyIter_Next(m_ptr)); - if (PyErr_Occurred()) { + if (value.ptr() == nullptr && PyErr_Occurred()) { throw error_already_set(); } } From 964c49978f7e7227f2968c359f4f05255d2b54f4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 12 Oct 2022 15:43:43 -0700 Subject: [PATCH 54/71] Minor `py::capsule` cleanup. No functional change. (#4238) Use `PyCapsule_Destructor` (part of the stable Python ABI) instead of spelling out the C `typedef`. The deprecation message is misleading. Replace with a message pointing to another existing ctor. Background: According to @wjakob the original motivation for deprecating the ctor (in PR #752) was to hide Python C API details, but PR #902 brought those back with a new ctor, it cannot be avoided. Having a `PyCapsule_Destructor` or a `void (*destructor)(void *)` are two separate and valid use cases. --- include/pybind11/pytypes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0dda48145..4b93d2018 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1809,16 +1809,16 @@ public: explicit capsule(const void *value, const char *name = nullptr, - void (*destructor)(PyObject *) = nullptr) + PyCapsule_Destructor destructor = nullptr) : object(PyCapsule_New(const_cast(value), name, destructor), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } } - PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input") - capsule(const void *value, void (*destruct)(PyObject *)) - : object(PyCapsule_New(const_cast(value), nullptr, destruct), stolen_t{}) { + PYBIND11_DEPRECATED("Please use the ctor with value, name, destructor args") + capsule(const void *value, PyCapsule_Destructor destructor) + : object(PyCapsule_New(const_cast(value), nullptr, destructor), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } From 5b5547bc1bcfdf58ff4febbb3a08f1764ac4ec7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:57:55 -0400 Subject: [PATCH 55/71] chore(deps): bump ilammy/msvc-dev-cmd from 1.11.0 to 1.12.0 (#4242) Bumps [ilammy/msvc-dev-cmd](https://github.com/ilammy/msvc-dev-cmd) from 1.11.0 to 1.12.0. - [Release notes](https://github.com/ilammy/msvc-dev-cmd/releases) - [Commits](https://github.com/ilammy/msvc-dev-cmd/compare/v1.11.0...v1.12.0) --- updated-dependencies: - dependency-name: ilammy/msvc-dev-cmd dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38694bcfa..bc7175d82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -757,7 +757,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.11.0 + uses: ilammy/msvc-dev-cmd@v1.12.0 with: arch: x86 @@ -810,7 +810,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.11.0 + uses: ilammy/msvc-dev-cmd@v1.12.0 with: arch: x86 From b926396bdf42ef1e5f1f67f9d42277ce0c88adb8 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 17 Oct 2022 19:15:08 -0400 Subject: [PATCH 56/71] bugfix: py contains raises errors when appropiate (#4209) * bugfix: contains now throws an exception if the key is not hashable * Fix tests and improve robustness * Remove todo * Workaround PyPy corner case * PyPy xfail * Fix typo * fix xfail * Make clang-tidy happy * Remove redundant exc checking --- include/pybind11/pytypes.h | 12 ++++++++++-- tests/test_pytypes.cpp | 5 ++++- tests/test_pytypes.py | 25 +++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 4b93d2018..37fc49a0e 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1967,7 +1967,11 @@ public: void clear() /* py-non-const */ { PyDict_Clear(ptr()); } template bool contains(T &&key) const { - return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward(key)).ptr()) == 1; + auto result = PyDict_Contains(m_ptr, detail::object_or_cast(std::forward(key)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } private: @@ -2053,7 +2057,11 @@ public: bool empty() const { return size() == 0; } template bool contains(T &&val) const { - return PySet_Contains(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) == 1; + auto result = PySet_Contains(m_ptr, detail::object_or_cast(std::forward(val)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } }; diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index c95ff8230..99237ccef 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -183,7 +183,7 @@ TEST_SUBMODULE(pytypes, m) { return d2; }); m.def("dict_contains", - [](const py::dict &dict, py::object val) { return dict.contains(val); }); + [](const py::dict &dict, const py::object &val) { return dict.contains(val); }); m.def("dict_contains", [](const py::dict &dict, const char *val) { return dict.contains(val); }); @@ -538,6 +538,9 @@ TEST_SUBMODULE(pytypes, m) { m.def("hash_function", [](py::object obj) { return py::hash(std::move(obj)); }); + m.def("obj_contains", + [](py::object &obj, const py::object &key) { return obj.contains(key); }); + m.def("test_number_protocol", [](const py::object &a, const py::object &b) { py::list l; l.append(a.equal(b)); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 070849fc5..3ed7b9c94 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -168,6 +168,31 @@ def test_dict(capture, doc): assert m.dict_keyword_constructor() == {"x": 1, "y": 2, "z": 3} +class CustomContains: + d = {"key": None} + + def __contains__(self, m): + return m in self.d + + +@pytest.mark.parametrize( + "arg,func", + [ + (set(), m.anyset_contains), + (dict(), m.dict_contains), + (CustomContains(), m.obj_contains), + ], +) +@pytest.mark.xfail("env.PYPY and sys.pypy_version_info < (7, 3, 10)", strict=False) +def test_unhashable_exceptions(arg, func): + class Unhashable: + __hash__ = None + + with pytest.raises(TypeError) as exc_info: + func(arg, Unhashable()) + assert "unhashable type:" in str(exc_info.value) + + def test_tuple(): assert m.tuple_no_args() == () assert m.tuple_ssize_t() == () From 895fc66320beee40e5bc2caf4ff585ce8a067a8b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 20 Oct 2022 05:49:52 -0700 Subject: [PATCH 57/71] ci: update PGI build (old one no longer signed) (#4260) * Simply replace "22.3" with "22.9" to see what happens. * Remove PYBIND11_TEST_FILTER to see what happens. * Revert "Remove PYBIND11_TEST_FILTER to see what happens." This reverts commit 0cba2cef0c5c5bcbf57cad219c7d5dfb0cda241c. * Remove only test_smart_ptr.cpp to see what happens. * Revert "Remove only test_smart_ptr.cpp to see what happens." This reverts commit 8e9df22c85292003abcab50260393e547567fc18. * Remove only test_virtual_functions.cpp to see what happens. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc7175d82..2951a0ad1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: runs-on: ubuntu-latest - name: "🐍 3 • CentOS7 / PGI 22.3 • x64" + name: "🐍 3 • CentOS7 / PGI 22.9 • x64" container: centos:7 steps: @@ -401,7 +401,7 @@ jobs: run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils - name: Install NVidia HPC SDK - run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.3 + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 # On CentOS 7, we have to filter a few tests (compiler internal error) # and allow deeper template recursion (not needed on CentOS 8 with a newer @@ -411,12 +411,12 @@ jobs: shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.3 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=11 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ - -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" # Building before installing Pip should produce a warning but not an error - name: Build From 412918d12c12dca5acca813b37328bb280b68660 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:35:18 -0400 Subject: [PATCH 58/71] feat: add entrypoint for cmake modules dir (#4258) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- setup.py | 2 ++ tests/extra_python_package/test_files.py | 2 ++ tools/setup_main.py.in | 3 +++ 3 files changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 68573519c..9097440bf 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,8 @@ with remove_output("pybind11/include", "pybind11/share"): stdout=sys.stdout, stderr=sys.stderr, ) + if not global_sdist: + Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 8e1ddd850..caf320671 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -160,6 +160,7 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") + files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -246,6 +247,7 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } + files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 65198bdb6..92ba04a95 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -36,6 +36,9 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", + ], + "cmake.modules": [ + "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From a8f211073bb4020f351e3ca8e241a3b8f865be2d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 20 Oct 2022 10:58:04 -0400 Subject: [PATCH 59/71] docs: update changelog (#4265) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- docs/changelog.rst | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c8ff423f..1199ecded 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,107 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Version 2.10.1 (Oct 2?, 2022) +----------------------------- + + +Changes: + + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 `_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 `_ + +Bug fixes: + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 `_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 `_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 `_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 `_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 `_ + released with v2.10.0. + `#4079 `_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 `_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 `_ + +* Further improve another error in exception handling. + `#4232 `_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 `_ + + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 `_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 `_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 `_ + +* Optimize iterator advancement in C++ bindings. + `#4237 `_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 `_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 `_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 `_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 `_ + + +Build system improvements: + +* Experimental support for ``cmake.modules`` entrypoint. + `#4258 `_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 `_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 `_ + +* Followup to `#3948 `_, fixing vcpkg again. + `#4123 `_ + Version 2.10.0 (Jul 15, 2022) ----------------------------- From 251516bce69f42007d403eebf526194b7e2a803b Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 12:51:26 -0400 Subject: [PATCH 60/71] Cleanup casters to release none() to avoid ref counting (#4269) --- include/pybind11/cast.h | 6 +++--- include/pybind11/functional.h | 2 +- include/pybind11/stl.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index b89e4946a..5699d4fb0 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -248,7 +248,7 @@ public: return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; @@ -291,7 +291,7 @@ public: if (ptr) { return capsule(ptr).release(); } - return none().inc_ref(); + return none().release(); } template @@ -537,7 +537,7 @@ public: static handle cast(const CharT *src, return_value_policy policy, handle parent) { if (src == nullptr) { - return pybind11::none().inc_ref(); + return pybind11::none().release(); } return StringCaster::cast(StringType(src), policy, parent); } diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 4034990d8..102d1a938 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -110,7 +110,7 @@ public: template static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) { if (!f_) { - return none().inc_ref(); + return none().release(); } auto result = f_.template target(); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 8f243502e..2d144b598 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -311,7 +311,7 @@ struct optional_caster { template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!src) { - return none().inc_ref(); + return none().release(); } if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); From eaa5f7bd40b0339a78d97b448b78af158b20c15e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 21 Oct 2022 17:25:53 -0400 Subject: [PATCH 61/71] Revert "feat: add entrypoint for cmake modules dir" (#4270) * Revert "feat: add entrypoint for cmake modules dir (#4258)" This reverts commit 1d4a65e2f145a25b66e44e9d5edae9ec11abbc73. * docs: revert changelog mention too --- docs/changelog.rst | 3 --- setup.py | 2 -- tests/extra_python_package/test_files.py | 2 -- tools/setup_main.py.in | 3 --- 4 files changed, 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1199ecded..df3181c52 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -103,9 +103,6 @@ Performance and style: Build system improvements: -* Experimental support for ``cmake.modules`` entrypoint. - `#4258 `_ - * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ diff --git a/setup.py b/setup.py index 9097440bf..68573519c 100644 --- a/setup.py +++ b/setup.py @@ -144,8 +144,6 @@ with remove_output("pybind11/include", "pybind11/share"): stdout=sys.stdout, stderr=sys.stderr, ) - if not global_sdist: - Path("pybind11/share/cmake/pybind11/__init__.py").touch() txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index caf320671..8e1ddd850 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -160,7 +160,6 @@ def test_build_sdist(monkeypatch, tmpdir): files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") - files.add("pybind11/share/cmake/pybind11/__init__.py") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: @@ -247,7 +246,6 @@ def tests_build_wheel(monkeypatch, tmpdir): "dist-info/entry_points.txt", "dist-info/top_level.txt", } - files.add("pybind11/share/cmake/pybind11/__init__.py") with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 92ba04a95..65198bdb6 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -36,9 +36,6 @@ setup( ], "pipx.run": [ "pybind11 = pybind11.__main__:main", - ], - "cmake.modules": [ - "pybind11 = pybind11.share.cmake.pybind11", ] }, cmdclass=cmdclass From 305c471131a385e067738f589dde0f8f1da5f683 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Fri, 21 Oct 2022 18:04:01 -0400 Subject: [PATCH 62/71] fix: Revert pfect args make iterator (#4234) * Revert "chore: perfectly forward all make_iterator args (#3980)" This reverts commit 8da58da53996e9aa40d051b3b84cb3220fdbbb58. * Redo unrelated optimization in commit * Add tests for ambiguous overloads --- include/pybind11/pybind11.h | 22 ++++++++-------------- tests/test_sequences_and_iterators.cpp | 19 +++++++++++++++++++ tests/test_sequences_and_iterators.py | 8 ++++++++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index fb4b7578d..e662236d0 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2333,7 +2333,7 @@ template -iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values @@ -2359,7 +2359,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) Policy); } - return cast(state{std::forward(first), std::forward(last), true}); + return cast(state{first, last, true}); } PYBIND11_NAMESPACE_END(detail) @@ -2370,15 +2370,13 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2388,15 +2386,13 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2406,15 +2402,13 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index b867f49a2..1de65edbf 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) { []() { return py::make_iterator(list); }); m.def("make_iterator_2", []() { return py::make_iterator(list); }); + + // test_iterator on c arrays + // #4100: ensure lvalue required as increment operand + class CArrayHolder { + public: + CArrayHolder(double x, double y, double z) { + values[0] = x; + values[1] = y; + values[2] = z; + }; + double values[3]; + }; + + py::class_(m, "CArrayHolder") + .def(py::init()) + .def( + "__iter__", + [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); }, + py::keep_alive<0, 1>()); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index 062e3b3d3..de486e3e8 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -241,3 +241,11 @@ def test_iterator_rvp(): assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) + + +def test_carray_iterator(): + """#4100: Check for proper iterator overload with C-Arrays""" + args_gt = list(float(i) for i in range(3)) + arr_h = m.CArrayHolder(*args_gt) + args = list(arr_h) + assert args_gt == args From ce63bcb9808d6bb1e7e6dddb4fde081b10d24877 Mon Sep 17 00:00:00 2001 From: Lalaland Date: Sat, 22 Oct 2022 16:52:35 -0700 Subject: [PATCH 63/71] Fix casts to void* (#4275) * Fix casts to void* * Improve tests * style: pre-commit fixes * remove c style cast Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Aaron Gokaslan --- include/pybind11/cast.h | 6 ++---- tests/test_class.cpp | 18 +++++++++++++++++- tests/test_class.py | 2 ++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 5699d4fb0..430c62f35 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1179,11 +1179,9 @@ enable_if_t::value, T> cast_safe(object &&) pybind11_fail("Internal error: cast_safe fallback invoked"); } template -enable_if_t>::value, void> cast_safe(object &&) {} +enable_if_t::value, void> cast_safe(object &&) {} template -enable_if_t, - std::is_same>>::value, - T> +enable_if_t, std::is_void>::value, T> cast_safe(object &&o) { return pybind11::cast(std::move(o)); } diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 18c8d358b..fc1c17b17 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -384,6 +384,8 @@ TEST_SUBMODULE(class_, m) { protected: virtual int foo() const { return value; } + virtual void *void_foo() { return static_cast(&value); } + virtual void *get_self() { return static_cast(this); } private: int value = 42; @@ -392,6 +394,8 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } + void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); } + void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); } }; class PublicistB : public ProtectedB { @@ -401,11 +405,23 @@ TEST_SUBMODULE(class_, m) { // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) ~PublicistB() override{}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; + using ProtectedB::get_self; + using ProtectedB::void_foo; }; + m.def("read_foo", [](const void *original) { + const int *ptr = reinterpret_cast(original); + return *ptr; + }); + + m.def("pointers_equal", + [](const void *original, const void *comparison) { return original == comparison; }); + py::class_(m, "ProtectedB") .def(py::init<>()) - .def("foo", &PublicistB::foo); + .def("foo", &PublicistB::foo) + .def("void_foo", &PublicistB::void_foo) + .def("get_self", &PublicistB::get_self); // test_brace_initialization struct BraceInitialization { diff --git a/tests/test_class.py b/tests/test_class.py index 47ba450fc..7c1ed2060 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -313,6 +313,8 @@ def test_bind_protected_functions(): b = m.ProtectedB() assert b.foo() == 42 + assert m.read_foo(b.void_foo()) == 42 + assert m.pointers_equal(b.get_self(), b) class C(m.ProtectedB): def __init__(self): From 0e82c360942907f5a2f379e64e0d211aaff80774 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 23 Oct 2022 00:32:17 -0400 Subject: [PATCH 64/71] fix: add flag for overriding classic Python search values (#4195) * fix: PyPy needs to overrite broken FindPythonInterp values Signed-off-by: Henry Schreiner * fix: add flag to opt-in to new (cross-compile) behavior Signed-off-by: Henry Schreiner * Apply suggestions from code review Signed-off-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index a5a628fd7..bbbd9f9ec 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0) return() endif() +option( + PYBIND11_PYTHONLIBS_OVERWRITE + "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values." + ON) # Can manually set values when cross-compiling macro(_PYBIND11_GET_IF_UNDEF lst index name) - if(NOT DEFINED "${name}") + if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}") list(GET "${lst}" "${index}" "${name}") endif() endmacro() From 1f187d9a35642f04bfa2e95aa16a81bb2d2220fe Mon Sep 17 00:00:00 2001 From: Vemund Handeland Date: Sun, 23 Oct 2022 20:57:45 +0200 Subject: [PATCH 65/71] Fix char8_t support (#4278) Standard library macro __cpp_lib_char8_t is only available after including standard header --- include/pybind11/detail/common.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6da7ef859..b43100b95 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -205,10 +205,6 @@ # endif #endif -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif - #include #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." @@ -259,6 +255,11 @@ # endif #endif +// Must be after including or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the From 738a6f83432e45c09bdd6b284c42f442cbec8bdc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 26 Oct 2022 10:41:51 -0400 Subject: [PATCH 66/71] ci: move to final release of 3.11 (#4286) Signed-off-by: Henry Schreiner Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2951a0ad1..b56e8f447 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - '3.6' - '3.9' - '3.10' - - '3.11-dev' + - '3.11' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -119,7 +119,7 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -146,7 +146,7 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build build2 --target cpptest # Third build - C++17 mode with unstable ABI @@ -186,7 +186,7 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - - python-version: "3.11-dev" + - python-version: "3.11" python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" From 5b395c9b44d9bac2082bd71a1d2ed3cbb4af35db Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Sat, 29 Oct 2022 11:12:24 -0400 Subject: [PATCH 67/71] fix: improve bytes to str decoding error handling (#4294) * (bugfix): Improve bytes to str decoding error handling * regroup test * Further broaden tests * Add another decode error test * Fix bug in tests * Reviewer suggestions --- include/pybind11/pytypes.h | 9 +++++++++ tests/test_pytypes.cpp | 5 +++++ tests/test_pytypes.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 37fc49a0e..80a2e2228 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1432,6 +1432,9 @@ public: str(const char *c, const SzType &n) : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1441,6 +1444,9 @@ public: // NOLINTNEXTLINE(google-explicit-constructor) str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1598,6 +1604,9 @@ inline str::str(const bytes &b) { } auto obj = reinterpret_steal(PyUnicode_FromStringAndSize(buffer, length)); if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } m_ptr = obj.release().ptr(); diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 99237ccef..ea8d03958 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -206,7 +206,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; }); m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; }); m.def("str_from_string", []() { return py::str(std::string("baz")); }); + m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); }); + m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); + m.def("str_from_bytes_input", + [](const py::bytes &encoded_str) { return py::str(encoded_str); }); + m.def("str_from_object", [](const py::object &obj) { return py::str(obj); }); m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); }); m.def("str_from_handle", [](py::handle h) { return py::str(h); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 3ed7b9c94..079ee7ca5 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -244,6 +244,20 @@ def test_str(doc): m.str_from_string_from_str(ucs_surrogates_str) +@pytest.mark.parametrize( + "func", + [ + m.str_from_bytes_input, + m.str_from_cstr_input, + m.str_from_std_string_input, + ], +) +def test_surrogate_pairs_unicode_error(func): + input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") + with pytest.raises(UnicodeDecodeError): + func(input_str) + + def test_bytes(doc): assert m.bytes_from_char_ssize_t().decode() == "green" assert m.bytes_from_char_size_t().decode() == "purple" From 15fde1def2a8b620aea5fd557b71adb29e73b3d6 Mon Sep 17 00:00:00 2001 From: Chekov2k Date: Sun, 30 Oct 2022 15:57:23 +0000 Subject: [PATCH 68/71] Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` option (cmake, C++ define) (#4216) * Add option to force the use of the PYPY GIL scoped acquire/release logic to support nested gil access, see https://github.com/pybind/pybind11/issues/1276 and https://github.com/pytorch/pytorch/issues/83101 * Apply suggestions from code review * Update CMakeLists.txt * docs: update upgrade guide * Update docs/upgrade.rst * All bells & whistles. * Add Reminder to common.h, so that we will not forget to purge `!WITH_THREAD` branches when dropping Python 3.6 * New sentence instead of semicolon. * Temporarily pull in snapshot of PR #4246 * Add `test_release_acquire` * Add more unit tests for nested gil locking * Add test_report_builtins_internals_keys * Very minor enhancement: sort list only after filtering. * Revert change in docs/upgrade.rst * Add test_multi_acquire_release_cross_module, while also forcing unique PYBIND11_INTERNALS_VERSION for cross_module_gil_utils.cpp * Hopefully fix apparently new ICC error. ``` 2022-10-28T07:57:54.5187728Z -- The CXX compiler identification is Intel 2021.7.0.20220726 ... 2022-10-28T07:58:53.6758994Z icpc: remark #10441: The Intel(R) C++ Compiler Classic (ICC) is deprecated and will be removed from product release in the second half of 2023. The Intel(R) oneAPI DPC++/C++ Compiler (ICX) is the recommended compiler moving forward. Please transition to use this compiler. Use '-diag-disable=10441' to disable this message. 2022-10-28T07:58:54.5801597Z In file included from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/type_caster_base.h(15), 2022-10-28T07:58:54.5803794Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../cast.h(15), 2022-10-28T07:58:54.5805740Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/../attr.h(14), 2022-10-28T07:58:54.5809556Z from /home/runner/work/pybind11/pybind11/include/pybind11/detail/class.h(12), 2022-10-28T07:58:54.5812154Z from /home/runner/work/pybind11/pybind11/include/pybind11/pybind11.h(13), 2022-10-28T07:58:54.5948523Z from /home/runner/work/pybind11/pybind11/tests/cross_module_gil_utils.cpp(13): 2022-10-28T07:58:54.5949009Z /home/runner/work/pybind11/pybind11/include/pybind11/detail/../detail/internals.h(177): error #2282: unrecognized GCC pragma 2022-10-28T07:58:54.5949374Z PYBIND11_TLS_KEY_INIT(tstate) 2022-10-28T07:58:54.5949579Z ^ 2022-10-28T07:58:54.5949695Z ``` * clang-tidy fixes * Workaround for PYPY WIN exitcode None * Revert "Temporarily pull in snapshot of PR #4246" This reverts commit 23ac16e859150f27fda25ca865cabcb4444e0770. * Another workaround for PYPY WIN exitcode None * Clean up how the tests are run "run in process" Part 1: uniformity * Clean up how the tests are run "run in process" Part 2: use `@pytest.mark.parametrize` and clean up the naming. * Skip some tests `#if defined(THREAD_SANITIZER)` (tested with TSAN using the Google-internal toolchain). * Run all tests again but ignore ThreadSanitizer exitcode 66 (this is less likely to mask unrelated ThreadSanitizer issues in the future). * bug fix: missing common.h include before using `PYBIND11_SIMPLE_GIL_MANAGEMENT` For the tests in the github CI this does not matter, because `PYBIND11_SIMPLE_GIL_MANAGEMENT` is always defined from the command line, but when monkey-patching common.h locally, it matters. * if process.exitcode is None: assert t_delta > 9.9 * More sophisiticated `_run_in_process()` implementation, clearly reporting `DEADLOCK`, additionally exercised via added `intentional_deadlock()` * Wrap m.intentional_deadlock in a Python function, for `ForkingPickler` compatibility. ``` > ForkingPickler(file, protocol).dump(obj) E TypeError: cannot pickle 'PyCapsule' object ``` Observed with all Windows builds including mingw but not PyPy, and macos-latest with Python 3.9, 3.10, 3.11 but not 3.6. * Add link to potential solution for WOULD-BE-NICE-TO-HAVE feature. * Add `SKIP_IF_DEADLOCK = True` option, to not pollute the CI results with expected `DEADLOCK` failures while we figure out what to do about them. * Add COPY-PASTE-THIS: gdb ... command (to be used for debugging the detected deadlock) * style: pre-commit fixes * Do better than automatic pre-commit fixes. * Add `PYBIND11_SIMPLE_GIL_MANAGEMENT` to `pytest_report_header()` (so that we can easily know when harvesting deadlock information from the CI logs). Co-authored-by: Arnim Balzer Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 + CMakeLists.txt | 6 + include/pybind11/detail/common.h | 5 + include/pybind11/detail/internals.h | 16 +- include/pybind11/gil.h | 53 ++++-- tests/conftest.py | 1 + tests/cross_module_gil_utils.cpp | 67 +++++++- tests/pybind11_tests.cpp | 6 + tests/test_embed/test_interpreter.cpp | 1 - tests/test_gil_scoped.cpp | 107 +++++++++++- tests/test_gil_scoped.py | 227 +++++++++++++++++++++----- 11 files changed, 433 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56e8f447..a11cae1ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,10 +102,12 @@ jobs: run: python -m pip install pytest-github-actions-annotate-failures # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -129,10 +131,12 @@ jobs: run: git clean -fdx # Second build - C++17 mode and in a build directory + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. - name: Configure C++17 run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 diff --git a/CMakeLists.txt b/CMakeLists.txt index ee0975bc1..3be6f7a49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,10 +91,16 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index b43100b95..a3e0bc9b3 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -206,6 +206,7 @@ #endif #include +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." #endif @@ -229,6 +230,10 @@ # undef copysign #endif +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d47084e26..7de779434 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -9,6 +9,12 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" #include @@ -49,7 +55,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // `Py_LIMITED_API` anyway. # if PYBIND11_INTERNALS_VERSION > 4 # define PYBIND11_TLS_KEY_REF Py_tss_t & -# ifdef __GNUC__ +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer // for every field. # define PYBIND11_TLS_KEY_INIT(var) \ @@ -169,10 +175,12 @@ struct internals { PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) # endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 @@ -408,6 +416,10 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { @@ -417,6 +429,8 @@ PYBIND11_NOINLINE internals &get_internals() { ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif error_scope err_scope; PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 1ef5f0a8c..cb0028d50 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -10,7 +10,10 @@ #pragma once #include "detail/common.h" -#include "detail/internals.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked(); PYBIND11_NAMESPACE_END(detail) -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -62,11 +67,11 @@ public: if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); } else { @@ -87,20 +92,20 @@ public: PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif PyThreadState_Clear(tstate); if (active) { PyThreadState_DeleteCurrent(); @@ -178,12 +183,14 @@ private: bool disassoc; bool active = true; }; -#elif defined(PYPY_VERSION) + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + class gil_scoped_acquire { PyGILState_STATE state; public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire() : state{PyGILState_Ensure()} {} gil_scoped_acquire(const gil_scoped_acquire &) = delete; gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } @@ -194,19 +201,39 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release() : state{PyEval_SaveThread()} {} gil_scoped_release(const gil_scoped_release &) = delete; gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; -#else + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; + class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; -#endif + +#endif // WITH_THREAD PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/conftest.py b/tests/conftest.py index 02ce263af..f5ddb9f12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -210,4 +210,5 @@ def pytest_report_header(config): f" {pybind11_tests.compiler_info}" f" {pybind11_tests.cpp_std}" f" {pybind11_tests.PYBIND11_INTERNALS_ID}" + f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}" ) diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 1436c35d6..7c20849dd 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -6,9 +6,15 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance. #include #include +#include +#include // This file mimics a DSO that makes pybind11 calls but does not define a // PYBIND11_MODULE. The purpose is to test that such a DSO can create a @@ -21,8 +27,54 @@ namespace { namespace py = pybind11; + void gil_acquire() { py::gil_scoped_acquire gil; } +std::string gil_multi_acquire_release(unsigned bits) { + if ((bits & 0x1u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x2u) != 0u) { + py::gil_scoped_release gil; + } + if ((bits & 0x4u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x8u) != 0u) { + py::gil_scoped_release gil; + } + return PYBIND11_INTERNALS_ID; +} + +struct CustomAutoGIL { + CustomAutoGIL() : gstate(PyGILState_Ensure()) {} + ~CustomAutoGIL() { PyGILState_Release(gstate); } + + PyGILState_STATE gstate; +}; +struct CustomAutoNoGIL { + CustomAutoNoGIL() : save(PyEval_SaveThread()) {} + ~CustomAutoNoGIL() { PyEval_RestoreThread(save); } + + PyThreadState *save; +}; + +template +void gil_acquire_inner() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; +} + +template +void gil_acquire_nested() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; + auto thread = std::thread(&gil_acquire_inner); + thread.join(); +} + constexpr char kModuleName[] = "cross_module_gil_utils"; struct PyModuleDef moduledef = { @@ -30,6 +82,9 @@ struct PyModuleDef moduledef = { } // namespace +#define ADD_FUNCTION(Name, ...) \ + PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); + extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); @@ -37,8 +92,16 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { if (m != nullptr) { static_assert(sizeof(&gil_acquire) == sizeof(void *), "Function pointer must have the same size as void*"); - PyModule_AddObject( - m, "gil_acquire_funcaddr", PyLong_FromVoidPtr(reinterpret_cast(&gil_acquire))); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) } return m; diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index aa3095594..624034648 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + true; +#else + false; +#endif bind_ConstructorStats(m); diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 6299293b9..44dcd1fdb 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -293,7 +293,6 @@ TEST_CASE("Threads") { { py::gil_scoped_release gil_release{}; - REQUIRE(has_pybind11_internals_static()); auto threads = std::vector(); for (auto i = 0; i < num_threads; ++i) { diff --git a/tests/test_gil_scoped.cpp b/tests/test_gil_scoped.cpp index 97efdc161..f136086e8 100644 --- a/tests/test_gil_scoped.cpp +++ b/tests/test_gil_scoped.cpp @@ -11,6 +11,13 @@ #include "pybind11_tests.h" +#include +#include + +#define CROSS_MODULE(Function) \ + auto cm = py::module_::import("cross_module_gil_utils"); \ + auto target = reinterpret_cast(PyLong_AsVoidPtr(cm.attr(Function).ptr())); + class VirtClass { public: virtual ~VirtClass() = default; @@ -28,6 +35,16 @@ class PyVirtClass : public VirtClass { }; TEST_SUBMODULE(gil_scoped, m) { + m.attr("defined_THREAD_SANITIZER") = +#if defined(THREAD_SANITIZER) + true; +#else + false; +#endif + + m.def("intentional_deadlock", + []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); }); + py::class_(m, "VirtClass") .def(py::init<>()) .def("virtual_func", &VirtClass::virtual_func) @@ -37,11 +54,91 @@ TEST_SUBMODULE(gil_scoped, m) { m.def("test_callback_std_func", [](const std::function &func) { func(); }); m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); }); m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); }); - m.def("test_cross_module_gil", []() { - auto cm = py::module_::import("cross_module_gil_utils"); - auto gil_acquire = reinterpret_cast( - PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); + m.def("test_cross_module_gil_released", []() { + CROSS_MODULE("gil_acquire_funcaddr") py::gil_scoped_release gil_release; - gil_acquire(); + target(); + }); + m.def("test_cross_module_gil_acquired", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_custom_released", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_custom_acquired", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_released", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_custom_released", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_custom_acquired", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_released", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_release_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired; + return py::str(obj); + }); + m.def("test_nested_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired_outer; + py::gil_scoped_acquire gil_acquired_inner; + return py::str(obj); + }); + m.def("test_multi_acquire_release_cross_module", [](unsigned bits) { + py::set internals_ids; + internals_ids.add(PYBIND11_INTERNALS_ID); + { + py::gil_scoped_release gil_released; + auto thread_f = [bits, &internals_ids]() { + py::gil_scoped_acquire gil_acquired; + auto cm = py::module_::import("cross_module_gil_utils"); + auto target = reinterpret_cast( + PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr())); + std::string cm_internals_id = target(bits >> 3); + internals_ids.add(cm_internals_id); + }; + if ((bits & 0x1u) != 0u) { + thread_f(); + } + if ((bits & 0x2u) != 0u) { + std::thread non_python_thread(thread_f); + non_python_thread.join(); + } + if ((bits & 0x4u) != 0u) { + thread_f(); + } + } + return internals_ids; }); } diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 52374b0cc..e890a7b0c 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -1,45 +1,199 @@ import multiprocessing +import sys import threading +import time + +import pytest from pybind11_tests import gil_scoped as m +class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + +def test_callback_py_obj(): + m.test_callback_py_obj(lambda: None) + + +def test_callback_std_func(): + m.test_callback_std_func(lambda: None) + + +def test_callback_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_virtual_func(extended) + + +def test_callback_pure_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_pure_virtual_func(extended) + + +def test_cross_module_gil_released(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil_released() # Should not raise a SIGSEGV + + +def test_cross_module_gil_acquired(): + """Makes sure that the GIL can be acquired by another module from a GIL-acquired state.""" + m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV + + +def test_cross_module_gil_inner_custom_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_released() + + +def test_cross_module_gil_inner_custom_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_acquired() + + +def test_cross_module_gil_inner_pybind11_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_released() + + +def test_cross_module_gil_inner_pybind11_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_acquired() + + +def test_cross_module_gil_nested_custom_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_released() + + +def test_cross_module_gil_nested_custom_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_acquired() + + +def test_cross_module_gil_nested_pybind11_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_released() + + +def test_cross_module_gil_nested_pybind11_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_acquired() + + +def test_release_acquire(): + assert m.test_release_acquire(0xAB) == "171" + + +def test_nested_acquire(): + assert m.test_nested_acquire(0xAB) == "171" + + +def test_multi_acquire_release_cross_module(): + for bits in range(16 * 8): + internals_ids = m.test_multi_acquire_release_cross_module(bits) + assert len(internals_ids) == 2 if bits % 8 else 1 + + +# Intentionally putting human review in the loop here, to guard against accidents. +VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical). +ALL_BASIC_TESTS = ( + test_callback_py_obj, + test_callback_std_func, + test_callback_virtual_func, + test_callback_pure_virtual_func, + test_cross_module_gil_released, + test_cross_module_gil_acquired, + test_cross_module_gil_inner_custom_released, + test_cross_module_gil_inner_custom_acquired, + test_cross_module_gil_inner_pybind11_released, + test_cross_module_gil_inner_pybind11_acquired, + test_cross_module_gil_nested_custom_released, + test_cross_module_gil_nested_custom_acquired, + test_cross_module_gil_nested_pybind11_released, + test_cross_module_gil_nested_pybind11_acquired, + test_release_acquire, + test_nested_acquire, + test_multi_acquire_release_cross_module, +) + + +def test_all_basic_tests_completeness(): + num_found = 0 + for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items(): + if not key.startswith("test_"): + continue + assert value in ALL_BASIC_TESTS + num_found += 1 + assert len(ALL_BASIC_TESTS) == num_found + + +def _intentional_deadlock(): + m.intentional_deadlock() + + +ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) +SKIP_IF_DEADLOCK = True # See PR #4216 + + def _run_in_process(target, *args, **kwargs): - """Runs target in process and returns its exitcode after 10s (None if still alive).""" + if len(args) == 0: + test_fn = target + else: + test_fn = args[0] + # Do not need to wait much, 10s should be more than enough. + timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: + t_start = time.time() process.start() - # Do not need to wait much, 10s should be more than enough. - process.join(timeout=10) + if timeout >= 100: # For debugging. + print( + "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs) + ) + print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True) + process.join(timeout=timeout) + if timeout >= 100: + print("\nprocess.pid JOINED", process.pid, flush=True) + t_delta = time.time() - t_start + if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754 + # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output. + # Maybe this could work: + # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e + pytest.skip( + "ThreadSanitizer: starting new threads after multi-threaded fork is not supported." + ) + elif test_fn is _intentional_deadlock: + assert process.exitcode is None + return 0 + elif process.exitcode is None: + assert t_delta > 0.9 * timeout + msg = "DEADLOCK, most likely, exactly what this test is meant to detect." + if SKIP_IF_DEADLOCK: + pytest.skip(msg) + raise RuntimeError(msg) return process.exitcode finally: if process.is_alive(): process.terminate() -def _python_to_cpp_to_python(): - """Calls different C++ functions that come back to Python.""" - - class ExtendedVirtClass(m.VirtClass): - def virtual_func(self): - pass - - def pure_virtual_func(self): - pass - - extended = ExtendedVirtClass() - m.test_callback_py_obj(lambda: None) - m.test_callback_std_func(lambda: None) - m.test_callback_virtual_func(extended) - m.test_callback_pure_virtual_func(extended) - - -def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): - """Calls different C++ functions that come back to Python, from Python threads.""" +def _run_in_threads(test_fn, num_threads, parallel): threads = [] for _ in range(num_threads): - thread = threading.Thread(target=_python_to_cpp_to_python) + thread = threading.Thread(target=test_fn) thread.daemon = True thread.start() if parallel: @@ -51,43 +205,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_parallel(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_sequential(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert ( - _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 - ) + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_process(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. This test is for completion, but it was never an issue. """ - assert _run_in_process(_python_to_cpp_to_python) == 0 - - -def test_cross_module_gil(): - """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" - m.test_cross_module_gil() # Should not raise a SIGSEGV + assert _run_in_process(test_fn) == 0 From 3fb36a99f68c082ce9fd58cbb3e6058ababd3b3e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 09:18:05 -0700 Subject: [PATCH 69/71] fix: unicode surrogate character in Python exception message. (#4297) * Fix & test for issue #4288 (unicode surrogate character in Python exception message). * DRY `message_unavailable_exc` * fix: add a constexpr Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Henry Schreiner Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 22 ++++++++++++++++++++-- tests/test_exceptions.cpp | 5 ----- tests/test_exceptions.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80a2e2228..91cbaaba2 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,11 +501,29 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; } else { - result = value_str.cast(); + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } } } else { result = ""; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3583f22a5..f57e09506 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 70b6ffea9..7eb1a9d62 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": From f2ee641ee3c613c07c84cd4a473e1398002ded7f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:11:23 -0400 Subject: [PATCH 70/71] docs: prepare for 2.10.1 release (#4279) * docs: prepare for 2.10.1 release Signed-off-by: Henry Schreiner * Update changelog.rst * docs: update changelog with final list of PRs Signed-off-by: Henry Schreiner * Update docs/changelog.rst * chore: one more changelog bump Signed-off-by: Henry Schreiner --- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df3181c52..7d6d0c0f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,19 +10,18 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. - IN DEVELOPMENT -------------- Changes will be summarized here periodically. -Version 2.10.1 (Oct 2?, 2022) +Version 2.10.1 (Oct 31, 2022) ----------------------------- +This is the first version to fully support embedding the newly released Python 3.11. Changes: - * Allow ``pybind11::capsule`` constructor to take null destructor pointers. `#4221 `_ @@ -30,8 +29,40 @@ Changes: (established behavior). `#4119 `_ +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + Bug fixes: +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + * Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. `#4188 `_ @@ -100,9 +131,15 @@ Performance and style: * Optimize unpacking_collector when processing ``arg_v`` arguments. `#4219 `_ +* Optimize casting C++ object to ``None``. + `#4269 `_ + Build system improvements: +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ From 80dc998efced8ceb2be59756668a7e90e8bef917 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:44:51 -0400 Subject: [PATCH 71/71] chore: bump versions for 2.10.1 Signed-off-by: Henry Schreiner --- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a3e0bc9b3..f34fd279c 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_MINOR 10 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020A0100 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5c..3d8c2cd36 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.10.1" version_info = tuple(_to_int(s) for s in __version__.split("."))