From 3a2c96bd6f95f6b183c342380558238583415ab1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 09:18:05 -0700 Subject: [PATCH 01/17] fix: unicode surrogate character in Python exception message. (#4297) * Fix & test for issue #4288 (unicode surrogate character in Python exception message). * DRY `message_unavailable_exc` * fix: add a constexpr Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Henry Schreiner Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 22 ++++++++++++++++++++-- tests/test_exceptions.cpp | 5 ----- tests/test_exceptions.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80a2e2228..91cbaaba2 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,11 +501,29 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; } else { - result = value_str.cast(); + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } } } else { result = ""; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3583f22a5..f57e09506 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 70b6ffea9..7eb1a9d62 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": From b1bd7f2600d650bf59c88800c637703dd89317f3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 10:36:26 -0700 Subject: [PATCH 02/17] fix: define (non-empty) `PYBIND11_EXPORT_EXCEPTION` only under macOS. (#4298) Background: #2999, #4105, #4283, #4284 In a nutshell: * Only macOS actually needs `PYBIND11_EXPORT_EXCEPTION` (#4284). * Evidently (#4283), under macOS `PYBIND11_EXPORT_EXCEPTION` does not run the risk of introducing ODR violations, * but evidently (#4283) under Linux it does, in the presumably rare/unusual situation that `RTLD_GLOBAL` is used. * Windows does no have the equivalent of `RTLD_GLOBAL`, therefore `PYBIND11_EXPORT_EXCEPTION` has no practical benefit, on the contrary, noisy warning suppression pragmas are needed, therefore it is best left empty. --- docs/advanced/exceptions.rst | 9 ++++++--- include/pybind11/detail/common.h | 18 +++--------------- include/pybind11/pytypes.h | 9 --------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 2211caf5d..53981dc08 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -177,9 +177,12 @@ section. may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. - Note that ``libc++`` and ``libstdc++`` `behave differently `_ - with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI boundaries need to be explicitly exported, as exercised in ``tests/test_exceptions.h``. - See also: "Problems with C++ exceptions" under `GCC Wiki `_. + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + `_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki `_. Local vs Global Exception Translators diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a3e0bc9b3..9b74323f6 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -96,13 +96,10 @@ #endif #if !defined(PYBIND11_EXPORT_EXCEPTION) -# ifdef __MINGW32__ -// workaround for: -// error: 'dllexport' implies default visibility, but xxx has already been declared with a -// different visibility -# define PYBIND11_EXPORT_EXCEPTION -# else +# if defined(__apple_build_version__) # define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION # endif #endif @@ -904,12 +901,6 @@ using expand_side_effects = bool[]; PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// C++ bindings of builtin Python exceptions class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: @@ -917,9 +908,6 @@ public: /// Set the error using the Python C API virtual void set_error() const = 0; }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 91cbaaba2..80b49ec39 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -623,12 +623,6 @@ inline std::string error_string() { PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275 4251) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// Fetch and hold an error which was already set in Python. An instance of this is typically /// thrown to propagate python-side errors back through C++ which can either be caught manually or /// else falls back to the function dispatcher (which then raises the captured error back to @@ -688,9 +682,6 @@ private: /// crashes (undefined behavior) if the Python interpreter is finalizing. static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif /// Replaces the current Python error indicator with the chosen error, performing a /// 'raise from' to indicate that the chosen error was caused by the original error. From 252ed8fb52e46eb3ec3e3b8c621ca9f79b53b26a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:11:23 -0400 Subject: [PATCH 03/17] docs: prepare for 2.10.1 release (#4279) * docs: prepare for 2.10.1 release Signed-off-by: Henry Schreiner * Update changelog.rst * docs: update changelog with final list of PRs Signed-off-by: Henry Schreiner * Update docs/changelog.rst * chore: one more changelog bump Signed-off-by: Henry Schreiner --- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df3181c52..7d6d0c0f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,19 +10,18 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. - IN DEVELOPMENT -------------- Changes will be summarized here periodically. -Version 2.10.1 (Oct 2?, 2022) +Version 2.10.1 (Oct 31, 2022) ----------------------------- +This is the first version to fully support embedding the newly released Python 3.11. Changes: - * Allow ``pybind11::capsule`` constructor to take null destructor pointers. `#4221 `_ @@ -30,8 +29,40 @@ Changes: (established behavior). `#4119 `_ +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + Bug fixes: +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + * Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. `#4188 `_ @@ -100,9 +131,15 @@ Performance and style: * Optimize unpacking_collector when processing ``arg_v`` arguments. `#4219 `_ +* Optimize casting C++ object to ``None``. + `#4269 `_ + Build system improvements: +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_ From 2441d25b26af3e7b89d521a218694b604ec716e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 00:10:13 -0400 Subject: [PATCH 04/17] chore(deps): update pre-commit hooks (#4302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.38.2 → v3.2.0](https://github.com/asottile/pyupgrade/compare/v2.38.2...v3.2.0) - [github.com/psf/black: 22.8.0 → 22.10.0](https://github.com/psf/black/compare/22.8.0...22.10.0) - [github.com/PyCQA/pylint: v2.15.3 → v2.15.5](https://github.com/PyCQA/pylint/compare/v2.15.3...v2.15.5) - [github.com/pre-commit/mirrors-mypy: v0.981 → v0.982](https://github.com/pre-commit/mirrors-mypy/compare/v0.981...v0.982) - [github.com/codespell-project/codespell: v2.2.1 → v2.2.2](https://github.com/codespell-project/codespell/compare/v2.2.1...v2.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 742e7a30a..a60d84f22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # Upgrade old Python syntax - repo: https://github.com/asottile/pyupgrade - rev: "v2.38.2" + rev: "v3.2.0" hooks: - id: pyupgrade args: [--py36-plus] @@ -54,7 +54,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "22.8.0" # Keep in sync with blacken-docs + rev: "22.10.0" # Keep in sync with blacken-docs hooks: - id: black @@ -116,7 +116,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.15.3" + rev: "v2.15.5" hooks: - id: pylint files: ^pybind11 @@ -132,7 +132,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.981" + rev: "v0.982" hooks: - id: mypy args: [] @@ -152,7 +152,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.1" + rev: "v2.2.2" hooks: - id: codespell exclude: ".supp$" From 0176632e8cef72a4223f2df54d6dfb9e552ec71f Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 1 Nov 2022 12:19:34 -0400 Subject: [PATCH 05/17] chore: sync blacken-docs hook with black (#4304) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a60d84f22..fd079dd03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==22.8.0 # keep in sync with black hook + - black==22.10.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks From ee2b5226295d67b690faddd446a329bb2840a1a8 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 2 Nov 2022 11:32:53 -0700 Subject: [PATCH 06/17] Fix functional.h bug + introduce test to verify that it is fixed (#4254) * Illustrate bug in functional.h * style: pre-commit fixes * Make functional casting more robust / add workaround * Make function_record* casting even more robust * See if this fixes PyPy issue * It still fails on PyPy sadly * Do not make new CTOR just yet * Fix test * Add name to ensure correctness * style: pre-commit fixes * Clean up tests + remove ifdef guards * Add comments * Improve comments, error handling, and safety * Fix compile error * Fix magic logic * Extract helper function * Fix func signature * move to local internals * style: pre-commit fixes * Switch to simpler design * style: pre-commit fixes * Move to function_record * style: pre-commit fixes * Switch to internals, update tests and docs * Fix lint * Oops, forgot to resolve last comment * Fix typo * Update in response to comments * Implement suggestion to improve test * Update comment * Simple fixes Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Aaron Gokaslan --- include/pybind11/detail/internals.h | 31 ++++++++++++++++++++ include/pybind11/functional.h | 11 ++++++-- include/pybind11/pybind11.h | 44 ++++++++++++++++++++++------- tests/test_callbacks.cpp | 37 ++++++++++++++++++++++++ tests/test_callbacks.py | 13 +++++++++ 5 files changed, 124 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 7de779434..6fd61098c 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -43,6 +43,8 @@ using ExceptionTranslator = void (*)(std::exception_ptr); PYBIND11_NAMESPACE_BEGIN(detail) +constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; + // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); @@ -182,6 +184,16 @@ struct internals { # endif // PYBIND11_INTERNALS_VERSION > 4 // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; + +# if PYBIND11_INTERNALS_VERSION > 4 + // Note that we have to use a std::string to allocate memory to ensure a unique address + // We want unique addresses since we use pointer equality to compare function records + std::string function_record_capsule_name = internals_function_record_capsule_name; +# endif + + internals() = default; + internals(const internals &other) = delete; + internals &operator=(const internals &other) = delete; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_FREE(loader_life_support_tls_key); @@ -548,6 +560,25 @@ const char *c_str(Args &&...args) { return strings.front().c_str(); } +inline const char *get_function_record_capsule_name() { +#if PYBIND11_INTERNALS_VERSION > 4 + return get_internals().function_record_capsule_name.c_str(); +#else + return nullptr; +#endif +} + +// Determine whether or not the following capsule contains a pybind11 function record. +// Note that we use `internals` to make sure that only ABI compatible records are touched. +// +// This check is currently used in two places: +// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ +// - The sibling feature of cpp_function to allow overloads +inline bool is_function_record_capsule(const capsule &cap) { + // Pointer equality as we rely on internals() to ensure unique pointers + return cap.name() == get_function_record_capsule_name(); +} + PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 102d1a938..87ec4d10c 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -48,9 +48,16 @@ public: */ if (auto cfunc = func.cpp_function()) { auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); - if (isinstance(cfunc_self)) { + if (cfunc_self == nullptr) { + PyErr_Clear(); + } else if (isinstance(cfunc_self)) { auto c = reinterpret_borrow(cfunc_self); - auto *rec = (function_record *) c; + + function_record *rec = nullptr; + // Check that we can safely reinterpret the capsule into a function_record + if (detail::is_function_record_capsule(c)) { + rec = c.get_pointer(); + } while (rec != nullptr) { if (rec->is_stateless diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index e662236d0..76d6eadca 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -468,13 +468,20 @@ protected: if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); - capsule rec_capsule = isinstance(self) ? reinterpret_borrow(self) - : capsule(self); - chain = (detail::function_record *) rec_capsule; - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) { + if (!isinstance(self)) { chain = nullptr; + } else { + auto rec_capsule = reinterpret_borrow(self); + if (detail::is_function_record_capsule(rec_capsule)) { + chain = rec_capsule.get_pointer(); + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ + if (!chain->scope.is(rec->scope)) { + chain = nullptr; + } + } else { + chain = nullptr; + } } } // Don't trigger for things like the default __init__, which are wrapper_descriptors @@ -496,6 +503,7 @@ protected: capsule rec_capsule(unique_rec.release(), [](void *ptr) { destruct((detail::function_record *) ptr); }); + rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; @@ -661,10 +669,13 @@ protected: /// Main dispatch logic for calls to functions bound using pybind11 static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { using namespace detail; + assert(isinstance(self)); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + const function_record *overloads = reinterpret_cast( + PyCapsule_GetPointer(self, get_function_record_capsule_name())), *it = overloads; + assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right overload */ @@ -1871,9 +1882,22 @@ private: static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow( - PyCFunction_GET_SELF(h.ptr())) - : nullptr; + if (!h) { + return nullptr; + } + + handle func_self = PyCFunction_GET_SELF(h.ptr()); + if (!func_self) { + throw error_already_set(); + } + if (!isinstance(func_self)) { + return nullptr; + } + auto cap = reinterpret_borrow(func_self); + if (!detail::is_function_record_capsule(cap)) { + return nullptr; + } + return cap.get_pointer(); } }; diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index 92b8053de..2fd05dec7 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -240,4 +240,41 @@ TEST_SUBMODULE(callbacks, m) { f(); } }); + + auto *custom_def = []() { + static PyMethodDef def; + def.ml_name = "example_name"; + def.ml_doc = "Example doc"; + def.ml_meth = [](PyObject *, PyObject *args) -> PyObject * { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("Invalid number of arguments for example_name"); + } + PyObject *first = PyTuple_GetItem(args, 0); + if (!PyLong_Check(first)) { + throw std::runtime_error("Invalid argument to example_name"); + } + auto result = py::cast(PyLong_AsLong(first) * 9); + return result.release().ptr(); + }; + def.ml_flags = METH_VARARGS; + return &def; + }(); + + // rec_capsule with name that has the same value (but not pointer) as our internal one + // This capsule should be detected by our code as foreign and not inspected as the pointers + // shouldn't match + constexpr const char *rec_capsule_name + = pybind11::detail::internals_function_record_capsule_name; + py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); }); + rec_capsule.set_name(rec_capsule_name); + m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); + + // This test requires a new ABI version to pass +#if PYBIND11_INTERNALS_VERSION > 4 + // rec_capsule with nullptr name + py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); + m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); +#else + m.add_object("custom_function2", py::none()); +#endif } diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 0b1047bbf..57b659988 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -193,3 +193,16 @@ def test_callback_num_times(): if len(rates) > 1: print("Min Mean Max") print(f"{min(rates):6.3f} {sum(rates) / len(rates):6.3f} {max(rates):6.3f}") + + +def test_custom_func(): + assert m.custom_function(4) == 36 + assert m.roundtrip(m.custom_function)(4) == 36 + + +@pytest.mark.skipif( + m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low" +) +def test_custom_func2(): + assert m.custom_function2(3) == 27 + assert m.roundtrip(m.custom_function2)(3) == 27 From 1f04cc7062e33481c62c78231e9561b318bca67b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 10 Nov 2022 08:33:26 -0800 Subject: [PATCH 07/17] Add windows_clang to ci.yml (#4323) * Add windows_clang to ci.yml (previously tested under PRs #4321, #4319) * Add `pip install --upgrade pip`, Show env, cosmetic changes Already tested under PR #4321 --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a11cae1ab..adbbf626f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -967,3 +967,73 @@ jobs: - name: Interface test C++17 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target test_cmake_build + + windows_clang: + + strategy: + matrix: + os: [windows-latest] + python: ['3.10'] + + runs-on: "${{ matrix.os }}" + + name: "🐍 ${{ matrix.python }} • ${{ matrix.os }} • clang-latest" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.13 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show Clang++ version + run: clang++ --version + + - name: Show CMake version + run: cmake --version + + # TODO: WERROR=ON + - name: Configure Clang + run: > + cmake -G Ninja -S . -B . + -DCMAKE_VERBOSE_MAKEFILE=ON + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx From 88b019a8a5e116d7e4a4ffae6399a426364d4bcd Mon Sep 17 00:00:00 2001 From: gitartpiano <51239761+gitartpiano@users.noreply.github.com> Date: Fri, 11 Nov 2022 19:52:57 -0600 Subject: [PATCH 08/17] fix pybind11Tools.cmake typo causing Unknown arguments (#4327) * fix pybind11Tools.cmake typo causing Unknown arguments CMake Error at pybind11/tools/pybind11Tools.cmake:217 (if): if given arguments: "NOT" "MSVC" "AND" "NOT" "TEST" "MATCHES" "DEBUG|RELWITHDEBINFO" Unknown arguments specified https://github.com/pybind/pybind11/issues/4325 * Apply the same fix in tools/pybind11NewTools.cmake Co-authored-by: Ralf W. Grosse-Kunstleve --- tools/pybind11NewTools.cmake | 2 +- tools/pybind11Tools.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 9e13daf1a..5a6a0cb8b 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -235,7 +235,7 @@ function(pybind11_add_module target_name) # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 0dc61d399..66ad00a47 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -214,7 +214,7 @@ function(pybind11_add_module target_name) # Use case-insensitive comparison to match the result of $ string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) pybind11_strip(${target_name}) endif() From 296615ad34f9d8f680efbab22553881ad9843063 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 12 Nov 2022 12:24:19 -0800 Subject: [PATCH 09/17] Add macos_brew_install_llvm to ci.yml (#4326) * Add macos_brew_install_llvm to ci.yml Added block transferred from PR #4324 * `test_cross_module_exception_translator` xfail 'Homebrew Clang' * Add `pip install numpy scipy` (tested already under PR #4324). --- .github/workflows/ci.yml | 67 ++++++++++++++++++++++++++++++++++++++++ tests/test_exceptions.py | 5 +-- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adbbf626f..bd99ddd33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1037,3 +1037,70 @@ jobs: - name: Clean directory run: git clean -fdx + + macos_brew_install_llvm: + name: "macos-latest • brew install llvm" + runs-on: macos-latest + + env: + # https://apple.stackexchange.com/questions/227026/how-to-install-recent-clang-with-homebrew + LDFLAGS: '-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib' + + steps: + - name: Update PATH + run: echo "/usr/local/opt/llvm/bin" >> $GITHUB_PATH + + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Show Clang++ version before brew install llvm + run: clang++ --version + + - name: brew install llvm + run: brew install llvm + + - name: Show Clang++ version after brew install llvm + run: clang++ --version + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.13 + + - name: Run pip installs + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r tests/requirements.txt + python3 -m pip install numpy + python3 -m pip install scipy + + - name: Show CMake version + run: cmake --version + + - name: CMake Configure + run: > + cmake -S . -B . + -DCMAKE_VERBOSE_MAKEFILE=ON + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 7eb1a9d62..0d2c80814 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -4,6 +4,7 @@ import pytest import env import pybind11_cross_module_tests as cm +import pybind11_tests # noqa: F401 from pybind11_tests import exceptions as m @@ -72,9 +73,9 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "env.PYPY and env.MACOS", + "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))", raises=RuntimeError, - reason="Expected failure with PyPy and libc++ (Issue #2847 & PR #2999)", + reason="See Issue #2847, PR #2999, PR #4324", ) def test_cross_module_exception_translator(): with pytest.raises(KeyError): From 48949222c637da9fc72b0ed6526ae1b40bb55237 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 22 Nov 2022 15:14:49 -0800 Subject: [PATCH 10/17] Use `PyEval_InitThreads()` as intended (#4350) * Use `PyEval_InitThreads()` as intended (actually matters only for Python 3.6). * Add `if defined(WITH_THREAD)` condition. https://docs.python.org/3.6/c-api/init.html#c.PyEval_InitThreads > This function is not available when thread support is disabled at compile time. * Fix oversight pointed out by @EricCousineau-TRI: Remove condition that is always false. --- include/pybind11/detail/internals.h | 3 --- include/pybind11/embed.h | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 6fd61098c..ef1849fbe 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -468,9 +468,6 @@ PYBIND11_NOINLINE internals &get_internals() { internals_ptr = new internals(); #if defined(WITH_THREAD) -# if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); -# endif PyThreadState *tstate = PyThreadState_Get(); if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 0ac609e0f..5b77594b3 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -118,6 +118,9 @@ inline void initialize_interpreter(bool init_signal_handlers = true, #if PY_VERSION_HEX < 0x030B0000 Py_InitializeEx(init_signal_handlers ? 1 : 0); +# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +# endif // Before it was special-cased in python 3.8, passing an empty or null argv // caused a segfault, so we have to reimplement the special case ourselves. From 9c18a74e377dece2f2acd22c2c6e63ecb2a59c77 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 22 Nov 2022 17:17:02 -0800 Subject: [PATCH 11/17] Use `multiprocessing` `start_method` `"forkserver"` (#4306) * Use `multiprocessing` `start_method` `"forkserver"` Alternative to PR #4305 * Add link to comment under PR #4105 * Unconditionally `pytest.skip("DEADLOCK")` for PyPy Windows * Remove `SKIP_IF_DEADLOCK` entirely, for simplicity. Hopefully this PR will resolve the deadlocks for good. * Add "In a nutshell" comment, in response to request by @EricCousineau-TRI --- tests/conftest.py | 12 ++++++++++++ tests/test_gil_scoped.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f5ddb9f12..96dffc81c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ Adds docstring and exceptions message sanitizers. import contextlib import difflib import gc +import multiprocessing +import os import re import textwrap @@ -15,6 +17,16 @@ import pytest # Early diagnostic for failed imports import pybind11_tests +if os.name != "nt": + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 + # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. + # It is actually a well-known pitfall, unfortunately without guard rails. + # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, + # visit the issuecomment link above for details). + # Windows does not have fork() and the associated pitfall, therefore it is best left + # running with defaults. + multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index e890a7b0c..6af6a472d 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -5,6 +5,7 @@ import time import pytest +import env from pybind11_tests import gil_scoped as m @@ -144,7 +145,6 @@ def _intentional_deadlock(): ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) -SKIP_IF_DEADLOCK = True # See PR #4216 def _run_in_process(target, *args, **kwargs): @@ -181,7 +181,7 @@ def _run_in_process(target, *args, **kwargs): elif process.exitcode is None: assert t_delta > 0.9 * timeout msg = "DEADLOCK, most likely, exactly what this test is meant to detect." - if SKIP_IF_DEADLOCK: + if env.PYPY and env.WIN: pytest.skip(msg) raise RuntimeError(msg) return process.exitcode From 9907bedce517ea291c70c6a7b5dfec9e594f41df Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 26 Nov 2022 07:15:54 +0800 Subject: [PATCH 12/17] fix(.github): fix bug-report issue template (#4363) --- .github/ISSUE_TEMPLATE/bug-report.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index e6494ba6a..4f1e78f33 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -21,10 +21,11 @@ body: - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). required: false - - type: Input + - type: input id: version attributes: label: What version (or hash if on master) of pybind11 are you using? + validations: required: true - type: textarea @@ -52,7 +53,7 @@ body: starting point for working out fixes. render: text - - type: Input + - type: input id: regression attributes: label: Is this a regression? Put the last known working version here if it is. From 06003e82b3ff48337b71b310b46c3d8b15ca6d5a Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 28 Nov 2022 07:39:38 -0800 Subject: [PATCH 13/17] Introduce a new style of warning suppression based on push/pop (#4285) * Introduce a new warning suppression system * Switch to better name * Nits --- include/pybind11/cast.h | 11 ++-- include/pybind11/detail/common.h | 82 +++++++++++++++++++++----- include/pybind11/detail/init.h | 9 ++- include/pybind11/eigen/matrix.h | 31 +++------- include/pybind11/eigen/tensor.h | 35 ++++------- include/pybind11/numpy.h | 15 ++--- include/pybind11/pybind11.h | 38 +++++------- include/pybind11/pytypes.h | 16 +++-- tests/test_builtin_casters.cpp | 9 ++- tests/test_class.cpp | 4 +- tests/test_constants_and_functions.cpp | 34 ++++------- tests/test_eigen_matrix.cpp | 4 +- tests/test_eigen_tensor.inl | 8 ++- tests/test_embed/catch.cpp | 4 +- tests/test_embed/test_interpreter.cpp | 4 +- tests/test_kwargs_and_defaults.cpp | 9 ++- tests/test_operator_overloading.cpp | 23 +++----- 17 files changed, 173 insertions(+), 163 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 430c62f35..8f92df8ab 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -29,6 +29,9 @@ #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template @@ -389,7 +392,7 @@ struct string_caster { // For UTF-8 we avoid the need for a temporary `bytes` object by using // `PyUnicode_AsUTF8AndSize`. - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { + if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); @@ -416,7 +419,7 @@ struct string_caster { = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { + if (UTF_N > 8) { buffer++; length--; } @@ -572,7 +575,7 @@ public: // figure out how long the first encoded character is in bytes to distinguish between these // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as // those can fit into a single char value. - if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast(value[0]); // low bits only: 0-127 // 0b110xxxxx - start of 2-byte sequence @@ -598,7 +601,7 @@ public: // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a // surrogate pair with total length 2 instantly indicates a range error (but not a "your // string was too long" error). - else if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 16) && str_len == 2) { + else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast(value[0]); if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9b74323f6..b58dc3afa 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -17,8 +17,69 @@ // Additional convention: 0xD = dev #define PYBIND11_VERSION_HEX 0x020B00D1 -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(# __VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH + +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions // requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute @@ -151,9 +212,9 @@ /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# pragma warning(push) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# pragma warning(disable : 4505) # if defined(_DEBUG) && !defined(Py_DEBUG) // Workaround for a VS 2022 issue. // NOTE: This workaround knowingly violates the Python.h include order requirement: @@ -236,7 +297,7 @@ # define _DEBUG # undef PYBIND11_DEBUG_MARKER # endif -# pragma warning(pop) +PYBIND11_WARNING_POP #endif #include @@ -1136,17 +1197,6 @@ constexpr # define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) #endif -#if defined(_MSC_VER) // All versions (as of July 2021). - -// warning C4127: Conditional expression is constant -constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } - -# define PYBIND11_SILENCE_MSVC_C4127(...) ::pybind11::detail::silence_msvc_c4127(__VA_ARGS__) - -#else -# define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ -#endif - #if defined(__clang__) \ && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ available. */ \ diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 05f4fe54a..0938c9bde 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -115,7 +118,7 @@ template void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not // that succeeds, we still need to destroy the original cpp pointer (either the // moved away leftover, if the alias construction works, or the value itself if we @@ -156,7 +159,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { auto *ptr = holder_helper>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); } @@ -174,7 +177,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); static_assert(std::is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias) { + if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); } else { v_h.value_ptr() = new Cpp(std::move(result)); diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index c30dac241..34fe329a8 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -16,29 +16,15 @@ https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4127) // C4127: conditional expression is constant -# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #include #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit // move constructors that break things. We could detect this an explicitly copy, but an extra copy @@ -48,6 +34,8 @@ static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + // Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: using EigenDStride = Eigen::Stride; template @@ -189,8 +177,7 @@ struct EigenProps { EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); - if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) - || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) { return false; } @@ -203,7 +190,7 @@ struct EigenProps { stride = a.strides(0) / static_cast(sizeof(Scalar)); if (vector) { // Eigen type is a compile-time vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { + if (fixed && size != n) { return false; // Vector size mismatch } return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; @@ -220,7 +207,7 @@ struct EigenProps { } return {1, n, stride}; } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { + if (fixed_rows && rows != n) { return false; } return {n, 1, stride}; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index a823c0f39..a129f9929 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -13,28 +13,23 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); #endif -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4554) // Tensor.h warning -# pragma warning(disable : 4127) // Tensor.h warning -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif +// Disable warnings for Eigen +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4554) +PYBIND11_WARNING_DISABLE_MSVC(4127) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) inline bool is_tensor_aligned(const void *data) { @@ -138,10 +133,8 @@ struct get_tensor_descriptor { // // We need to disable the type-limits warning for the inner loop when size = 0. -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wtype-limits") template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { @@ -165,9 +158,7 @@ Eigen::DSizes get_shape_for_array(const array &arr) { return result; } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP template struct type_caster::ValidType> { @@ -389,7 +380,7 @@ struct type_caster, constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; - if (PYBIND11_SILENCE_MSVC_C4127(is_aligned) && !is_tensor_aligned(arr.data())) { + if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } @@ -399,7 +390,7 @@ struct type_caster, return false; } - if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable) && !arr.writeable()) { + if (needs_writeable && !arr.writeable()) { return false; } diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 369e18649..ea64aa7e7 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -36,6 +36,8 @@ static_assert(std::is_signed::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) @@ -875,7 +877,7 @@ public: */ template detail::unchecked_mutable_reference mutable_unchecked() & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -893,7 +895,7 @@ public: */ template detail::unchecked_reference unchecked() const & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -1865,9 +1867,10 @@ private: } auto result = returned_array::create(trivial, shape); + + PYBIND11_WARNING_PUSH #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wreturn-std-move" + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") #endif if (size == 0) { @@ -1883,9 +1886,7 @@ private: } return result; -#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic pop -#endif + PYBIND11_WARNING_POP } template diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 76d6eadca..10b27254e 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -35,6 +35,8 @@ # include #endif +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning This warning is about ABI compatibility, not code health. It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if @@ -43,11 +45,10 @@ No other GCC version generates this warning. */ #if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnoexcept-type" +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") #endif -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) @@ -177,22 +178,22 @@ protected: auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(capture) <= sizeof(rec->data))) { + if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif + new ((capture *) &rec->data) capture{std::forward(f)}; -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif + // UB without std::launder, but without breaking ABI and/or // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible::value) { @@ -202,9 +203,7 @@ protected: data->~capture(); }; } -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP } else { rec->data[0] = new capture{std::forward(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; @@ -1841,8 +1840,7 @@ private: if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible()); v_h.set_holder_constructed(); - } else if (PYBIND11_SILENCE_MSVC_C4127(detail::always_construct_holder::value) - || inst->owned) { + } else if (detail::always_construct_holder::value || inst->owned) { new (std::addressof(v_h.holder())) holder_type(v_h.value_ptr()); v_h.set_holder_constructed(); } @@ -2876,7 +2874,3 @@ inline function get_overload(const T *this_ptr, const char *name) { PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic pop // -Wnoexcept-type -#endif diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80b49ec39..f913565d3 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -33,6 +33,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ class handle; class object; @@ -884,10 +886,8 @@ object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(push) -# pragma warning(disable : 4522) // warning C4522: multiple assignment operators specified -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template class accessor : public object_api> { using key_type = typename Policy::key_type; @@ -951,9 +951,7 @@ private: key_type key; mutable object cache; }; -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(pop) -#endif +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -1693,7 +1691,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template Unsigned as_unsigned(PyObject *o) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(Unsigned) <= sizeof(unsigned long))) { + if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } @@ -1710,7 +1708,7 @@ public: template ::value, int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(T) <= sizeof(long))) { + if (sizeof(T) <= sizeof(long)) { if (std::is_signed::value) { m_ptr = PyLong_FromLong((long) value); } else { diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index 6ff63edfc..0623b85dc 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -73,6 +73,9 @@ PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) TEST_SUBMODULE(builtin_casters, m) { + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4127) + // test_simple_string m.def("string_roundtrip", [](const char *s) { return s; }); @@ -86,7 +89,7 @@ TEST_SUBMODULE(builtin_casters, m) { std::wstring wstr; wstr.push_back(0x61); // a wstr.push_back(0x2e18); // ⸘ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // 𝐀, utf16 @@ -113,7 +116,7 @@ TEST_SUBMODULE(builtin_casters, m) { // Under Python 2.7, invalid unicode UTF-32 characters didn't appear to trigger // UnicodeDecodeError m.def("bad_utf32_string", [=]() { return std::u32string({a32, char32_t(0xd800), z32}); }); - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { m.def("bad_wchar_string", [=]() { return std::wstring({wchar_t(0x61), wchar_t(0xd800)}); }); @@ -384,4 +387,6 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("takes_const_ref", [](const ConstRefCasted &x) { return x.tag; }); m.def("takes_const_ref_wrap", [](std::reference_wrapper x) { return x.get().tag; }); + + PYBIND11_WARNING_POP } diff --git a/tests/test_class.cpp b/tests/test_class.cpp index fc1c17b17..9ea10fae1 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -22,10 +22,8 @@ #include -#if defined(_MSC_VER) -# pragma warning(disable : 4324) +PYBIND11_WARNING_DISABLE_MSVC(4324) // warning C4324: structure was padded due to alignment specifier -#endif // test_brace_initialization struct NoBraceInitialization { diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index 1918a429c..922375c5e 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -52,15 +52,12 @@ int f1(int x) noexcept { return x + 1; } #endif int f2(int x) noexcept(true) { return x + 2; } int f3(int x) noexcept(false) { return x + 3; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") +PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int f4(int x) throw() { return x + 4; } // Deprecated equivalent to noexcept(true) -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP struct C { int m1(int x) noexcept { return x - 1; } int m2(int x) const noexcept { return x - 2; } @@ -68,17 +65,14 @@ struct C { int m4(int x) const noexcept(true) { return x - 4; } int m5(int x) noexcept(false) { return x - 5; } int m6(int x) const noexcept(false) { return x - 6; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") + PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int m7(int x) throw() { return x - 7; } // NOLINTNEXTLINE(modernize-use-noexcept) int m8(int x) const throw() { return x - 8; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP }; } // namespace test_exc_sp @@ -126,14 +120,12 @@ TEST_SUBMODULE(constants_and_functions, m) { .def("m8", &C::m8); m.def("f1", f1); m.def("f2", f2); -#if defined(__INTEL_COMPILER) -# pragma warning push -# pragma warning disable 878 // incompatible exception specifications -#endif + + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_INTEL(878) // incompatible exception specifications m.def("f3", f3); -#if defined(__INTEL_COMPILER) -# pragma warning pop -#endif + PYBIND11_WARNING_POP + m.def("f4", f4); // test_function_record_leaks diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index 71e41d198..554cc4d7f 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -13,9 +13,7 @@ #include "constructor_stats.h" #include "pybind11_tests.h" -#if defined(_MSC_VER) -# pragma warning(disable : 4996) // C4996: std::unary_negation is deprecated -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) #include diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl index 09b35fa13..f46eb1803 100644 --- a/tests/test_eigen_tensor.inl +++ b/tests/test_eigen_tensor.inl @@ -9,7 +9,9 @@ #include "pybind11_tests.h" -namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE { +PYBIND11_NAMESPACE_BEGIN(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) template void reset_tensor(M &x) { @@ -90,7 +92,7 @@ struct CustomExample { template void init_tensor_module(pybind11::module &m) { const char *needed_options = ""; - if (PYBIND11_SILENCE_MSVC_C4127(Options == Eigen::ColMajor)) { + if (Options == Eigen::ColMajor) { needed_options = "F"; } else { needed_options = "C"; @@ -330,4 +332,4 @@ void test_module(py::module_ &m) { init_tensor_module(c_style); } -} // namespace PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE +PYBIND11_NAMESPACE_END(PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE) diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp index a03a8b37c..558a7a35e 100644 --- a/tests/test_embed/catch.cpp +++ b/tests/test_embed/catch.cpp @@ -3,11 +3,9 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) // Catch uses _ internally, which breaks gettext style defines #ifdef _ diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 44dcd1fdb..8cfd13fe5 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,10 +1,8 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) #include #include diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 2f3cabaf0..77e72c0c7 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -44,14 +44,13 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { + PYBIND11_WARNING_PUSH + #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wreturn-std-move" + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") #endif return args; -#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic pop -#endif + PYBIND11_WARNING_POP }); m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(args, kwargs); diff --git a/tests/test_operator_overloading.cpp b/tests/test_operator_overloading.cpp index a4b895a89..112a363b4 100644 --- a/tests/test_operator_overloading.cpp +++ b/tests/test_operator_overloading.cpp @@ -132,22 +132,18 @@ struct hash { // Not a good abs function, but easy to test. std::string abs(const Vector2 &) { return "abs(Vector2)"; } -// MSVC & Intel warns about unknown pragmas, and warnings are errors. -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push // clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to // `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). -// Here, we suppress the warning using `#pragma diagnostic`. +// Here, we suppress the warning // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). -# if defined(__APPLE__) && defined(__clang__) -# if (__clang_major__ >= 10) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif -# elif defined(__clang__) -# if (__clang_major__ >= 7) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif +#if defined(__APPLE__) && defined(__clang__) +# if (__clang_major__ >= 10) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") +# endif +#elif defined(__clang__) +# if (__clang_major__ >= 7) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") # endif #endif @@ -283,6 +279,3 @@ TEST_SUBMODULE(operators, m) { m.def("get_unhashable_HashMe_set", []() { return std::unordered_set{{"one"}}; }); } -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif From 88699849264150f513dafb98d9ec1bb31a7fa904 Mon Sep 17 00:00:00 2001 From: Arman Date: Thu, 1 Dec 2022 07:17:59 +0200 Subject: [PATCH 14/17] scoped_interpreter. overloaded constructor: PyConfig param (#4330) * scoped_interpreter overloaded ctor: PyConfig param * style: pre-commit fixes * refact: some logics extracted into funcs (precheck_interpreter, _initialize_interpreter); config_guard * style: pre-commit fixes * refact: scoped_config, some funcs hidden in detail ns * refact: macro PYBIND11_PYCONFIG_SUPPORT_PY_VERSION + undef * feat: PYBIND11_PYCONFIG_SUPPORT_PY_VERSION set to 3.8 * tests: Custom PyConfig * ci: python 3.6 -> 3.8 * ci: reverted py 38 back to 36; refact: initialize_interpreter overloads * style: pre-commit fixes * fix: readability-implicit-bool-conversion * refact: each initialize_interpreter overloads in pybind11 ns * Move `initialize_interpreter_pre_pyconfig()` into the `detail` namespace. Move the `PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX` define down to where it is used for the first time, and check if it is defined already, so that it is possible to customize from the compilation command line, just in case there is some unforeseen issue for Python 3.8, 3.9, 3.10. * tests: Add program dir to path, Custom PyConfig with argv * refact: clang-formatted * tests: Add-program-dir-to-path covers both scoped_interpreter overloads * tests: Add-program-dir-to-path fixed * tests: Add-program-dir-to-path py_version dependant validation Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/embed.h | 119 +++++++++++++++++--------- tests/test_embed/test_interpreter.cpp | 66 ++++++++++++++ 2 files changed, 143 insertions(+), 42 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 5b77594b3..749c75beb 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -86,37 +86,22 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -PYBIND11_NAMESPACE_END(detail) - -/** \rst - Initialize the Python interpreter. No other pybind11 or CPython API functions can be - called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional `init_signal_handlers` parameter can be used to skip the registration of - signal handlers (see the `Python documentation`_ for details). Calling this function - again after the interpreter has already been initialized is a fatal error. - - If initializing the Python interpreter fails, then the program is terminated. (This - is controlled by the CPython runtime and is an exception to pybind11's normal behavior - of throwing exceptions on errors.) - - The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are - used to populate ``sys.argv`` and ``sys.path``. - See the |PySys_SetArgvEx documentation|_ for details. - - .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx - .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation - .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx - \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true, - int argc = 0, - const char *const *argv = nullptr, - bool add_program_dir_to_path = true) { +inline void precheck_interpreter() { if (Py_IsInitialized() != 0) { pybind11_fail("The interpreter is already running"); } +} -#if PY_VERSION_HEX < 0x030B0000 +#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) +# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) +#endif +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, + int argc, + const char *const *argv, + bool add_program_dir_to_path) { + detail::precheck_interpreter(); Py_InitializeEx(init_signal_handlers ? 1 : 0); # if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 PyEval_InitThreads(); @@ -150,26 +135,30 @@ inline void initialize_interpreter(bool init_signal_handlers = true, auto *pysys_argv = widened_argv.get(); PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -#else - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; - config.install_signal_handlers = init_signal_handlers ? 1 : 0; +} +#endif - PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); - if (PyStatus_Exception(status)) { +PYBIND11_NAMESPACE_END(detail) + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + detail::precheck_interpreter(); + PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast(argv)); + if (PyStatus_Exception(status) != 0) { // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. - PyConfig_Clear(&config); - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to prepare CPython"); + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to prepare CPython"); } - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to init CPython"); + status = Py_InitializeFromConfig(config); + if (PyStatus_Exception(status) != 0) { + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to init CPython"); } if (add_program_dir_to_path) { PyRun_SimpleString("import sys, os.path; " @@ -177,6 +166,43 @@ inline void initialize_interpreter(bool init_signal_handlers = true, "os.path.abspath(os.path.dirname(sys.argv[0])) " "if sys.argv and os.path.exists(sys.argv[0]) else '')"); } + PyConfig_Clear(config); +} +#endif + +/** \rst + Initialize the Python interpreter. No other pybind11 or CPython API functions can be + called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. + + If initializing the Python interpreter fails, then the program is terminated. (This + is controlled by the CPython runtime and is an exception to pybind11's normal behavior + of throwing exceptions on errors.) + + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx + \endrst */ +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + detail::initialize_interpreter_pre_pyconfig( + init_signal_handlers, argc, argv, add_program_dir_to_path); +#else + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + config.isolated = 0; + config.use_environment = 1; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif } @@ -264,6 +290,15 @@ public: initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + explicit scoped_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(config, argc, argv, add_program_dir_to_path); + } +#endif + scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } scoped_interpreter &operator=(const scoped_interpreter &) = delete; diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 8cfd13fe5..5c306d363 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -166,6 +166,72 @@ TEST_CASE("There can be only one interpreter") { py::initialize_interpreter(); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Custom PyConfig") { + py::finalize_interpreter(); + PyConfig config; + PyConfig_InitPythonConfig(&config); + REQUIRE_NOTHROW(py::scoped_interpreter{&config}); + { + py::scoped_interpreter p{&config}; + REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast() == 42); + } + py::initialize_interpreter(); +} + +TEST_CASE("Custom PyConfig with argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope{&config, 1, argv}; + std::free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} +#endif + +TEST_CASE("Add program dir to path") { + static auto get_sys_path_size = []() -> size_t { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); + }; + static auto validate_path_len = [](size_t default_len) { +#if PY_VERSION_HEX < 0x030A0000 + // It seems a value remains in sys.path + // left by the previous call of scoped_interpreter ctor. + REQUIRE(get_sys_path_size() > default_len); +#else + REQUIRE(get_sys_path_size() == default_len + 1); +#endif + }; + py::finalize_interpreter(); + + size_t sys_path_default_size = 0; + { + py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; + sys_path_default_size = get_sys_path_size(); + } + { + py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path + validate_path_len(sys_path_default_size); + } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path + validate_path_len(sys_path_default_size); + } +#endif + py::initialize_interpreter(); +} + bool has_pybind11_internals_builtin() { auto builtins = py::handle(PyEval_GetBuiltins()); return builtins.contains(PYBIND11_INTERNALS_ID); From b14d58b6151b3cc7a9d9b2bf90d636940935db5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 23:30:36 -0800 Subject: [PATCH 15/17] chore(deps): bump pypa/gh-action-pypi-publish from 1.5.1 to 1.5.2 (#4370) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.5.1 to 1.5.2. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.5.1...v1.5.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index f03a39701..b787187ee 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.5.2 with: password: ${{ secrets.pypi_password }} packages_dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.5.1 + uses: pypa/gh-action-pypi-publish@v1.5.2 with: password: ${{ secrets.pypi_password_global }} packages_dir: global/ From 358ba459d2f05321a15555c3b57c606fe2597ec7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 1 Dec 2022 09:25:30 -0800 Subject: [PATCH 16/17] Fix test added with PR #4330 (#4372) --- tests/test_embed/test_interpreter.cpp | 47 ++++++++++++++------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 5c306d363..10b20f371 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -14,6 +14,11 @@ PYBIND11_WARNING_DISABLE_MSVC(4996) namespace py = pybind11; using namespace py::literals; +size_t get_sys_path_size() { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); +} + class Widget { public: explicit Widget(std::string message) : message(std::move(message)) {} @@ -196,41 +201,39 @@ TEST_CASE("Custom PyConfig with argv") { } #endif -TEST_CASE("Add program dir to path") { - static auto get_sys_path_size = []() -> size_t { - auto sys_path = py::module::import("sys").attr("path"); - return py::len(sys_path); - }; - static auto validate_path_len = [](size_t default_len) { -#if PY_VERSION_HEX < 0x030A0000 - // It seems a value remains in sys.path - // left by the previous call of scoped_interpreter ctor. - REQUIRE(get_sys_path_size() > default_len); -#else - REQUIRE(get_sys_path_size() == default_len + 1); -#endif - }; +TEST_CASE("Add program dir to path pre-PyConfig") { py::finalize_interpreter(); - - size_t sys_path_default_size = 0; + size_t path_size_add_program_dir_to_path_false = 0; { py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; - sys_path_default_size = get_sys_path_size(); + path_size_add_program_dir_to_path_false = get_sys_path_size(); } { - py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path - validate_path_len(sys_path_default_size); + py::scoped_interpreter scoped_interp{}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); } + py::initialize_interpreter(); +} + #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Add program dir to path using PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; { PyConfig config; PyConfig_InitPythonConfig(&config); - py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path - validate_path_len(sys_path_default_size); + py::scoped_interpreter scoped_interp{&config, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); } -#endif py::initialize_interpreter(); } +#endif bool has_pybind11_internals_builtin() { auto builtins = py::handle(PyEval_GetBuiltins()); From e133c33d5c6b19acab55fb1d6c10331d918bd830 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 1 Dec 2022 15:15:47 -0500 Subject: [PATCH 17/17] chore: Convert direct multiprocessing.set_start_method("forkserver") call to a pytest fixture. (#4377) * chore: convert multiprocessing set_spawn to fixture in pytest * Switch to early return --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 96dffc81c..402fd4b25 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,12 @@ import pytest # Early diagnostic for failed imports import pybind11_tests -if os.name != "nt": + +@pytest.fixture(scope="session", autouse=True) +def always_forkserver_on_unix(): + if os.name == "nt": + return + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. # It is actually a well-known pitfall, unfortunately without guard rails. @@ -27,6 +32,7 @@ if os.name != "nt": # running with defaults. multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+")