From 3a2c96bd6f95f6b183c342380558238583415ab1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 09:18:05 -0700 Subject: [PATCH 1/3] fix: unicode surrogate character in Python exception message. (#4297) * Fix & test for issue #4288 (unicode surrogate character in Python exception message). * DRY `message_unavailable_exc` * fix: add a constexpr Co-authored-by: Aaron Gokaslan * style: pre-commit fixes Co-authored-by: Henry Schreiner Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pytypes.h | 22 ++++++++++++++++++++-- tests/test_exceptions.cpp | 5 ----- tests/test_exceptions.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 80a2e2228..91cbaaba2 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -501,11 +501,29 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; } else { - result = value_str.cast(); + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } } } else { result = ""; diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 3583f22a5..f57e09506 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 70b6ffea9..7eb1a9d62 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": From b1bd7f2600d650bf59c88800c637703dd89317f3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 31 Oct 2022 10:36:26 -0700 Subject: [PATCH 2/3] fix: define (non-empty) `PYBIND11_EXPORT_EXCEPTION` only under macOS. (#4298) Background: #2999, #4105, #4283, #4284 In a nutshell: * Only macOS actually needs `PYBIND11_EXPORT_EXCEPTION` (#4284). * Evidently (#4283), under macOS `PYBIND11_EXPORT_EXCEPTION` does not run the risk of introducing ODR violations, * but evidently (#4283) under Linux it does, in the presumably rare/unusual situation that `RTLD_GLOBAL` is used. * Windows does no have the equivalent of `RTLD_GLOBAL`, therefore `PYBIND11_EXPORT_EXCEPTION` has no practical benefit, on the contrary, noisy warning suppression pragmas are needed, therefore it is best left empty. --- docs/advanced/exceptions.rst | 9 ++++++--- include/pybind11/detail/common.h | 18 +++--------------- include/pybind11/pytypes.h | 9 --------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 2211caf5d..53981dc08 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -177,9 +177,12 @@ section. may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. - Note that ``libc++`` and ``libstdc++`` `behave differently `_ - with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI boundaries need to be explicitly exported, as exercised in ``tests/test_exceptions.h``. - See also: "Problems with C++ exceptions" under `GCC Wiki `_. + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + `_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki `_. Local vs Global Exception Translators diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a3e0bc9b3..9b74323f6 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -96,13 +96,10 @@ #endif #if !defined(PYBIND11_EXPORT_EXCEPTION) -# ifdef __MINGW32__ -// workaround for: -// error: 'dllexport' implies default visibility, but xxx has already been declared with a -// different visibility -# define PYBIND11_EXPORT_EXCEPTION -# else +# if defined(__apple_build_version__) # define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION # endif #endif @@ -904,12 +901,6 @@ using expand_side_effects = bool[]; PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// C++ bindings of builtin Python exceptions class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: @@ -917,9 +908,6 @@ public: /// Set the error using the Python C API virtual void set_error() const = 0; }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 91cbaaba2..80b49ec39 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -623,12 +623,6 @@ inline std::string error_string() { PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275 4251) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// Fetch and hold an error which was already set in Python. An instance of this is typically /// thrown to propagate python-side errors back through C++ which can either be caught manually or /// else falls back to the function dispatcher (which then raises the captured error back to @@ -688,9 +682,6 @@ private: /// crashes (undefined behavior) if the Python interpreter is finalizing. static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif /// Replaces the current Python error indicator with the chosen error, performing a /// 'raise from' to indicate that the chosen error was caused by the original error. From 252ed8fb52e46eb3ec3e3b8c621ca9f79b53b26a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 31 Oct 2022 14:11:23 -0400 Subject: [PATCH 3/3] docs: prepare for 2.10.1 release (#4279) * docs: prepare for 2.10.1 release Signed-off-by: Henry Schreiner * Update changelog.rst * docs: update changelog with final list of PRs Signed-off-by: Henry Schreiner * Update docs/changelog.rst * chore: one more changelog bump Signed-off-by: Henry Schreiner --- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df3181c52..7d6d0c0f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,19 +10,18 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. - IN DEVELOPMENT -------------- Changes will be summarized here periodically. -Version 2.10.1 (Oct 2?, 2022) +Version 2.10.1 (Oct 31, 2022) ----------------------------- +This is the first version to fully support embedding the newly released Python 3.11. Changes: - * Allow ``pybind11::capsule`` constructor to take null destructor pointers. `#4221 `_ @@ -30,8 +29,40 @@ Changes: (established behavior). `#4119 `_ +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + Bug fixes: +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + * Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. `#4188 `_ @@ -100,9 +131,15 @@ Performance and style: * Optimize unpacking_collector when processing ``arg_v`` arguments. `#4219 `_ +* Optimize casting C++ object to ``None``. + `#4269 `_ + Build system improvements: +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + * Include a pkg-config file when installing pybind11, such as in the Python package. `#4077 `_