From 2af163d9c7d5c0276b7b2385034c1dd9181b187c Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 6 Jul 2022 16:35:12 -0400 Subject: [PATCH 1/3] Fix: 3.11 beta support (#3923) * Placeholder commit for 3.11 testing * Does this fix it? * Try suggestion * Placeholder commit for 3.11 testing * Does this fix it? * Try suggestion * fix: try using modern init for embedded interp Signed-off-by: Henry Schreiner * fix: error message changed in 3.11 * fix: apply logic in Python manually Signed-off-by: Henry Schreiner * fix autodetect dynamic attrs in 3.11 * fix: include error message if possible in error Signed-off-by: Henry Schreiner * ci: enable standard Python 3.11 testing Signed-off-by: Henry Schreiner * Make dynamic attrs condtiion exclusive to ver. Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 5 +- .github/workflows/upstream.yml | 2 +- include/pybind11/attr.h | 8 ++- include/pybind11/detail/class.h | 4 ++ include/pybind11/embed.h | 88 ++++++++++++++++++---------- setup.cfg | 1 + tests/requirements.txt | 2 +- tests/test_methods_and_attributes.py | 30 +++++++--- 8 files changed, 92 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a69438bd..787fa3c50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - '3.6' - '3.9' - '3.10' + - '3.11-dev' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -185,8 +186,8 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - # - python-version: "3.11-dev" - # python-debug: false + - python-version: "3.11-dev" + python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" runs-on: ubuntu-latest diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 424fce79c..a40a6c7d7 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -14,7 +14,7 @@ env: jobs: standard: - name: "🐍 3.11 dev • ubuntu-latest • x64" + name: "🐍 3.11 latest internals • ubuntu-latest • x64" runs-on: ubuntu-latest if: "contains(github.event.pull_request.labels.*.name, 'python dev')" diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 14600e9bb..db7cd8eff 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -345,9 +345,11 @@ struct type_record { bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) { - dynamic_attr = true; - } +#if PY_VERSION_HEX < 0x030B0000 + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif if (caster) { base_info->implicit_casts.emplace_back(type, caster); diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 1ddf1393b..c09f70ef0 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -545,8 +545,12 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; +#if PY_VERSION_HEX < 0x030B0000 type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it +#else + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; +#endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 98cc3e466..d6999cd77 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -86,37 +86,6 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -/// Python 2.x/3.x-compatible version of `PySys_SetArgv` -inline void set_interpreter_argv(int argc, const char *const *argv, bool add_program_dir_to_path) { - // Before it was special-cased in python 3.8, passing an empty or null argv - // caused a segfault, so we have to reimplement the special case ourselves. - bool special_case = (argv == nullptr || argc <= 0); - - const char *const empty_argv[]{"\0"}; - const char *const *safe_argv = special_case ? empty_argv : argv; - if (special_case) { - argc = 1; - } - - auto argv_size = static_cast(argc); - // SetArgv* on python 3 takes wchar_t, so we have to convert. - std::unique_ptr widened_argv(new wchar_t *[argv_size]); - std::vector> widened_argv_entries; - widened_argv_entries.reserve(argv_size); - for (size_t ii = 0; ii < argv_size; ++ii) { - widened_argv_entries.emplace_back(widen_chars(safe_argv[ii])); - if (!widened_argv_entries.back()) { - // A null here indicates a character-encoding failure or the python - // interpreter out of memory. Give up. - return; - } - widened_argv[ii] = widened_argv_entries.back().get(); - } - - auto *pysys_argv = widened_argv.get(); - PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -} - PYBIND11_NAMESPACE_END(detail) /** \rst @@ -146,9 +115,64 @@ inline void initialize_interpreter(bool init_signal_handlers = true, pybind11_fail("The interpreter is already running"); } +#if PY_VERSION_HEX < 0x030B0000 + Py_InitializeEx(init_signal_handlers ? 1 : 0); - detail::set_interpreter_argv(argc, argv, add_program_dir_to_path); + // Before it was special-cased in python 3.8, passing an empty or null argv + // caused a segfault, so we have to reimplement the special case ourselves. + bool special_case = (argv == nullptr || argc <= 0); + + const char *const empty_argv[]{"\0"}; + const char *const *safe_argv = special_case ? empty_argv : argv; + if (special_case) { + argc = 1; + } + + auto argv_size = static_cast(argc); + // SetArgv* on python 3 takes wchar_t, so we have to convert. + std::unique_ptr widened_argv(new wchar_t *[argv_size]); + std::vector> widened_argv_entries; + widened_argv_entries.reserve(argv_size); + for (size_t ii = 0; ii < argv_size; ++ii) { + widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii])); + if (!widened_argv_entries.back()) { + // A null here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + return; + } + widened_argv[ii] = widened_argv_entries.back().get(); + } + + auto *pysys_argv = widened_argv.get(); + + PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); +#else + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + + PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); + if (PyStatus_Exception(status)) { + // A failure here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + PyConfig_Clear(&config); + throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg + : "Failed to prepare CPython"); + } + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg + : "Failed to init CPython"); + } + if (add_program_dir_to_path) { + PyRun_SimpleString("import sys, os.path; " + "sys.path.insert(0, " + "os.path.abspath(os.path.dirname(sys.argv[0])) " + "if sys.argv and os.path.exists(sys.argv[0]) else '')"); + } +#endif } /** \rst diff --git a/setup.cfg b/setup.cfg index 4fd1e7a6f..36fbfed4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython diff --git a/tests/requirements.txt b/tests/requirements.txt index 1159287fa..04aafa8cf 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ -build==0.7.0 +build==0.8.0 numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7" numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 54597fa78..0a2ae1239 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -1,9 +1,21 @@ +import sys + import pytest import env # noqa: F401 from pybind11_tests import ConstructorStats from pybind11_tests import methods_and_attributes as m +NO_GETTER_MSG = ( + "unreadable attribute" if sys.version_info < (3, 11) else "object has no getter" +) +NO_SETTER_MSG = ( + "can't set attribute" if sys.version_info < (3, 11) else "object has no setter" +) +NO_DELETER_MSG = ( + "can't delete attribute" if sys.version_info < (3, 11) else "object has no deleter" +) + def test_methods_and_attributes(): instance1 = m.ExampleMandA() @@ -102,32 +114,32 @@ def test_properties(): with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_writeonly # unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly = 4 assert instance.def_property_readonly == 4 with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_impossible # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) with pytest.raises(AttributeError) as excinfo: instance.def_property_impossible = 5 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) def test_static_properties(): assert m.TestProperties.def_readonly_static == 1 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_readonly_static = 2 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_readwrite_static = 2 assert m.TestProperties.def_readwrite_static == 2 with pytest.raises(AttributeError) as excinfo: dummy = m.TestProperties.def_writeonly_static # unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_writeonly_static = 3 assert m.TestProperties.def_readonly_static == 3 @@ -135,14 +147,14 @@ def test_static_properties(): assert m.TestProperties.def_property_readonly_static == 3 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_property_readonly_static = 99 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_property_static = 4 assert m.TestProperties.def_property_static == 4 with pytest.raises(AttributeError) as excinfo: dummy = m.TestProperties.def_property_writeonly_static - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_property_writeonly_static = 5 assert m.TestProperties.def_property_static == 5 @@ -160,7 +172,7 @@ def test_static_properties(): with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_writeonly_static # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly_static = 4 assert instance.def_property_static == 4 @@ -180,7 +192,7 @@ def test_static_properties(): properties_override = m.TestPropertiesOverride() with pytest.raises(AttributeError) as excinfo: del properties_override.def_readonly - assert "can't delete attribute" in str(excinfo.value) + assert NO_DELETER_MSG in str(excinfo.value) def test_static_cls(): From 0ab1fcfb1c6c696422cc072ba7b9f50a11ed23c2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 6 Jul 2022 16:36:05 -0400 Subject: [PATCH 2/3] docs: update changelog (#4042) Signed-off-by: Henry Schreiner --- docs/changelog.rst | 151 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 13 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 67ac79cdd..9d0f217ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,44 +14,164 @@ IN DEVELOPMENT Removed support for Python 2.7, Python 3.5, and MSVC 2015. Support for MSVC 2017 is limited due to availability of CI runners; we highly recommend MSVC -2019 or 2022 be used. +2019 or 2022 be used. Initial support added for Python 3.11. + New features: +* ``py::anyset`` & ``py::frozenset`` were added, with copying (cast) to + ``std::set`` (similar to ``set``). + `#3901 `_ + +* Support bytearray casting to string. + `#3707 `_ + * ``type_caster`` was added. ``std::monostate`` is a tag type that allows ``std::variant`` to act as an optional, or allows default construction of a ``std::variant`` holding a non-default constructible type. `#3818 `_ -* Support bytearray casting to string. - `#3707 `_ +* ``pybind11::capsule::set_name`` added to mutate the name of the capsule instance. + `#3866 `_ + +* NumPy: dtype constructor from type number added, accessors corresponding to + Python API ``dtype.num``, ``dtype.byteorder``, ``dtype.flags`` and + ``dtype.alignment`` added. + `#3868 `_ + Changes: -* Python 2 support was removed completely. +* Python 3.6 is now the minimum supported version. `#3688 `_ + `#3719 `_ * The minimum version for MSVC is now 2017. `#3722 `_ +* Fix issues with CPython 3.11 betas and add to supported test matrix. + `#3923 `_ + +* ``error_already_set`` is now safer and more performant, especially for + exceptions with long tracebacks, by delaying computation. + `#1895 `_ + * Improve exception handling in python ``str`` bindings. `#3826 `_ * The bindings for capsules now have more consistent exception handling. `#3825 `_ -* Fix exception handling when ``pybind11::weakref()`` fails. - `#3739 `_ +* ``PYBIND11_OBJECT_CVT`` and ``PYBIND11_OBJECT_CVT_DEFAULT`` macro can now be + used to define classes in namespaces other than pybind11. + `#3797 `_ + +* Error printing code now uses ``PYBIND11_DETAILED_ERROR_MESSAGES`` instead of + requiring ``NDEBUG``, allowing use with release builds if desired. + `#3913 `_ Bug fixes: -* ``PYBIND11_OBJECT_CVT`` and ``PYBIND11_OBJECT_CVT_DEFAULT`` macro can be used - to define classes in namespaces other than pybind11. - `#3797 `_ +* Fix exception handling when ``pybind11::weakref()`` fails. + `#3739 `_ + +* ``module_::def_submodule`` was missing proper error handling. This is fixed now. + `#3973 `_ + +* The behavior or ``error_already_set`` was made safer and the highly opaque + "Unknown internal error occurred" message was replaced with a more helpful + message. + `#3982 `_ + +* ``error_already_set::what()`` now handles non-normalized exceptions correctly. + `#3971 `_ + +* Support older C++ compilers where filesystem is not yet part of the standard + library and is instead included in ``std::experimental::filesystem``. + `#3840 `_ + +* Fix ``-Wfree-nonheap-object`` warnings produced by GCC by avoiding returning + pointers to static objects with ``return_value_policy::take_ownership``. + `#3946 `_ + +* Fix cast from pytype rvalue to another pytype. + `#3949 `_ + +* ``pybind11::detail::get_internals()`` is now resilient to in-flight Python + exceptions. + `#3981 `_ + +* Arrays with a dimension of size 0 are now properly converted to dynamic Eigen + matrices (more common in NumPy 1.23). + `#4038 `_ + +* Avoid catching unrelated errors when importing NumPy. + `#3974 `_ + +Performance and style: + +* Added an accessor overload of ``(object &&key)`` to reference steal the + object when using python types as keys. This prevents unnecessary reference + count overhead for attr, dictionary, tuple, and sequence look ups. Added + additional regression tests. Fixed a performance bug the caused accessor + assignments to potentially perform unnecessary copies. + `#3970 `_ + +* Perfect forward all args of ``make_iterator``. + `#3980 `_ + +* Avoid potential bug in pycapsule destructor by adding an ``error_guard`` to + one of the dtors. + `#3958 `_ + +* Optimize dictionary access in ``strip_padding`` for numpy. + `#3994 `_ + +* ``stl_bind.h`` bindings now take slice args as a const-ref. + `#3852 `_ + +* Made slice constructor more consistent, and improve performance of some + casters by allowing reference stealing. + `#3845 `_ + +* Change numpy dtype from_args method to use const ref. + `#3878 `_ + +* Follow rule of three to ensure ``PyErr_Restore`` is called only once. + `#3872 `_ + +* Added missing perfect forwarding for ``make_iterator`` functions. + `#3860 `_ + +* Optimize c++ to python function casting by using the rvalue caster. + `#3966 `_ + +* Avoid potential implicit copy/assignment constructors causing double free in + ``strdup_gaurd``. + `#3905 `_ + +* Enable clang-tidy checks ``misc-definitions-in-headers``, + ``modernize-loop-convert``, and ``modernize-use-nullptr``. + `#3881 `_ + `#3988 `_ + Build system improvements: +* CMake: Fix file extension on Windows with cp36 and cp37 using FindPython. + `#3919 `_ + +* CMake: Support multiple Python targets (such as on vcpkg). + `#3948 `_ + +* CMake: Fix issue with NVCC on Windows. + `#3947 `_ + +* CMake: Drop the bitness check on cross compiles (like targeting WebAssembly + via Emscripten). + `#3959 `_ + * Add MSVC builds in debug mode to CI. `#3784 `_ @@ -59,15 +179,20 @@ Build system improvements: `#3732 `_, `#3741 `_ -* Avoid ``setup.py `` usage in internal tests. - `#3734 `_ - Backend and tidying up: -* Remove idioms in code comments. Use inclusive language. +* Remove idioms in code comments. Use more inclusive language. `#3809 `_ +* ``#include `` was removed from the ``pybind11/stl.h`` header. Your + project may break if it has a transitive dependency on this include. The fix + is to "Include What You Use". + `#3928 `_ + +* Avoid ``setup.py `` usage in internal tests. + `#3734 `_ + Version 2.9.2 (Mar 29, 2022) ---------------------------- From cd08869df14d02afd55519ea92c79b42b8e48c1c Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 6 Jul 2022 14:29:20 -0700 Subject: [PATCH 3/3] PYBIND11_NAMESPACE consistency fixes. (#4043) --- include/pybind11/cast.h | 8 ++++---- include/pybind11/detail/init.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9a971704e..782fb20d4 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -846,7 +846,7 @@ struct always_construct_holder { /// Create a specialization for custom holder types (silently ignores std::shared_ptr) #define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ - namespace pybind11 { \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template \ struct always_construct_holder : always_construct_holder { \ @@ -855,7 +855,7 @@ struct always_construct_holder { class type_caster::value>> \ : public type_caster_holder {}; \ } \ - } + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) // PYBIND11_DECLARE_HOLDER_TYPE holder types: template @@ -1650,12 +1650,12 @@ handle type::handle_of() { } #define PYBIND11_MAKE_OPAQUE(...) \ - namespace pybind11 { \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template <> \ class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> {}; \ } \ - } + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) /// Lets you pass a type containing a `,` through a macro parameter without needing a separate /// typedef, e.g.: diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index e1e665a69..05f4fe54a 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -425,4 +425,4 @@ struct pickle_factory { PYBIND11_NAMESPACE_END(initimpl) PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(pybind11) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)