From 4f29b8a45b8d1e52d06d920f7a8140811a925fd9 Mon Sep 17 00:00:00 2001 From: Liam Keegan Date: Thu, 19 Aug 2021 20:42:55 +0200 Subject: [PATCH 01/11] ci: extend msys2 mingw CI (#3207) * extend msys2 CI - add 32-bit job - add c++11/17 c++/interface tests copied from standard ci - add numpy/scipy * account for padding of PartialStruct in numpy dtypes test with mingw32 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * msys2 ci: add c++14 tests Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 81 +++++++++++++++++++++++++++++++------- tests/test_numpy_dtypes.py | 14 +++++-- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be3fa723c..a3664ddf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -861,32 +861,85 @@ jobs: run: cmake --build build -t check mingw: + name: "🐍 3 • windows-latest • ${{ matrix.sys }}" runs-on: windows-latest defaults: run: shell: msys2 {0} + strategy: + fail-fast: false + matrix: + include: + - { sys: mingw64, env: x86_64 } + - { sys: mingw32, env: i686 } steps: - uses: msys2/setup-msys2@v2 with: + msystem: ${{matrix.sys}} install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-python-pip - mingw-w64-x86_64-cmake - mingw-w64-x86_64-make - mingw-w64-x86_64-python-pytest - mingw-w64-x86_64-eigen3 - mingw-w64-x86_64-boost - mingw-w64-x86_64-catch + git + mingw-w64-${{matrix.env}}-gcc + mingw-w64-${{matrix.env}}-python-pip + mingw-w64-${{matrix.env}}-python-numpy + mingw-w64-${{matrix.env}}-python-scipy + mingw-w64-${{matrix.env}}-cmake + mingw-w64-${{matrix.env}}-make + mingw-w64-${{matrix.env}}-python-pytest + mingw-w64-${{matrix.env}}-eigen3 + mingw-w64-${{matrix.env}}-boost + mingw-w64-${{matrix.env}}-catch - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - - name: Configure + - name: Configure C++11 # LTO leads to many undefined reference like # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) - run: cmake -G "MinGW Makefiles" -S . -B build + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -S . -B build - - name: Build + - name: Build C++11 run: cmake --build build -j 2 - - name: Python tests - run: cmake --build build --target pytest + - name: Python tests C++11 + run: cmake --build build --target pytest -j 2 + + - name: C++11 tests + run: cmake --build build --target cpptest -j 2 + + - name: Interface test C++11 + run: cmake --build build --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++14 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -S . -B build2 + + - name: Build C++14 + run: cmake --build build2 -j 2 + + - name: Python tests C++14 + run: cmake --build build2 --target pytest -j 2 + + - name: C++14 tests + run: cmake --build build2 --target cpptest -j 2 + + - name: Interface test C++14 + run: cmake --build build2 --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++17 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -S . -B build3 + + - name: Build C++17 + run: cmake --build build3 -j 2 + + - name: Python tests C++17 + run: cmake --build build3 --target pytest -j 2 + + - name: C++17 tests + run: cmake --build build3 --target cpptest -j 2 + + - name: Interface test C++17 + run: cmake --build build3 --target test_cmake_build diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index 6ea064d5b..06e578329 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -63,14 +63,20 @@ def partial_ld_offset(): def partial_dtype_fmt(): ld = np.dtype("longdouble") partial_ld_off = partial_ld_offset() - return dt_fmt().format(ld.itemsize, partial_ld_off, partial_ld_off + ld.itemsize) + partial_size = partial_ld_off + ld.itemsize + partial_end_padding = partial_size % np.dtype("uint64").alignment + return dt_fmt().format( + ld.itemsize, partial_ld_off, partial_size + partial_end_padding + ) def partial_nested_fmt(): ld = np.dtype("longdouble") partial_nested_off = 8 + 8 * (ld.alignment > 8) partial_ld_off = partial_ld_offset() - partial_nested_size = partial_nested_off * 2 + partial_ld_off + ld.itemsize + partial_size = partial_ld_off + ld.itemsize + partial_end_padding = partial_size % np.dtype("uint64").alignment + partial_nested_size = partial_nested_off * 2 + partial_size + partial_end_padding return "{{'names':['a'], 'formats':[{}], 'offsets':[{}], 'itemsize':{}}}".format( partial_dtype_fmt(), partial_nested_off, partial_nested_size ) @@ -91,10 +97,12 @@ def test_format_descriptors(): ldbl_fmt = ("4x" if ld.alignment > 4 else "") + ld.char ss_fmt = "^T{?:bool_:3xI:uint_:f:float_:" + ldbl_fmt + ":ldbl_:}" dbl = np.dtype("double") + end_padding = ld.itemsize % np.dtype("uint64").alignment partial_fmt = ( "^T{?:bool_:3xI:uint_:f:float_:" + str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) - + "xg:ldbl_:}" + + "xg:ldbl_:" + + (str(end_padding) + "x}" if end_padding > 0 else "}") ) nested_extra = str(max(8, ld.alignment)) assert m.print_format_descriptors() == [ From b3d18f382f3bc37ec4d95cfe2045db72cf0e950d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 15:22:12 -0400 Subject: [PATCH 02/11] [pre-commit.ci] pre-commit autoupdate (#3213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.3 → v2.24.0](https://github.com/asottile/pyupgrade/compare/v2.23.3...v2.24.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a5900df16..96ddcf746 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^noxfile.py$ - repo: https://github.com/asottile/pyupgrade - rev: v2.23.3 + rev: v2.24.0 hooks: - id: pyupgrade From fdac5fbf7c1c3b63b6f3067734a10db0b409e28c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 23 Aug 2021 15:05:54 -0700 Subject: [PATCH 03/11] chore: support targeting different Python versions with nox (#3214) --- .github/CONTRIBUTING.md | 4 ++-- noxfile.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ff1997f98..39c32b2ac 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -68,8 +68,8 @@ nox -l # Run linters nox -s lint -# Run tests -nox -s tests +# Run tests on Python 3.9 +nox -s tests-3.9 # Build and preview docs nox -s docs -- serve diff --git a/noxfile.py b/noxfile.py index 55b1d180b..234179821 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,6 +2,8 @@ import nox nox.options.sessions = ["lint", "tests", "tests_packaging"] +PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] + @nox.session(reuse_venv=True) def lint(session: nox.Session) -> None: @@ -12,7 +14,7 @@ def lint(session: nox.Session) -> None: session.run("pre-commit", "run", "-a") -@nox.session +@nox.session(python=PYTHON_VERISONS) def tests(session: nox.Session) -> None: """ Run the tests (requires a compiler). From 6cbabc4b8c86a838216cb9d17dce14167222e6f7 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 23 Aug 2021 18:42:19 -0400 Subject: [PATCH 04/11] maint(clang-tidy): Enable cpp-coreguideline slicing checks (#3210) * maint(clang-tidy): add a clang-tidy slicing check * Add self + touch up readme * Fix typo --- .clang-tidy | 1 + README.rst | 4 ++-- include/pybind11/pybind11.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index c83b9b2f5..cefffba1e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -3,6 +3,7 @@ FormatStyle: file Checks: ' *bugprone*, cppcoreguidelines-init-variables, +cppcoreguidelines-slicing, clang-analyzer-optin.cplusplus.VirtualCall, llvm-namespace-comment, misc-misplaced-const, diff --git a/README.rst b/README.rst index 57eb06e55..7ce57b03a 100644 --- a/README.rst +++ b/README.rst @@ -134,9 +134,9 @@ About This project was created by `Wenzel Jakob `_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, -Sylvain Corlay, Eric Cousineau, Ralf Grosse-Kunstleve, Trent Houliston, Axel +Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, -Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim +Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. We thank Google for a generous financial contribution to the continuous diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 7b7b3ca71..47b042147 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1664,7 +1664,7 @@ inline str enum_name(handle arg) { } struct enum_base { - enum_base(handle base, handle parent) : m_base(base), m_parent(parent) { } + enum_base(const handle &base, const handle &parent) : m_base(base), m_parent(parent) { } PYBIND11_NOINLINE void init(bool is_arithmetic, bool is_convertible) { m_base.attr("__entries") = dict(); From c8ce4b8df854a630a5f02192305c183299778b84 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 23 Aug 2021 17:30:01 -0700 Subject: [PATCH 05/11] Clone of @virtuald's PR #2112 with minor enhancements. (#3215) * Add py::raise_from to enable chaining exceptions on Python 3.3+ * Use 'raise from' in initialization * Documenting the exact base version of _PyErr_FormatVFromCause, adding back `assert`s. Co-authored-by: Dustin Spicuzza --- docs/advanced/exceptions.rst | 28 ++++++++++++++++++ include/pybind11/detail/common.h | 15 ++++++++++ include/pybind11/pytypes.h | 41 +++++++++++++++++++++++++++ tests/test_embed/test_interpreter.cpp | 16 +++++++++++ tests/test_exceptions.cpp | 20 +++++++++++++ tests/test_exceptions.py | 16 +++++++++++ 6 files changed, 136 insertions(+) diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 738fb0ba9..2aaa0ad32 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -323,6 +323,34 @@ Alternately, to ignore the error, call `PyErr_Clear Any Python error must be thrown or cleared, or Python/pybind11 will be left in an invalid state. +Chaining exceptions ('raise from') +================================== + +In Python 3.3 a mechanism for indicating that exceptions were caused by other +exceptions was introduced: + +.. code-block:: py + + try: + print(1 / 0) + except Exception as exc: + raise RuntimeError("could not divide by zero") from exc + +To do a similar thing in pybind11, you can use the ``py::raise_from`` function. It +sets the current python error indicator, so to continue propagating the exception +you should ``throw py::error_already_set()`` (Python 3 only). + +.. code-block:: cpp + + try { + py::eval("print(1 / 0")); + } catch (py::error_already_set &e) { + py::raise_from(e, PyExc_RuntimeError, "could not divide by zero"); + throw py::error_already_set(); + } + +.. versionadded:: 2.8 + .. _unraisable_exceptions: Handling unraisable exceptions diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index cb52aa274..5050da802 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -315,6 +315,19 @@ extern "C" { } \ } +#if PY_VERSION_HEX >= 0x03030000 + +#define PYBIND11_CATCH_INIT_EXCEPTIONS \ + catch (pybind11::error_already_set &e) { \ + pybind11::raise_from(e, PyExc_ImportError, "initialization failed"); \ + return nullptr; \ + } catch (const std::exception &e) { \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } \ + +#else + #define PYBIND11_CATCH_INIT_EXCEPTIONS \ catch (pybind11::error_already_set &e) { \ PyErr_SetString(PyExc_ImportError, e.what()); \ @@ -324,6 +337,8 @@ extern "C" { return nullptr; \ } \ +#endif + /** \rst ***Deprecated in favor of PYBIND11_MODULE*** diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index dc1607ff2..85f6a40a4 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -382,6 +382,47 @@ private: # pragma warning(pop) #endif +#if PY_VERSION_HEX >= 0x03030000 + +/// Replaces the current Python error indicator with the chosen error, performing a +/// 'raise from' to indicate that the chosen error was caused by the original error. +inline void raise_from(PyObject *type, const char *message) { + // Based on _PyErr_FormatVFromCause: + // https://github.com/python/cpython/blob/467ab194fc6189d9f7310c89937c51abeac56839/Python/errors.c#L405 + // See https://github.com/pybind/pybind11/pull/2112 for details. + PyObject *exc = nullptr, *val = nullptr, *val2 = nullptr, *tb = nullptr; + + assert(PyErr_Occurred()); + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != nullptr) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + assert(!PyErr_Occurred()); + + PyErr_SetString(type, message); + + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); +} + +/// Sets the current Python error indicator with the chosen error, performing a 'raise from' +/// from the error contained in error_already_set to indicate that the chosen error was +/// caused by the original error. After this function is called error_already_set will +/// no longer contain an error. +inline void raise_from(error_already_set& err, PyObject *type, const char *message) { + err.restore(); + raise_from(type, message); +} + +#endif + /** \defgroup python_builtins _ Unless stated otherwise, the following C++ functions behave the same as their Python counterparts. diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index cd50e952f..b40ff4817 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -74,8 +74,24 @@ TEST_CASE("Import error handling") { REQUIRE_NOTHROW(py::module_::import("widget_module")); REQUIRE_THROWS_WITH(py::module_::import("throw_exception"), "ImportError: C++ Error"); +#if PY_VERSION_HEX >= 0x03030000 + REQUIRE_THROWS_WITH(py::module_::import("throw_error_already_set"), + Catch::Contains("ImportError: initialization failed")); + + auto locals = py::dict("is_keyerror"_a=false, "message"_a="not set"); + py::exec(R"( + try: + import throw_error_already_set + except ImportError as e: + is_keyerror = type(e.__cause__) == KeyError + message = str(e.__cause__) + )", py::globals(), locals); + REQUIRE(locals["is_keyerror"].cast() == true); + REQUIRE(locals["message"].cast() == "'missing'"); +#else REQUIRE_THROWS_WITH(py::module_::import("throw_error_already_set"), Catch::Contains("ImportError: KeyError")); +#endif } TEST_CASE("There can be only one interpreter") { diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index e28f0bb79..4805a0249 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -262,4 +262,24 @@ TEST_SUBMODULE(exceptions, m) { m.def("simple_bool_passthrough", [](bool x) {return x;}); m.def("throw_should_be_translated_to_key_error", []() { throw shared_exception(); }); + +#if PY_VERSION_HEX >= 0x03030000 + + m.def("raise_from", []() { + PyErr_SetString(PyExc_ValueError, "inner"); + py::raise_from(PyExc_ValueError, "outer"); + throw py::error_already_set(); + }); + + m.def("raise_from_already_set", []() { + try { + PyErr_SetString(PyExc_ValueError, "inner"); + throw py::error_already_set(); + } catch (py::error_already_set& e) { + py::raise_from(e, PyExc_ValueError, "outer"); + throw py::error_already_set(); + } + }); + +#endif } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index d1edc39f0..3821eadaa 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -24,6 +24,22 @@ def test_error_already_set(msg): assert msg(excinfo.value) == "foo" +@pytest.mark.skipif("env.PY2") +def test_raise_from(msg): + with pytest.raises(ValueError) as excinfo: + m.raise_from() + assert msg(excinfo.value) == "outer" + assert msg(excinfo.value.__cause__) == "inner" + + +@pytest.mark.skipif("env.PY2") +def test_raise_from_already_set(msg): + with pytest.raises(ValueError) as excinfo: + m.raise_from_already_set() + assert msg(excinfo.value) == "outer" + assert msg(excinfo.value.__cause__) == "inner" + + def test_cross_module_exceptions(msg): with pytest.raises(RuntimeError) as excinfo: cm.raise_runtime_error() From 031a700dfd611efc3949d8427d8c87e91a7b6998 Mon Sep 17 00:00:00 2001 From: Jouke Witteveen Date: Thu, 26 Aug 2021 17:04:22 +0200 Subject: [PATCH 06/11] Add make_simple_namespace function and tests (#2840) Co-authored-by: Jouke Witteveen --- docs/advanced/pycpp/object.rst | 43 +++++++++++++++++++++++++++++++++- include/pybind11/cast.h | 10 ++++++++ tests/test_pytypes.cpp | 13 ++++++++++ tests/test_pytypes.py | 13 ++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/docs/advanced/pycpp/object.rst b/docs/advanced/pycpp/object.rst index 6c7525cea..6fa8d0708 100644 --- a/docs/advanced/pycpp/object.rst +++ b/docs/advanced/pycpp/object.rst @@ -20,6 +20,47 @@ Available types include :class:`handle`, :class:`object`, :class:`bool_`, Be sure to review the :ref:`pytypes_gotchas` before using this heavily in your C++ API. +.. _instantiating_compound_types: + +Instantiating compound Python types from C++ +============================================ + +Dictionaries can be initialized in the :class:`dict` constructor: + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::dict d("spam"_a=py::none(), "eggs"_a=42); + +A tuple of python objects can be instantiated using :func:`py::make_tuple`: + +.. code-block:: cpp + + py::tuple tup = py::make_tuple(42, py::none(), "spam"); + +Each element is converted to a supported Python type. + +A `simple namespace`_ can be instantiated using +:func:`py::make_simple_namespace`: + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::object ns = py::make_simple_namespace("spam"_a=py::none(), "eggs"_a=42); + +Attributes on a namespace can be modified with the :func:`py::delattr`, +:func:`py::getattr`, and :func:`py::setattr` functions. Simple namespaces can +be useful as lightweight stand-ins for class instances. + +.. note:: + + ``make_simple_namespace`` is not available in Python 2. + +.. versionchanged:: 2.8 + ``make_simple_namespace`` added. + +.. _simple namespace: https://docs.python.org/3/library/types.html#types.SimpleNamespace + .. _casting_back_and_forth: Casting back and forth @@ -30,7 +71,7 @@ types to Python, which can be done using :func:`py::cast`: .. code-block:: cpp - MyClass *cls = ..; + MyClass *cls = ...; py::object obj = py::cast(cls); The reverse direction uses the following syntax: diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 718dc2de8..79bf506d8 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1018,6 +1018,16 @@ template = 0x03030000 +template ()>> +object make_simple_namespace(Args&&... args_) { + PyObject *ns = _PyNamespace_New(dict(std::forward(args_)...).ptr()); + if (!ns) throw error_already_set(); + return reinterpret_steal(ns); +} +#endif + /// \ingroup annotations /// Annotation for arguments struct arg { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index d70536d3f..15d007a43 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -70,6 +70,19 @@ TEST_SUBMODULE(pytypes, m) { m.def("dict_contains", [](const py::dict &dict, const char *val) { return dict.contains(val); }); + // test_tuple + m.def("get_tuple", []() { return py::make_tuple(42, py::none(), "spam"); }); + +#if PY_VERSION_HEX >= 0x03030000 + // test_simple_namespace + m.def("get_simple_namespace", []() { + auto ns = py::make_simple_namespace("attr"_a=42, "x"_a="foo", "wrong"_a=1); + py::delattr(ns, "wrong"); + py::setattr(ns, "right", py::int_(2)); + return ns; + }); +#endif + // test_str m.def("str_from_string", []() { return py::str(std::string("baz")); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 8a11b1872..f873658ab 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -99,6 +99,19 @@ def test_dict(capture, doc): assert m.dict_keyword_constructor() == {"x": 1, "y": 2, "z": 3} +def test_tuple(): + assert m.get_tuple() == (42, None, "spam") + + +@pytest.mark.skipif("env.PY2") +def test_simple_namespace(): + ns = m.get_simple_namespace() + assert ns.attr == 42 + assert ns.x == "foo" + assert ns.right == 2 + assert not hasattr(ns, "wrong") + + def test_str(doc): assert m.str_from_string().encode().decode() == "baz" assert m.str_from_bytes().encode().decode() == "boo" From 59ad1e7d05d96ccc4e7090d5da7a328e4512cb7a Mon Sep 17 00:00:00 2001 From: Nick Cullen Date: Thu, 26 Aug 2021 17:12:35 +0200 Subject: [PATCH 07/11] reshape for numpy arrays (#984) * reshape * more tests * Update numpy.h * Update test_numpy_array.py * Update numpy.h * Update numpy.h * Update test_numpy_array.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix merge bug * Make clang-tidy happy * Add xfail for PyPy * Fix casting issue * Address reviews on additional tests * Fix ordering * Do a little more reordering * Fix typo * Try improving tests * Fix error in reshape * Add one more reshape test * streamlining new tests; removing a few stray msg Co-authored-by: ncullen93 Co-authored-by: NC Cullen Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf Grosse-Kunstleve --- include/pybind11/numpy.h | 21 +++++++++++++++++++-- tests/test_numpy_array.cpp | 7 +++++++ tests/test_numpy_array.py | 31 ++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7717059f4..0d0cbdfa8 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -198,6 +198,8 @@ struct npy_api { // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); PyObject* (*PyArray_Resize_)(PyObject*, PyArray_Dims*, int, int); + PyObject* (*PyArray_Newshape_)(PyObject*, PyArray_Dims*, int); + private: enum functions { API_PyArray_GetNDArrayCFeatureVersion = 211, @@ -212,10 +214,11 @@ private: API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 96, + API_PyArray_Newshape = 135, + API_PyArray_Squeeze = 136, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, - API_PyArray_Squeeze = 136, API_PyArray_SetBaseObject = 282 }; @@ -243,11 +246,13 @@ private: DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); DECL_NPY_API(PyArray_DescrNewFromType); + DECL_NPY_API(PyArray_Newshape); + DECL_NPY_API(PyArray_Squeeze); DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); - DECL_NPY_API(PyArray_Squeeze); DECL_NPY_API(PyArray_SetBaseObject); + #undef DECL_NPY_API return api; } @@ -785,6 +790,18 @@ public: if (isinstance(new_array)) { *this = std::move(new_array); } } + /// Optional `order` parameter omitted, to be added as needed. + array reshape(ShapeContainer new_shape) { + detail::npy_api::PyArray_Dims d + = {reinterpret_cast(new_shape->data()), int(new_shape->size())}; + auto new_array + = reinterpret_steal(detail::npy_api::get().PyArray_Newshape_(m_ptr, &d, 0)); + if (!new_array) { + throw error_already_set(); + } + return new_array; + } + /// Ensure that the argument is a NumPy array /// In case of an error, nullptr is returned and the Python error is cleared. static array ensure(handle h, int ExtraFlags = 0) { diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 5c22a3d25..4ccfd279b 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -405,6 +405,13 @@ TEST_SUBMODULE(numpy_array, sm) { return a; }); + sm.def("reshape_initializer_list", [](py::array_t a, size_t N, size_t M, size_t O) { + return a.reshape({N, M, O}); + }); + sm.def("reshape_tuple", [](py::array_t a, const std::vector &new_shape) { + return a.reshape(new_shape); + }); + sm.def("index_using_ellipsis", [](const py::array &a) { return a[py::make_tuple(0, py::ellipsis(), 0)]; }); diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 69ba9d495..e96454be4 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -411,7 +411,7 @@ def test_array_unchecked_fixed_dims(msg): assert m.proxy_auxiliaries2_const_ref(z1) -def test_array_unchecked_dyn_dims(msg): +def test_array_unchecked_dyn_dims(): z1 = np.array([[1, 2], [3, 4]], dtype="float64") m.proxy_add2_dyn(z1, 10) assert np.all(z1 == [[11, 12], [13, 14]]) @@ -444,7 +444,7 @@ def test_initializer_list(): assert m.array_initializer_list4().shape == (1, 2, 3, 4) -def test_array_resize(msg): +def test_array_resize(): a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="float64") m.array_reshape2(a) assert a.size == 9 @@ -470,12 +470,37 @@ def test_array_resize(msg): @pytest.mark.xfail("env.PYPY") -def test_array_create_and_resize(msg): +def test_array_create_and_resize(): a = m.create_and_resize(2) assert a.size == 4 assert np.all(a == 42.0) +def test_reshape_initializer_list(): + a = np.arange(2 * 7 * 3) + 1 + x = m.reshape_initializer_list(a, 2, 7, 3) + assert x.shape == (2, 7, 3) + assert list(x[1][4]) == [34, 35, 36] + with pytest.raises(ValueError) as excinfo: + m.reshape_initializer_list(a, 1, 7, 3) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape (1,7,3)" + + +def test_reshape_tuple(): + a = np.arange(3 * 7 * 2) + 1 + x = m.reshape_tuple(a, (3, 7, 2)) + assert x.shape == (3, 7, 2) + assert list(x[1][4]) == [23, 24] + y = m.reshape_tuple(x, (x.size,)) + assert y.shape == (42,) + with pytest.raises(ValueError) as excinfo: + m.reshape_tuple(a, (3, 7, 1)) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape (3,7,1)" + with pytest.raises(ValueError) as excinfo: + m.reshape_tuple(a, ()) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape ()" + + def test_index_using_ellipsis(): a = m.index_using_ellipsis(np.zeros((5, 6, 7))) assert a.shape == (6,) From db44afa33b21af09f81c70ee6449c4a4d93fbeca Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 26 Aug 2021 10:52:13 -0700 Subject: [PATCH 08/11] tests: fix pytest usage on Python 3.10 (#3221) --- tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 00cb5f119..069122b88 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,11 +2,11 @@ numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6" numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6" -numpy==1.20.0; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10" +numpy==1.21.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10" +numpy==1.21.2; platform_python_implementation!="PyPy" and sys_platform=="linux" and python_version=="3.10" pytest==4.6.9; python_version<"3.5" pytest==6.1.2; python_version=="3.5" -pytest==6.2.1; python_version>="3.6" and python_version<="3.9" -pytest @ git+https://github.com/pytest-dev/pytest@c117bc350ec1e570672fda3b2ad234fd52e72b53; python_version>="3.10" +pytest==6.2.4; python_version>="3.6" pytest-timeout scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6" scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" From 503ff2a6fbb498138c0b7e85419de491c9860a93 Mon Sep 17 00:00:00 2001 From: Nick Cullen Date: Thu, 26 Aug 2021 23:11:01 +0200 Subject: [PATCH 09/11] view for numpy arrays (#987) * reshape * more tests * Update numpy.h * Update test_numpy_array.py * array view * test * Update test_numpy_array.cpp * Update numpy.h * Update numpy.h * Update test_numpy_array.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix merge bug * Make clang-tidy happy * Add xfail for PyPy * Fix casting issue * Fix formatting * Apply clang-tidy * Address reviews on additional tests * Fix ordering * Do a little more reordering * Fix typo * Try improving tests * Fix error in reshape * Add one more reshape test * Fix bugs and add test * Relax test * streamlining new tests; removing a few stray msg * Fix style revert * Fix clang-tidy * Misc tweaks: * Comment: matching style in file (///), responsibility sentence, consistent punctuation. * Replacing `unsigned char` with `uint8_t` for max consistency. * Removing `1` from `array_view1` because there is only one. * Partial clang-format-diff. Co-authored-by: ncullen93 Co-authored-by: NC Cullen Co-authored-by: Aaron Gokaslan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf Grosse-Kunstleve --- include/pybind11/numpy.h | 18 ++++++++++++++++++ tests/test_numpy_array.cpp | 3 +++ tests/test_numpy_array.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 0d0cbdfa8..fa128efdd 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -199,6 +199,7 @@ struct npy_api { int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); PyObject* (*PyArray_Resize_)(PyObject*, PyArray_Dims*, int, int); PyObject* (*PyArray_Newshape_)(PyObject*, PyArray_Dims*, int); + PyObject* (*PyArray_View_)(PyObject*, PyObject*, PyObject*); private: enum functions { @@ -216,6 +217,7 @@ private: API_PyArray_DescrNewFromType = 96, API_PyArray_Newshape = 135, API_PyArray_Squeeze = 136, + API_PyArray_View = 137, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, @@ -248,6 +250,7 @@ private: DECL_NPY_API(PyArray_DescrNewFromType); DECL_NPY_API(PyArray_Newshape); DECL_NPY_API(PyArray_Squeeze); + DECL_NPY_API(PyArray_View); DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); @@ -802,6 +805,21 @@ public: return new_array; } + /// Create a view of an array in a different data type. + /// This function may fundamentally reinterpret the data in the array. + /// It is the responsibility of the caller to ensure that this is safe. + /// Only supports the `dtype` argument, the `type` argument is omitted, + /// to be added as needed. + array view(const std::string &dtype) { + auto &api = detail::npy_api::get(); + auto new_view = reinterpret_steal(api.PyArray_View_( + m_ptr, dtype::from_args(pybind11::str(dtype)).release().ptr(), nullptr)); + if (!new_view) { + throw error_already_set(); + } + return new_view; + } + /// Ensure that the argument is a NumPy array /// In case of an error, nullptr is returned and the Python error is cleared. static array ensure(handle h, int ExtraFlags = 0) { diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 4ccfd279b..30a71acc9 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -405,6 +405,9 @@ TEST_SUBMODULE(numpy_array, sm) { return a; }); + sm.def("array_view", + [](py::array_t a, const std::string &dtype) { return a.view(dtype); }); + sm.def("reshape_initializer_list", [](py::array_t a, size_t N, size_t M, size_t O) { return a.reshape({N, M, O}); }); diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index e96454be4..e4138f023 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -476,6 +476,21 @@ def test_array_create_and_resize(): assert np.all(a == 42.0) +def test_array_view(): + a = np.ones(100 * 4).astype("uint8") + a_float_view = m.array_view(a, "float32") + assert a_float_view.shape == (100 * 1,) # 1 / 4 bytes = 8 / 32 + + a_int16_view = m.array_view(a, "int16") # 1 / 2 bytes = 16 / 32 + assert a_int16_view.shape == (100 * 2,) + + +def test_array_view_invalid(): + a = np.ones(100 * 4).astype("uint8") + with pytest.raises(TypeError): + m.array_view(a, "deadly_dtype") + + def test_reshape_initializer_list(): a = np.arange(2 * 7 * 3) + 1 x = m.reshape_initializer_list(a, 2, 7, 3) From 930bb16c797af642ed4d216cd8972bbd3276dceb Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 26 Aug 2021 17:12:54 -0400 Subject: [PATCH 10/11] Call PySys_SetArgv when initializing interpreter. (#2341) * Call PySys_SetArgv when initializing interpreter. * Document argc/argv parameters in initialize_interpreter. * Remove manual memory management from set_interpreter_argv in favor of smart pointers. * Use size_t for indexers in set_interpreter_argv. * Minimize macros for flow control in set_interpreter_argv. * Fix 'unused variable' warning on Py2 * whitespace * Define wide_char_arg_deleter outside set_interpreter_argv. * Do sys.path workaround in C++ rather than eval. * Factor out wchar conversion to a separate function. * Restore widened_argv variable declaration. * Fix undeclared widened_arg variable on some paths. * Use delete[] to match new wchar_t[]. * Fix compiler errors * Use PY_VERSION_HEX for a cleaner CVE-2008-5983 mode check. * Fix typo * Use explicit type for deleter so delete[] works cross-compiler. * Always use PySys_SetArgvEx because pybind11 doesn't support pythons that don't include it. * Remove pointless ternary operator. * Use unique_ptr.reset instead of a second initialization. * Rename add_program_dir_to_path parameter to clarify intent. * Add defined() check before evaluating HAVE_BROKEN_MBSTOWCS. * Apply clang-tidy fixes * Pre-commit * refactor: use const for set_interpreter_argv * Try to fix const issue and allocate vector properly * fix: copy strings on Python 2 * Applying clang-format-diff relative to master. The only manual change is an added empty line between pybind11 and system `#include`s. ``` git diff -U0 --no-color master | python3 $HOME/clone/llvm-project/clang/tools/clang-format/clang-format-diff.py -p1 -style=file -i ``` Co-authored-by: Boris Staletic Co-authored-by: Aaron Gokaslan Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/embed.h | 101 ++++++++++++++++++++++++-- tests/test_embed/test_interpreter.cpp | 24 ++++++ tests/test_embed/test_interpreter.py | 5 ++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 6e777830f..7b5d7cd24 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -12,6 +12,9 @@ #include "pybind11.h" #include "eval.h" +#include +#include + #if defined(PYPY_VERSION) # error Embedding the interpreter is not supported with PyPy #endif @@ -83,29 +86,106 @@ struct embedded_module { } }; +struct wide_char_arg_deleter { + void operator()(wchar_t *ptr) const { +#if PY_VERSION_HEX >= 0x030500f0 + // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale + PyMem_RawFree(ptr); +#else + delete[] ptr; +#endif + } +}; + +inline wchar_t *widen_chars(const char *safe_arg) { +#if PY_VERSION_HEX >= 0x030500f0 + wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr); +#else + wchar_t *widened_arg = nullptr; +# if defined(HAVE_BROKEN_MBSTOWCS) && HAVE_BROKEN_MBSTOWCS + size_t count = strlen(safe_arg); +# else + size_t count = mbstowcs(nullptr, safe_arg, 0); +# endif + if (count != static_cast(-1)) { + widened_arg = new wchar_t[count + 1]; + mbstowcs(widened_arg, safe_arg, count + 1); + } +#endif + return widened_arg; +} + +/// Python 2.x/3.x-compatible version of `PySys_SetArgv` +inline void set_interpreter_argv(int argc, const char *const *argv, bool add_program_dir_to_path) { + // Before it was special-cased in python 3.8, passing an empty or null argv + // caused a segfault, so we have to reimplement the special case ourselves. + bool special_case = (argv == nullptr || argc <= 0); + + const char *const empty_argv[]{"\0"}; + const char *const *safe_argv = special_case ? empty_argv : argv; + if (special_case) + argc = 1; + + auto argv_size = static_cast(argc); +#if PY_MAJOR_VERSION >= 3 + // SetArgv* on python 3 takes wchar_t, so we have to convert. + std::unique_ptr widened_argv(new wchar_t *[argv_size]); + std::vector> widened_argv_entries; + widened_argv_entries.reserve(argv_size); + for (size_t ii = 0; ii < argv_size; ++ii) { + widened_argv_entries.emplace_back(widen_chars(safe_argv[ii])); + if (!widened_argv_entries.back()) { + // A null here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + return; + } + widened_argv[ii] = widened_argv_entries.back().get(); + } + + auto pysys_argv = widened_argv.get(); +#else + // python 2.x + std::vector strings{safe_argv, safe_argv + argv_size}; + std::vector char_strings{argv_size}; + for (std::size_t i = 0; i < argv_size; ++i) + char_strings[i] = &strings[i][0]; + char **pysys_argv = char_strings.data(); +#endif + + PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); +} + PYBIND11_NAMESPACE_END(detail) /** \rst Initialize the Python interpreter. No other pybind11 or CPython API functions can be called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional parameter can be used to skip the registration of signal handlers (see the - `Python documentation`_ for details). Calling this function again after the interpreter - has already been initialized is a fatal error. + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. If initializing the Python interpreter fails, then the program is terminated. (This is controlled by the CPython runtime and is an exception to pybind11's normal behavior of throwing exceptions on errors.) + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true) { +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { if (Py_IsInitialized() != 0) pybind11_fail("The interpreter is already running"); Py_InitializeEx(init_signal_handlers ? 1 : 0); - // Make .py files in the working directory available by default - module_::import("sys").attr("path").cast().append("."); + detail::set_interpreter_argv(argc, argv, add_program_dir_to_path); } /** \rst @@ -167,6 +247,8 @@ inline void finalize_interpreter() { Scope guard version of `initialize_interpreter` and `finalize_interpreter`. This a move-only guard and only a single instance can exist. + See `initialize_interpreter` for a discussion of its constructor arguments. + .. code-block:: cpp #include @@ -178,8 +260,11 @@ inline void finalize_interpreter() { \endrst */ class scoped_interpreter { public: - scoped_interpreter(bool init_signal_handlers = true) { - initialize_interpreter(init_signal_handlers); + scoped_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } scoped_interpreter(const scoped_interpreter &) = delete; diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index b40ff4817..78b64be6b 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -23,6 +23,7 @@ public: std::string the_message() const { return message; } virtual int the_answer() const = 0; + virtual std::string argv0() const = 0; private: std::string message; @@ -32,6 +33,7 @@ class PyWidget final : public Widget { using Widget::Widget; int the_answer() const override { PYBIND11_OVERRIDE_PURE(int, Widget, the_answer); } + std::string argv0() const override { PYBIND11_OVERRIDE_PURE(std::string, Widget, argv0); } }; PYBIND11_EMBEDDED_MODULE(widget_module, m) { @@ -299,3 +301,25 @@ TEST_CASE("Reload module from file") { result = module_.attr("test")().cast(); REQUIRE(result == 2); } + +TEST_CASE("sys.argv gets initialized properly") { + py::finalize_interpreter(); + { + py::scoped_interpreter default_scope; + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0().empty()); + } + + { + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope(true, 1, argv); + free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} diff --git a/tests/test_embed/test_interpreter.py b/tests/test_embed/test_interpreter.py index 6174ede44..5ab55a4b3 100644 --- a/tests/test_embed/test_interpreter.py +++ b/tests/test_embed/test_interpreter.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys + from widget_module import Widget @@ -8,3 +10,6 @@ class DerivedWidget(Widget): def the_answer(self): return 42 + + def argv0(self): + return sys.argv[0] From cb60ed49e46fb31cc578b86b2178d8ce574617ad Mon Sep 17 00:00:00 2001 From: Ye Zhihao Date: Fri, 27 Aug 2021 05:34:24 +0800 Subject: [PATCH 11/11] Fix enum value's __int__ returning non-int when underlying type is bool or of char type (#1334) * Use equivalent_integer for enum's Scalar decision * Add test for char underlying enum * Support translating bool type in enum's Scalar * Add test for bool underlying enum * Fix comment in test * Switch from `PYBIND11_CPP20` macro to `PYBIND11_HAS_U8STRING` * Refine tests Co-authored-by: Aaron Gokaslan --- include/pybind11/pybind11.h | 21 +++++++++++-- tests/test_enum.cpp | 61 +++++++++++++++++++++++++++++++++++++ tests/test_enum.py | 28 +++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 47b042147..89f9cbd9b 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1814,6 +1814,19 @@ struct enum_base { handle m_parent; }; +template struct equivalent_integer {}; +template <> struct equivalent_integer { using type = int8_t; }; +template <> struct equivalent_integer { using type = uint8_t; }; +template <> struct equivalent_integer { using type = int16_t; }; +template <> struct equivalent_integer { using type = uint16_t; }; +template <> struct equivalent_integer { using type = int32_t; }; +template <> struct equivalent_integer { using type = uint32_t; }; +template <> struct equivalent_integer { using type = int64_t; }; +template <> struct equivalent_integer { using type = uint64_t; }; + +template +using equivalent_integer_t = typename equivalent_integer::value, sizeof(IntLike)>::type; + PYBIND11_NAMESPACE_END(detail) /// Binds C++ enumerations and enumeration classes to Python @@ -1824,13 +1837,17 @@ public: using Base::attr; using Base::def_property_readonly; using Base::def_property_readonly_static; - using Scalar = typename std::underlying_type::type; + using Underlying = typename std::underlying_type::type; + // Scalar is the integer representation of underlying type + using Scalar = detail::conditional_t, std::is_same + >::value, detail::equivalent_integer_t, Underlying>; template enum_(const handle &scope, const char *name, const Extra&... extra) : class_(scope, name, extra...), m_base(*this, scope) { constexpr bool is_arithmetic = detail::any_of...>::value; - constexpr bool is_convertible = std::is_convertible::value; + constexpr bool is_convertible = std::is_convertible::value; m_base.init(is_arithmetic, is_convertible); def(init([](Scalar i) { return static_cast(i); }), arg("value")); diff --git a/tests/test_enum.cpp b/tests/test_enum.cpp index 315308920..40c48d412 100644 --- a/tests/test_enum.cpp +++ b/tests/test_enum.cpp @@ -84,4 +84,65 @@ TEST_SUBMODULE(enums, m) { .value("ONE", SimpleEnum::THREE) .export_values(); }); + + // test_enum_scalar + enum UnscopedUCharEnum : unsigned char {}; + enum class ScopedShortEnum : short {}; + enum class ScopedLongEnum : long {}; + enum UnscopedUInt64Enum : std::uint64_t {}; + static_assert(py::detail::all_of< + std::is_same::Scalar, unsigned char>, + std::is_same::Scalar, short>, + std::is_same::Scalar, long>, + std::is_same::Scalar, std::uint64_t> + >::value, "Error during the deduction of enum's scalar type with normal integer underlying"); + + // test_enum_scalar_with_char_underlying + enum class ScopedCharEnum : char { Zero, Positive }; + enum class ScopedWCharEnum : wchar_t { Zero, Positive }; + enum class ScopedChar32Enum : char32_t { Zero, Positive }; + enum class ScopedChar16Enum : char16_t { Zero, Positive }; + + // test the scalar of char type enums according to chapter 'Character types' + // from https://en.cppreference.com/w/cpp/language/types + static_assert(py::detail::any_of< + std::is_same::Scalar, signed char>, // e.g. gcc on x86 + std::is_same::Scalar, unsigned char> // e.g. arm linux + >::value, "char should be cast to either signed char or unsigned char"); + static_assert( + sizeof(py::enum_::Scalar) == 2 || + sizeof(py::enum_::Scalar) == 4 + , "wchar_t should be either 16 bits (Windows) or 32 (everywhere else)"); + static_assert(py::detail::all_of< + std::is_same::Scalar, std::uint_least32_t>, + std::is_same::Scalar, std::uint_least16_t> + >::value, "char32_t, char16_t (and char8_t)'s size, signedness, and alignment is determined"); +#if defined(PYBIND11_HAS_U8STRING) + enum class ScopedChar8Enum : char8_t { Zero, Positive }; + static_assert(std::is_same::Scalar, unsigned char>::value); +#endif + + // test_char_underlying_enum + py::enum_(m, "ScopedCharEnum") + .value("Zero", ScopedCharEnum::Zero) + .value("Positive", ScopedCharEnum::Positive); + py::enum_(m, "ScopedWCharEnum") + .value("Zero", ScopedWCharEnum::Zero) + .value("Positive", ScopedWCharEnum::Positive); + py::enum_(m, "ScopedChar32Enum") + .value("Zero", ScopedChar32Enum::Zero) + .value("Positive", ScopedChar32Enum::Positive); + py::enum_(m, "ScopedChar16Enum") + .value("Zero", ScopedChar16Enum::Zero) + .value("Positive", ScopedChar16Enum::Positive); + + // test_bool_underlying_enum + enum class ScopedBoolEnum : bool { FALSE, TRUE }; + + // bool is unsigned (std::is_signed returns false) and 1-byte long, so represented with u8 + static_assert(std::is_same::Scalar, std::uint8_t>::value, ""); + + py::enum_(m, "ScopedBoolEnum") + .value("FALSE", ScopedBoolEnum::FALSE) + .value("TRUE", ScopedBoolEnum::TRUE); } diff --git a/tests/test_enum.py b/tests/test_enum.py index 62f9426ee..11cab6ddf 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -218,10 +218,16 @@ def test_binary_operators(): def test_enum_to_int(): m.test_enum_to_int(m.Flags.Read) m.test_enum_to_int(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_int(m.ScopedCharEnum.Positive) + m.test_enum_to_int(m.ScopedBoolEnum.TRUE) m.test_enum_to_uint(m.Flags.Read) m.test_enum_to_uint(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_uint(m.ScopedCharEnum.Positive) + m.test_enum_to_uint(m.ScopedBoolEnum.TRUE) m.test_enum_to_long_long(m.Flags.Read) m.test_enum_to_long_long(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_long_long(m.ScopedCharEnum.Positive) + m.test_enum_to_long_long(m.ScopedBoolEnum.TRUE) def test_duplicate_enum_name(): @@ -230,6 +236,28 @@ def test_duplicate_enum_name(): assert str(excinfo.value) == 'SimpleEnum: element "ONE" already exists!' +def test_char_underlying_enum(): # Issue #1331/PR #1334: + assert type(m.ScopedCharEnum.Positive.__int__()) is int + assert int(m.ScopedChar16Enum.Zero) == 0 # int() call should successfully return + assert hash(m.ScopedChar32Enum.Positive) == 1 + assert m.ScopedCharEnum.Positive.__getstate__() == 1 # return type is long in py2.x + assert m.ScopedWCharEnum(1) == m.ScopedWCharEnum.Positive + with pytest.raises(TypeError): + # Enum should construct with a int, even with char underlying type + m.ScopedWCharEnum("0") + + +def test_bool_underlying_enum(): + assert type(m.ScopedBoolEnum.TRUE.__int__()) is int + assert int(m.ScopedBoolEnum.FALSE) == 0 + assert hash(m.ScopedBoolEnum.TRUE) == 1 + assert m.ScopedBoolEnum.TRUE.__getstate__() == 1 + assert m.ScopedBoolEnum(1) == m.ScopedBoolEnum.TRUE + # Enum could construct with a bool + # (bool is a strict subclass of int, and False will be converted to 0) + assert m.ScopedBoolEnum(False) == m.ScopedBoolEnum.FALSE + + def test_docstring_signatures(): for enum_type in [m.ScopedEnum, m.UnscopedEnum]: for attr in enum_type.__dict__.values():