From 0dbda6e80ba41df50edaef40475011d0395aba16 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 4 Sep 2020 20:02:05 -0400 Subject: [PATCH] feat: py::pos_only (#2459) * feat: py::pos_only * fix: review points from @YannickJadoul * fix: review points from @bstaletic * refactor: kwonly -> kw_only --- docs/advanced/functions.rst | 26 +++++++-- docs/changelog.rst | 4 +- include/pybind11/attr.h | 34 +++++++----- include/pybind11/cast.h | 7 ++- include/pybind11/pybind11.h | 40 ++++++++++++-- tests/test_kwargs_and_defaults.cpp | 45 ++++++++++------ tests/test_kwargs_and_defaults.py | 84 +++++++++++++++++++++++------- 7 files changed, 182 insertions(+), 58 deletions(-) diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 3e33c9cf7..2814adfba 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -378,17 +378,35 @@ argument in a function definition: f(1, b=2) # good f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given -Pybind11 provides a ``py::kwonly`` object that allows you to implement +Pybind11 provides a ``py::kw_only`` object that allows you to implement the same behaviour by specifying the object between positional and keyword-only argument annotations when registering the function: .. code-block:: cpp m.def("f", [](int a, int b) { /* ... */ }, - py::arg("a"), py::kwonly(), py::arg("b")); + py::arg("a"), py::kw_only(), py::arg("b")); -Note that, as in Python, you cannot combine this with a ``py::args`` argument. -This feature does *not* require Python 3 to work. +Note that you currently cannot combine this with a ``py::args`` argument. This +feature does *not* require Python 3 to work. + +.. versionadded:: 2.6 + +Positional-only arguments +========================= + +Python 3.8 introduced a new positional-only argument syntax, using ``/`` in the +function definition (note that this has been a convention for CPython +positional arguments, such as in ``pow()``, since Python 2). You can +do the same thing in any version of Python using ``py::pos_only()``: + +.. code-block:: cpp + + m.def("f", [](int a, int b) { /* ... */ }, + py::arg("a"), py::pos_only(), py::arg("b")); + +You now cannot give argument ``a`` by keyword. This can be combined with +keyword-only arguments, as well. .. versionadded:: 2.6 diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e15621d2..36381eb53 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,9 +11,11 @@ v2.6.0 (IN PROGRESS) See :ref:`upgrade-guide-2.6` for help upgrading to the new version. -* Keyword only argument supported in Python 2 or 3 with ``py::kwonly()``. +* Keyword-only argument supported in Python 2 or 3 with ``py::kw_only()``. `#2100 `_ +* Positional-only argument supported in Python 2 or 3 with ``py::pos_only()``. + * Perfect forwarding support for methods. `#2048 `_ diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 54065fc9e..aedbf62f1 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -138,7 +138,7 @@ struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), is_operator(false), is_method(false), - has_args(false), has_kwargs(false), has_kwonly_args(false) { } + has_args(false), has_kwargs(false), has_kw_only_args(false) { } /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -185,14 +185,17 @@ struct function_record { /// True if the function has a '**kwargs' argument bool has_kwargs : 1; - /// True once a 'py::kwonly' is encountered (any following args are keyword-only) - bool has_kwonly_args : 1; + /// True once a 'py::kw_only' is encountered (any following args are keyword-only) + bool has_kw_only_args : 1; /// Number of arguments (including py::args and/or py::kwargs, if present) std::uint16_t nargs; /// Number of trailing arguments (counted in `nargs`) that are keyword-only - std::uint16_t nargs_kwonly = 0; + std::uint16_t nargs_kw_only = 0; + + /// Number of leading arguments (counted in `nargs`) that are positional-only + std::uint16_t nargs_pos_only = 0; /// Python method object PyMethodDef *def = nullptr; @@ -366,10 +369,10 @@ template <> struct process_attribute : process_attribu static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } }; -inline void process_kwonly_arg(const arg &a, function_record *r) { +inline void process_kw_only_arg(const arg &a, function_record *r) { if (!a.name || strlen(a.name) == 0) - pybind11_fail("arg(): cannot specify an unnamed argument after an kwonly() annotation"); - ++r->nargs_kwonly; + pybind11_fail("arg(): cannot specify an unnamed argument after an kw_only() annotation"); + ++r->nargs_kw_only; } /// Process a keyword argument attribute (*without* a default value) @@ -379,7 +382,7 @@ template <> struct process_attribute : process_attribute_default { r->args.emplace_back("self", nullptr, handle(), true /*convert*/, false /*none not allowed*/); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); - if (r->has_kwonly_args) process_kwonly_arg(a, r); + if (r->has_kw_only_args) process_kw_only_arg(a, r); } }; @@ -412,14 +415,21 @@ template <> struct process_attribute : process_attribute_default { } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); - if (r->has_kwonly_args) process_kwonly_arg(a, r); + if (r->has_kw_only_args) process_kw_only_arg(a, r); } }; /// Process a keyword-only-arguments-follow pseudo argument -template <> struct process_attribute : process_attribute_default { - static void init(const kwonly &, function_record *r) { - r->has_kwonly_args = true; +template <> struct process_attribute : process_attribute_default { + static void init(const kw_only &, function_record *r) { + r->has_kw_only_args = true; + } +}; + +/// Process a positional-only-argument maker +template <> struct process_attribute : process_attribute_default { + static void init(const pos_only &, function_record *r) { + r->nargs_pos_only = static_cast(r->args.size()); } }; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 5711004df..bae69c8ac 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1910,7 +1910,12 @@ public: /// \ingroup annotations /// Annotation indicating that all following arguments are keyword-only; the is the equivalent of an /// unnamed '*' argument (in Python 3) -struct kwonly {}; +struct kw_only {}; + +/// \ingroup annotations +/// Annotation indicating that all previous arguments are positional-only; the is the equivalent of an +/// unnamed '/' argument (in Python 3.8) +struct pos_only {}; template arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward(value)}; } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3a7d7b884..602d5790a 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -187,11 +187,13 @@ protected: process_attributes::init(extra..., rec); { - constexpr bool has_kwonly_args = any_of...>::value, + constexpr bool has_kw_only_args = any_of...>::value, + has_pos_only_args = any_of...>::value, has_args = any_of...>::value, has_arg_annotations = any_of...>::value; - static_assert(has_arg_annotations || !has_kwonly_args, "py::kwonly requires the use of argument annotations"); - static_assert(!(has_args && has_kwonly_args), "py::kwonly cannot be combined with a py::args argument"); + static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations"); + static_assert(has_arg_annotations || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings and aligning the annotations to the argument)"); + static_assert(!(has_args && has_kw_only_args), "py::kw_only cannot be combined with a py::args argument"); } /* Generate a readable signature describing the function's arguments and return value types */ @@ -257,7 +259,10 @@ protected: // Write arg name for everything except *args and **kwargs. if (*(pc + 1) == '*') continue; - + // Separator for keyword-only arguments, placed before the kw + // arguments start + if (rec->nargs_kw_only > 0 && arg_index + rec->nargs_kw_only == args) + signature += "*, "; if (arg_index < rec->args.size() && rec->args[arg_index].name) { signature += rec->args[arg_index].name; } else if (arg_index == 0 && rec->is_method) { @@ -272,6 +277,10 @@ protected: signature += " = "; signature += rec->args[arg_index].descr; } + // Separator for positional-only arguments (placed after the + // argument, rather than before like * + if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only) + signature += ", /"; arg_index++; } else if (c == '%') { const std::type_info *t = types[type_index++]; @@ -297,6 +306,7 @@ protected: signature += c; } } + if (arg_index != args || types[type_index] != nullptr) pybind11_fail("Internal error while parsing type signature (2)"); @@ -512,7 +522,7 @@ protected: size_t num_args = func.nargs; // Number of positional arguments that we need if (func.has_args) --num_args; // (but don't count py::args if (func.has_kwargs) --num_args; // or py::kwargs) - size_t pos_args = num_args - func.nargs_kwonly; + size_t pos_args = num_args - func.nargs_kw_only; if (!func.has_args && n_args_in > pos_args) continue; // Too many positional arguments for this overload @@ -561,6 +571,26 @@ protected: // We'll need to copy this if we steal some kwargs for defaults dict kwargs = reinterpret_borrow(kwargs_in); + // 1.5. Fill in any missing pos_only args from defaults if they exist + if (args_copied < func.nargs_pos_only) { + for (; args_copied < func.nargs_pos_only; ++args_copied) { + const auto &arg = func.args[args_copied]; + handle value; + + if (arg.value) { + value = arg.value; + } + if (value) { + call.args.push_back(value); + call.args_convert.push_back(arg.convert); + } else + break; + } + + if (args_copied < func.nargs_pos_only) + continue; // Not enough defaults to fill the positional arguments + } + // 2. Check kwargs and, failing that, defaults that may help complete the list if (args_copied < num_args) { bool copied_kwargs = false; diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 64bc2377b..641ec88c4 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -95,28 +95,39 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // m.def("bad_args7", [](py::kwargs, py::kwargs) {}); // test_keyword_only_args - m.def("kwonly_all", [](int i, int j) { return py::make_tuple(i, j); }, - py::kwonly(), py::arg("i"), py::arg("j")); - m.def("kwonly_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, - py::arg(), py::kwonly(), py::arg("j"), py::arg("k")); - m.def("kwonly_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, - py::arg() = 3, "j"_a = 4, py::kwonly(), "k"_a = 5, "z"_a); - m.def("kwonly_mixed", [](int i, int j) { return py::make_tuple(i, j); }, - "i"_a, py::kwonly(), "j"_a); - m.def("kwonly_plus_more", [](int i, int j, int k, py::kwargs kwargs) { + m.def("kw_only_all", [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), py::arg("i"), py::arg("j")); + m.def("kw_only_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg(), py::kw_only(), py::arg("j"), py::arg("k")); + m.def("kw_only_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, + py::arg() = 3, "j"_a = 4, py::kw_only(), "k"_a = 5, "z"_a); + m.def("kw_only_mixed", [](int i, int j) { return py::make_tuple(i, j); }, + "i"_a, py::kw_only(), "j"_a); + m.def("kw_only_plus_more", [](int i, int j, int k, py::kwargs kwargs) { return py::make_tuple(i, j, k, kwargs); }, - py::arg() /* positional */, py::arg("j") = -1 /* both */, py::kwonly(), py::arg("k") /* kw-only */); + py::arg() /* positional */, py::arg("j") = -1 /* both */, py::kw_only(), py::arg("k") /* kw-only */); - m.def("register_invalid_kwonly", [](py::module m) { - m.def("bad_kwonly", [](int i, int j) { return py::make_tuple(i, j); }, - py::kwonly(), py::arg() /* invalid unnamed argument */, "j"_a); + m.def("register_invalid_kw_only", [](py::module m) { + m.def("bad_kw_only", [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), py::arg() /* invalid unnamed argument */, "j"_a); }); + // test_positional_only_args + m.def("pos_only_all", [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), py::arg("j"), py::pos_only()); + m.def("pos_only_mix", [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), py::pos_only(), py::arg("j")); + m.def("pos_kw_only_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), py::pos_only(), py::arg("j"), py::kw_only(), py::arg("k")); + m.def("pos_only_def_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), py::arg("j") = 2, py::pos_only(), py::arg("k") = 3); + + // These should fail to compile: - // argument annotations are required when using kwonly -// m.def("bad_kwonly1", [](int) {}, py::kwonly()); - // can't specify both `py::kwonly` and a `py::args` argument -// m.def("bad_kwonly2", [](int i, py::args) {}, py::kwonly(), "i"_a); + // argument annotations are required when using kw_only +// m.def("bad_kw_only1", [](int) {}, py::kw_only()); + // can't specify both `py::kw_only` and a `py::args` argument +// m.def("bad_kw_only2", [](int i, py::args) {}, py::kw_only(), "i"_a); // test_function_signatures (along with most of the above) struct KWClass { void foo(int, float) {} }; diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 5257e0cd3..2a81dbdc5 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -112,43 +112,91 @@ def test_mixed_args_and_kwargs(msg): def test_keyword_only_args(msg): - assert m.kwonly_all(i=1, j=2) == (1, 2) - assert m.kwonly_all(j=1, i=2) == (2, 1) + assert m.kw_only_all(i=1, j=2) == (1, 2) + assert m.kw_only_all(j=1, i=2) == (2, 1) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_all(i=1) == (1,) + assert m.kw_only_all(i=1) == (1,) assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_all(1, 2) == (1, 2) + assert m.kw_only_all(1, 2) == (1, 2) assert "incompatible function arguments" in str(excinfo.value) - assert m.kwonly_some(1, k=3, j=2) == (1, 2, 3) + assert m.kw_only_some(1, k=3, j=2) == (1, 2, 3) - assert m.kwonly_with_defaults(z=8) == (3, 4, 5, 8) - assert m.kwonly_with_defaults(2, z=8) == (2, 4, 5, 8) - assert m.kwonly_with_defaults(2, j=7, k=8, z=9) == (2, 7, 8, 9) - assert m.kwonly_with_defaults(2, 7, z=9, k=8) == (2, 7, 8, 9) + assert m.kw_only_with_defaults(z=8) == (3, 4, 5, 8) + assert m.kw_only_with_defaults(2, z=8) == (2, 4, 5, 8) + assert m.kw_only_with_defaults(2, j=7, k=8, z=9) == (2, 7, 8, 9) + assert m.kw_only_with_defaults(2, 7, z=9, k=8) == (2, 7, 8, 9) - assert m.kwonly_mixed(1, j=2) == (1, 2) - assert m.kwonly_mixed(j=2, i=3) == (3, 2) - assert m.kwonly_mixed(i=2, j=3) == (2, 3) + assert m.kw_only_mixed(1, j=2) == (1, 2) + assert m.kw_only_mixed(j=2, i=3) == (3, 2) + assert m.kw_only_mixed(i=2, j=3) == (2, 3) - assert m.kwonly_plus_more(4, 5, k=6, extra=7) == (4, 5, 6, {'extra': 7}) - assert m.kwonly_plus_more(3, k=5, j=4, extra=6) == (3, 4, 5, {'extra': 6}) - assert m.kwonly_plus_more(2, k=3, extra=4) == (2, -1, 3, {'extra': 4}) + assert m.kw_only_plus_more(4, 5, k=6, extra=7) == (4, 5, 6, {'extra': 7}) + assert m.kw_only_plus_more(3, k=5, j=4, extra=6) == (3, 4, 5, {'extra': 6}) + assert m.kw_only_plus_more(2, k=3, extra=4) == (2, -1, 3, {'extra': 4}) with pytest.raises(TypeError) as excinfo: - assert m.kwonly_mixed(i=1) == (1,) + assert m.kw_only_mixed(i=1) == (1,) assert "incompatible function arguments" in str(excinfo.value) with pytest.raises(RuntimeError) as excinfo: - m.register_invalid_kwonly(m) + m.register_invalid_kw_only(m) assert msg(excinfo.value) == """ - arg(): cannot specify an unnamed argument after an kwonly() annotation + arg(): cannot specify an unnamed argument after an kw_only() annotation """ +def test_positional_only_args(msg): + assert m.pos_only_all(1, 2) == (1, 2) + assert m.pos_only_all(2, 1) == (2, 1) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_all(i=1, j=2) + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_only_mix(1, 2) == (1, 2) + assert m.pos_only_mix(2, j=1) == (2, 1) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_mix(i=1, j=2) + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_kw_only_mix(1, 2, k=3) == (1, 2, 3) + assert m.pos_kw_only_mix(1, j=2, k=3) == (1, 2, 3) + + with pytest.raises(TypeError) as excinfo: + m.pos_kw_only_mix(i=1, j=2, k=3) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + m.pos_kw_only_mix(1, 2, 3) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_def_mix() + assert "incompatible function arguments" in str(excinfo.value) + + assert m.pos_only_def_mix(1) == (1, 2, 3) + assert m.pos_only_def_mix(1, 4) == (1, 4, 3) + assert m.pos_only_def_mix(1, 4, 7) == (1, 4, 7) + assert m.pos_only_def_mix(1, 4, k=7) == (1, 4, 7) + + with pytest.raises(TypeError) as excinfo: + m.pos_only_def_mix(1, j=4) + assert "incompatible function arguments" in str(excinfo.value) + + +def test_signatures(): + assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ + assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ + assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ + assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" == m.pos_kw_only_mix.__doc__ + + @pytest.mark.xfail("env.PYPY and env.PY2", reason="PyPy2 doesn't double count") def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular