From be0d804523c01809292f93f0bbf6adbfcf5cd0fa Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 23 Dec 2017 18:56:07 -0400 Subject: [PATCH] Support keyword-only arguments This adds support for a `py::args_kw_only()` annotation that can be specified between `py::arg` annotations to indicate that any following arguments are keyword-only. This allows you to write: m.def("f", [](int a, int b) { /* ... */ }, py::arg("a"), py::args_kw_only(), py::arg("b")); and have it work like Python 3's: def f(a, *, b): # ... with respect to how `a` and `b` arguments are accepted (that is, `a` can be positional or by keyword; `b` can only be specified by keyword). --- docs/advanced/functions.rst | 28 ++++++++++++++++++++++ include/pybind11/attr.h | 30 ++++++++++++++++++++--- include/pybind11/cast.h | 5 ++++ include/pybind11/pybind11.h | 25 +++++++++++++------- tests/test_kwargs_and_defaults.cpp | 24 +++++++++++++++++++ tests/test_kwargs_and_defaults.py | 38 ++++++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 11 deletions(-) diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 3e1a3ff0e..c7790533b 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -362,6 +362,34 @@ like so: py::class_("MyClass") .def("myFunction", py::arg("arg") = (SomeType *) nullptr); +Keyword-only arguments +====================== + +Python 3 introduced keyword-only arguments by specifying an unnamed ``*`` +argument in a function definition: + +.. code-block:: python + + def f(a, *, b): # a can be positional or via keyword; b must be via keyword + pass + + f(a=1, b=2) # good + f(b=2, a=1) # good + f(1, b=2) # good + f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given + +Pybind11 provides a ``py::args_kw_only`` object that allows you to implement +the same behaviour by specifying the object between positional and keyword-only +argument annotations when registering the function: + +.. code-block:: cpp + + m.def("f", [](int a, int b) { /* ... */ }, + py::arg("a"), py::args_kw_only(), py::arg("b")); + +Note that, as in Python, you cannot combine this with a ``py::args`` argument. +This feature does *not* require Python 3 to work. + .. _nonconverting_arguments: Non-converting arguments diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 744284a83..70325510d 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -137,7 +137,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), has_args(false), has_kwargs(false), is_method(false) { } + is_operator(false), is_method(false), + has_args(false), has_kwargs(false), has_kw_only_args(false) { } /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -175,18 +176,24 @@ struct function_record { /// True if this is an operator (__add__), etc. bool is_operator : 1; + /// True if this is a method + bool is_method : 1; + /// True if the function has a '*args' argument bool has_args : 1; /// True if the function has a '**kwargs' argument bool has_kwargs : 1; - /// True if this is a method - bool is_method : 1; + /// True once a 'py::args_kw_only' is encountered (any following args are keyword-only) + bool has_kw_only_args : 1; /// Number of arguments (including py::args and/or py::kwargs, if present) std::uint16_t nargs; + /// Number of trailing arguments (counted in `nargs`) that are keyword-only + std::uint16_t nargs_kwonly = 0; + /// Python method object PyMethodDef *def = nullptr; @@ -359,12 +366,20 @@ template <> struct process_attribute : process_attribu static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } }; +inline void process_kwonly_arg(const arg &a, function_record *r) { + if (!a.name || strlen(a.name) == 0) + pybind11_fail("arg(): cannot specify an unnamed argument after an args_kw_only() annotation"); + ++r->nargs_kwonly; +} + /// Process a keyword argument attribute (*without* a default value) template <> struct process_attribute : process_attribute_default { static void init(const arg &a, function_record *r) { if (r->is_method && r->args.empty()) r->args.emplace_back("self", nullptr, handle(), true /*convert*/, false /*none not allowed*/); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); + + if (r->has_kw_only_args) process_kwonly_arg(a, r); } }; @@ -396,6 +411,15 @@ template <> struct process_attribute : process_attribute_default { #endif } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); + + if (r->has_kw_only_args) process_kwonly_arg(a, r); + } +}; + +/// Process a keyword-only-arguments-follow pseudo argument +template <> struct process_attribute : process_attribute_default { + static void init(const args_kw_only &, function_record *r) { + r->has_kw_only_args = true; } }; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 0edc33686..2ea1c635b 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1887,6 +1887,11 @@ public: #endif }; +/// \ingroup annotations +/// Annotation indicating that all following arguments are keyword-only; the is the equivalent of an +/// unnamed '*' argument (in Python 3) +struct args_kw_only {}; + template arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward(value)}; } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index d95d61f7b..ff586033d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -168,6 +168,14 @@ protected: /* Process any user-provided function attributes */ process_attributes::init(extra..., rec); + { + constexpr bool has_kw_only_args = any_of...>::value, + has_args = any_of...>::value, + has_arg_annotations = any_of...>::value; + static_assert(has_arg_annotations || !has_kw_only_args, "py::args_kw_only requires the use of argument annotations"); + static_assert(!(has_args && has_kw_only_args), "py::args_kw_only cannot be combined with a py::args argument"); + } + /* Generate a readable signature describing the function's arguments and return value types */ static constexpr auto signature = _("(") + cast_in::arg_names + _(") -> ") + cast_out::name; PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); @@ -483,15 +491,16 @@ protected: */ const function_record &func = *it; - size_t pos_args = func.nargs; // Number of positional arguments that we need - if (func.has_args) --pos_args; // (but don't count py::args - if (func.has_kwargs) --pos_args; // or py::kwargs) + size_t num_args = func.nargs; // Number of positional arguments that we need + if (func.has_args) --num_args; // (but don't count py::args + if (func.has_kwargs) --num_args; // or py::kwargs) + size_t pos_args = num_args - func.nargs_kwonly; if (!func.has_args && n_args_in > pos_args) - continue; // Too many arguments for this overload + continue; // Too many positional arguments for this overload if (n_args_in < pos_args && func.args.size() < pos_args) - continue; // Not enough arguments given, and not enough defaults to fill in the blanks + continue; // Not enough positional arguments given, and not enough defaults to fill in the blanks function_call call(func, parent); @@ -535,10 +544,10 @@ protected: dict kwargs = reinterpret_borrow(kwargs_in); // 2. Check kwargs and, failing that, defaults that may help complete the list - if (args_copied < pos_args) { + if (args_copied < num_args) { bool copied_kwargs = false; - for (; args_copied < pos_args; ++args_copied) { + for (; args_copied < num_args; ++args_copied) { const auto &arg = func.args[args_copied]; handle value; @@ -564,7 +573,7 @@ protected: break; } - if (args_copied < pos_args) + if (args_copied < num_args) continue; // Not enough arguments, defaults, or kwargs to fill the positional arguments } diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 6563fb9ad..6c4971404 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -94,6 +94,30 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { // m.def("bad_args6", [](py::args, py::args) {}); // m.def("bad_args7", [](py::kwargs, py::kwargs) {}); + // test_keyword_only_args + m.def("kwonly_all", [](int i, int j) { return py::make_tuple(i, j); }, + py::args_kw_only(), py::arg("i"), py::arg("j")); + m.def("kwonly_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg(), py::args_kw_only(), py::arg("j"), py::arg("k")); + m.def("kwonly_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, + py::arg() = 3, "j"_a = 4, py::args_kw_only(), "k"_a = 5, "z"_a); + m.def("kwonly_mixed", [](int i, int j) { return py::make_tuple(i, j); }, + "i"_a, py::args_kw_only(), "j"_a); + m.def("kwonly_plus_more", [](int i, int j, int k, py::kwargs kwargs) { + return py::make_tuple(i, j, k, kwargs); }, + py::arg() /* positional */, py::arg("j") = -1 /* both */, py::args_kw_only(), py::arg("k") /* kw-only */); + + m.def("register_invalid_kwonly", [](py::module m) { + m.def("bad_kwonly", [](int i, int j) { return py::make_tuple(i, j); }, + py::args_kw_only(), py::arg() /* invalid unnamed argument */, "j"_a); + }); + + // These should fail to compile: + // argument annotations are required when using args_kw_only +// m.def("bad_kwonly1", [](int) {}, py::args_kw_only()); + // can't specify both `py::args_kw_only` and a `py::args` argument +// m.def("bad_kwonly2", [](int i, py::args) {}, py::args_kw_only(), "i"_a); + // test_function_signatures (along with most of the above) struct KWClass { void foo(int, float) {} }; py::class_(m, "KWClass") diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 27a05a024..fa90140fd 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -107,6 +107,44 @@ def test_mixed_args_and_kwargs(msg): """ # noqa: E501 line too long +def test_keyword_only_args(msg): + assert m.kwonly_all(i=1, j=2) == (1, 2) + assert m.kwonly_all(j=1, i=2) == (2, 1) + + with pytest.raises(TypeError) as excinfo: + assert m.kwonly_all(i=1) == (1,) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + assert m.kwonly_all(1, 2) == (1, 2) + assert "incompatible function arguments" in str(excinfo.value) + + assert m.kwonly_some(1, k=3, j=2) == (1, 2, 3) + + assert m.kwonly_with_defaults(z=8) == (3, 4, 5, 8) + assert m.kwonly_with_defaults(2, z=8) == (2, 4, 5, 8) + assert m.kwonly_with_defaults(2, j=7, k=8, z=9) == (2, 7, 8, 9) + assert m.kwonly_with_defaults(2, 7, z=9, k=8) == (2, 7, 8, 9) + + assert m.kwonly_mixed(1, j=2) == (1, 2) + assert m.kwonly_mixed(j=2, i=3) == (3, 2) + assert m.kwonly_mixed(i=2, j=3) == (2, 3) + + assert m.kwonly_plus_more(4, 5, k=6, extra=7) == (4, 5, 6, {'extra': 7}) + assert m.kwonly_plus_more(3, k=5, j=4, extra=6) == (3, 4, 5, {'extra': 6}) + assert m.kwonly_plus_more(2, k=3, extra=4) == (2, -1, 3, {'extra': 4}) + + with pytest.raises(TypeError) as excinfo: + assert m.kwonly_mixed(i=1) == (1,) + assert "incompatible function arguments" in str(excinfo.value) + + with pytest.raises(RuntimeError) as excinfo: + m.register_invalid_kwonly(m) + assert msg(excinfo.value) == """ + arg(): cannot specify an unnamed argument after an args_kw_only() annotation + """ + + def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular arguments"""