From c743e1b1b4c1ea4a0c5c1aee9802f02533633e2a Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Mon, 29 Aug 2016 03:05:42 +0200 Subject: [PATCH] Support keyword arguments and generalized unpacking in C++ A Python function can be called with the syntax: ```python foo(a1, a2, *args, ka=1, kb=2, **kwargs) ``` This commit adds support for the equivalent syntax in C++: ```c++ foo(a1, a2, *args, "ka"_a=1, "kb"_a=2, **kwargs) ``` In addition, generalized unpacking is implemented, as per PEP 448, which allows calls with multiple * and ** unpacking: ```python bar(*args1, 99, *args2, 101, **kwargs1, kz=200, **kwargs2) ``` and ```c++ bar(*args1, 99, *args2, 101, **kwargs1, "kz"_a=200, **kwargs2) ``` --- include/pybind11/attr.h | 8 -- include/pybind11/cast.h | 187 +++++++++++++++++++++++++++-- include/pybind11/common.h | 32 +++++ include/pybind11/pytypes.h | 13 +- tests/pybind11_tests.h | 1 + tests/test_callbacks.cpp | 62 +++++++++- tests/test_callbacks.py | 35 ++++++ tests/test_kwargs_and_defaults.cpp | 1 - 8 files changed, 321 insertions(+), 18 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index ba4dd841b..638ebad72 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -296,9 +296,6 @@ template struct process_attribute struct process_attributes { static void init(const Args&... args, function_record *r) { @@ -319,11 +316,6 @@ template struct process_attributes { } }; -/// Compile-time integer sum -constexpr size_t constexpr_sum() { return 0; } -template -constexpr size_t constexpr_sum(T n, Ts... ns) { return n + constexpr_sum(ns...); } - /// Check the number of named arguments at compile time template ::value...), diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index cbb0ae5b8..1101178ed 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -57,6 +57,7 @@ PYBIND11_NOINLINE inline internals &get_internals() { } catch (const index_error &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; } catch (const key_error &e) { PyErr_SetString(PyExc_KeyError, e.what()); return; } catch (const value_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const type_error &e) { PyErr_SetString(PyExc_TypeError, e.what()); return; } catch (const stop_iteration &e) { PyErr_SetString(PyExc_StopIteration, e.what()); return; } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; @@ -308,6 +309,7 @@ protected: }; template class type_caster : public type_caster_base { }; +template using make_caster = type_caster>; template class type_caster> : public type_caster_base { public: @@ -947,13 +949,184 @@ template object handle::operator()(Args&&... args) const { - tuple args_tuple = pybind11::make_tuple(std::forward(args)...); - object result(PyObject_CallObject(m_ptr, args_tuple.ptr()), false); - if (!result) - throw error_already_set(); - return result; +NAMESPACE_BEGIN(detail) +NAMESPACE_BEGIN(constexpr_impl) +/// Implementation details for constexpr functions +constexpr int first(int i) { return i; } +template +constexpr int first(int i, T v, Ts... vs) { return v ? i : first(i + 1, vs...); } + +constexpr int last(int /*i*/, int result) { return result; } +template +constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); } +NAMESPACE_END(constexpr_impl) + +/// Return the index of the first type in Ts which satisfies Predicate +template class Predicate, typename... Ts> +constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate::value...); } + +/// Return the index of the last type in Ts which satisfies Predicate +template class Predicate, typename... Ts> +constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate::value...); } + +/// Helper class which collects only positional arguments for a Python function call. +/// A fancier version below can collect any argument, but this one is optimal for simple calls. +template +class simple_collector { +public: + template + simple_collector(Ts &&...values) + : m_args(pybind11::make_tuple(std::forward(values)...)) { } + + const tuple &args() const & { return m_args; } + dict kwargs() const { return {}; } + + tuple args() && { return std::move(m_args); } + + /// Call a Python function and pass the collected arguments + object call(PyObject *ptr) const { + auto result = object(PyObject_CallObject(ptr, m_args.ptr()), false); + if (!result) + throw error_already_set(); + return result; + } + +private: + tuple m_args; +}; + +/// Helper class which collects positional, keyword, * and ** arguments for a Python function call +template +class unpacking_collector { +public: + template + unpacking_collector(Ts &&...values) { + // Tuples aren't (easily) resizable so a list is needed for collection, + // but the actual function call strictly requires a tuple. + auto args_list = list(); + int _[] = { 0, (process(args_list, std::forward(values)), 0)... }; + ignore_unused(_); + + m_args = object(PyList_AsTuple(args_list.ptr()), false); + } + + const tuple &args() const & { return m_args; } + const dict &kwargs() const & { return m_kwargs; } + + tuple args() && { return std::move(m_args); } + dict kwargs() && { return std::move(m_kwargs); } + + /// Call a Python function and pass the collected arguments + object call(PyObject *ptr) const { + auto result = object(PyObject_Call(ptr, m_args.ptr(), m_kwargs.ptr()), false); + if (!result) + throw error_already_set(); + return result; + } + +private: + template + void process(list &args_list, T &&x) { + auto o = object(detail::make_caster::cast(std::forward(x), policy, nullptr), false); + if (!o) { +#if defined(NDEBUG) + argument_cast_error(); +#else + argument_cast_error(std::to_string(args_list.size()), type_id()); +#endif + } + args_list.append(o); + } + + void process(list &args_list, detail::args_proxy ap) { + for (const auto &a : ap) { + args_list.append(a.cast()); + } + } + + template + void process(list &/*args_list*/, arg_t &&a) { + if (m_kwargs[a.name]) { +#if defined(NDEBUG) + multiple_values_error(); +#else + multiple_values_error(a.name); +#endif + } + auto o = object(detail::make_caster::cast(*a.value, policy, nullptr), false); + if (!o) { +#if defined(NDEBUG) + argument_cast_error(); +#else + argument_cast_error(a.name, type_id()); +#endif + } + m_kwargs[a.name] = o; + } + + void process(list &/*args_list*/, detail::kwargs_proxy kp) { + for (const auto &k : dict(kp, true)) { + if (m_kwargs[k.first]) { +#if defined(NDEBUG) + multiple_values_error(); +#else + multiple_values_error(k.first.str()); +#endif + } + m_kwargs[k.first] = k.second; + } + } + + [[noreturn]] static void multiple_values_error() { + throw type_error("Got multiple values for keyword argument " + "(compile in debug mode for details)"); + } + + [[noreturn]] static void multiple_values_error(std::string name) { + throw type_error("Got multiple values for keyword argument '" + name + "'"); + } + + [[noreturn]] static void argument_cast_error() { + throw cast_error("Unable to convert call argument to Python object " + "(compile in debug mode for details)"); + } + + [[noreturn]] static void argument_cast_error(std::string name, std::string type) { + throw cast_error("Unable to convert call argument '" + name + + "' of type '" + type + "' to Python object"); + } + +private: + tuple m_args; + dict m_kwargs; +}; + +/// Collect only positional arguments for a Python function call +template ::value>> +simple_collector collect_arguments(Args &&...args) { + return {std::forward(args)...}; +} + +/// Collect all arguments, including keywords and unpacking (only instantiated when needed) +template ::value>> +unpacking_collector collect_arguments(Args &&...args) { + // Following argument order rules for generalized unpacking according to PEP 448 + static_assert( + constexpr_last() < constexpr_first() + && constexpr_last() < constexpr_first(), + "Invalid function call: positional args must precede keywords and ** unpacking; " + "* unpacking must precede ** unpacking" + ); + return {std::forward(args)...}; +} + +NAMESPACE_END(detail) + +template +object handle::operator()(Args &&...args) const { + return detail::collect_arguments(std::forward(args)...).call(m_ptr); } template struct intrinsic_type { typedef type template struct intrinsic_type { typedef typename intrinsic_type::type type; }; template struct intrinsic_type { typedef typename intrinsic_type::type type; }; template struct intrinsic_type { typedef typename intrinsic_type::type type; }; +template using intrinsic_t = typename intrinsic_type::type; /// Helper type to replace 'void' in some expressions struct void_type { }; +/// from __cpp_future__ import (convenient aliases from C++14/17) +template using bool_constant = std::integral_constant; +template using negation = bool_constant; +template using enable_if_t = typename std::enable_if::type; +template using conditional_t = typename std::conditional::type; + +/// Compile-time integer sum +constexpr size_t constexpr_sum() { return 0; } +template +constexpr size_t constexpr_sum(T n, Ts... ns) { return size_t{n} + constexpr_sum(ns...); } + +/// Return true if all/any Ts satify Predicate +#if !defined(_MSC_VER) +template class Predicate, typename... Ts> +using all_of_t = bool_constant<(constexpr_sum(Predicate::value...) == sizeof...(Ts))>; +template class Predicate, typename... Ts> +using any_of_t = bool_constant<(constexpr_sum(Predicate::value...) > 0)>; +#else +// MSVC workaround (2015 Update 3 has issues with some member type aliases and constexpr) +template class P, typename...> struct all_of_t : std::true_type { }; +template class P, typename T, typename... Ts> +struct all_of_t : conditional_t::value, all_of_t, std::false_type> { }; +template class P, typename...> struct any_of_t : std::false_type { }; +template class P, typename T, typename... Ts> +struct any_of_t : conditional_t::value, std::true_type, any_of_t> { }; +#endif + +/// Ignore that a variable is unused in compiler warnings +inline void ignore_unused(const int *) { } + NAMESPACE_END(detail) #define PYBIND11_RUNTIME_EXCEPTION(name) \ @@ -345,6 +376,7 @@ PYBIND11_RUNTIME_EXCEPTION(stop_iteration) PYBIND11_RUNTIME_EXCEPTION(index_error) PYBIND11_RUNTIME_EXCEPTION(key_error) PYBIND11_RUNTIME_EXCEPTION(value_error) +PYBIND11_RUNTIME_EXCEPTION(type_error) PYBIND11_RUNTIME_EXCEPTION(cast_error) /// Thrown when pybind11::cast or handle::call fail due to a type casting error PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 463f713b8..5f60895f9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -16,7 +16,7 @@ NAMESPACE_BEGIN(pybind11) /* A few forward declarations */ -class object; class str; class object; class dict; class iterator; +class object; class str; class iterator; struct arg; template struct arg_t; namespace detail { class accessor; class args_proxy; class kwargs_proxy; } @@ -250,6 +250,17 @@ public: kwargs_proxy operator*() const { return kwargs_proxy(*this); } }; +/// Python argument categories (using PEP 448 terms) +template using is_keyword = std::is_base_of; +template using is_s_unpacking = std::is_same; // * unpacking +template using is_ds_unpacking = std::is_same; // ** unpacking +template using is_positional = bool_constant< + !is_keyword::value && !is_s_unpacking::value && !is_ds_unpacking::value +>; +template using is_keyword_or_ds = bool_constant< + is_keyword::value || is_ds_unpacking::value +>; + NAMESPACE_END(detail) #define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, CvtStmt) \ diff --git a/tests/pybind11_tests.h b/tests/pybind11_tests.h index 8af3154ef..cf3cb36ef 100644 --- a/tests/pybind11_tests.h +++ b/tests/pybind11_tests.h @@ -8,6 +8,7 @@ using std::cout; using std::endl; namespace py = pybind11; +using namespace pybind11::literals; class test_initializer { public: diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index 8e0a6cc7b..bfb932691 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -71,6 +71,9 @@ struct Payload { } }; +/// Something to trigger a conversion error +struct Unregistered {}; + test_initializer callbacks([](py::module &m) { m.def("test_callback1", &test_callback1); m.def("test_callback2", &test_callback2); @@ -78,8 +81,65 @@ test_initializer callbacks([](py::module &m) { m.def("test_callback4", &test_callback4); m.def("test_callback5", &test_callback5); - /* Test cleanup of lambda closure */ + // Test keyword args and generalized unpacking + m.def("test_tuple_unpacking", [](py::function f) { + auto t1 = py::make_tuple(2, 3); + auto t2 = py::make_tuple(5, 6); + return f("positional", 1, *t1, 4, *t2); + }); + m.def("test_dict_unpacking", [](py::function f) { + auto d1 = py::dict(); + d1["key"] = py::cast("value"); + d1["a"] = py::cast(1); + auto d2 = py::dict(); + auto d3 = py::dict(); + d3["b"] = py::cast(2); + return f("positional", 1, **d1, **d2, **d3); + }); + + m.def("test_keyword_args", [](py::function f) { + return f("x"_a=10, "y"_a=20); + }); + + m.def("test_unpacking_and_keywords1", [](py::function f) { + auto args = py::make_tuple(2); + auto kwargs = py::dict(); + kwargs["d"] = py::cast(4); + return f(1, *args, "c"_a=3, **kwargs); + }); + + m.def("test_unpacking_and_keywords2", [](py::function f) { + auto kwargs1 = py::dict(); + kwargs1["a"] = py::cast(1); + auto kwargs2 = py::dict(); + kwargs2["c"] = py::cast(3); + kwargs2["d"] = py::cast(4); + return f("positional", *py::make_tuple(1), 2, *py::make_tuple(3, 4), 5, + "key"_a="value", **kwargs1, "b"_a=2, **kwargs2, "e"_a=5); + }); + + m.def("test_unpacking_error1", [](py::function f) { + auto kwargs = py::dict(); + kwargs["x"] = py::cast(3); + return f("x"_a=1, "y"_a=2, **kwargs); // duplicate ** after keyword + }); + + m.def("test_unpacking_error2", [](py::function f) { + auto kwargs = py::dict(); + kwargs["x"] = py::cast(3); + return f(**kwargs, "x"_a=1); // duplicate keyword after ** + }); + + m.def("test_arg_conversion_error1", [](py::function f) { + f(234, Unregistered(), "kw"_a=567); + }); + + m.def("test_arg_conversion_error2", [](py::function f) { + f(234, "expected_name"_a=Unregistered(), "kw"_a=567); + }); + + /* Test cleanup of lambda closure */ m.def("test_cleanup", []() -> std::function { Payload p; diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index d6e72f333..8f867d43a 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -27,6 +27,41 @@ def test_callbacks(): assert f(number=43) == 44 +def test_keyword_args_and_generalized_unpacking(): + from pybind11_tests import (test_tuple_unpacking, test_dict_unpacking, test_keyword_args, + test_unpacking_and_keywords1, test_unpacking_and_keywords2, + test_unpacking_error1, test_unpacking_error2, + test_arg_conversion_error1, test_arg_conversion_error2) + + def f(*args, **kwargs): + return args, kwargs + + assert test_tuple_unpacking(f) == (("positional", 1, 2, 3, 4, 5, 6), {}) + assert test_dict_unpacking(f) == (("positional", 1), {"key": "value", "a": 1, "b": 2}) + assert test_keyword_args(f) == ((), {"x": 10, "y": 20}) + assert test_unpacking_and_keywords1(f) == ((1, 2), {"c": 3, "d": 4}) + assert test_unpacking_and_keywords2(f) == ( + ("positional", 1, 2, 3, 4, 5), + {"key": "value", "a": 1, "b": 2, "c": 3, "d": 4, "e": 5} + ) + + with pytest.raises(TypeError) as excinfo: + test_unpacking_error1(f) + assert "Got multiple values for keyword argument" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + test_unpacking_error2(f) + assert "Got multiple values for keyword argument" in str(excinfo.value) + + with pytest.raises(RuntimeError) as excinfo: + test_arg_conversion_error1(f) + assert "Unable to convert call argument" in str(excinfo.value) + + with pytest.raises(RuntimeError) as excinfo: + test_arg_conversion_error2(f) + assert "Unable to convert call argument" in str(excinfo.value) + + def test_lambda_closure_cleanup(): from pybind11_tests import test_cleanup, payload_cstats diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index bd244983f..d35ca02ac 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -56,7 +56,6 @@ test_initializer arg_keywords_and_defaults([](py::module &m) { m.def("args_function", &args_function); m.def("args_kwargs_function", &args_kwargs_function); - using namespace py::literals; m.def("kw_func_udl", &kw_func, "x"_a, "y"_a=300); m.def("kw_func_udl_z", &kw_func, "x"_a, "y"_a=0);