Support keyword arguments and generalized unpacking in C++

A Python function can be called with the syntax:
```python
foo(a1, a2, *args, ka=1, kb=2, **kwargs)
```
This commit adds support for the equivalent syntax in C++:
```c++
foo(a1, a2, *args, "ka"_a=1, "kb"_a=2, **kwargs)
```

In addition, generalized unpacking is implemented, as per PEP 448,
which allows calls with multiple * and ** unpacking:
```python
bar(*args1, 99, *args2, 101, **kwargs1, kz=200, **kwargs2)
```
and
```c++
bar(*args1, 99, *args2, 101, **kwargs1, "kz"_a=200, **kwargs2)
```

include/pybind11/attr.h

@@ -296,9 +296,6 @@ template <int Nurse, int Patient> struct process_attribute<keep_alive<Nurse, Pat
     static void postcall(handle args, handle ret) { keep_alive_impl(Nurse, Patient, args, ret); }
 };
 
-/// Ignore that a variable is unused in compiler warnings
-inline void ignore_unused(const int *) { }
-
 /// Recursively iterate over variadic template arguments
 template <typename... Args> struct process_attributes {
     static void init(const Args&... args, function_record *r) {
@@ -319,11 +316,6 @@ template <typename... Args> struct process_attributes {
     }
 };
 
-/// Compile-time integer sum
-constexpr size_t constexpr_sum() { return 0; }
-template <typename T, typename... Ts>
-constexpr size_t constexpr_sum(T n, Ts... ns) { return n + constexpr_sum(ns...); }
-
 /// Check the number of named arguments at compile time
 template <typename... Extra,
           size_t named = constexpr_sum(std::is_base_of<arg, Extra>::value...),

include/pybind11/cast.h

@@ -57,6 +57,7 @@ PYBIND11_NOINLINE inline internals &get_internals() {
                 } catch (const index_error &e)           { PyErr_SetString(PyExc_IndexError,    e.what()); return;
                 } catch (const key_error &e)             { PyErr_SetString(PyExc_KeyError,      e.what()); return;
                 } catch (const value_error &e)           { PyErr_SetString(PyExc_ValueError,    e.what()); return;
+                } catch (const type_error &e)            { PyErr_SetString(PyExc_TypeError,     e.what()); return;
                 } catch (const stop_iteration &e)        { PyErr_SetString(PyExc_StopIteration, e.what()); return;
                 } catch (const std::bad_alloc &e)        { PyErr_SetString(PyExc_MemoryError,   e.what()); return;
                 } catch (const std::domain_error &e)     { PyErr_SetString(PyExc_ValueError,    e.what()); return;
@@ -308,6 +309,7 @@ protected:
 };
 
 template <typename type, typename SFINAE = void> class type_caster : public type_caster_base<type> { };
+template <typename type> using make_caster = type_caster<intrinsic_t<type>>;
 
 template <typename type> class type_caster<std::reference_wrapper<type>> : public type_caster_base<type> {
 public:
@@ -947,15 +949,186 @@ template <return_value_policy policy = return_value_policy::automatic_reference,
     return result;
 }
 
-template <return_value_policy policy,
-          typename... Args> object handle::operator()(Args&&... args) const {
-    tuple args_tuple = pybind11::make_tuple<policy>(std::forward<Args>(args)...);
-    object result(PyObject_CallObject(m_ptr, args_tuple.ptr()), false);
+NAMESPACE_BEGIN(detail)
+NAMESPACE_BEGIN(constexpr_impl)
+/// Implementation details for constexpr functions
+constexpr int first(int i) { return i; }
+template <typename T, typename... Ts>
+constexpr int first(int i, T v, Ts... vs) { return v ? i : first(i + 1, vs...); }
+
+constexpr int last(int /*i*/, int result) { return result; }
+template <typename T, typename... Ts>
+constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); }
+NAMESPACE_END(constexpr_impl)
+
+/// Return the index of the first type in Ts which satisfies Predicate<T>
+template <template<typename> class Predicate, typename... Ts>
+constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate<Ts>::value...); }
+
+/// Return the index of the last type in Ts which satisfies Predicate<T>
+template <template<typename> class Predicate, typename... Ts>
+constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate<Ts>::value...); }
+
+/// Helper class which collects only positional arguments for a Python function call.
+/// A fancier version below can collect any argument, but this one is optimal for simple calls.
+template <return_value_policy policy>
+class simple_collector {
+public:
+    template <typename... Ts>
+    simple_collector(Ts &&...values)
+        : m_args(pybind11::make_tuple<policy>(std::forward<Ts>(values)...)) { }
+
+    const tuple &args() const & { return m_args; }
+    dict kwargs() const { return {}; }
+
+    tuple args() && { return std::move(m_args); }
+
+    /// Call a Python function and pass the collected arguments
+    object call(PyObject *ptr) const {
+        auto result = object(PyObject_CallObject(ptr, m_args.ptr()), false);
         if (!result)
             throw error_already_set();
         return result;
     }
 
+private:
+    tuple m_args;
+};
+
+/// Helper class which collects positional, keyword, * and ** arguments for a Python function call
+template <return_value_policy policy>
+class unpacking_collector {
+public:
+    template <typename... Ts>
+    unpacking_collector(Ts &&...values) {
+        // Tuples aren't (easily) resizable so a list is needed for collection,
+        // but the actual function call strictly requires a tuple.
+        auto args_list = list();
+        int _[] = { 0, (process(args_list, std::forward<Ts>(values)), 0)... };
+        ignore_unused(_);
+
+        m_args = object(PyList_AsTuple(args_list.ptr()), false);
+    }
+
+    const tuple &args() const & { return m_args; }
+    const dict &kwargs() const & { return m_kwargs; }
+
+    tuple args() && { return std::move(m_args); }
+    dict kwargs() && { return std::move(m_kwargs); }
+
+    /// Call a Python function and pass the collected arguments
+    object call(PyObject *ptr) const {
+        auto result = object(PyObject_Call(ptr, m_args.ptr(), m_kwargs.ptr()), false);
+        if (!result)
+            throw error_already_set();
+        return result;
+    }
+
+private:
+    template <typename T>
+    void process(list &args_list, T &&x) {
+        auto o = object(detail::make_caster<T>::cast(std::forward<T>(x), policy, nullptr), false);
+        if (!o) {
+#if defined(NDEBUG)
+            argument_cast_error();
+#else
+            argument_cast_error(std::to_string(args_list.size()), type_id<T>());
+#endif
+        }
+        args_list.append(o);
+    }
+
+    void process(list &args_list, detail::args_proxy ap) {
+        for (const auto &a : ap) {
+            args_list.append(a.cast<object>());
+        }
+    }
+
+    template <typename T>
+    void process(list &/*args_list*/, arg_t<T> &&a) {
+        if (m_kwargs[a.name]) {
+#if defined(NDEBUG)
+            multiple_values_error();
+#else
+            multiple_values_error(a.name);
+#endif
+        }
+        auto o = object(detail::make_caster<T>::cast(*a.value, policy, nullptr), false);
+        if (!o) {
+#if defined(NDEBUG)
+            argument_cast_error();
+#else
+            argument_cast_error(a.name, type_id<T>());
+#endif
+        }
+        m_kwargs[a.name] = o;
+    }
+
+    void process(list &/*args_list*/, detail::kwargs_proxy kp) {
+        for (const auto &k : dict(kp, true)) {
+            if (m_kwargs[k.first]) {
+#if defined(NDEBUG)
+                multiple_values_error();
+#else
+                multiple_values_error(k.first.str());
+#endif
+            }
+            m_kwargs[k.first] = k.second;
+        }
+    }
+
+    [[noreturn]] static void multiple_values_error() {
+        throw type_error("Got multiple values for keyword argument "
+                         "(compile in debug mode for details)");
+    }
+
+    [[noreturn]] static void multiple_values_error(std::string name) {
+        throw type_error("Got multiple values for keyword argument '" + name + "'");
+    }
+
+    [[noreturn]] static void argument_cast_error() {
+        throw cast_error("Unable to convert call argument to Python object "
+                         "(compile in debug mode for details)");
+    }
+
+    [[noreturn]] static void argument_cast_error(std::string name, std::string type) {
+        throw cast_error("Unable to convert call argument '" + name
+                         + "' of type '" + type + "' to Python object");
+    }
+
+private:
+    tuple m_args;
+    dict m_kwargs;
+};
+
+/// Collect only positional arguments for a Python function call
+template <return_value_policy policy, typename... Args,
+          typename = enable_if_t<all_of_t<is_positional, Args...>::value>>
+simple_collector<policy> collect_arguments(Args &&...args) {
+    return {std::forward<Args>(args)...};
+}
+
+/// Collect all arguments, including keywords and unpacking (only instantiated when needed)
+template <return_value_policy policy, typename... Args,
+          typename = enable_if_t<!all_of_t<is_positional, Args...>::value>>
+unpacking_collector<policy> collect_arguments(Args &&...args) {
+    // Following argument order rules for generalized unpacking according to PEP 448
+    static_assert(
+        constexpr_last<is_positional, Args...>() < constexpr_first<is_keyword_or_ds, Args...>()
+        && constexpr_last<is_s_unpacking, Args...>() < constexpr_first<is_ds_unpacking, Args...>(),
+        "Invalid function call: positional args must precede keywords and ** unpacking; "
+        "* unpacking must precede ** unpacking"
+    );
+    return {std::forward<Args>(args)...};
+}
+
+NAMESPACE_END(detail)
+
+template <return_value_policy policy, typename... Args>
+object handle::operator()(Args &&...args) const {
+    return detail::collect_arguments<policy>(std::forward<Args>(args)...).call(m_ptr);
+}
+
 template <return_value_policy policy,
           typename... Args> object handle::call(Args &&... args) const {
     return operator()<policy>(std::forward<Args>(args)...);

include/pybind11/common.h

@@ -326,10 +326,41 @@ template <typename T> struct intrinsic_type<T&>                   { typedef type
 template <typename T> struct intrinsic_type<T&&>                  { typedef typename intrinsic_type<T>::type type; };
 template <typename T, size_t N> struct intrinsic_type<const T[N]> { typedef typename intrinsic_type<T>::type type; };
 template <typename T, size_t N> struct intrinsic_type<T[N]>       { typedef typename intrinsic_type<T>::type type; };
+template <typename T> using intrinsic_t = typename intrinsic_type<T>::type;
 
 /// Helper type to replace 'void' in some expressions
 struct void_type { };
 
+/// from __cpp_future__ import (convenient aliases from C++14/17)
+template <bool B> using bool_constant = std::integral_constant<bool, B>;
+template <class T> using negation = bool_constant<!T::value>;
+template <bool B, typename T = void> using enable_if_t = typename std::enable_if<B, T>::type;
+template <bool B, typename T, typename F> using conditional_t = typename std::conditional<B, T, F>::type;
+
+/// Compile-time integer sum
+constexpr size_t constexpr_sum() { return 0; }
+template <typename T, typename... Ts>
+constexpr size_t constexpr_sum(T n, Ts... ns) { return size_t{n} + constexpr_sum(ns...); }
+
+/// Return true if all/any Ts satify Predicate<T>
+#if !defined(_MSC_VER)
+template <template<typename> class Predicate, typename... Ts>
+using all_of_t = bool_constant<(constexpr_sum(Predicate<Ts>::value...) == sizeof...(Ts))>;
+template <template<typename> class Predicate, typename... Ts>
+using any_of_t = bool_constant<(constexpr_sum(Predicate<Ts>::value...) > 0)>;
+#else
+// MSVC workaround (2015 Update 3 has issues with some member type aliases and constexpr)
+template <template<typename> class P, typename...> struct all_of_t : std::true_type { };
+template <template<typename> class P, typename T, typename... Ts>
+struct all_of_t<P, T, Ts...> : conditional_t<P<T>::value, all_of_t<P, Ts...>, std::false_type> { };
+template <template<typename> class P, typename...> struct any_of_t : std::false_type { };
+template <template<typename> class P, typename T, typename... Ts>
+struct any_of_t<P, T, Ts...> : conditional_t<P<T>::value, std::true_type, any_of_t<P, Ts...>> { };
+#endif
+
+/// Ignore that a variable is unused in compiler warnings
+inline void ignore_unused(const int *) { }
+
 NAMESPACE_END(detail)
 
 #define PYBIND11_RUNTIME_EXCEPTION(name) \
@@ -345,6 +376,7 @@ PYBIND11_RUNTIME_EXCEPTION(stop_iteration)
 PYBIND11_RUNTIME_EXCEPTION(index_error)
 PYBIND11_RUNTIME_EXCEPTION(key_error)
 PYBIND11_RUNTIME_EXCEPTION(value_error)
+PYBIND11_RUNTIME_EXCEPTION(type_error)
 PYBIND11_RUNTIME_EXCEPTION(cast_error) /// Thrown when pybind11::cast or handle::call fail due to a type casting error
 PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally
 

include/pybind11/pytypes.h

@@ -16,7 +16,7 @@
 NAMESPACE_BEGIN(pybind11)
 
 /* A few forward declarations */
-class object; class str; class object; class dict; class iterator;
+class object; class str; class iterator;
 struct arg; template <typename T> struct arg_t;
 namespace detail { class accessor; class args_proxy; class kwargs_proxy; }
 
@@ -250,6 +250,17 @@ public:
     kwargs_proxy operator*() const { return kwargs_proxy(*this); }
 };
 
+/// Python argument categories (using PEP 448 terms)
+template <typename T> using is_keyword = std::is_base_of<arg, T>;
+template <typename T> using is_s_unpacking = std::is_same<args_proxy, T>; // * unpacking
+template <typename T> using is_ds_unpacking = std::is_same<kwargs_proxy, T>; // ** unpacking
+template <typename T> using is_positional = bool_constant<
+    !is_keyword<T>::value && !is_s_unpacking<T>::value && !is_ds_unpacking<T>::value
+>;
+template <typename T> using is_keyword_or_ds = bool_constant<
+    is_keyword<T>::value || is_ds_unpacking<T>::value
+>;
+
 NAMESPACE_END(detail)
 
 #define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, CvtStmt) \

tests/pybind11_tests.h

@@ -8,6 +8,7 @@ using std::cout;
 using std::endl;
 
 namespace py = pybind11;
+using namespace pybind11::literals;
 
 class test_initializer {
 public:

tests/test_callbacks.cpp

@@ -71,6 +71,9 @@ struct Payload {
     }
 };
 
+/// Something to trigger a conversion error
+struct Unregistered {};
+
 test_initializer callbacks([](py::module &m) {
     m.def("test_callback1", &test_callback1);
     m.def("test_callback2", &test_callback2);
@@ -78,8 +81,65 @@ test_initializer callbacks([](py::module &m) {
     m.def("test_callback4", &test_callback4);
     m.def("test_callback5", &test_callback5);
 
-    /* Test cleanup of lambda closure */
+    // Test keyword args and generalized unpacking
+    m.def("test_tuple_unpacking", [](py::function f) {
+        auto t1 = py::make_tuple(2, 3);
+        auto t2 = py::make_tuple(5, 6);
+        return f("positional", 1, *t1, 4, *t2);
+    });
 
+    m.def("test_dict_unpacking", [](py::function f) {
+        auto d1 = py::dict();
+        d1["key"] = py::cast("value");
+        d1["a"] = py::cast(1);
+        auto d2 = py::dict();
+        auto d3 = py::dict();
+        d3["b"] = py::cast(2);
+        return f("positional", 1, **d1, **d2, **d3);
+    });
+
+    m.def("test_keyword_args", [](py::function f) {
+        return f("x"_a=10, "y"_a=20);
+    });
+
+    m.def("test_unpacking_and_keywords1", [](py::function f) {
+        auto args = py::make_tuple(2);
+        auto kwargs = py::dict();
+        kwargs["d"] = py::cast(4);
+        return f(1, *args, "c"_a=3, **kwargs);
+    });
+
+    m.def("test_unpacking_and_keywords2", [](py::function f) {
+        auto kwargs1 = py::dict();
+        kwargs1["a"] = py::cast(1);
+        auto kwargs2 = py::dict();
+        kwargs2["c"] = py::cast(3);
+        kwargs2["d"] = py::cast(4);
+        return f("positional", *py::make_tuple(1), 2, *py::make_tuple(3, 4), 5,
+                 "key"_a="value", **kwargs1, "b"_a=2, **kwargs2, "e"_a=5);
+    });
+
+    m.def("test_unpacking_error1", [](py::function f) {
+        auto kwargs = py::dict();
+        kwargs["x"] = py::cast(3);
+        return f("x"_a=1, "y"_a=2, **kwargs); // duplicate ** after keyword
+    });
+
+    m.def("test_unpacking_error2", [](py::function f) {
+        auto kwargs = py::dict();
+        kwargs["x"] = py::cast(3);
+        return f(**kwargs, "x"_a=1); // duplicate keyword after **
+    });
+
+    m.def("test_arg_conversion_error1", [](py::function f) {
+        f(234, Unregistered(), "kw"_a=567);
+    });
+
+    m.def("test_arg_conversion_error2", [](py::function f) {
+        f(234, "expected_name"_a=Unregistered(), "kw"_a=567);
+    });
+
+    /* Test cleanup of lambda closure */
     m.def("test_cleanup", []() -> std::function<void(void)> {
         Payload p;
 

tests/test_callbacks.py

@@ -27,6 +27,41 @@ def test_callbacks():
     assert f(number=43) == 44
 
 
+def test_keyword_args_and_generalized_unpacking():
+    from pybind11_tests import (test_tuple_unpacking, test_dict_unpacking, test_keyword_args,
+                                test_unpacking_and_keywords1, test_unpacking_and_keywords2,
+                                test_unpacking_error1, test_unpacking_error2,
+                                test_arg_conversion_error1, test_arg_conversion_error2)
+
+    def f(*args, **kwargs):
+        return args, kwargs
+
+    assert test_tuple_unpacking(f) == (("positional", 1, 2, 3, 4, 5, 6), {})
+    assert test_dict_unpacking(f) == (("positional", 1), {"key": "value", "a": 1, "b": 2})
+    assert test_keyword_args(f) == ((), {"x": 10, "y": 20})
+    assert test_unpacking_and_keywords1(f) == ((1, 2), {"c": 3, "d": 4})
+    assert test_unpacking_and_keywords2(f) == (
+        ("positional", 1, 2, 3, 4, 5),
+        {"key": "value", "a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
+    )
+
+    with pytest.raises(TypeError) as excinfo:
+        test_unpacking_error1(f)
+    assert "Got multiple values for keyword argument" in str(excinfo.value)
+
+    with pytest.raises(TypeError) as excinfo:
+        test_unpacking_error2(f)
+    assert "Got multiple values for keyword argument" in str(excinfo.value)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        test_arg_conversion_error1(f)
+    assert "Unable to convert call argument" in str(excinfo.value)
+
+    with pytest.raises(RuntimeError) as excinfo:
+        test_arg_conversion_error2(f)
+    assert "Unable to convert call argument" in str(excinfo.value)
+
+
 def test_lambda_closure_cleanup():
     from pybind11_tests import test_cleanup, payload_cstats
 

tests/test_kwargs_and_defaults.cpp

@@ -56,7 +56,6 @@ test_initializer arg_keywords_and_defaults([](py::module &m) {
     m.def("args_function", &args_function);
     m.def("args_kwargs_function", &args_kwargs_function);
 
-    using namespace py::literals;
     m.def("kw_func_udl", &kw_func, "x"_a, "y"_a=300);
     m.def("kw_func_udl_z", &kw_func, "x"_a, "y"_a=0);