From 66c9a40213dab8d51ae5933a409c5e970b754f61 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Sun, 17 Jan 2016 22:36:36 +0100 Subject: [PATCH] Much more efficient generation of function signatures, updated docs This modification taps into some newer C++14 features (if present) to generate function signatures considerably more efficiently at compile time rather than at run time. With this change, pybind11 binaries are now *2.1 times* smaller compared to the Boost.Python baseline in the benchmark. Compilation times get a nice improvement as well. Visual Studio 2015 unfortunately doesn't implement 'constexpr' well enough yet to support this change and uses a runtime fallback. --- CMakeLists.txt | 21 +- README.md | 17 +- docs/benchmark.py | 2 +- docs/benchmark.rst | 43 ++- docs/cmake.rst | 12 +- docs/intro.rst | 17 +- docs/pybind11_vs_boost_python1.svg | 438 ++++++++++++++--------------- docs/pybind11_vs_boost_python2.svg | 134 ++++----- example/example11.cpp | 1 + example/example11.py | 19 +- example/example11.ref | 26 +- include/pybind11/cast.h | 138 ++------- include/pybind11/common.h | 12 +- include/pybind11/complex.h | 2 +- include/pybind11/descr.h | 161 +++++++++++ include/pybind11/functional.h | 7 +- include/pybind11/pybind11.h | 207 +++++++++----- include/pybind11/stl.h | 24 +- include/pybind11/typeid.h | 10 +- 19 files changed, 731 insertions(+), 560 deletions(-) create mode 100644 include/pybind11/descr.h diff --git a/CMakeLists.txt b/CMakeLists.txt index df08a397e..4d411f54f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,16 @@ endif() find_package(PythonInterp ${PYTHONLIBS_VERSION_STRING} EXACT REQUIRED) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - # Enable C++11 mode on C++ / Clang - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG) + CHECK_CXX_COMPILER_FLAG("-std=c++11" HAS_CPP11_FLAG) + + if (HAS_CPP14_FLAG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") + elseif (HAS_CPP11_FLAG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + else() + message(FATAL_ERROR "Unsupported compiler -- pybind11 requires C++11 support!") + endif() # Enable link time optimization and set the default symbol # visibility to hidden (very important to obtain small binaries) @@ -74,14 +82,15 @@ include_directories(include) set(PYBIND11_HEADERS include/pybind11/cast.h include/pybind11/common.h + include/pybind11/complex.h + include/pybind11/descr.h + include/pybind11/functional.h + include/pybind11/numpy.h include/pybind11/operators.h include/pybind11/pybind11.h include/pybind11/pytypes.h - include/pybind11/typeid.h - include/pybind11/numpy.h - include/pybind11/complex.h include/pybind11/stl.h - include/pybind11/functional.h + include/pybind11/typeid.h ) # Create the binding library diff --git a/README.md b/README.md index f84154c29..96560e46e 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,12 @@ become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. The core -header files only require ~2K lines of code and depend on Python (2.7 or 3.x) +header files only require ~3K lines of code and depend on Python (2.7 or 3.x) and the C++ standard library. This compact implementation was possible thanks -to some of the new C++11 language features (tuples, lambda functions and -variadic templates). Since its creation, this library has grown beyond -Boost.Python in many ways, leading to dramatically simpler binding code in many -common situations. +to some of the new C++11 language features (specifically: tuples, lambda +functions and variadic templates). Since its creation, this library has grown +beyond Boost.Python in many ways, leading to dramatically simpler binding code +in many common situations. Tutorial and reference documentation is provided at [http://pybind11.readthedocs.org/en/latest](http://pybind11.readthedocs.org/en/latest). @@ -71,6 +71,13 @@ In addition to the core functionality, pybind11 provides some extra goodies: - Everything is contained in just a few header files; there is no need to link against any additional libraries. +- Binaries are generally smaller by a factor of 2 or more compared to + equivalent bindings generated by Boost.Python. + +- When supported by the compiler, two new C++14 features (relaxed constexpr, + return value deduction) such as are used to do additional work at compile + time, leading to smaller binaries. + ### License pybind11 is provided under a BSD-style license that can be found in the diff --git a/docs/benchmark.py b/docs/benchmark.py index efa82847b..6f02e92ff 100644 --- a/docs/benchmark.py +++ b/docs/benchmark.py @@ -81,7 +81,7 @@ for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: f.write(codegen(nclasses)) n1 = dt.datetime.now() os.system("g++ -Os -shared -rdynamic -undefined dynamic_lookup " - "-fvisibility=hidden -std=c++11 test.cpp -I include " + "-fvisibility=hidden -std=c++14 test.cpp -I include " "-I /System/Library/Frameworks/Python.framework/Headers -o test.so") n2 = dt.datetime.now() elapsed = (n2 - n1).total_seconds() diff --git a/docs/benchmark.rst b/docs/benchmark.rst index c06bfb395..c31e2902d 100644 --- a/docs/benchmark.rst +++ b/docs/benchmark.rst @@ -4,9 +4,12 @@ Benchmark The following is the result of a synthetic benchmark comparing both compilation time and module size of pybind11 against Boost.Python. -A python script (see the ``docs/benchmark.py`` file) was used to generate a -set of dummy classes whose count increases for each successive benchmark -(between 1 and 512 classes in powers of two). Each class has four methods with +Setup +----- + +A python script (see the ``docs/benchmark.py`` file) was used to generate a set +of files with dummy classes whose count increases for each successive benchmark +(between 1 and 2048 classes in powers of two). Each class has four methods with a randomly generated signature with a return value and four arguments. (There was no particular reason for this setup other than the desire to generate many unique function signatures whose count could be controlled in a simple way.) @@ -43,28 +46,38 @@ compilation was done with .. code-block:: bash - Apple LLVM version 7.0.0 (clang-700.0.72) + Apple LLVM version 7.0.2 (clang-700.1.81) and the following compilation flags .. code-block:: bash - g++ -Os -shared -rdynamic -undefined dynamic_lookup -fvisibility=hidden -std=c++11 + g++ -Os -shared -rdynamic -undefined dynamic_lookup -fvisibility=hidden -std=c++14 + +Compilation time +---------------- The following log-log plot shows how the compilation time grows for an -increasing number of class and function declarations. pybind11 includes fewer -headers, which initially leads to shorter compilation times, but the -performance is ultimately very similar (pybind11 is 1 second faster for the -largest file, which is less than 1% of the total compilation time). +increasing number of class and function declarations. pybind11 includes many +fewer headers, which initially leads to shorter compilation times, but the +performance is ultimately fairly similar (pybind11 is 19.8 seconds faster for +the largest largest file with 2048 classes and a total of 8192 methods -- a +modest **1.2x** speedup relative to Boost.Python, which required 116.35 +seconds). .. image:: pybind11_vs_boost_python1.svg -Differences between the two libraries become more pronounced when considering -the file size of the generated Python plugin. Note that the plot below does not -include the size of the Boost.Python shared library, hence Boost actually has a -slight advantage. +Module size +----------- + +Differences between the two libraries become much more pronounced when +considering the file size of the generated Python plugin: for the largest file, +the binary generated by Boost.Python required 16.8 MiB, which was **2.17 +times** / **9.1 megabytes** larger than the output generated by pybind11. For +very small inputs, Boost.Python has an edge in the plot below -- however, note +that it stores many definitions in an external library, whose size was not +included here, hence the comparison is slightly shifted in Boost.Python's +favor. .. image:: pybind11_vs_boost_python2.svg -Despite this, the libraries procuced by Boost.Python for more than a few -functions are consistently larger by a factor of 1.75. diff --git a/docs/cmake.rst b/docs/cmake.rst index 348c54a64..c44366f22 100644 --- a/docs/cmake.rst +++ b/docs/cmake.rst @@ -39,8 +39,16 @@ and that the pybind11 repository is located in a subdirectory named :file:`pybin # find_package(PythonInterp ${PYTHONLIBS_VERSION_STRING} EXACT REQUIRED) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") - # Enable C++11 mode on C++ / Clang - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG) + CHECK_CXX_COMPILER_FLAG("-std=c++11" HAS_CPP11_FLAG) + + if (HAS_CPP14_FLAG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") + elseif (HAS_CPP11_FLAG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + else() + message(FATAL_ERROR "Unsupported compiler -- at least C++11 support is needed!") + endif() # Enable link time optimization and set the default symbol # visibility to hidden (very important to obtain small binaries) diff --git a/docs/intro.rst b/docs/intro.rst index 730d19dfa..291469b1b 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -18,12 +18,12 @@ become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. The core -header files only require ~2K lines of code and depend on Python (2.7 or 3.x) +header files only require ~3K lines of code and depend on Python (2.7 or 3.x) and the C++ standard library. This compact implementation was possible thanks -to some of the new C++11 language features (tuples, lambda functions and -variadic templates). Since its creation, this library has grown beyond -Boost.Python in many ways, leading to dramatically simpler binding code in many -common situations. +to some of the new C++11 language features (specifically: tuples, lambda +functions and variadic templates). Since its creation, this library has grown +beyond Boost.Python in many ways, leading to dramatically simpler binding code +in many common situations. Core features ************* @@ -64,3 +64,10 @@ In addition to the core functionality, pybind11 provides some extra goodies: - Everything is contained in just a few header files; there is no need to link against any additional libraries. + +- Binaries are generally smaller by a factor of 2 or more compared to + equivalent bindings generated by Boost.Python. + +- When supported by the compiler, two new C++14 features (relaxed constexpr, + return value deduction) such as are used to do additional work at compile + time, leading to smaller binaries. diff --git a/docs/pybind11_vs_boost_python1.svg b/docs/pybind11_vs_boost_python1.svg index 8ffacf1ab..5bf950e6f 100644 --- a/docs/pybind11_vs_boost_python1.svg +++ b/docs/pybind11_vs_boost_python1.svg @@ -1,5 +1,5 @@ - + @@ -12,30 +12,33 @@ - + - + - + - + - + - + - + - + + + + @@ -151,289 +154,274 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - - + + + - + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - - + + - + - - + + - + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + - - + + - - - - - - + + + + + + - + - - - - - + + + + + - + - - - - - - + + + + + + - + - - - - - - - - + + + + + + + + diff --git a/docs/pybind11_vs_boost_python2.svg b/docs/pybind11_vs_boost_python2.svg index dbaf9aa08..5ed6530ca 100644 --- a/docs/pybind11_vs_boost_python2.svg +++ b/docs/pybind11_vs_boost_python2.svg @@ -163,7 +163,7 @@ - + @@ -177,13 +177,13 @@ - + - + - + @@ -195,16 +195,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -250,54 +250,54 @@ - + - - + + - + - + - - + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + @@ -324,33 +324,33 @@ - - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/example/example11.cpp b/example/example11.cpp index 522485cb8..9599f940a 100644 --- a/example/example11.cpp +++ b/example/example11.cpp @@ -14,4 +14,5 @@ void kw_func(int x, int y) { std::cout << "kw_func(x=" << x << ", y=" << y << ") void init_ex11(py::module &m) { m.def("kw_func", &kw_func, py::arg("x"), py::arg("y")); m.def("kw_func2", &kw_func, py::arg("x") = 100, py::arg("y") = 200); + m.def("kw_func3", [](const char *) { }, py::arg("data") = std::string("Hello world!")); } diff --git a/example/example11.py b/example/example11.py index 7fc1b4816..7d0217a23 100755 --- a/example/example11.py +++ b/example/example11.py @@ -1,19 +1,19 @@ #!/usr/bin/env python from __future__ import print_function -import sys, pydoc +import sys +import pydoc + sys.path.append('.') -import example - -from example import kw_func -from example import kw_func2 +from example import kw_func, kw_func2, kw_func3 print(pydoc.render_doc(kw_func, "Help on %s")) print(pydoc.render_doc(kw_func2, "Help on %s")) +print(pydoc.render_doc(kw_func3, "Help on %s")) kw_func(5, 10) -kw_func(5, y = 10) -kw_func(y = 10, x = 5) +kw_func(5, y=10) +kw_func(y=10, x=5) kw_func2() @@ -24,3 +24,8 @@ kw_func2(y=10) kw_func2(5, 10) kw_func2(x=5, y=10) + +try: + kw_func2(x=5, y=10, z=12) +except Exception as e: + print("Caught expected exception: " + str(e)) diff --git a/example/example11.ref b/example/example11.ref index 728e804dc..b7719b132 100644 --- a/example/example11.ref +++ b/example/example11.ref @@ -1,3 +1,18 @@ +Help on built-in function kw_func + +kkww__ffuunncc(...) + Signature : (x : int, y : int) -> None + +Help on built-in function kw_func2 + +kkww__ffuunncc22(...) + Signature : (x : int = 100L, y : int = 200L) -> None + +Help on built-in function kw_func3 + +kkww__ffuunncc33(...) + Signature : (data : str = u'Hello world!') -> None + kw_func(x=5, y=10) kw_func(x=5, y=10) kw_func(x=5, y=10) @@ -7,13 +22,6 @@ kw_func(x=5, y=200) kw_func(x=100, y=10) kw_func(x=5, y=10) kw_func(x=5, y=10) -Help on built-in function kw_func - -kkww__ffuunncc(...) method of builtins.PyCapsule instance - Signature : (x : int, y : int) -> None - -Help on built-in function kw_func2 - -kkww__ffuunncc22(...) method of builtins.PyCapsule instance - Signature : (x : int = 100, y : int = 200) -> None +Caught expected exception: Incompatible function arguments. The following argument types are supported: + 1. (x : int = 100L, y : int = 200L) -> None diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 0c7efc6d3..da0831179 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -12,8 +12,8 @@ #include "pytypes.h" #include "typeid.h" +#include "descr.h" #include -#include #include NAMESPACE_BEGIN(pybind11) @@ -25,80 +25,6 @@ NAMESPACE_BEGIN(detail) #define PYBIND11_AS_STRING PyString_AsString #endif -/** Linked list descriptor type for function signatures (produces smaller binaries - compared to a previous solution using std::string and operator +=) */ -class descr { -public: - struct entry { - const std::type_info *type = nullptr; - const char *str = nullptr; - entry *next = nullptr; - entry(const std::type_info *type) : type(type) { } - entry(const char *str) : str(str) { } - }; - - descr() { } - descr(descr &&d) : first(d.first), last(d.last) { d.first = d.last = nullptr; } - PYBIND11_NOINLINE descr(const char *str) { first = last = new entry { str }; } - PYBIND11_NOINLINE descr(const std::type_info &type) { first = last = new entry { &type }; } - - PYBIND11_NOINLINE void operator+(const char *str) { - entry *next = new entry { str }; - last->next = next; - last = next; - } - - PYBIND11_NOINLINE void operator+(const std::type_info *type) { - entry *next = new entry { type }; - last->next = next; - last = next; - } - - PYBIND11_NOINLINE void operator+=(descr &&other) { - last->next = other.first; - while (last->next) - last = last->next; - other.first = other.last = nullptr; - } - - PYBIND11_NOINLINE friend descr operator+(descr &&l, descr &&r) { - descr result(std::move(l)); - result += std::move(r); - return result; - } - - PYBIND11_NOINLINE std::string str() const { - std::string result; - auto const& registered_types = get_internals().registered_types; - for (entry *it = first; it != nullptr; it = it->next) { - if (it->type) { - auto it2 = registered_types.find(it->type); - if (it2 != registered_types.end()) { - result += it2->second.type->tp_name; - } else { - std::string tname(it->type->name()); - detail::clean_type_id(tname); - result += tname; - } - } else { - result += it->str; - } - } - return result; - } - - PYBIND11_NOINLINE ~descr() { - while (first) { - entry *tmp = first->next; - delete first; - first = tmp; - } - } - - entry *first = nullptr; - entry *last = nullptr; -}; - class type_caster_custom { public: PYBIND11_NOINLINE type_caster_custom(const std::type_info *type_info) { @@ -195,7 +121,7 @@ protected: /// Generic type caster for objects stored on the heap template class type_caster : public type_caster_custom { public: - static descr name() { return typeid(type); } + static PYBIND11_DESCR name() { return type_descr(_()); } type_caster() : type_caster_custom(&typeid(type)) { } @@ -224,7 +150,7 @@ protected: protected: \ type value; \ public: \ - static descr name() { return py_name; } \ + static PYBIND11_DESCR name() { return type_descr(py_name); } \ static PyObject *cast(const type *src, return_value_policy policy, PyObject *parent) { \ return cast(*src, policy, parent); \ } \ @@ -296,9 +222,10 @@ public: return cast(*src, policy, parent); } - static descr name() { - return std::is_floating_point::value ? "float" : "int"; - } + template ::value, int>::type = 0> + static PYBIND11_DESCR name() { return type_descr(_("int")); } + template ::value, int>::type = 0> + static PYBIND11_DESCR name() { return type_descr(_("float")); } operator T*() { return &value; } operator T&() { return value; } @@ -314,7 +241,7 @@ public: Py_INCREF(Py_None); return Py_None; } - PYBIND11_TYPE_CASTER(void_type, "None"); + PYBIND11_TYPE_CASTER(void_type, _("None")); }; template <> class type_caster : public type_caster { }; @@ -332,7 +259,7 @@ public: Py_INCREF(result); return result; } - PYBIND11_TYPE_CASTER(bool, "bool"); + PYBIND11_TYPE_CASTER(bool, _("bool")); }; template <> class type_caster { @@ -352,7 +279,7 @@ public: static PyObject *cast(const std::string &src, return_value_policy /* policy */, PyObject * /* parent */) { return PyUnicode_FromString(src.c_str()); } - PYBIND11_TYPE_CASTER(std::string, "str"); + PYBIND11_TYPE_CASTER(std::string, _("str")); }; template <> class type_caster { @@ -379,7 +306,7 @@ public: return PyUnicode_DecodeLatin1(str, 1, nullptr); } - static descr name() { return "str"; } + static PYBIND11_DESCR name() { return type_descr(_("str")); } operator char*() { return (char *) value.c_str(); } operator char() { if (value.length() > 0) return value[0]; else return '\0'; } @@ -411,13 +338,10 @@ public: return tuple; } - static descr name() { - class descr result("("); - result += std::move(type_caster::type>::name()); - result += ", "; - result += std::move(type_caster::type>::name()); - result += ")"; - return result; + static PYBIND11_DESCR name() { + return type_descr( + _("(") + type_caster::type>::name() + + _(", ") + type_caster::type>::name() + _(")")); } operator type() { @@ -441,31 +365,11 @@ public: return cast(src, policy, parent, typename make_index_sequence::type()); } - static descr name(const std::list &args = std::list()) { - std::array type_names {{ - type_caster::type>::name()... - }}; - auto it = args.begin(); - class descr result("("); - for (int i=0; iname; - result += " : "; - } - result += std::move(type_names[i]); - if (it != args.end()) { - if (it->descr) { - result += " = "; - result += it->descr; - } - ++it; - } - if (i+1 < size) - result += ", "; - ++it; - } - result += ")"; - return result; + static PYBIND11_DESCR name() { + return type_descr( + _("(") + + detail::concat(type_caster::type>::name()...) + + _(")")); } template typename std::enable_if::value, ReturnValue>::type call(Func &&f) { @@ -576,7 +480,7 @@ public: src.inc_ref(); return (PyObject *) src.ptr(); } - PYBIND11_TYPE_CASTER(type, typeid(type)); + PYBIND11_TYPE_CASTER(type, _()); }; NAMESPACE_END(detail) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index 3ed473cdc..d5d10e822 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -167,16 +167,12 @@ struct overload_hash { /// Stores information about a keyword argument struct argument_entry { - char *name; ///< Argument name - char *descr; ///< Human-readable version of the argument value - PyObject *value; ///< Associated Python object + const char *name; ///< Argument name + const char *descr; ///< Human-readable version of the argument value + PyObject *value; ///< Associated Python object - argument_entry(char *name, char *descr, PyObject *value) + argument_entry(const char *name, const char *descr, PyObject *value) : name(name), descr(descr), value(value) { } - - ~argument_entry() { - free(name); free(descr); Py_XDECREF(value); - } }; /// Internal data struture used to track registered instances and types diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index ce3276c05..be0614c59 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -34,7 +34,7 @@ public: return PyComplex_FromDoubles((double) src.real(), (double) src.imag()); } - PYBIND11_TYPE_CASTER(std::complex, "complex"); + PYBIND11_TYPE_CASTER(std::complex, _("complex")); }; NAMESPACE_END(detail) NAMESPACE_END(pybind11) diff --git a/include/pybind11/descr.h b/include/pybind11/descr.h new file mode 100644 index 000000000..ed44b2626 --- /dev/null +++ b/include/pybind11/descr.h @@ -0,0 +1,161 @@ +/* + pybind11/descr.h: Helper type for concatenating type signatures + either at runtime (C++11) or compile time (C++14) + + Copyright (c) 2015 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "common.h" + +NAMESPACE_BEGIN(pybind11) +NAMESPACE_BEGIN(detail) + + +#if defined(__clang__) +# if __has_feature(cxx_return_type_deduction) && __has_feature(cxx_relaxed_constexpr) +# define PYBIND11_CPP14 +# endif +#elif defined(__GNUG__) +# if __cpp_constexpr >= 201304 && __cpp_decltype_auto >= 201304 +# define PYBIND11_CPP14 +# endif +#endif + +#if defined(PYBIND11_CPP14) /* Concatenate type signatures at compile time using C++14 */ + +template class descr { + template friend class descr; +public: + constexpr descr(char const (&text) [Size1+1], const std::type_info * const (&types)[Size2+1]) + : descr(text, types, + typename make_index_sequence::type(), + typename make_index_sequence::type()) { } + + constexpr const char *text() const { return m_text; } + constexpr const std::type_info * const * types() const { return m_types; } + + template + constexpr descr operator+(const descr &other) const { + return concat(other, + typename make_index_sequence::type(), + typename make_index_sequence::type(), + typename make_index_sequence::type(), + typename make_index_sequence::type()); + } + +protected: + template + constexpr descr( + char const (&text) [Size1+1], + const std::type_info * const (&types) [Size2+1], + index_sequence, index_sequence) + : m_text{text[Indices1]..., '\0'}, + m_types{types[Indices2]..., nullptr } {} + + template + constexpr descr + concat(const descr &other, + index_sequence, index_sequence, + index_sequence, index_sequence) const { + return descr( + { m_text[Indices1]..., other.m_text[OtherIndices1]..., '\0' }, + { m_types[Indices2]..., other.m_types[OtherIndices2]..., nullptr } + ); + } + +protected: + char m_text[Size1 + 1]; + const std::type_info * m_types[Size2 + 1]; +}; + +template constexpr descr _(char const(&text)[Size]) { + return descr(text, { nullptr }); +} + +template constexpr descr<1, 1> _() { + return descr<1, 1>({ '%', '\0' }, { &typeid(Type), nullptr }); +} + +inline constexpr descr<0, 0> concat() { return _(""); } +template auto constexpr concat(descr descr) { return descr; } +template auto constexpr concat(descr descr, Args&&... args) { return descr + _(", ") + concat(args...); } +template auto constexpr type_descr(descr descr) { return _("{") + descr + _("}"); } + +#define PYBIND11_DESCR constexpr auto + +#else /* Simpler C++11 implementation based on run-time memory allocation and copying */ + +class descr { +public: + PYBIND11_NOINLINE descr(const char *text, const std::type_info * const * types) { + size_t nChars = len(text), nTypes = len(types); + m_text = new char[nChars]; + m_types = new const std::type_info *[nTypes]; + memcpy(m_text, text, nChars * sizeof(char)); + memcpy(m_types, types, nTypes * sizeof(const std::type_info *)); + } + + PYBIND11_NOINLINE descr friend operator+(descr &&d1, descr &&d2) { + descr r; + + size_t nChars1 = len(d1.m_text), nTypes1 = len(d1.m_types); + size_t nChars2 = len(d2.m_text), nTypes2 = len(d2.m_types); + + r.m_text = new char[nChars1 + nChars2 - 1]; + r.m_types = new const std::type_info *[nTypes1 + nTypes2 - 1]; + memcpy(r.m_text, d1.m_text, (nChars1-1) * sizeof(char)); + memcpy(r.m_text + nChars1 - 1, d2.m_text, nChars2 * sizeof(char)); + memcpy(r.m_types, d1.m_types, (nTypes1-1) * sizeof(std::type_info *)); + memcpy(r.m_types + nTypes1 - 1, d2.m_types, nTypes2 * sizeof(std::type_info *)); + + delete[] d1.m_text; delete[] d1.m_types; + delete[] d2.m_text; delete[] d2.m_types; + + return r; + } + + char *text() { return m_text; } + const std::type_info * * types() { return m_types; } + +protected: + PYBIND11_NOINLINE descr() { } + + template static size_t len(const T *ptr) { // return length including null termination + const T *it = ptr; + while (*it++ != (T) 0) + ; + return it - ptr; + } + + const std::type_info **m_types = nullptr; + char *m_text = nullptr; +}; + +/* The 'PYBIND11_NOINLINE inline' combinations below are intentional to get the desired linkage while producing as little object code as possible */ + +PYBIND11_NOINLINE inline descr _(const char *text) { + const std::type_info *types[1] = { nullptr }; + return descr(text, types); +} + +template PYBIND11_NOINLINE descr _() { + const std::type_info *types[2] = { &typeid(Type), nullptr }; + return descr("%", types); +} + +PYBIND11_NOINLINE inline descr concat() { return _(""); } +PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; } +template PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward(args)...); } +PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); } + +#define PYBIND11_DESCR descr +#endif + +NAMESPACE_END(detail) +NAMESPACE_END(pybind11) diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 34d1c6bc9..18056ce4a 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -39,11 +39,10 @@ public: return f.ptr(); } - - PYBIND11_TYPE_CASTER(type, detail::descr("function<") + - type_caster>::name() + detail::descr(" -> ") + + PYBIND11_TYPE_CASTER(type, _("function<") + + type_caster>::name() + _(" -> ") + type_caster::type>::name() + - detail::descr(">")); + _(">")); }; NAMESPACE_END(detail) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 34383bcab..0c3d75614 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -24,7 +24,6 @@ #endif #include "cast.h" -#include NAMESPACE_BEGIN(pybind11) @@ -39,10 +38,10 @@ struct arg { /// Annotation for keyword arguments with default values template struct arg_t : public arg { - arg_t(const char *name, const T &value, const char *value_str = nullptr) - : arg(name), value(value), value_str(value_str) {} + arg_t(const char *name, const T &value, const char *descr = nullptr) + : arg(name), value(value), descr(descr) { } T value; - const char *value_str; + const char *descr; }; template arg_t arg::operator=(const T &value) { return arg_t(name, value); } @@ -65,15 +64,17 @@ private: /// Linked list of function overloads struct function_entry { /// Function name and user-specified documentation string - const char *name = nullptr, *doc = nullptr; + char *name = nullptr, *doc = nullptr; /* why no C++ strings? They generate heavier code.. */ + /// Human-readable version of the function signature + char *signature = nullptr; /// List of registered keyword arguments - std::list args; - /// Pointer to lambda function which converts arguments and performs the call + std::vector args; + /// Pointer to lambda function which converts arguments and performs the actual call PyObject * (*impl) (function_entry *, PyObject *, PyObject *) = nullptr; /// Storage for the wrapped function pointer and captured data, if any void *data = nullptr; /// Pointer to custom destructor for 'data' (if needed) - void (*free) (void *ptr) = nullptr; + void (*free_data) (void *ptr) = nullptr; /// Return value policy associated with this function return_value_policy policy = return_value_policy::automatic; /// True if name == '__init__' @@ -86,12 +87,6 @@ private: PyObject *sibling = nullptr; /// Pointer to next overload function_entry *next = nullptr; - - ~function_entry() { - delete def; - if (free) - free(data); - } }; function_entry *m_entry; @@ -115,59 +110,56 @@ private: (void) unused; } - static void process_extra(const char *doc, function_entry *entry) { entry->doc = doc; } - static void process_extra(const pybind11::doc &d, function_entry *entry) { entry->doc = d.value; } - static void process_extra(const pybind11::name &n, function_entry *entry) { entry->name = n.value; } + static void process_extra(const char *doc, function_entry *entry) { entry->doc = (char *) doc; } + static void process_extra(const pybind11::doc &d, function_entry *entry) { entry->doc = (char *) d.value; } + static void process_extra(const pybind11::name &n, function_entry *entry) { entry->name = (char *) n.value; } static void process_extra(const pybind11::return_value_policy p, function_entry *entry) { entry->policy = p; } static void process_extra(const pybind11::sibling s, function_entry *entry) { entry->sibling = s.value; } static void process_extra(const pybind11::is_method &m, function_entry *entry) { entry->class_ = m.class_; } static void process_extra(const pybind11::arg &a, function_entry *entry) { if (entry->class_ && entry->args.empty()) - entry->args.emplace_back(strdup("self"), nullptr, nullptr); - entry->args.emplace_back(strdup(a.name), nullptr, nullptr); + entry->args.emplace_back("self", nullptr, nullptr); + entry->args.emplace_back(a.name, nullptr, nullptr); } template static void process_extra(const pybind11::arg_t &a, function_entry *entry) { if (entry->class_ && entry->args.empty()) - entry->args.emplace_back(strdup("self"), nullptr, nullptr); + entry->args.emplace_back("self", nullptr, nullptr); PyObject *obj = detail::type_caster::type>::cast( a.value, return_value_policy::automatic, nullptr); - entry->args.emplace_back( - strdup(a.name), - strdup(a.value_str != nullptr ? a.value_str : - (const char *) ((object) handle(obj).attr("__repr__")).call().str()), - obj - ); + if (obj == nullptr) + throw std::runtime_error("arg(): could not convert default keyword " + "argument into a Python object (type not " + "registered yet?)"); + + entry->args.emplace_back(a.name, a.descr, obj); } public: cpp_function() { } /// Vanilla function pointers - template - cpp_function(Return (*f)(Arg...), Extra&&... extra) { + template + cpp_function(Return (*f)(Args...), Extra&&... extra) { + using detail::descr; m_entry = new function_entry(); m_entry->data = (void *) f; - typedef arg_value_caster cast_in; + typedef arg_value_caster cast_in; typedef return_value_caster cast_out; m_entry->impl = [](function_entry *entry, PyObject *pyArgs, PyObject *parent) -> PyObject * { cast_in args; if (!args.load(pyArgs, true)) return (PyObject *) 1; /* Special return code: try next overload */ - return cast_out::cast(args.template call((Return (*)(Arg...)) entry->data), entry->policy, parent); + return cast_out::cast(args.template call((Return (*)(Args...)) entry->data), entry->policy, parent); }; process_extras(std::make_tuple(std::forward(extra)...), m_entry); - - detail::descr d = cast_in::name(m_entry->args); - d += " -> "; - d += std::move(cast_out::name()); - - initialize(d, sizeof...(Arg)); + PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name(); + initialize(signature.text(), signature.types(), sizeof...(Args)); } /// Delegating helper constructor to deal with lambda functions @@ -197,21 +189,21 @@ public: private: /// Functors, lambda functions, etc. - template - void initialize(Func &&f, Return (*)(Arg...), Extra&&... extra) { - struct capture { - typename std::remove_reference::type f; - }; + template + void initialize(Func &&f, Return (*)(Args...), Extra&&... extra) { + using detail::descr; + + struct capture { typename std::remove_reference::type f; }; m_entry = new function_entry(); m_entry->data = new capture { std::forward(f) }; if (!std::is_trivially_destructible::value) - m_entry->free = [](void *ptr) { delete (capture *) ptr; }; + m_entry->free_data = [](void *ptr) { delete (capture *) ptr; }; else - m_entry->free = operator delete; + m_entry->free_data = operator delete; - typedef arg_value_caster cast_in; + typedef arg_value_caster cast_in; typedef return_value_caster cast_out; m_entry->impl = [](function_entry *entry, PyObject *pyArgs, PyObject *parent) -> PyObject *{ @@ -222,12 +214,8 @@ private: }; process_extras(std::make_tuple(std::forward(extra)...), m_entry); - - detail::descr d = cast_in::name(m_entry->args); - d += " -> "; - d += std::move(cast_out::name()); - - initialize(d, sizeof...(Arg)); + PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name(); + initialize(signature.text(), signature.types(), sizeof...(Args)); } static PyObject *dispatcher(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -250,17 +238,17 @@ private: PyTuple_SET_ITEM(args_, i, item); } int arg_ctr = 0; - for (auto const &it : it->args) { + for (auto const &it2 : it->args) { int index = arg_ctr++; if (PyTuple_GET_ITEM(args_, index)) continue; PyObject *value = nullptr; if (kwargs) - value = PyDict_GetItemString(kwargs, it.name); + value = PyDict_GetItemString(kwargs, it2.name); if (value) kwargs_consumed++; - else if (it.value) - value = it.value; + else if (it2.value) + value = it2.value; if (value) { Py_INCREF(value); PyTuple_SET_ITEM(args_, index, value); @@ -301,7 +289,7 @@ private: int ctr = 0; for (function_entry *it2 = overloads; it2 != nullptr; it2 = it2->next) { msg += " "+ std::to_string(++ctr) + ". "; - //msg += it2->signature; XXX + msg += it2->signature; msg += "\n"; } PyErr_SetString(PyExc_TypeError, msg.c_str()); @@ -309,7 +297,7 @@ private: } else if (result == nullptr) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; - //msg += it->signature; + msg += it->signature; PyErr_SetString(PyExc_TypeError, msg.c_str()); return nullptr; } else { @@ -327,18 +315,88 @@ private: static void destruct(function_entry *entry) { while (entry) { function_entry *next = entry->next; + delete entry->def; + if (entry->free_data) + entry->free_data(entry->data); + std::free((char *) entry->name); + std::free((char *) entry->doc); + std::free((char *) entry->signature); + for (auto &arg: entry->args) { + std::free((char *) arg.name); + std::free((char *) arg.descr); + Py_XDECREF(arg.value); + } delete entry; entry = next; } } - void initialize(const detail::descr &, int args) { - if (m_entry->name == nullptr) - m_entry->name = ""; + void initialize(const char *text, const std::type_info * const * types, int args) { + /* Create copies of all referenced C-style strings */ + m_entry->name = strdup(m_entry->name ? m_entry->name : ""); + if (m_entry->doc) m_entry->doc = strdup(m_entry->doc); + for (auto &a: m_entry->args) { + if (a.name) + a.name = strdup(a.name); + if (a.descr) + a.descr = strdup(a.descr); + else if (a.value) + a.descr = strdup(((object) handle(a.value).attr("__repr__")).call().str()); + } + auto const ®istered_types = detail::get_internals().registered_types; + + /* Generate a proper function signature */ + std::string signature; + size_t type_depth = 0, char_index = 0, type_index = 0, arg_index = 0; + while (true) { + char c = text[char_index++]; + if (c == '\0') + break; + + if (c == '{') { + if (type_depth == 1 && arg_index < m_entry->args.size()) { + signature += m_entry->args[arg_index].name; + signature += " : "; + } + ++type_depth; + } else if (c == '}') { + --type_depth; + if (type_depth == 1 && arg_index < m_entry->args.size()) { + if (m_entry->args[arg_index].descr) { + signature += " = "; + signature += m_entry->args[arg_index].descr; + } + arg_index++; + } + } else if (c == '%') { + const std::type_info *t = types[type_index++]; + if (!t) + throw std::runtime_error("Internal error while generating type signature (1)"); + auto it = registered_types.find(t); + if (it != registered_types.end()) { + signature += it->second.type->tp_name; + } else { + std::string tname(t->name()); + detail::clean_type_id(tname); + signature += tname; + } + } else { + signature += c; + } + } + if (type_depth != 0 && types[type_index ] != nullptr) + throw std::runtime_error("Internal error while generating type signature (2)"); + + #if !defined(PYBIND11_CPP14) + delete[] types; + delete[] text; + #endif #if PY_MAJOR_VERSION < 3 - if (strcmp(m_entry->name, "__next__") == 0) - m_entry->name = "next"; + if (strcmp(m_entry->name, "__next__") == 0) { + free(m_entry->name); + m_entry->name = strdup("next"); + } #endif if (!m_entry->args.empty() && (int) m_entry->args.size() != args) @@ -348,7 +406,8 @@ private: " pybind11::arg entries were specified!"); m_entry->is_constructor = !strcmp(m_entry->name, "__init__"); - //m_entry->signature = descr.str(); // XXX + m_entry->signature = strdup(signature.c_str()); + m_entry->args.shrink_to_fit(); #if PY_MAJOR_VERSION < 3 if (m_entry->sibling && PyMethod_Check(m_entry->sibling)) @@ -360,10 +419,10 @@ private: capsule entry_capsule(PyCFunction_GetSelf(m_entry->sibling), true); s_entry = (function_entry *) entry_capsule; if (s_entry->class_ != m_entry->class_) - s_entry = nullptr; /* Method override */ + s_entry = nullptr; /* Overridden method, don't append to parent class overloads */ } - if (!s_entry) { + if (!s_entry) { /* No existing overload was found, create a function object */ m_entry->def = new PyMethodDef(); memset(m_entry->def, 0, sizeof(PyMethodDef)); m_entry->def->ml_name = m_entry->name; @@ -385,19 +444,24 @@ private: std::string signatures; int index = 0; function_entry *it = entry; - while (it) { /* Create pydoc entry */ + while (it) { /* Create pydoc entry including all function signatures and docstrings of the overload chain */ if (s_entry) signatures += std::to_string(++index) + ". "; - //signatures += "Signature : " + std::string(it->signature) + "\n"; XXX - if (it->doc && strlen(it->doc) > 0) - signatures += "\n" + std::string(it->doc) + "\n"; + signatures += "Signature : "; + signatures += it->signature; + signatures += "\n"; + if (it->doc && strlen(it->doc) > 0) { + signatures += "\n"; + signatures += it->doc; + signatures += "\n"; + } if (it->next) signatures += "\n"; it = it->next; } PyCFunctionObject *func = (PyCFunctionObject *) m_ptr; if (func->m_ml->ml_doc) - std::free((char *) func->m_ml->ml_doc); + free((char *) func->m_ml->ml_doc); func->m_ml->ml_doc = strdup(signatures.c_str()); if (entry->class_) { #if PY_MAJOR_VERSION >= 3 @@ -510,7 +574,7 @@ public: type->ht_type.tp_base = (PyTypeObject *) parent; if (doc) { size_t size = strlen(doc)+1; - type->ht_type.tp_doc = (char *)PyObject_MALLOC(size); + type->ht_type.tp_doc = (char *) PyObject_MALLOC(size); memcpy((void *) type->ht_type.tp_doc, doc, size); } Py_XINCREF(parent); @@ -522,7 +586,8 @@ public: /* Needed by pydoc */ attr("__module__") = scope_name; - auto &type_info = detail::get_internals().registered_types[tinfo]; + auto ®istered_types = get_internals().registered_types; + auto &type_info = registered_types[tinfo]; type_info.type = (PyTypeObject *) m_ptr; type_info.type_size = type_size; type_info.init_holder = init_holder; diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index c4f4b196c..3bff58216 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -45,7 +45,7 @@ public: static PyObject *cast(const type &src, return_value_policy policy, PyObject *parent) { object list(PyList_New(src.size()), false); if (!list) - return nullptr; + return nullptr; size_t index = 0; for (auto const &value: src) { object value_ (value_conv::cast(value, policy, parent), false); @@ -55,23 +55,23 @@ public: } return list.release(); } - PYBIND11_TYPE_CASTER(type, detail::descr("list<") + value_conv::name() + detail::descr(">")); + PYBIND11_TYPE_CASTER(type, _("list<") + value_conv::name() + _(">")); }; -template struct type_caster> { - typedef std::set type; - typedef type_caster value_conv; +template struct type_caster> { + typedef std::set type; + typedef type_caster key_conv; public: bool load(PyObject *src, bool convert) { pybind11::set s(src, true); if (!s.check()) return false; value.clear(); - value_conv conv; + key_conv conv; for (const object &o: s) { if (!conv.load((PyObject *) o.ptr(), convert)) return false; - value.insert((Value) conv); + value.insert((Key) conv); } return true; } @@ -79,15 +79,15 @@ public: static PyObject *cast(const type &src, return_value_policy policy, PyObject *parent) { object set(PySet_New(nullptr), false); if (!set) - return nullptr; + return nullptr; for (auto const &value: src) { - object value_(value_conv::cast(value, policy, parent), false); + object value_(key_conv::cast(value, policy, parent), false); if (!value_ || PySet_Add(set.ptr(), value_.ptr()) != 0) return nullptr; } return set.release(); } - PYBIND11_TYPE_CASTER(type, detail::descr("set<") + value_conv::name() + detail::descr(">")); + PYBIND11_TYPE_CASTER(type, _("set<") + key_conv::name() + _(">")); }; template struct type_caster> { @@ -116,7 +116,7 @@ public: static PyObject *cast(const type &src, return_value_policy policy, PyObject *parent) { object dict(PyDict_New(), false); if (!dict) - return nullptr; + return nullptr; for (auto const &kv: src) { object key(key_conv::cast(kv.first, policy, parent), false); object value(value_conv::cast(kv.second, policy, parent), false); @@ -126,7 +126,7 @@ public: return dict.release(); } - PYBIND11_TYPE_CASTER(type, detail::descr("dict<") + key_conv::name() + detail::descr(", ") + value_conv::name() + detail::descr(">")); + PYBIND11_TYPE_CASTER(type, _("dict<") + key_conv::name() + _(", ") + value_conv::name() + _(">")); }; NAMESPACE_END(detail) diff --git a/include/pybind11/typeid.h b/include/pybind11/typeid.h index 551895fcc..2d7c39bbd 100644 --- a/include/pybind11/typeid.h +++ b/include/pybind11/typeid.h @@ -33,11 +33,11 @@ inline void clean_type_id(std::string &name) { abi::__cxa_demangle(name.c_str(), nullptr, nullptr, &status), std::free }; if (status == 0) name = res.get(); - #else - detail::erase_all(name, "class "); - detail::erase_all(name, "struct "); - detail::erase_all(name, "enum "); - #endif +#else + detail::erase_all(name, "class "); + detail::erase_all(name, "struct "); + detail::erase_all(name, "enum "); +#endif detail::erase_all(name, "pybind11::"); } NAMESPACE_END(detail)