diff --git a/.travis.yml b/.travis.yml index b9c5ab540..d4bdafd7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,13 @@ matrix: - os: osx osx_image: xcode7.3 env: PYTHON=3.5 CPP=14 CLANG + # Test a PyPy 2.7 nightly build + - os: linux + env: PYPY=1 PYTHON=2.7 CPP=11 GCC=4.8 + addons: + apt: + sources: [ubuntu-toolchain-r-test, kubuntu-backports] + packages: [g++-4.8, cmake] # A barebones build makes sure everything still works without optional deps (numpy/scipy/eigen) # and also tests the automatic discovery functions in CMake (Python version, C++ standard). - os: linux @@ -70,9 +77,18 @@ before_install: fi if [ -n "$CPP" ]; then export CPP=-std=c++$CPP; fi if [ "${PYTHON:0:1}" = "3" ]; then export PY=3; fi + if [ -n "$PYPY" ]; then + curl http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-linux64.tar.bz2 | tar -xj + export PYPY_BINARY=$(echo `pwd`/pypy-c-jit*/bin/pypy) + export CMAKE_EXTRA_ARGS="-DPYTHON_EXECUTABLE:FILEPATH=$PYPY_BINARY" + fi if [ -n "$DEBUG" ]; then export CMAKE_EXTRA_ARGS="-DCMAKE_BUILD_TYPE=Debug"; fi - | - # Initialize enviornment + # Initialize environment + if [ -n "$PYPY" ]; then + $PYPY_BINARY -m ensurepip + $PYPY_BINARY -m pip install pytest + fi if [ -n "$DOCKER" ]; then docker pull $DOCKER export containerid=$(docker run --detach --tty \ diff --git a/README.md b/README.md index cfe3f1519..96feb52ff 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~2.5K lines of code and depend on -Python (2.7 or 3.x) and the C++ standard library. This compact implementation +Python (2.7 or 3.x, or PyPy2.7 >= 5.5) and the C++ standard library. This compact implementation was possible thanks to some of the new C++11 language features (specifically: tuples, lambda functions and variadic templates). Since its creation, this library has grown beyond Boost.Python in many ways, leading to dramatically @@ -58,12 +58,15 @@ pybind11 can map the following core C++ features to Python ## Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- pybind11 uses C++11 move constructors and move assignment operators whenever - possible to efficiently transfer custom data types. +- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.5) are supported with an + implementation-agnostic interface. - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting Python function object. +- pybind11 uses C++11 move constructors and move assignment operators whenever + possible to efficiently transfer custom data types. + - It's easy to expose the internal storage of custom data types through Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -100,7 +103,7 @@ In addition to the core functionality, pybind11 provides some extra goodies: ## About -This project was created by [Wenzel Jakob](https://www.mitsuba-renderer.org/~wenzel/). +This project was created by [Wenzel Jakob](http://rgl.epfl.ch/people/wjakob). Significant features and/or improvements to the code were contributed by Jonas Adler, Sylvain Corlay, diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 4a423b578..e20895e6d 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -422,15 +422,24 @@ The section on :ref:`properties` discussed the creation of instance properties that are implemented in terms of C++ getters and setters. Static properties can also be created in a similar way to expose getters and -setters of static class attributes. It is important to note that the implicit -``self`` argument also exists in this case and is used to pass the Python -``type`` subclass instance. This parameter will often not be needed by the C++ -side, and the following example illustrates how to instantiate a lambda getter -function that ignores it: +setters of static class attributes. Two things are important to note: + +1. Static properties are implemented by instrumenting the *metaclass* of the + class in question -- however, this requires the class to have a modifiable + metaclass in the first place. pybind11 provides a ``py::metaclass()`` + annotation that must be specified in the ``class_`` constructor, or any + later method calls to ``def_{property_,∅}_{readwrite,readonly}_static`` will + fail (see the example below). + +2. For static properties defined in terms of setter and getter functions, note + that the implicit ``self`` argument also exists in this case and is used to + pass the Python ``type`` subclass instance. This parameter will often not be + needed by the C++ side, and the following example illustrates how to + instantiate a lambda getter function that ignores it: .. code-block:: cpp - py::class_(m, "Foo") + py::class_(m, "Foo", py::metaclass()) .def_property_readonly_static("foo", [](py::object /* self */) { return Foo(); }); Operator overloading diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index 8b46b7c83..111ff0e3c 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -33,7 +33,7 @@ completely avoid copy operations with Python expressions like .. code-block:: cpp - py::class_(m, "Matrix") + py::class_(m, "Matrix", py::buffer_protocol()) .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( m.data(), /* Pointer to buffer */ @@ -46,9 +46,12 @@ completely avoid copy operations with Python expressions like ); }); -The snippet above binds a lambda function, which can create ``py::buffer_info`` -description records on demand describing a given matrix. The contents of -``py::buffer_info`` mirror the Python buffer protocol specification. +Supporting the buffer protocol in a new type involves specifying the special +``py::buffer_protocol()`` tag in the ``py::class_`` constructor and calling the +``def_buffer()`` method with a lambda function that creates a +``py::buffer_info`` description record on demand describing a given matrix +instance. The contents of ``py::buffer_info`` mirror the Python buffer protocol +specification. .. code-block:: cpp @@ -77,7 +80,7 @@ buffer objects (e.g. a NumPy matrix). typedef Matrix::Scalar Scalar; constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; - py::class_(m, "Matrix") + py::class_(m, "Matrix", py::buffer_protocol()) .def("__init__", [](Matrix &m, py::buffer b) { typedef Eigen::Stride Strides; diff --git a/docs/intro.rst b/docs/intro.rst index 429a01cdd..f22eeedb9 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -51,6 +51,9 @@ Goodies ******* In addition to the core functionality, pybind11 provides some extra goodies: +- Python 2.7, 3.x, and PyPy (PyPy2.7 >= 5.5) are supported with an + implementation-agnostic interface. + - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting Python function object. diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 448612c52..0676d5da6 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -47,6 +47,12 @@ struct multiple_inheritance { }; /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class struct dynamic_attr { }; +/// Annotation which enables the buffer protocol for a type +struct buffer_protocol { }; + +/// Annotation which requests that a special metaclass is created for a type +struct metaclass { }; + /// Annotation to mark enums as an arithmetic type struct arithmetic { }; @@ -136,7 +142,9 @@ struct function_record { /// Special data structure which (temporarily) holds metadata about a bound class struct type_record { - PYBIND11_NOINLINE type_record() { } + PYBIND11_NOINLINE type_record() + : multiple_inheritance(false), dynamic_attr(false), + buffer_protocol(false), metaclass(false) { } /// Handle to the parent scope handle scope; @@ -166,10 +174,16 @@ struct type_record { const char *doc = nullptr; /// Multiple inheritance marker - bool multiple_inheritance = false; + bool multiple_inheritance : 1; /// Does the class manage a __dict__? - bool dynamic_attr = false; + bool dynamic_attr : 1; + + /// Does the class implement the buffer protocol? + bool buffer_protocol : 1; + + /// Does the class require its own metaclass? + bool metaclass : 1; PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) { auto base_info = detail::get_type_info(*base, false); @@ -309,6 +323,16 @@ struct process_attribute : process_attribute_default static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } }; +template <> +struct process_attribute : process_attribute_default { + static void init(const buffer_protocol &, type_record *r) { r->buffer_protocol = true; } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const metaclass &, type_record *r) { r->metaclass = true; } +}; + /// Process an 'arithmetic' attribute for enums (does nothing here) template <> diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 336308420..c7be40a2d 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -108,14 +108,10 @@ PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool t } PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { - const auto type = detail::get_type_handle(tp, false); + handle type = detail::get_type_handle(tp, false); if (!type) return false; - - const auto result = PyObject_IsInstance(obj.ptr(), type.ptr()); - if (result == -1) - throw error_already_set(); - return result != 0; + return isinstance(obj, type); } PYBIND11_NOINLINE inline std::string error_string() { @@ -141,6 +137,7 @@ PYBIND11_NOINLINE inline std::string error_string() { PyException_SetTraceback(scope.value, scope.trace); #endif +#if !defined(PYPY_VERSION) if (scope.trace) { PyTracebackObject *trace = (PyTracebackObject *) scope.trace; @@ -160,6 +157,7 @@ PYBIND11_NOINLINE inline std::string error_string() { } trace = trace->tb_next; } +#endif return errorString; } @@ -176,7 +174,9 @@ PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail: } inline PyThreadState *get_thread_state_unchecked() { -#if PY_VERSION_HEX < 0x03000000 +#if defined(PYPY_VERSION) + return PyThreadState_GET(); +#elif PY_VERSION_HEX < 0x03000000 return _PyThreadState_Current; #elif PY_VERSION_HEX < 0x03050000 return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); @@ -224,7 +224,7 @@ public: /* If this is a python class, also check the parents recursively */ auto const &type_dict = get_internals().registered_types_py; - bool new_style_class = PyType_Check(tobj); + bool new_style_class = PyType_Check((PyObject *) tobj); if (type_dict.find(tobj) == type_dict.end() && new_style_class && tobj->tp_bases) { auto parents = reinterpret_borrow(tobj->tp_bases); for (handle parent : parents) { @@ -662,10 +662,10 @@ public: #if PY_MAJOR_VERSION >= 3 buffer = PyUnicode_AsWideCharString(load_src.ptr(), &length); #else - temp = reinterpret_steal( - sizeof(wchar_t) == sizeof(short) - ? PyUnicode_AsUTF16String(load_src.ptr()) - : PyUnicode_AsUTF32String(load_src.ptr())); + temp = reinterpret_steal(PyUnicode_AsEncodedString( + load_src.ptr(), sizeof(wchar_t) == sizeof(short) + ? "utf16" : "utf32", nullptr)); + if (temp) { int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), (char **) &buffer, &length); if (err == -1) { buffer = nullptr; } // TypeError @@ -868,7 +868,7 @@ public: /* If this is a python class, also check the parents recursively */ auto const &type_dict = get_internals().registered_types_py; - bool new_style_class = PyType_Check(tobj); + bool new_style_class = PyType_Check((PyObject *) tobj); if (type_dict.find(tobj) == type_dict.end() && new_style_class && tobj->tp_bases) { auto parents = reinterpret_borrow(tobj->tp_bases); for (handle parent : parents) { diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index 204427d77..5b2b98272 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -95,8 +95,15 @@ object eval_file(str fname, object global = object(), object local = object()) { pybind11_fail("File \"" + fname_str + "\" could not be opened!"); } +#if PY_VERSION_HEX < 0x03000000 && defined(PYPY_VERSION) + PyObject *result = PyRun_File(f, fname_str.c_str(), start, global.ptr(), + local.ptr()); + (void) closeFile; +#else PyObject *result = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), local.ptr(), closeFile); +#endif + if (!result) throw error_already_set(); return reinterpret_steal(result); diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 7f1ffc1d7..2dd52d3ec 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -39,7 +39,7 @@ public: captured variables), in which case the roundtrip can be avoided. */ if (PyCFunction_Check(src_.ptr())) { - auto c = reinterpret_borrow(PyCFunction_GetSelf(src_.ptr())); + auto c = reinterpret_borrow(PyCFunction_GET_SELF(src_.ptr())); auto rec = (function_record *) c; if (rec && rec->is_stateless && rec->data[1] == &typeid(function_type)) { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index c5655d348..84b60afd8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -221,6 +221,11 @@ protected: if (!t) pybind11_fail("Internal error while parsing type signature (1)"); if (auto tinfo = detail::get_type_info(*t)) { +#if defined(PYPY_VERSION) + signature += handle((PyObject *) tinfo->type) + .attr("__module__") + .cast() + "."; +#endif signature += tinfo->type->tp_name; } else { std::string tname(t->name()); @@ -261,7 +266,7 @@ protected: detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto rec_capsule = reinterpret_borrow(PyCFunction_GetSelf(rec->sibling.ptr())); + auto rec_capsule = reinterpret_borrow(PyCFunction_GET_SELF(rec->sibling.ptr())); chain = (detail::function_record *) rec_capsule; /* Never append a method to an overload chain of a parent class; instead, hide the parent's overloads in this case */ @@ -602,9 +607,8 @@ public: NAMESPACE_BEGIN(detail) extern "C" inline PyObject *get_dict(PyObject *op, void *) { PyObject *&dict = *_PyObject_GetDictPtr(op); - if (!dict) { + if (!dict) dict = PyDict_New(); - } Py_XINCREF(dict); return dict; } @@ -660,21 +664,57 @@ protected: object scope_qualname; if (rec->scope && hasattr(rec->scope, "__qualname__")) scope_qualname = rec->scope.attr("__qualname__"); - object ht_qualname; - if (scope_qualname) { + object ht_qualname, ht_qualname_meta; + if (scope_qualname) ht_qualname = reinterpret_steal(PyUnicode_FromFormat( "%U.%U", scope_qualname.ptr(), name.ptr())); - } else { + else ht_qualname = name; - } + if (rec->metaclass) + ht_qualname_meta = reinterpret_steal( + PyUnicode_FromFormat("%U__Meta", ht_qualname.ptr())); #endif +#if !defined(PYPY_VERSION) + std::string full_name = (scope_module ? ((std::string) pybind11::str(scope_module) + "." + rec->name) + : std::string(rec->name)); +#else + std::string full_name = std::string(rec->name); +#endif + + /* Create a custom metaclass if requested (used for static properties) */ + object metaclass; + if (rec->metaclass) { + std::string meta_name_ = full_name + "__Meta"; + object meta_name = reinterpret_steal(PYBIND11_FROM_STRING(meta_name_.c_str())); + metaclass = reinterpret_steal(PyType_Type.tp_alloc(&PyType_Type, 0)); + if (!metaclass || !name) + pybind11_fail("generic_type::generic_type(): unable to create metaclass!"); + + /* Danger zone: from now (and until PyType_Ready), make sure to + issue no Python C API calls which could potentially invoke the + garbage collector (the GC will call type_traverse(), which will in + turn find the newly constructed type in an invalid state) */ + + auto type = (PyHeapTypeObject*) metaclass.ptr(); + type->ht_name = meta_name.release().ptr(); + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + /* Qualified names for Python >= 3.3 */ + type->ht_qualname = ht_qualname.release().ptr(); +#endif + type->ht_type.tp_name = strdup(meta_name_.c_str()); + type->ht_type.tp_base = &PyType_Type; + type->ht_type.tp_flags |= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE) & + ~Py_TPFLAGS_HAVE_GC; + + if (PyType_Ready(&type->ht_type) < 0) + pybind11_fail("generic_type::generic_type(): failure in PyType_Ready() for metaclass!"); + } + size_t num_bases = rec->bases.size(); auto bases = tuple(rec->bases); - std::string full_name = (scope_module ? ((std::string) pybind11::str(scope_module) + "." + rec->name) - : std::string(rec->name)); - char *tp_doc = nullptr; if (rec->doc && options::show_user_defined_docstrings()) { /* Allocate memory for docstring (using PyObject_MALLOC, since @@ -720,6 +760,9 @@ protected: type->ht_qualname = ht_qualname.release().ptr(); #endif + /* Metaclass */ + PYBIND11_OB_TYPE(type->ht_type) = (PyTypeObject *) metaclass.release().ptr(); + /* Supported protocols */ type->ht_type.tp_as_number = &type->as_number; type->ht_type.tp_as_sequence = &type->as_sequence; @@ -750,14 +793,23 @@ protected: type->ht_type.tp_clear = clear; } + if (rec->buffer_protocol) { + type->ht_type.tp_as_buffer = &type->as_buffer; +#if PY_MAJOR_VERSION < 3 + type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; +#endif + type->as_buffer.bf_getbuffer = getbuffer; + type->as_buffer.bf_releasebuffer = releasebuffer; + } + type->ht_type.tp_doc = tp_doc; + m_ptr = type_holder.ptr(); + if (PyType_Ready(&type->ht_type) < 0) pybind11_fail(std::string(rec->name) + ": PyType_Ready failed (" + detail::error_string() + ")!"); - m_ptr = type_holder.ptr(); - if (scope_module) // Needed by pydoc attr("__module__") = scope_module; @@ -782,43 +834,14 @@ protected: } } - /// Allocate a metaclass on demand (for static properties) - handle metaclass() { - auto &ht_type = ((PyHeapTypeObject *) m_ptr)->ht_type; - auto &ob_type = PYBIND11_OB_TYPE(ht_type); - - if (ob_type == &PyType_Type) { - std::string name_ = std::string(ht_type.tp_name) + "__Meta"; -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - auto ht_qualname = reinterpret_steal(PyUnicode_FromFormat("%U__Meta", attr("__qualname__").ptr())); -#endif - auto name = reinterpret_steal(PYBIND11_FROM_STRING(name_.c_str())); - auto type_holder = reinterpret_steal(PyType_Type.tp_alloc(&PyType_Type, 0)); - if (!type_holder || !name) - pybind11_fail("generic_type::metaclass(): unable to create type object!"); - - auto type = (PyHeapTypeObject*) type_holder.ptr(); - type->ht_name = name.release().ptr(); - -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - /* Qualified names for Python >= 3.3 */ - type->ht_qualname = ht_qualname.release().ptr(); -#endif - type->ht_type.tp_name = strdup(name_.c_str()); - type->ht_type.tp_base = ob_type; - type->ht_type.tp_flags |= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE) & - ~Py_TPFLAGS_HAVE_GC; - - if (PyType_Ready(&type->ht_type) < 0) - pybind11_fail("generic_type::metaclass(): PyType_Ready failed!"); - - ob_type = (PyTypeObject *) type_holder.release().ptr(); - } - return handle((PyObject *) ob_type); - } - static int init(void *self, PyObject *, PyObject *) { - std::string msg = std::string(Py_TYPE(self)->tp_name) + ": No constructor defined!"; + PyTypeObject *type = Py_TYPE(self); + std::string msg; +#if defined(PYPY_VERSION) + msg += handle((PyObject *) type).attr("__module__").cast() + "."; +#endif + msg += type->tp_name; + msg += ": No constructor defined!"; PyErr_SetString(PyExc_TypeError, msg.c_str()); return -1; } @@ -853,9 +876,8 @@ protected: PyObject_ClearWeakRefs((PyObject *) self); PyObject **dict_ptr = _PyObject_GetDictPtr((PyObject *) self); - if (dict_ptr) { + if (dict_ptr) Py_CLEAR(*dict_ptr); - } } Py_TYPE(self)->tp_free((PyObject*) self); } @@ -876,13 +898,15 @@ protected: buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { PyHeapTypeObject *type = (PyHeapTypeObject*) m_ptr; - type->ht_type.tp_as_buffer = &type->as_buffer; -#if PY_MAJOR_VERSION < 3 - type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; -#endif - type->as_buffer.bf_getbuffer = getbuffer; - type->as_buffer.bf_releasebuffer = releasebuffer; auto tinfo = detail::get_type_info(&type->ht_type); + + if (!type->ht_type.tp_as_buffer) + pybind11_fail( + "To be able to register buffer protocol support for the type '" + + std::string(tinfo->type->tp_name) + + "' the associated class<>(..) invocation must " + "include the pybind11::buffer_protocol() annotation!"); + tinfo->get_buffer = get_buffer; tinfo->get_buffer_data = get_buffer_data; } @@ -890,6 +914,8 @@ protected: static int getbuffer(PyObject *obj, Py_buffer *view, int flags) { auto tinfo = detail::get_type_info(Py_TYPE(obj)); if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) { + if (view) + view->obj = nullptr; PyErr_SetString(PyExc_BufferError, "generic_type::getbuffer(): Internal error"); return -1; } @@ -915,6 +941,31 @@ protected: } static void releasebuffer(PyObject *, Py_buffer *view) { delete (buffer_info *) view->internal; } + + void def_property_static_impl(const char *name, + handle fget, handle fset, + detail::function_record *rec_fget) { + pybind11::str doc_obj = pybind11::str( + (rec_fget->doc && pybind11::options::show_user_defined_docstrings()) + ? rec_fget->doc : ""); + const auto property = reinterpret_steal( + PyObject_CallFunctionObjArgs((PyObject *) &PyProperty_Type, fget.ptr() ? fget.ptr() : Py_None, + fset.ptr() ? fset.ptr() : Py_None, Py_None, doc_obj.ptr(), nullptr)); + if (rec_fget->is_method && rec_fget->scope) { + attr(name) = property; + } else { + auto mclass = handle((PyObject *) PYBIND11_OB_TYPE(*((PyTypeObject *) m_ptr))); + + if ((PyTypeObject *) mclass.ptr() == &PyType_Type) + pybind11_fail( + "Adding static properties to the type '" + + std::string(((PyTypeObject *) m_ptr)->tp_name) + + "' requires the type to have a custom metaclass. Please " + "ensure that one is created by supplying the pybind11::metaclass() " + "annotation to the associated class_<>(..) invocation."); + mclass.attr(name) = property; + } + } }; NAMESPACE_END(detail) @@ -1118,14 +1169,7 @@ public: rec_fset->doc = strdup(rec_fset->doc); } } - pybind11::str doc_obj = pybind11::str((rec_fget->doc && pybind11::options::show_user_defined_docstrings()) ? rec_fget->doc : ""); - const auto property = reinterpret_steal( - PyObject_CallFunctionObjArgs((PyObject *) &PyProperty_Type, fget.ptr() ? fget.ptr() : Py_None, - fset.ptr() ? fset.ptr() : Py_None, Py_None, doc_obj.ptr(), nullptr)); - if (rec_fget->is_method && rec_fget->scope) - attr(name) = property; - else - metaclass().attr(name) = property; + def_property_static_impl(name, fget, fset, rec_fget); return *this; } @@ -1185,7 +1229,7 @@ private: static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow(PyCFunction_GetSelf(h.ptr())) + return h ? (detail::function_record *) reinterpret_borrow(PyCFunction_GET_SELF(h.ptr())) : nullptr; } }; @@ -1255,12 +1299,29 @@ public: /// Export enumeration entries into the parent scope enum_ &export_values() { +#if !defined(PYPY_VERSION) PyObject *dict = ((PyTypeObject *) this->m_ptr)->tp_dict; PyObject *key, *value; ssize_t pos = 0; - while (PyDict_Next(dict, &pos, &key, &value)) + + while (PyDict_Next(dict, &pos, &key, &value)) { if (PyObject_IsInstance(value, this->m_ptr)) m_parent.attr(key) = value; + } +#else + /* PyPy's cpyext still has difficulties with the above + CPython API calls; emulate using Python code. */ + dict d; d["t"] = *this; d["p"] = m_parent; + PyObject *result = PyRun_String( + "for k, v in t.__dict__.items():\n" + " if isinstance(v, t):\n" + " setattr(p, k, v)\n", + Py_file_input, d.ptr(), d.ptr()); + if (result == nullptr) + throw error_already_set(); + Py_DECREF(result); +#endif + return *this; } @@ -1521,7 +1582,7 @@ void print(Args &&...args) { detail::print(c.args(), c.kwargs()); } -#if defined(WITH_THREAD) +#if defined(WITH_THREAD) && !defined(PYPY_VERSION) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -1644,6 +1705,20 @@ private: PyThreadState *tstate; bool disassoc; }; +#elif defined(PYPY_VERSION) +class gil_scoped_acquire { + PyGILState_STATE state; +public: + gil_scoped_acquire() { state = PyGILState_Ensure(); } + ~gil_scoped_acquire() { PyGILState_Release(state); } +}; + +class gil_scoped_release { + PyThreadState *state; +public: + gil_scoped_release() { state = PyEval_SaveThread(); } + ~gil_scoped_release() { PyEval_RestoreThread(state); } +}; #else class gil_scoped_acquire { }; class gil_scoped_release { }; @@ -1658,10 +1733,10 @@ error_already_set::~error_already_set() { } inline function get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { - handle py_object = detail::get_object_handle(this_ptr, this_type); - if (!py_object) + handle self = detail::get_object_handle(this_ptr, this_type); + if (!self) return function(); - handle type = py_object.get_type(); + handle type = self.get_type(); auto key = std::make_pair(type.ptr(), name); /* Cache functions that aren't overloaded in Python to avoid @@ -1670,22 +1745,47 @@ inline function get_type_overload(const void *this_ptr, const detail::type_info if (cache.find(key) != cache.end()) return function(); - function overload = getattr(py_object, name, function()); + function overload = getattr(self, name, function()); if (overload.is_cpp_function()) { cache.insert(key); return function(); } - /* Don't call dispatch code if invoked from overridden function */ + /* Don't call dispatch code if invoked from overridden function. + Unfortunately this doesn't work on PyPy. */ +#if !defined(PYPY_VERSION) PyFrameObject *frame = PyThreadState_Get()->frame; if (frame && (std::string) str(frame->f_code->co_name) == name && frame->f_code->co_argcount > 0) { PyFrame_FastToLocals(frame); PyObject *self_caller = PyDict_GetItem( frame->f_locals, PyTuple_GET_ITEM(frame->f_code->co_varnames, 0)); - if (self_caller == py_object.ptr()) + if (self_caller == self.ptr()) return function(); } +#else + /* PyPy currently doesn't provide a detailed cpyext emulation of + frame objects, so we have to emulate this using Python. This + is going to be slow..*/ + dict d; d["self"] = self; d["name"] = pybind11::str(name); + PyObject *result = PyRun_String( + "import inspect\n" + "frame = inspect.currentframe()\n" + "if frame is not None:\n" + " frame = frame.f_back\n" + " if frame is not None and str(frame.f_code.co_name) == name and " + "frame.f_code.co_argcount > 0:\n" + " self_caller = frame.f_locals[frame.f_code.co_varnames[0]]\n" + " if self_caller == self:\n" + " self = None\n", + Py_file_input, d.ptr(), d.ptr()); + if (result == nullptr) + throw error_already_set(); + if ((handle) d["self"] == Py_None) + return function(); + Py_DECREF(result); +#endif + return overload; } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 975395866..a89aad78d 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -165,6 +165,13 @@ bool isinstance(handle obj) { return detail::isinstance_generic(obj, typeid(T)); template <> inline bool isinstance(handle obj) = delete; template <> inline bool isinstance(handle obj) { return obj.ptr() != nullptr; } +inline bool isinstance(handle obj, handle type) { + const auto result = PyObject_IsInstance(obj.ptr(), type.ptr()); + if (result == -1) + throw error_already_set(); + return result != 0; +} + inline bool hasattr(handle obj, handle name) { return PyObject_HasAttr(obj.ptr(), name.ptr()) == 1; } diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index d4b0fc914..4b557bd16 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -139,7 +139,7 @@ public: auto value_ = reinterpret_steal(value_conv::cast(value, policy, parent)); if (!value_) return handle(); - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference } return l.release(); } @@ -192,7 +192,7 @@ public: auto value_ = reinterpret_steal(value_conv::cast(value, policy, parent)); if (!value_) return handle(); - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference } return l.release(); } diff --git a/tests/conftest.py b/tests/conftest.py index d4335fc6d..b69fd6cb2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ import difflib import re import sys import contextlib +import platform +import gc _unicode_marker = re.compile(r'u(\'[^\']*\')') _long_marker = re.compile(r'([0-9])L') @@ -176,6 +178,13 @@ def suppress(exception): pass +def gc_collect(): + ''' Run the garbage collector twice (needed when running + reference counting tests with PyPy) ''' + gc.collect() + gc.collect() + + def pytest_namespace(): """Add import suppression and test requirements to `pytest` namespace""" try: @@ -190,6 +199,7 @@ def pytest_namespace(): from pybind11_tests import have_eigen except ImportError: have_eigen = False + pypy = platform.python_implementation() == "PyPy" skipif = pytest.mark.skipif return { @@ -200,6 +210,8 @@ def pytest_namespace(): reason="eigen and/or numpy are not installed"), 'requires_eigen_and_scipy': skipif(not have_eigen or not scipy, reason="eigen and/or scipy are not installed"), + 'unsupported_on_pypy': skipif(pypy, reason="unsupported on PyPy"), + 'gc_collect': gc_collect } diff --git a/tests/constructor_stats.h b/tests/constructor_stats.h index eb3e49cab..de5c133c1 100644 --- a/tests/constructor_stats.h +++ b/tests/constructor_stats.h @@ -85,27 +85,51 @@ public: created(inst); copy_constructions++; } + void move_created(void *inst) { created(inst); move_constructions++; } + void default_created(void *inst) { created(inst); default_constructions++; } + void created(void *inst) { ++_instances[inst]; - }; + } + void destroyed(void *inst) { if (--_instances[inst] < 0) - throw std::runtime_error("cstats.destroyed() called with unknown instance; potential double-destruction or a missing cstats.created()"); + throw std::runtime_error("cstats.destroyed() called with unknown " + "instance; potential double-destruction " + "or a missing cstats.created()"); + } + + static void gc() { + // Force garbage collection to ensure any pending destructors are invoked: +#if defined(PYPY_VERSION) + PyObject *globals = PyEval_GetGlobals(); + PyObject *result = PyRun_String( + "import gc\n" + "for i in range(2):" + " gc.collect()\n", + Py_file_input, globals, globals); + if (result == nullptr) + throw py::error_already_set(); + Py_DECREF(result); +#else + py::module::import("gc").attr("collect")(); +#endif } int alive() { - // Force garbage collection to ensure any pending destructors are invoked: - py::module::import("gc").attr("collect")(); + gc(); int total = 0; - for (const auto &p : _instances) if (p.second > 0) total += p.second; + for (const auto &p : _instances) + if (p.second > 0) + total += p.second; return total; } @@ -134,6 +158,9 @@ public: // Gets constructor stats from a C++ type template static ConstructorStats& get() { +#if defined(PYPY_VERSION) + gc(); +#endif return get(typeid(T)); } diff --git a/tests/test_alias_initialization.py b/tests/test_alias_initialization.py index 0ed9d2f79..fb90cfc7b 100644 --- a/tests/test_alias_initialization.py +++ b/tests/test_alias_initialization.py @@ -1,10 +1,11 @@ -import gc +import pytest def test_alias_delay_initialization1(capture): - """A only initializes its trampoline class when we inherit from it; if we just - create and use an A instance directly, the trampoline initialization is bypassed - and we only initialize an A() instead (for performance reasons). + """ + A only initializes its trampoline class when we inherit from it; if we just + create and use an A instance directly, the trampoline initialization is + bypassed and we only initialize an A() instead (for performance reasons). """ from pybind11_tests import A, call_f @@ -20,7 +21,7 @@ def test_alias_delay_initialization1(capture): a = A() call_f(a) del a - gc.collect() + pytest.gc_collect() assert capture == "A.f()" # Python version @@ -28,7 +29,7 @@ def test_alias_delay_initialization1(capture): b = B() call_f(b) del b - gc.collect() + pytest.gc_collect() assert capture == """ PyA.PyA() PyA.f() @@ -57,7 +58,7 @@ def test_alias_delay_initialization2(capture): a2 = A2() call_f(a2) del a2 - gc.collect() + pytest.gc_collect() assert capture == """ PyA2.PyA2() PyA2.f() @@ -70,7 +71,7 @@ def test_alias_delay_initialization2(capture): b2 = B2() call_f(b2) del b2 - gc.collect() + pytest.gc_collect() assert capture == """ PyA2.PyA2() PyA2.f() diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index c3a7a9e02..057250d29 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -75,7 +75,7 @@ private: }; test_initializer buffers([](py::module &m) { - py::class_ mtx(m, "Matrix"); + py::class_ mtx(m, "Matrix", py::buffer_protocol()); mtx.def(py::init()) /// Construct from a buffer diff --git a/tests/test_buffers.py b/tests/test_buffers.py index f0ea964d9..956839c1c 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -5,34 +5,6 @@ with pytest.suppress(ImportError): import numpy as np -@pytest.requires_numpy -def test_to_python(): - m = Matrix(5, 5) - - assert m[2, 3] == 0 - m[2, 3] = 4 - assert m[2, 3] == 4 - - m2 = np.array(m, copy=False) - assert m2.shape == (5, 5) - assert abs(m2).sum() == 4 - assert m2[2, 3] == 4 - m2[2, 3] = 5 - assert m2[2, 3] == 5 - - cstats = ConstructorStats.get(Matrix) - assert cstats.alive() == 1 - del m - assert cstats.alive() == 1 - del m2 # holds an m reference - assert cstats.alive() == 0 - assert cstats.values() == ["5x5 matrix"] - assert cstats.copy_constructions == 0 - # assert cstats.move_constructions >= 0 # Don't invoke any - assert cstats.copy_assignments == 0 - assert cstats.move_assignments == 0 - - @pytest.requires_numpy def test_from_python(): with pytest.raises(RuntimeError) as excinfo: @@ -55,3 +27,36 @@ def test_from_python(): # assert cstats.move_constructions >= 0 # Don't invoke any assert cstats.copy_assignments == 0 assert cstats.move_assignments == 0 + + +# PyPy: Memory leak in the "np.array(m, copy=False)" call +# https://bitbucket.org/pypy/pypy/issues/2444 +@pytest.unsupported_on_pypy +@pytest.requires_numpy +def test_to_python(): + m = Matrix(5, 5) + + assert m[2, 3] == 0 + m[2, 3] = 4 + assert m[2, 3] == 4 + + m2 = np.array(m, copy=False) + assert m2.shape == (5, 5) + assert abs(m2).sum() == 4 + assert m2[2, 3] == 4 + m2[2, 3] = 5 + assert m2[2, 3] == 5 + + cstats = ConstructorStats.get(Matrix) + assert cstats.alive() == 1 + del m + pytest.gc_collect() + assert cstats.alive() == 1 + del m2 # holds an m reference + pytest.gc_collect() + assert cstats.alive() == 0 + assert cstats.values() == ["5x5 matrix"] + assert cstats.copy_constructions == 0 + # assert cstats.move_constructions >= 0 # Don't invoke any + assert cstats.copy_assignments == 0 + assert cstats.move_assignments == 0 diff --git a/tests/test_issues.py b/tests/test_issues.py index 2098ff8a3..e60b5ca90 100644 --- a/tests/test_issues.py +++ b/tests/test_issues.py @@ -1,5 +1,4 @@ import pytest -import gc from pybind11_tests import ConstructorStats @@ -55,7 +54,7 @@ def test_shared_ptr_gc(): el = ElementList() for i in range(10): el.add(ElementA(i)) - gc.collect() + pytest.gc_collect() for i, v in enumerate(el.get()): assert i == v.value() @@ -130,13 +129,13 @@ def test_nested(): assert c.b.a.as_base().value == 42 del c - gc.collect() + pytest.gc_collect() del a # Should't delete while abase is still alive - gc.collect() + pytest.gc_collect() assert abase.value == 42 del abase, b - gc.collect() + pytest.gc_collect() def test_move_fallback(): diff --git a/tests/test_keep_alive.py b/tests/test_keep_alive.py index 0cef34658..bfd7d40c3 100644 --- a/tests/test_keep_alive.py +++ b/tests/test_keep_alive.py @@ -1,4 +1,4 @@ -import gc +import pytest def test_keep_alive_argument(capture): @@ -9,14 +9,14 @@ def test_keep_alive_argument(capture): assert capture == "Allocating parent." with capture: p.addChild(Child()) - gc.collect() + pytest.gc_collect() assert capture == """ Allocating child. Releasing child. """ with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -24,11 +24,11 @@ def test_keep_alive_argument(capture): assert capture == "Allocating parent." with capture: p.addChildKeepAlive(Child()) - gc.collect() + pytest.gc_collect() assert capture == "Allocating child." with capture: del p - gc.collect() + pytest.gc_collect() assert capture == """ Releasing parent. Releasing child. @@ -43,14 +43,14 @@ def test_keep_alive_return_value(capture): assert capture == "Allocating parent." with capture: p.returnChild() - gc.collect() + pytest.gc_collect() assert capture == """ Allocating child. Releasing child. """ with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -58,11 +58,11 @@ def test_keep_alive_return_value(capture): assert capture == "Allocating parent." with capture: p.returnChildKeepAlive() - gc.collect() + pytest.gc_collect() assert capture == "Allocating child." with capture: del p - gc.collect() + pytest.gc_collect() assert capture == """ Releasing parent. Releasing child. @@ -77,11 +77,11 @@ def test_return_none(capture): assert capture == "Allocating parent." with capture: p.returnNullChildKeepAliveChild() - gc.collect() + pytest.gc_collect() assert capture == "" with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." with capture: @@ -89,9 +89,9 @@ def test_return_none(capture): assert capture == "Allocating parent." with capture: p.returnNullChildKeepAliveParent() - gc.collect() + pytest.gc_collect() assert capture == "" with capture: del p - gc.collect() + pytest.gc_collect() assert capture == "Releasing parent." diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 11fc90091..824e74310 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -134,10 +134,9 @@ test_initializer methods_and_attributes([](py::module &m) { .def("overloaded_const", static_cast(&ExampleMandA::overloaded)) #endif .def("__str__", &ExampleMandA::toString) - .def_readwrite("value", &ExampleMandA::value) - ; + .def_readwrite("value", &ExampleMandA::value); - py::class_(m, "TestProperties") + py::class_(m, "TestProperties", py::metaclass()) .def(py::init<>()) .def_readonly("def_readonly", &TestProperties::value) .def_readwrite("def_readwrite", &TestProperties::value) @@ -160,7 +159,7 @@ test_initializer methods_and_attributes([](py::module &m) { auto static_set2 = [](py::object, int v) { TestPropRVP::sv2.value = v; }; auto rvp_copy = py::return_value_policy::copy; - py::class_(m, "TestPropRVP") + py::class_(m, "TestPropRVP", py::metaclass()) .def(py::init<>()) .def_property_readonly("ro_ref", &TestPropRVP::get1) .def_property_readonly("ro_copy", &TestPropRVP::get2, rvp_copy) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 2b0f8d571..1502f77f5 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -125,10 +125,19 @@ def test_property_rvalue_policy(): instance = TestPropRVP() o = instance.rvalue assert o.value == 1 + + +def test_property_rvalue_policy_static(): + """When returning an rvalue, the return value policy is automatically changed from + `reference(_internal)` to `move`. The following would not work otherwise. + """ + from pybind11_tests import TestPropRVP o = TestPropRVP.static_rvalue assert o.value == 1 +# https://bitbucket.org/pypy/pypy/issues/2447 +@pytest.unsupported_on_pypy def test_dynamic_attributes(): from pybind11_tests import DynamicClass, CppDerivedDynamicClass diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index 3cb12b68d..c57cb852a 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -10,7 +10,6 @@ #include "pybind11_tests.h" - struct Base1 { Base1(int i) : i(i) { } int foo() { return i; } diff --git a/tests/test_multiple_inheritance.py b/tests/test_multiple_inheritance.py index 581cf5687..c10298d70 100644 --- a/tests/test_multiple_inheritance.py +++ b/tests/test_multiple_inheritance.py @@ -1,5 +1,3 @@ - - def test_multiple_inheritance_cpp(): from pybind11_tests import MIType diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 1c218a10b..b96790c39 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -1,5 +1,4 @@ import pytest -import gc with pytest.suppress(ImportError): import numpy as np @@ -220,7 +219,7 @@ def test_numpy_view(capture): ac_view_2 = ac.numpy_view() assert np.all(ac_view_1 == np.array([1, 2], dtype=np.int32)) del ac - gc.collect() + pytest.gc_collect() assert capture == """ ArrayClass() ArrayClass::numpy_view() @@ -233,12 +232,14 @@ def test_numpy_view(capture): with capture: del ac_view_1 del ac_view_2 - gc.collect() + pytest.gc_collect() + pytest.gc_collect() assert capture == """ ~ArrayClass() """ +@pytest.unsupported_on_pypy @pytest.requires_numpy def test_cast_numpy_int64_to_uint64(): from pybind11_tests.array import function_taking_uint64 diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index e0d42391e..02ccb9633 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -1,4 +1,3 @@ - def test_operator_overloading(): from pybind11_tests import Vector2, Vector, ConstructorStats diff --git a/tests/test_pickling.py b/tests/test_pickling.py index 5e62e1fcc..548c618af 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -1,3 +1,5 @@ +import pytest + try: import cPickle as pickle # Use cPickle on Python 2.7 except ImportError: @@ -18,6 +20,7 @@ def test_roundtrip(): assert p2.extra2() == p.extra2() +@pytest.unsupported_on_pypy def test_roundtrip_with_dict(): from pybind11_tests import PickleableWithDict diff --git a/tests/test_python_types.cpp b/tests/test_python_types.cpp index ae77f8209..e1598e9ef 100644 --- a/tests/test_python_types.cpp +++ b/tests/test_python_types.cpp @@ -185,7 +185,7 @@ struct MoveOutContainer { test_initializer python_types([](py::module &m) { /* No constructor is explicitly defined below. An exception is raised when trying to construct it directly from Python */ - py::class_(m, "ExamplePythonTypes", "Example 2 documentation") + py::class_(m, "ExamplePythonTypes", "Example 2 documentation", py::metaclass()) .def("get_dict", &ExamplePythonTypes::get_dict, "Return a Python dictionary") .def("get_dict_2", &ExamplePythonTypes::get_dict_2, "Return a C++ dictionary") .def("get_list", &ExamplePythonTypes::get_list, "Return a Python list") @@ -212,8 +212,7 @@ test_initializer python_types([](py::module &m) { .def("test_print", &ExamplePythonTypes::test_print, "test the print function") .def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance") .def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member") - .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)") - ; + .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)"); m.def("test_print_function", []() { py::print("Hello, World!"); diff --git a/tests/test_python_types.py b/tests/test_python_types.py index 9fe1ef71e..347abaaee 100644 --- a/tests/test_python_types.py +++ b/tests/test_python_types.py @@ -132,8 +132,12 @@ def test_instance(capture): assert cstats.alive() == 0 -def test_docs(doc): +# PyPy does not seem to propagate the tp_docs field at the moment +def test_class_docs(doc): assert doc(ExamplePythonTypes) == "Example 2 documentation" + + +def test_method_docs(doc): assert doc(ExamplePythonTypes.get_dict) == """ get_dict(self: m.ExamplePythonTypes) -> dict diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index a9aecd67f..b11c699df 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -206,6 +206,9 @@ def test_inheriting_repeat(): assert obj.say_everything() == "BT -7" +# PyPy: Reference count > 1 causes call with noncopyable instance +# to fail in ncv1.print_nc() +@pytest.unsupported_on_pypy @pytest.mark.skipif(not hasattr(pybind11_tests, 'NCVirt'), reason="NCVirt test broken on ICPC") def test_move_support():