improved handling of shared/smart pointers

Previously, pybind11 required classes using std::shared_ptr<> to derive
from std::enable_shared_from_this<> (or compilation failures would ensue).

Everything now also works for classes that don't do this, assuming that
some basic rules are followed (e.g. never passing "raw" pointers of
instances manged by shared pointers). The safer
std::enable_shared_from_this<> approach continues to be supported.

docs/advanced.rst

@@ -521,13 +521,9 @@ following snippet causes ``std::shared_ptr`` to be used instead.
 
 .. code-block:: cpp
 
-    /// Type declaration
+    py::class_<Example, std::shared_ptr<Example> /* <- holder type */> obj(m, "Example");
-    class Example : public std::enable_shared_from_this<Example> /* <- important, see below */ {
-        // ...
-    };
 
-    /// .... code within PYBIND11_PLUGIN declaration .....
+Note that any particular class can only be associated with a single holder type.
-    py::class_<Example, std::shared_ptr<Example> /* <- important */> obj(m, "Example");
 
 To enable transparent conversions for functions that take shared pointers as an
 argument or that return them, a macro invocation similar to the following must
@@ -537,7 +533,7 @@ be declared at the top level before any binding code:
 
     PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>);
 
-.. warning::
+.. note::
 
     The first argument of :func:`PYBIND11_DECLARE_HOLDER_TYPE` should be a
     placeholder name that is used as a template parameter of the second
@@ -545,25 +541,74 @@ be declared at the top level before any binding code:
     both sides; also, don't use the name of a type that already exists in your
     codebase.
 
-.. warning::
+One potential stumbling block when using holder types is that they need to be
+applied consistently. Can you guess what's broken about the following binding
+code?
 
-   To ensure correct reference counting among Python and C++, the use of
+.. code-block:: cpp
-   ``std::shared_ptr<T>`` as a holder type requires that ``T`` inherits from
-   ``std::enable_shared_from_this<T>`` (see cppreference_ for details).
 
-If you encounter issues (failure to compile, ``bad_weak_ptr`` exceptions),
+    class Child { };
-please check that you really did all three steps:
 
-1. invoking the ``PYBIND11_DECLARE_HOLDER_TYPE`` macro in every file that
+    class Parent {
-   contains pybind11 code and uses your chosen smart pointer type.
+    public:
+       Parent() : child(std::make_shared<Child>()) { }
+       Child *get_child() { return child.get(); }  /* Hint: ** DON'T DO THIS ** */
+    private:
+        std::shared_ptr<Child> child;
+    };
 
-2. specifying the holder types to ``class_``.
+    PYBIND11_PLUGIN(example) {
+        py::module m("example");
 
-3. extending from ``std::enable_shared_from_this`` when using
+        py::class_<Child, std::shared_ptr<Child>>(m, "Child");
-   ``std::shared_ptr``.
+
+        py::class_<Parent, std::shared_ptr<Parent>>(m, "Parent")
+           .def(py::init<>())
+           .def("get_child", &Parent::get_child);
+
+        return m.ptr();
+    }
+
+The following Python code will cause undefined behavior (and likely a
+segmentation fault).
+
+.. code-block:: python
+
+   from example import Parent
+   print(Parent().get_child())
+
+The problem is that ``Parent::get_child()`` returns a pointer to an instance of
+``Child``, but the fact that this instance is already managed by
+``std::shared_ptr<...>`` is lost when passing raw pointers. In this case,
+pybind11 will create a second independent ``std::shared_ptr<...>`` that also
+claims ownership of the pointer. In the end, the object will be freed **twice**
+since these shared pointers have no way of knowing about each other.
+
+There are two ways to resolve this issue:
+
+1. For types that are managed by a smart pointer class, never use raw pointers
+   in function arguments or return values. In other words: always consistently
+   wrap pointers into their designated holder types (such as
+   ``std::shared_ptr<...>``). In this case, the signature of ``get_child()``
+   should be modified as follows:
+
+.. code-block:: cpp
+
+    std::shared_ptr<Child> get_child() { return child; }
+
+2. Adjust the definition of ``Child`` by specifying
+   ``std::enable_shared_from_this<T>`` (see cppreference_ for details) as a
+   base class. This adds a small bit of information to ``Child`` that allows
+   pybind11 to realize that there is already an existing
+   ``std::shared_ptr<...>`` and communicate with it. In this case, the
+   declaration of ``Child`` should look as follows:
 
 .. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this
 
+.. code-block:: cpp
+
+    class Child : public std::enable_shared_from_this<Child> { };
+
 .. seealso::
 
     The file :file:`example/example8.cpp` contains a complete example that

example/example8.cpp

@@ -11,19 +11,19 @@
 #include "example.h"
 #include "object.h"
 
-/// Object subclass
+/// Custom object with builtin reference counting (see 'object.h' for the implementation)
-class MyObject : public Object {
+class MyObject1 : public Object {
 public:
-    MyObject(int value) : value(value) {
+    MyObject1(int value) : value(value) {
         std::cout << toString() << " constructor" << std::endl;
     }
 
     std::string toString() const {
-        return "MyObject[" + std::to_string(value) + "]";
+        return "MyObject1[" + std::to_string(value) + "]";
     }
 
 protected:
-    virtual ~MyObject() {
+    virtual ~MyObject1() {
         std::cout << toString() << " destructor" << std::endl;
     }
 
@@ -31,7 +31,8 @@ private:
     int value;
 };
 
-class MyObject2 : public std::enable_shared_from_this<MyObject2> {
+/// Object managed by a std::shared_ptr<>
+class MyObject2 {
 public:
     MyObject2(int value) : value(value) {
         std::cout << toString() << " constructor" << std::endl;
@@ -49,52 +50,80 @@ private:
     int value;
 };
 
-/// Make pybind aware of the ref-counted wrapper type
+/// Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<>
+class MyObject3 : public std::enable_shared_from_this<MyObject3> {
+public:
+    MyObject3(int value) : value(value) {
+        std::cout << toString() << " constructor" << std::endl;
+    }
+
+    std::string toString() const {
+        return "MyObject3[" + std::to_string(value) + "]";
+    }
+
+    virtual ~MyObject3() {
+        std::cout << toString() << " destructor" << std::endl;
+    }
+
+private:
+    int value;
+};
+
+/// Make pybind aware of the ref-counted wrapper type (s)
 PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>);
 PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>);
 
-Object *make_object_1() { return new MyObject(1); }
+Object *make_object_1() { return new MyObject1(1); }
-ref<Object> make_object_2() { return new MyObject(2); }
+ref<Object> make_object_2() { return new MyObject1(2); }
-MyObject *make_myobject_4() { return new MyObject(4); }
-ref<MyObject> make_myobject_5() { return new MyObject(5); }
 
-MyObject2 *make_myobject2_1() { return new MyObject2(1); }
+MyObject1 *make_myobject1_1() { return new MyObject1(4); }
-std::shared_ptr<MyObject2> make_myobject2_2() { return std::make_shared<MyObject2>(2); }
+ref<MyObject1> make_myobject1_2() { return new MyObject1(5); }
+
+MyObject2 *make_myobject2_1() { return new MyObject2(6); }
+std::shared_ptr<MyObject2> make_myobject2_2() { return std::make_shared<MyObject2>(7); }
+
+MyObject3 *make_myobject3_1() { return new MyObject3(8); }
+std::shared_ptr<MyObject3> make_myobject3_2() { return std::make_shared<MyObject3>(9); }
 
 void print_object_1(const Object *obj) { std::cout << obj->toString() << std::endl; }
 void print_object_2(ref<Object> obj) { std::cout << obj->toString() << std::endl; }
 void print_object_3(const ref<Object> &obj) { std::cout << obj->toString() << std::endl; }
 void print_object_4(const ref<Object> *obj) { std::cout << (*obj)->toString() << std::endl; }
 
-void print_myobject_1(const MyObject *obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject1_1(const MyObject1 *obj) { std::cout << obj->toString() << std::endl; }
-void print_myobject_2(ref<MyObject> obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject1_2(ref<MyObject1> obj) { std::cout << obj->toString() << std::endl; }
-void print_myobject_3(const ref<MyObject> &obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject1_3(const ref<MyObject1> &obj) { std::cout << obj->toString() << std::endl; }
-void print_myobject_4(const ref<MyObject> *obj) { std::cout << (*obj)->toString() << std::endl; }
+void print_myobject1_4(const ref<MyObject1> *obj) { std::cout << (*obj)->toString() << std::endl; }
 
 void print_myobject2_1(const MyObject2 *obj) { std::cout << obj->toString() << std::endl; }
 void print_myobject2_2(std::shared_ptr<MyObject2> obj) { std::cout << obj->toString() << std::endl; }
 void print_myobject2_3(const std::shared_ptr<MyObject2> &obj) { std::cout << obj->toString() << std::endl; }
 void print_myobject2_4(const std::shared_ptr<MyObject2> *obj) { std::cout << (*obj)->toString() << std::endl; }
 
+void print_myobject3_1(const MyObject3 *obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject3_2(std::shared_ptr<MyObject3> obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject3_3(const std::shared_ptr<MyObject3> &obj) { std::cout << obj->toString() << std::endl; }
+void print_myobject3_4(const std::shared_ptr<MyObject3> *obj) { std::cout << (*obj)->toString() << std::endl; }
+
 void init_ex8(py::module &m) {
     py::class_<Object, ref<Object>> obj(m, "Object");
     obj.def("getRefCount", &Object::getRefCount);
 
-    py::class_<MyObject, ref<MyObject>>(m, "MyObject", obj)
+    py::class_<MyObject1, ref<MyObject1>>(m, "MyObject1", obj)
         .def(py::init<int>());
 
     m.def("make_object_1", &make_object_1);
     m.def("make_object_2", &make_object_2);
-    m.def("make_myobject_4", &make_myobject_4);
+    m.def("make_myobject1_1", &make_myobject1_1);
-    m.def("make_myobject_5", &make_myobject_5);
+    m.def("make_myobject1_2", &make_myobject1_2);
     m.def("print_object_1", &print_object_1);
     m.def("print_object_2", &print_object_2);
     m.def("print_object_3", &print_object_3);
     m.def("print_object_4", &print_object_4);
-    m.def("print_myobject_1", &print_myobject_1);
+    m.def("print_myobject1_1", &print_myobject1_1);
-    m.def("print_myobject_2", &print_myobject_2);
+    m.def("print_myobject1_2", &print_myobject1_2);
-    m.def("print_myobject_3", &print_myobject_3);
+    m.def("print_myobject1_3", &print_myobject1_3);
-    m.def("print_myobject_4", &print_myobject_4);
+    m.def("print_myobject1_4", &print_myobject1_4);
 
     py::class_<MyObject2, std::shared_ptr<MyObject2>>(m, "MyObject2")
         .def(py::init<int>());
@@ -105,5 +134,14 @@ void init_ex8(py::module &m) {
     m.def("print_myobject2_3", &print_myobject2_3);
     m.def("print_myobject2_4", &print_myobject2_4);
 
-    py::implicitly_convertible<py::int_, MyObject>();
+    py::class_<MyObject3, std::shared_ptr<MyObject3>>(m, "MyObject3")
+        .def(py::init<int>());
+    m.def("make_myobject3_1", &make_myobject3_1);
+    m.def("make_myobject3_2", &make_myobject3_2);
+    m.def("print_myobject3_1", &print_myobject3_1);
+    m.def("print_myobject3_2", &print_myobject3_2);
+    m.def("print_myobject3_3", &print_myobject3_3);
+    m.def("print_myobject3_4", &print_myobject3_4);
+
+    py::implicitly_convertible<py::int_, MyObject1>();
 }

example/example8.py

@@ -3,49 +3,68 @@ from __future__ import print_function
 import sys
 sys.path.append('.')
 
-from example import MyObject
+from example import MyObject1
+from example import MyObject2
+from example import MyObject3
+
 from example import make_object_1
 from example import make_object_2
-from example import make_myobject_4
+from example import make_myobject1_1
-from example import make_myobject_5
+from example import make_myobject1_2
 from example import make_myobject2_1
 from example import make_myobject2_2
+from example import make_myobject3_1
+from example import make_myobject3_2
+
 from example import print_object_1
 from example import print_object_2
 from example import print_object_3
 from example import print_object_4
-from example import print_myobject_1
+
-from example import print_myobject_2
+from example import print_myobject1_1
-from example import print_myobject_3
+from example import print_myobject1_2
-from example import print_myobject_4
+from example import print_myobject1_3
+from example import print_myobject1_4
+
 from example import print_myobject2_1
 from example import print_myobject2_2
 from example import print_myobject2_3
 from example import print_myobject2_4
 
-for o in [make_object_1(), make_object_2(), MyObject(3)]:
+from example import print_myobject3_1
+from example import print_myobject3_2
+from example import print_myobject3_3
+from example import print_myobject3_4
+
+for o in [make_object_1(), make_object_2(), MyObject1(3)]:
     print("Reference count = %i" % o.getRefCount())
     print_object_1(o)
     print_object_2(o)
     print_object_3(o)
     print_object_4(o)
 
-for o in [make_myobject_4(), make_myobject_5(), MyObject(6), 7]:
+for o in [make_myobject1_1(), make_myobject1_2(), MyObject1(6), 7]:
     print(o)
     if not isinstance(o, int):
         print_object_1(o)
         print_object_2(o)
         print_object_3(o)
         print_object_4(o)
-    print_myobject_1(o)
+    print_myobject1_1(o)
-    print_myobject_2(o)
+    print_myobject1_2(o)
-    print_myobject_3(o)
+    print_myobject1_3(o)
-    print_myobject_4(o)
+    print_myobject1_4(o)
 
-
+for o in [MyObject2(8), make_myobject2_1(), make_myobject2_2()]:
-for o in [make_myobject2_1(), make_myobject2_2()]:
     print(o)
     print_myobject2_1(o)
     print_myobject2_2(o)
     print_myobject2_3(o)
     print_myobject2_4(o)
+
+for o in [MyObject3(9), make_myobject3_1(), make_myobject3_2()]:
+    print(o)
+    print_myobject3_1(o)
+    print_myobject3_2(o)
+    print_myobject3_3(o)
+    print_myobject3_4(o)

example/example8.ref

@@ -1,275 +1,243 @@
-MyObject[1] constructor
+MyObject1[1] constructor
-Initialized ref from pointer 0x7ffdb2c16a50
+Initialized ref from pointer 0x1347ba0
-MyObject[2] constructor
+MyObject1[2] constructor
-Initialized ref from pointer 0x7ffdb2c167a0
+Initialized ref from pointer 0x12b9270
-Initialized ref from pointer 0x7ffdb2c167a0
+Initialized ref from ref 0x12b9270
-Destructing ref 0x7ffdb2c167a0
+Destructing ref 0x12b9270
-MyObject[3] constructor
+MyObject1[3] constructor
-Initialized ref from pointer 0x7ffdb2c17420
+Initialized ref from pointer 0x12a2a90
+MyObject1[1]
+Created empty ref
+Assigning ref 0x1347ba0
+Initialized ref from ref 0x1347ba0
+MyObject1[1]
+Destructing ref 0x1347ba0
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[1]
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[1]
+Destructing ref 0x1347ba0
+MyObject1[2]
+Created empty ref
+Assigning ref 0x12b9270
+Initialized ref from ref 0x12b9270
+MyObject1[2]
+Destructing ref 0x12b9270
+Destructing ref 0x12b9270
+Created empty ref
+Assigning ref 0x12b9270
+MyObject1[2]
+Destructing ref 0x12b9270
+Created empty ref
+Assigning ref 0x12b9270
+MyObject1[2]
+Destructing ref 0x12b9270
+MyObject1[3]
+Created empty ref
+Assigning ref 0x12a2a90
+Initialized ref from ref 0x12a2a90
+MyObject1[3]
+Destructing ref 0x12a2a90
+Destructing ref 0x12a2a90
+Created empty ref
+Assigning ref 0x12a2a90
+MyObject1[3]
+Destructing ref 0x12a2a90
+Created empty ref
+Assigning ref 0x12a2a90
+MyObject1[3]
+Destructing ref 0x12a2a90
+Destructing ref 0x12b9270
+MyObject1[2] destructor
+Destructing ref 0x1347ba0
+MyObject1[1] destructor
+MyObject1[4] constructor
+Initialized ref from pointer 0x1347ba0
+MyObject1[5] constructor
+Initialized ref from pointer 0x1299190
+Initialized ref from ref 0x1299190
+Destructing ref 0x1299190
+MyObject1[6] constructor
+Initialized ref from pointer 0x133e2f0
+Destructing ref 0x12a2a90
+MyObject1[3] destructor
+MyObject1[4]
+Created empty ref
+Assigning ref 0x1347ba0
+Initialized ref from ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+MyObject1[4]
+Created empty ref
+Assigning ref 0x1347ba0
+Initialized ref from ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+Created empty ref
+Assigning ref 0x1347ba0
+MyObject1[4]
+Destructing ref 0x1347ba0
+MyObject1[5]
+Created empty ref
+Assigning ref 0x1299190
+Initialized ref from ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+Destructing ref 0x1299190
+Created empty ref
+Assigning ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+Created empty ref
+Assigning ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+MyObject1[5]
+Created empty ref
+Assigning ref 0x1299190
+Initialized ref from ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+Destructing ref 0x1299190
+Created empty ref
+Assigning ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+Created empty ref
+Assigning ref 0x1299190
+MyObject1[5]
+Destructing ref 0x1299190
+MyObject1[6]
+Created empty ref
+Assigning ref 0x133e2f0
+Initialized ref from ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+Destructing ref 0x133e2f0
+Created empty ref
+Assigning ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+Created empty ref
+Assigning ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+MyObject1[6]
+Created empty ref
+Assigning ref 0x133e2f0
+Initialized ref from ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+Destructing ref 0x133e2f0
+Created empty ref
+Assigning ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+Created empty ref
+Assigning ref 0x133e2f0
+MyObject1[6]
+Destructing ref 0x133e2f0
+MyObject1[7] constructor
+Initialized ref from pointer 0x133f3a0
+MyObject1[7]
+Destructing ref 0x133f3a0
+MyObject1[7] destructor
+Created empty ref
+MyObject1[7] constructor
+Initialized ref from pointer 0x12a2a90
+Assigning ref 0x12a2a90
+Initialized ref from ref 0x12a2a90
+MyObject1[7]
+Destructing ref 0x12a2a90
+Destructing ref 0x12a2a90
+Destructing ref 0x12a2a90
+MyObject1[7] destructor
+Created empty ref
+MyObject1[7] constructor
+Initialized ref from pointer 0x133f3a0
+Assigning ref 0x133f3a0
+MyObject1[7]
+Destructing ref 0x133f3a0
+Destructing ref 0x133f3a0
+MyObject1[7] destructor
+Created empty ref
+MyObject1[7] constructor
+Initialized ref from pointer 0x12a2a90
+Assigning ref 0x12a2a90
+MyObject1[7]
+Destructing ref 0x12a2a90
+Destructing ref 0x12a2a90
+MyObject1[7] destructor
+Destructing ref 0x133e2f0
+MyObject1[6] destructor
+Destructing ref 0x1299190
+MyObject1[5] destructor
+Destructing ref 0x1347ba0
+MyObject1[4] destructor
+MyObject2[8] constructor
+MyObject2[6] constructor
+MyObject2[7] constructor
+MyObject2[8]
+MyObject2[8]
+MyObject2[8]
+MyObject2[8]
+MyObject2[6]
+MyObject2[6]
+MyObject2[6]
+MyObject2[6]
+MyObject2[7]
+MyObject2[7]
+MyObject2[7]
+MyObject2[7]
+MyObject2[6] destructor
+MyObject2[8] destructor
+MyObject3[9] constructor
+MyObject3[8] constructor
+MyObject3[9] constructor
+MyObject2[7] destructor
+MyObject3[9]
+MyObject3[9]
+MyObject3[9]
+MyObject3[9]
+MyObject3[8]
+MyObject3[8]
+MyObject3[8]
+MyObject3[8]
+MyObject3[9]
+MyObject3[9]
+MyObject3[9]
+MyObject3[9]
+MyObject3[8] destructor
+MyObject3[9] destructor
 Reference count = 1
-MyObject[1]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c16a50
-MyObject[1]
-Destructing ref 0x7ffdb2c16a50
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[1]
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[1]
-Destructing ref 0x7ffdb2c16a50
 Reference count = 1
-MyObject[2]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c167a0
-MyObject[2]
-Destructing ref 0x7ffdb2c167a0
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[2]
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[2]
-Destructing ref 0x7ffdb2c167a0
 Reference count = 1
-MyObject[3]
+<example.MyObject1 object at 0x7f830b500e68>
-Created empty ref
+<example.MyObject1 object at 0x7f830b4fc688>
-Initialized ref from pointer 0x7ffdb2c17420
+<example.MyObject1 object at 0x7f830b4fc5a8>
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c17420
-MyObject[3]
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c17420
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c17420
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-MyObject[3]
-Destructing ref 0x7ffdb2c17420
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c17420
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-MyObject[3]
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c167a0
-MyObject[2] destructor
-Destructing ref 0x7ffdb2c16a50
-MyObject[1] destructor
-MyObject[4] constructor
-Initialized ref from pointer 0x7ffdb2c16a50
-MyObject[5] constructor
-Initialized ref from pointer 0x7ffdb2c167a0
-Initialized ref from pointer 0x7ffdb2c167a0
-Destructing ref 0x7ffdb2c167a0
-MyObject[6] constructor
-Initialized ref from pointer 0x7ffdb2c161d0
-Destructing ref 0x7ffdb2c17420
-MyObject[3] destructor
-<example.MyObject object at 0x10cf67600>
-MyObject[4]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c16a50
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-MyObject[4]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c16a50
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c16a50
-Move-assigning ref 0x7ffdb2c16a50
-Destructing ref 0x0
-MyObject[4]
-Destructing ref 0x7ffdb2c16a50
-<example.MyObject object at 0x10cf7d210>
-MyObject[5]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c167a0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-MyObject[5]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c167a0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c167a0
-Move-assigning ref 0x7ffdb2c167a0
-Destructing ref 0x0
-MyObject[5]
-Destructing ref 0x7ffdb2c167a0
-<example.MyObject object at 0x10cf7d270>
-MyObject[6]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c161d0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
-Destructing ref 0x7ffdb2c161d0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
-MyObject[6]
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c161d0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
-Destructing ref 0x7ffdb2c161d0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
-Created empty ref
-Initialized ref from pointer 0x7ffdb2c161d0
-Move-assigning ref 0x7ffdb2c161d0
-Destructing ref 0x0
-MyObject[6]
-Destructing ref 0x7ffdb2c161d0
 7
-MyObject[7] constructor
+<example.MyObject2 object at 0x7f830b50b330>
-Initialized ref from pointer 0x7ffdb2c17420
+<example.MyObject2 object at 0x7f830b50bdb0>
-MyObject[7]
+<example.MyObject2 object at 0x7f83098f6330>
-Destructing ref 0x7ffdb2c17420
+<example.MyObject3 object at 0x7f830b50b330>
-MyObject[7] destructor
+<example.MyObject3 object at 0x7f830b50bdb0>
-Created empty ref
+<example.MyObject3 object at 0x7f83098f6370>
-MyObject[7] constructor
+MyObject3[9] destructor
-Initialized ref from pointer 0x7ffdb2c17420
-Initialized ref from pointer 0x7ffdb2c17420
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-Initialized ref from ref 0x7ffdb2c17420
-MyObject[7]
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c17420
-MyObject[7] destructor
-Created empty ref
-MyObject[7] constructor
-Initialized ref from pointer 0x7ffdb2c17420
-Initialized ref from pointer 0x7ffdb2c17420
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-MyObject[7]
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c17420
-MyObject[7] destructor
-Created empty ref
-MyObject[7] constructor
-Initialized ref from pointer 0x7ffdb2c17420
-Initialized ref from pointer 0x7ffdb2c17420
-Move-assigning ref 0x7ffdb2c17420
-Destructing ref 0x0
-MyObject[7]
-Destructing ref 0x7ffdb2c17420
-Destructing ref 0x7ffdb2c17420
-MyObject[7] destructor
-Destructing ref 0x7ffdb2c161d0
-MyObject[6] destructor
-Destructing ref 0x7ffdb2c167a0
-MyObject[5] destructor
-Destructing ref 0x7ffdb2c16a50
-MyObject[4] destructor
-MyObject2[1] constructor
-MyObject2[2] constructor
-<example.MyObject2 object at 0x10cf5c378>
-MyObject2[1]
-MyObject2[1]
-MyObject2[1]
-MyObject2[1]
-<example.MyObject2 object at 0x10cf5c340>
-MyObject2[2]
-MyObject2[2]
-MyObject2[2]
-MyObject2[2]
-MyObject2[1] destructor
-MyObject2[2] destructor

include/pybind11/cast.h

@@ -19,9 +19,9 @@
 NAMESPACE_BEGIN(pybind11)
 NAMESPACE_BEGIN(detail)
 
-class type_caster_custom {
+class type_caster_generic {
 public:
-    PYBIND11_NOINLINE type_caster_custom(const std::type_info *type_info) {
+    PYBIND11_NOINLINE type_caster_generic(const std::type_info *type_info) {
         auto & registered_types = get_internals().registered_types;
         auto it = registered_types.find(type_info);
         if (it != registered_types.end()) {
@@ -57,7 +57,9 @@ public:
     }
 
     PYBIND11_NOINLINE static PyObject *cast(const void *_src, return_value_policy policy, PyObject *parent,
-                                            const std::type_info *type_info, void *(*copy_constructor)(const void *)) {
+                                            const std::type_info *type_info,
+                                            void *(*copy_constructor)(const void *),
+                                            const void *existing_holder = nullptr) {
         void *src = const_cast<void *>(_src);
         if (src == nullptr) {
             Py_INCREF(Py_None);
@@ -100,7 +102,7 @@ public:
             Py_XINCREF(parent);
         }
         PyObject *inst_pyobj = (PyObject *) inst;
-        reg_type.init_holder(inst_pyobj);
+        reg_type.init_holder(inst_pyobj, existing_holder);
         if (!dont_cache)
             internals.registered_instances[inst->value] = inst_pyobj;
         return inst_pyobj;
@@ -113,20 +115,20 @@ protected:
 };
 
 /// Generic type caster for objects stored on the heap
-template <typename type, typename Enable = void> class type_caster : public type_caster_custom {
+template <typename type, typename Enable = void> class type_caster : public type_caster_generic {
 public:
     static PYBIND11_DESCR name() { return type_descr(_<type>()); }
 
-    type_caster() : type_caster_custom(&typeid(type)) { }
+    type_caster() : type_caster_generic(&typeid(type)) { }
 
     static PyObject *cast(const type &src, return_value_policy policy, PyObject *parent) {
         if (policy == return_value_policy::automatic)
             policy = return_value_policy::copy;
-        return type_caster_custom::cast(&src, policy, parent, &typeid(type), &copy_constructor);
+        return type_caster_generic::cast(&src, policy, parent, &typeid(type), &copy_constructor);
     }
 
     static PyObject *cast(const type *src, return_value_policy policy, PyObject *parent) {
-        return type_caster_custom::cast(src, policy, parent, &typeid(type), &copy_constructor);
+        return type_caster_generic::cast(src, policy, parent, &typeid(type), &copy_constructor);
     }
 
     operator type*() { return (type *) value; }
@@ -134,7 +136,7 @@ public:
 protected:
     template <typename T = type, typename std::enable_if<std::is_copy_constructible<T>::value, int>::type = 0>
     static void *copy_constructor(const void *arg) {
-        return new type(*((const type *)arg));
+        return new type(*((const type *) arg));
     }
     template <typename T = type, typename std::enable_if<!std::is_copy_constructible<T>::value, int>::type = 0>
     static void *copy_constructor(const void *) { return nullptr; }
@@ -433,34 +435,39 @@ protected:
 /// Type caster for holder types like std::shared_ptr, etc.
 template <typename type, typename holder_type> class type_caster_holder : public type_caster<type> {
 public:
-    typedef type_caster<type> parent;
+    using type_caster<type>::cast;
+    using type_caster<type>::typeinfo;
+    using type_caster<type>::value;
+    using type_caster<type>::temp;
+    using type_caster<type>::copy_constructor;
 
-    template <typename T = holder_type,
-              typename std::enable_if<std::is_same<std::shared_ptr<type>, T>::value, int>::type = 0>
     bool load(PyObject *src, bool convert) {
-        if (!parent::load(src, convert))
+        if (src == nullptr || typeinfo == nullptr)
             return false;
-        holder = holder_type(((type *) parent::value)->shared_from_this());
+        if (PyType_IsSubtype(Py_TYPE(src), typeinfo->type)) {
+            auto inst = (instance<type, holder_type> *) src;
+            value = inst->value;
+            holder = inst->holder;
             return true;
         }
-
+        if (convert) {
-    template <typename T = holder_type,
+            for (auto &converter : typeinfo->implicit_conversions) {
-              typename std::enable_if<!std::is_same<std::shared_ptr<type>, T>::value, int>::type = 0>
+                temp = object(converter(src, typeinfo->type), false);
-    bool load(PyObject *src, bool convert) {
+                if (load(temp.ptr(), false))
-        if (!parent::load(src, convert))
-            return false;
-        holder = holder_type((type *) parent::value);
                     return true;
             }
+        }
+        return false;
+    }
 
     explicit operator type*() { return this->value; }
     explicit operator type&() { return *(this->value); }
     explicit operator holder_type&() { return holder; }
     explicit operator holder_type*() { return &holder; }
 
-    using type_caster<type>::cast;
     static PyObject *cast(const holder_type &src, return_value_policy policy, PyObject *parent) {
-        return type_caster<type>::cast(src.get(), policy, parent);
+        return type_caster_generic::cast(
+            src.get(), policy, parent, &typeid(type), &copy_constructor, &src);
     }
 
 protected:

include/pybind11/common.h

@@ -177,7 +177,7 @@ template <typename type, typename holder_type = std::unique_ptr<type>> struct in
 struct type_info {
     PyTypeObject *type;
     size_t type_size;
-    void (*init_holder)(PyObject *);
+    void (*init_holder)(PyObject *, const void *);
     std::vector<PyObject *(*)(PyObject *, PyTypeObject *)> implicit_conversions;
     buffer_info *(*get_buffer)(PyObject *, void *) = nullptr;
     void *get_buffer_data = nullptr;

include/pybind11/pybind11.h

@@ -122,32 +122,32 @@ private:
     template <typename... T> using arg_value_caster =
         detail::type_caster<typename std::tuple<T...>>;
 
-    template <typename... T> static void process_extras(const std::tuple<T...> &args, function_entry *entry) {
+    template <typename... T> static void process_static(const std::tuple<T...> &args, function_entry *entry) {
-        process_extras(args, entry, typename detail::make_index_sequence<sizeof...(T)>::type());
+        process_static(args, entry, typename detail::make_index_sequence<sizeof...(T)>::type());
     }
 
-    template <typename... T, size_t ... Index> static void process_extras(const std::tuple<T...> &args,
+    template <typename... T, size_t ... Index> static void process_static(const std::tuple<T...> &args,
             function_entry *entry, detail::index_sequence<Index...>) {
-        int unused[] = { 0, (process_extra(std::get<Index>(args), entry), 0)... };
+        int unused[] = { 0, (process_static(std::get<Index>(args), entry), 0)... };
         (void) unused;
     }
 
     template <int Nurse, int Patient>
-    static void process_extra(const keep_alive<Nurse, Patient> &, function_entry *) { }
+    static void process_static(const keep_alive<Nurse, Patient> &, function_entry *) { }
-    static void process_extra(const char *doc, function_entry *entry) { entry->doc = (char *) doc; }
+    static void process_static(const char *doc, function_entry *entry) { entry->doc = (char *) doc; }
-    static void process_extra(const pybind11::doc &d, function_entry *entry) { entry->doc = (char *) d.value; }
+    static void process_static(const pybind11::doc &d, function_entry *entry) { entry->doc = (char *) d.value; }
-    static void process_extra(const pybind11::name &n, function_entry *entry) { entry->name = (char *) n.value; }
+    static void process_static(const pybind11::name &n, function_entry *entry) { entry->name = (char *) n.value; }
-    static void process_extra(const pybind11::return_value_policy p, function_entry *entry) { entry->policy = p; }
+    static void process_static(const pybind11::return_value_policy p, function_entry *entry) { entry->policy = p; }
-    static void process_extra(const pybind11::sibling s, function_entry *entry) { entry->sibling = s.value; }
+    static void process_static(const pybind11::sibling s, function_entry *entry) { entry->sibling = s.value; }
-    static void process_extra(const pybind11::is_method &m, function_entry *entry) { entry->class_ = m.class_; }
+    static void process_static(const pybind11::is_method &m, function_entry *entry) { entry->class_ = m.class_; }
-    static void process_extra(const pybind11::arg &a, function_entry *entry) {
+    static void process_static(const pybind11::arg &a, function_entry *entry) {
         if (entry->class_ && entry->args.empty())
             entry->args.emplace_back("self", nullptr, nullptr);
         entry->args.emplace_back(a.name, nullptr, nullptr);
     }
 
     template <typename T>
-    static void process_extra(const pybind11::arg_t<T> &a, function_entry *entry) {
+    static void process_static(const pybind11::arg_t<T> &a, function_entry *entry) {
         if (entry->class_ && entry->args.empty())
             entry->args.emplace_back("self", nullptr, nullptr);
 
@@ -184,7 +184,7 @@ public:
             return result;
         };
 
-        process_extras(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
+        process_static(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
         PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name();
         initialize(signature.text(), signature.types(), sizeof...(Args));
     }
@@ -243,7 +243,7 @@ private:
             return result;
         };
 
-        process_extras(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
+        process_static(std::make_tuple(std::forward<Extra>(extra)...), m_entry);
         PYBIND11_DESCR signature = cast_in::name() + detail::_(" -> ") + cast_out::name();
         initialize(signature.text(), signature.types(), sizeof...(Args));
     }
@@ -332,7 +332,7 @@ private:
                 const detail::type_info *type_info =
                     capsule(PyObject_GetAttrString((PyObject *) Py_TYPE(inst),
                                 const_cast<char *>("__pybind11__")), false);
-                type_info->init_holder(inst);
+                type_info->init_holder(inst, nullptr);
             }
             return result;
         }
@@ -558,7 +558,7 @@ public:
 
     custom_type(object &scope, const char *name_, const std::type_info *tinfo,
                 size_t type_size, size_t instance_size,
-                void (*init_holder)(PyObject *), const destructor &dealloc,
+                void (*init_holder)(PyObject *, const void *), const destructor &dealloc,
                 PyObject *parent, const char *doc) {
         PyHeapTypeObject *type = (PyHeapTypeObject*) PyType_Type.tp_alloc(&PyType_Type, 0);
 #if PY_MAJOR_VERSION >= 3
@@ -884,24 +884,37 @@ public:
         return *this;
     }
 private:
-    template <typename T = holder_type,
+    /// Initialize holder object, variant 1: object derives from enable_shared_from_this
-              typename std::enable_if<!std::is_same<std::shared_ptr<type>, T>::value, int>::type = 0>
+    template <typename T>
-    static void init_holder(PyObject *inst_) {
+    static void init_holder_helper(instance_type *inst, const holder_type * /* unused */, const std::enable_shared_from_this<T> * /* dummy */) {
-        instance_type *inst = (instance_type *) inst_;
-        new (&inst->holder) holder_type(inst->value);
-        inst->constructed = true;
-    }
-
-    template <typename T = holder_type,
-              typename std::enable_if<std::is_same<std::shared_ptr<type>, T>::value, int>::type = 0>
-    static void init_holder(PyObject *inst_) {
-        instance_type *inst = (instance_type *) inst_;
         try {
-            new (&inst->holder) holder_type(
+            new (&inst->holder) holder_type(inst->value->shared_from_this());
-                inst->value->shared_from_this());
         } catch (const std::bad_weak_ptr &) {
             new (&inst->holder) holder_type(inst->value);
         }
+    }
+
+    /// Initialize holder object, variant 2: try to construct from existing holder object, if possible
+    template <typename T = holder_type,
+              typename std::enable_if<std::is_copy_constructible<T>::value, int>::type = 0>
+    static void init_holder_helper(instance_type *inst, const holder_type *holder_ptr, const void * /* dummy */) {
+        if (holder_ptr)
+            new (&inst->holder) holder_type(*holder_ptr);
+        else
+            new (&inst->holder) holder_type(inst->value);
+    }
+
+    /// Initialize holder object, variant 3: holder is not copy constructible (e.g. unique_ptr), always initialize from raw pointer
+    template <typename T = holder_type,
+              typename std::enable_if<!std::is_copy_constructible<T>::value, int>::type = 0>
+    static void init_holder_helper(instance_type *inst, const holder_type * /* unused */, const void * /* dummy */) {
+        new (&inst->holder) holder_type(inst->value);
+    }
+
+    /// Initialize holder object of an instance, possibly given a pointer to an existing holder
+    static void init_holder(PyObject *inst_, const void *holder_ptr) {
+        auto inst = (instance_type *) inst_;
+        init_holder_helper(inst, (const holder_type *) holder_ptr, inst->value);
         inst->constructed = true;
     }
 
@@ -964,21 +977,21 @@ template <typename... Args> struct init {
 
 PYBIND11_NOINLINE inline void keep_alive_impl(int Nurse, int Patient, PyObject *arg, PyObject *ret) {
     /* Clever approach based on weak references taken from Boost.Python */
-    PyObject *nurse   =   Nurse > 0 ? PyTuple_GetItem(arg, Nurse - 1)   : ret;
+    handle nurse  (Nurse   > 0 ? PyTuple_GetItem(arg, Nurse   - 1) : ret);
-    PyObject *patient = Patient > 0 ? PyTuple_GetItem(arg, Patient - 1) : ret;
+    handle patient(Patient > 0 ? PyTuple_GetItem(arg, Patient - 1) : ret);
 
-    if (nurse == nullptr || patient == nullptr)
+    if (!nurse || !patient)
-        throw std::runtime_error("Could not activate keep_alive");
+        throw std::runtime_error("Could not activate keep_alive!");
 
     cpp_function disable_lifesupport(
-        [patient](handle weakref) { Py_DECREF(patient); weakref.dec_ref(); }
+        [patient](handle weakref) { patient.dec_ref(); weakref.dec_ref(); });
-   );
 
-    PyObject *weakref = PyWeakref_NewRef(nurse, disable_lifesupport.ptr());
+    weakref wr(nurse, disable_lifesupport);
-    if (weakref == nullptr)
+    if (!wr)
         throw std::runtime_error("Could not allocate weak reference!");
 
-    Py_INCREF(patient); /* reference patient and leak the weak reference */
+    patient.inc_ref(); /* reference patient and leak the weak reference */
+    (void) wr.release();
 }
 
 template <int Nurse, int Patient> struct process_dynamic<keep_alive<Nurse, Patient>> : public process_dynamic<void> {

include/pybind11/pytypes.h

@@ -334,6 +334,12 @@ public:
     operator double() const { return (double) PyFloat_AsDouble(m_ptr); }
 };
 
+class weakref : public object {
+public:
+    PYBIND11_OBJECT_DEFAULT(weakref, object, PyWeakref_Check)
+    weakref(handle obj, handle callback = handle()) : object(PyWeakref_NewRef(obj.ptr(), callback.ptr()), false) { }
+};
+
 class slice : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(slice, object, PySlice_Check)