convenience wrapper for constructing iterators (fixes #142)

README.md

@@ -45,6 +45,7 @@ pybind11 can map the following core C++ features to Python
 - Callbacks
 - Custom operators
 - STL data structures
+- Iterators and ranges
 - Smart pointers with reference counting like `std::shared_ptr`
 - Internal references with correct reference counting
 - C++ classes with virtual (and pure virtual) methods can be extended in Python
@@ -78,6 +79,9 @@ In addition to the core functionality, pybind11 provides some extra goodies:
   return value deduction) are used to precompute function signatures at compile
   time, leading to smaller binaries.
 
+- With little extra effort, C++ types can be pickled and unpickled similar to
+  regular Python objects.
+
 ## Supported compilers
 
 1. Clang/LLVM (any non-ancient version with C++11 support)

docs/advanced.rst

@@ -387,8 +387,8 @@ out of the box with just the core :file:`pybind11/pybind11.h` header.
     The file :file:`example/example2.cpp` contains a complete example that
     demonstrates how to pass STL data types in more detail.
 
-Binding sequence data types, the slicing protocol, etc.
+Binding sequence data types, iterators, the slicing protocol, etc.
-=======================================================
+==================================================================
 
 Please refer to the supplemental example for details.
 

docs/changelog.rst

@@ -7,8 +7,12 @@ Changelog
 ----------------------
 * For polymorphic types, use RTTI to try to return the closest type registered with pybind11.
 * Pickling support for serializing and unserializing C++ instances to a byte stream in Python
-* Added a variadic ``make_tuple()`` function
+* Added a convenience routine ``make_iterator()`` which turns a range indicated
-* Address a rare issue that could confuse the current virtual function dispatcher
+  by a pair of C++ iterators into a iterable Python object
+* Added ``len()`` and a variadic ``make_tuple()`` function
+* Addressed a rare issue that could confuse the current virtual function dispatcher
+* Added a ``get_include()`` function to the Python module that returns the path
+  of the directory containing the installed pybind11 header files
 * Documentation improvements: import issues, symbol visibility, pickling, limitations
 
 1.4 (April 7, 2016)

docs/intro.rst

@@ -37,6 +37,7 @@ The following core C++ features can be mapped to Python
 - Instance attributes and static attributes
 - Exceptions
 - Enumerations
+- Iterators and ranges
 - Callbacks
 - Custom operators
 - STL data structures
@@ -74,6 +75,9 @@ In addition to the core functionality, pybind11 provides some extra goodies:
   return value deduction) are used to precompute function signatures at compile
   time, leading to smaller binaries.
 
+- With little extra effort, C++ types can be pickled and unpickled similar to
+  regular Python objects.
+
 Supported compilers
 *******************
 

example/example6.cpp

@@ -101,28 +101,14 @@ public:
 
     size_t size() const { return m_size; }
 
+    const float *begin() const { return m_data; }
+    const float *end() const { return m_data+m_size; }
+
 private:
     size_t m_size;
     float *m_data;
 };
 
-namespace {
-    // Special iterator data structure for python
-    struct PySequenceIterator {
-        PySequenceIterator(const Sequence &seq, py::object ref) : seq(seq), ref(ref) { }
-
-        float next() {
-            if (index == seq.size())
-                throw py::stop_iteration();
-            return seq[index++];
-        }
-
-        const Sequence &seq;
-        py::object ref; // keep a reference
-        size_t index = 0;
-    };
-};
-
 void init_ex6(py::module &m) {
     py::class_<Sequence> seq(m, "Sequence");
 
@@ -141,7 +127,8 @@ void init_ex6(py::module &m) {
         })
        .def("__len__", &Sequence::size)
        /// Optional sequence protocol operations
-       .def("__iter__", [](py::object s) { return PySequenceIterator(s.cast<const Sequence &>(), s); })
+       .def("__iter__", [](const Sequence &s) { return py::make_iterator(s.begin(), s.end()); },
+                        py::keep_alive<0, 1>() /* Essential: keep object alive while iterator exists */)
        .def("__contains__", [](const Sequence &s, float v) { return s.contains(v); })
        .def("__reversed__", [](const Sequence &s) -> Sequence { return s.reversed(); })
        /// Slicing protocol (optional)
@@ -170,7 +157,30 @@ void init_ex6(py::module &m) {
        .def(py::self != py::self);
        // Could also define py::self + py::self for concatenation, etc.
 
+#if 0
+    // Obsolete: special data structure for exposing custom iterator types to python
+    // kept here for illustrative purposes because there might be some use cases which
+    // are not covered by the much simpler py::make_iterator
+
+    struct PySequenceIterator {
+        PySequenceIterator(const Sequence &seq, py::object ref) : seq(seq), ref(ref) { }
+
+        float next() {
+            if (index == seq.size())
+                throw py::stop_iteration();
+            return seq[index++];
+        }
+
+        const Sequence &seq;
+        py::object ref; // keep a reference
+        size_t index = 0;
+    };
+
     py::class_<PySequenceIterator>(seq, "Iterator")
         .def("__iter__", [](PySequenceIterator &it) -> PySequenceIterator& { return it; })
         .def("__next__", &PySequenceIterator::next);
+
+    On the actual Sequence object, the iterator would be constructed as follows:
+    .def("__iter__", [](py::object s) { return PySequenceIterator(s.cast<const Sequence &>(), s); })
+#endif
 }

include/pybind11/pybind11.h

@@ -546,9 +546,12 @@ protected:
         internals.registered_types_cpp[std::type_index(*(rec->type))] = tinfo;
         internals.registered_types_py[type] = tinfo;
 
-        auto scope_module = (object) rec->scope.attr("__module__");
+        object scope_module;
-        if (!scope_module)
+        if (rec->scope) {
-            scope_module = (object) rec->scope.attr("__name__");
+            scope_module = (object) rec->scope.attr("__module__");
+            if (!scope_module)
+                scope_module = (object) rec->scope.attr("__name__");
+        }
 
         std::string full_name = (scope_module ? ((std::string) scope_module.str() + "." + rec->name)
                                               : std::string(rec->name));
@@ -560,7 +563,9 @@ protected:
 
 #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
         /* Qualified names for Python >= 3.3 */
-        auto scope_qualname = (object) rec->scope.attr("__qualname__");
+        object scope_qualname;
+        if (rec->scope)
+            scope_qualname = (object) rec->scope.attr("__qualname__");
         if (scope_qualname) {
             type->ht_qualname = PyUnicode_FromFormat(
                 "%U.%U", scope_qualname.ptr(), name.ptr());
@@ -608,7 +613,8 @@ protected:
             attr("__module__") = scope_module;
 
         /* Register type with the parent scope */
-        rec->scope.attr(handle(type->ht_name)) = *this;
+        if (rec->scope)
+            rec->scope.attr(handle(type->ht_name)) = *this;
 
         type_holder.release();
     }
@@ -985,10 +991,28 @@ PYBIND11_NOINLINE inline void keep_alive_impl(int Nurse, int Patient, handle arg
     (void) wr.release();
 }
 
+template <typename Iterator> struct iterator_state { Iterator it, end; };
+
 NAMESPACE_END(detail)
 
 template <typename... Args> detail::init<Args...> init() { return detail::init<Args...>(); }
 
+template <typename Iterator, typename... Extra> iterator make_iterator(Iterator first, Iterator last, Extra&&... extra) {
+    typedef detail::iterator_state<Iterator> state;
+
+    if (!detail::get_type_info(typeid(state))) {
+        class_<state>(handle(), "")
+            .def("__iter__", [](state &s) -> state& { return s; })
+            .def("__next__", [](state &s) -> decltype(*std::declval<Iterator>()) & {
+                if (s.it == s.end)
+                    throw stop_iteration();
+                return *s.it++;
+            }, return_value_policy::reference_internal, std::forward<Extra>(extra)...);
+    }
+
+    return (iterator) cast(state { first, last });
+}
+
 template <typename InputType, typename OutputType> void implicitly_convertible() {
     auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * {
         if (!detail::type_caster<InputType>().load(obj, false))

include/pybind11/pytypes.h

@@ -86,22 +86,6 @@ public:
     }
 };
 
-class iterator : public object {
-public:
-    iterator(handle obj, bool borrowed = false) : object(obj, borrowed) { ++*this; }
-    iterator& operator++() {
-        if (ptr())
-            value = object(PyIter_Next(m_ptr), false);
-        return *this;
-    }
-    bool operator==(const iterator &it) const { return *it == **this; }
-    bool operator!=(const iterator &it) const { return *it != **this; }
-    const handle &operator*() const { return value; }
-    bool check() const { return PyIter_Check(ptr()); }
-private:
-    object value;
-};
-
 NAMESPACE_BEGIN(detail)
 inline handle get_function(handle value) {
     if (value) {
@@ -230,12 +214,6 @@ private:
 
 NAMESPACE_END(detail)
 
-inline detail::accessor handle::operator[](handle key) const { return detail::accessor(ptr(), key.ptr(), false); }
-inline detail::accessor handle::operator[](const char *key) const { return detail::accessor(ptr(), key, false); }
-inline detail::accessor handle::attr(handle key) const { return detail::accessor(ptr(), key.ptr(), true); }
-inline detail::accessor handle::attr(const char *key) const { return detail::accessor(ptr(), key, true); }
-inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr())); }
-inline iterator handle::end() const { return iterator(nullptr); }
 
 #define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, CvtStmt) \
     Name(const handle &h, bool borrowed) : Parent(h, borrowed) { CvtStmt; } \
@@ -252,6 +230,33 @@ inline iterator handle::end() const { return iterator(nullptr); }
     PYBIND11_OBJECT(Name, Parent, CheckFun) \
     Name() : Parent() { }
 
+class iterator : public object {
+public:
+    PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check)
+    iterator(handle obj, bool borrowed = false) : object(obj, borrowed) { }
+    iterator& operator++() {
+        if (ptr())
+            value = object(PyIter_Next(m_ptr), false);
+        return *this;
+    }
+    bool operator==(const iterator &it) const { return *it == **this; }
+    bool operator!=(const iterator &it) const { return *it != **this; }
+    const handle &operator*() const {
+        if (m_ptr && !value)
+            value = object(PyIter_Next(m_ptr), false);
+        return value;
+    }
+private:
+    mutable object value;
+};
+
+inline detail::accessor handle::operator[](handle key) const { return detail::accessor(ptr(), key.ptr(), false); }
+inline detail::accessor handle::operator[](const char *key) const { return detail::accessor(ptr(), key, false); }
+inline detail::accessor handle::attr(handle key) const { return detail::accessor(ptr(), key.ptr(), true); }
+inline detail::accessor handle::attr(const char *key) const { return detail::accessor(ptr(), key, true); }
+inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr())); }
+inline iterator handle::end() const { return iterator(nullptr); }
+
 class str : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(str, object, PyUnicode_Check)