diff --git a/.gitignore b/.gitignore index 16701940f..d1676c9b3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ CMakeFiles Makefile cmake_install.cmake .DS_Store -/example/example.so +/example/example*.so /example/example.cpython*.so /example/example.pyd /example/example*.dll @@ -31,3 +31,4 @@ MANIFEST .DS_Store /dist /build +/cmake/ diff --git a/README.md b/README.md index f9a8ed521..b2129c1c9 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,9 @@ Tomasz Miąsko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, -Boris Schäling, and -Pim Schellart. +Boris Schäling, +Pim Schellart, +Ivan Smirnov. ### License diff --git a/docs/advanced.rst b/docs/advanced.rst index c68c33aff..04dde6e06 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like py::class_(m, "Matrix") .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value, /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::format(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }); @@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix). py::buffer_info info = b.request(); /* Some sanity checks ... */ - if (info.format != py::format_descriptor::value) + if (info.format != py::format_descriptor::format()) throw std::runtime_error("Incompatible format: expected a double array!"); if (info.ndim != 2) @@ -1299,7 +1299,7 @@ as follows: m.data(), /* Pointer to buffer */ sizeof(Scalar), /* Size of one scalar */ /* Python struct-style format descriptor */ - py::format_descriptor::value, + py::format_descriptor::format(), /* Number of dimensions */ 2, /* Buffer dimensions */ @@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted into an array satisfying the specified requirements instead of trying the next function overload. +NumPy structured types +====================== + +In order for ``py::array_t`` to work with structured (record) types, we first need +to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE`` +macro which expects the type followed by field names: + +.. code-block:: cpp + + struct A { + int x; + double y; + }; + + struct B { + int z; + A a; + }; + + PYBIND11_NUMPY_DTYPE(A, x, y); + PYBIND11_NUMPY_DTYPE(B, z, a); + + /* now both A and B can be used as template arguments to py::array_t */ + Vectorizing functions ===================== @@ -1433,17 +1457,11 @@ simply using ``vectorize``). if (buf1.ndim != 1 || buf2.ndim != 1) throw std::runtime_error("Number of dimensions must be one"); - if (buf1.shape[0] != buf2.shape[0]) + if (buf1.size != buf2.size) throw std::runtime_error("Input shapes must match"); - auto result = py::array(py::buffer_info( - nullptr, /* Pointer to data (nullptr -> ask NumPy to allocate!) */ - sizeof(double), /* Size of one item */ - py::format_descriptor::value, /* Buffer format */ - buf1.ndim, /* How many dimensions? */ - { buf1.shape[0] }, /* Number of elements for each dimension */ - { sizeof(double) } /* Strides for each dimension */ - )); + /* No pointer is passed, so NumPy will allocate the buffer */ + auto result = py::array_t(buf1.size); auto buf3 = result.request(); @@ -1830,4 +1848,3 @@ is always ``none``). // Evaluate the statements in an separate Python file on disk py::eval_file("script.py", scope); - diff --git a/docs/changelog.rst b/docs/changelog.rst index 86ad18a8f..def3ceb71 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released) * ``make_iterator()`` improvements for better compatibility with various types (now uses prefix increment operator) * ``arg()`` now accepts a wider range of argument types for default values +* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro. +* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type. +* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers. +* Format descriptor strings should now be accessed via ``format_descriptor::format()`` + (for compatibility purposes, the old syntax ``format_descriptor::value`` will still + work for non-structured data types). +* Added a class wrapping NumPy array descriptors: ``dtype``. +* Added buffer/NumPy support for ``char[N]`` and ``std::array`` types. +* ``array`` gained new constructors accepting dtype objects. +* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and + strides; if strides are not provided, they are deduced assuming C-contiguity. + Also added simplified constructors for 1-dimensional case. +* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``. + This will do the UTF-8 decoding/encoding as required. +* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers, + and from char pointers and length. +* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``. * Various minor improvements of library internals (no user-visible changes) 1.8.1 (July 12, 2016) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 22d44c808..2cc8f8326 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES example-stl-binder-vector.cpp example-eval.cpp example-custom-exceptions.cpp + example-numpy-dtypes.cpp issues.cpp ) @@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES}) string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}") add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME}) endforeach() - diff --git a/example/example-buffers.cpp b/example/example-buffers.cpp index 17c8d271f..fa3178b51 100644 --- a/example/example-buffers.cpp +++ b/example/example-buffers.cpp @@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) { /// Construct from a buffer .def("__init__", [](Matrix &v, py::buffer b) { py::buffer_info info = b.request(); - if (info.format != py::format_descriptor::value || info.ndim != 2) + if (info.format != py::format_descriptor::format() || info.ndim != 2) throw std::runtime_error("Incompatible buffer format!"); new (&v) Matrix(info.shape[0], info.shape[1]); memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols()); @@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) { /// Provide buffer access .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - sizeof(float), /* Size of one scalar */ - py::format_descriptor::value, /* Python struct-style format descriptor */ - 2, /* Number of dimensions */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ + m.data(), /* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + py::format_descriptor::format(), /* Python struct-style format descriptor */ + 2, /* Number of dimensions */ + { m.rows(), m.cols() }, /* Buffer dimensions */ + { sizeof(float) * m.rows(), /* Strides (in bytes) for each index */ sizeof(float) } ); }) diff --git a/example/example-numpy-dtypes.cpp b/example/example-numpy-dtypes.cpp new file mode 100644 index 000000000..e5292bba0 --- /dev/null +++ b/example/example-numpy-dtypes.cpp @@ -0,0 +1,280 @@ +/* + example/example-numpy-dtypes.cpp -- Structured and compound NumPy dtypes + + Copyright (c) 2016 Ivan Smirnov + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include "example.h" + +#include +#include +#include + +#ifdef __GNUC__ +#define PYBIND11_PACKED(cls) cls __attribute__((__packed__)) +#else +#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop)) +#endif + +namespace py = pybind11; + +struct SimpleStruct { + bool x; + uint32_t y; + float z; +}; + +std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { + return os << "s:" << v.x << "," << v.y << "," << v.z; +} + +PYBIND11_PACKED(struct PackedStruct { + bool x; + uint32_t y; + float z; +}); + +std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { + return os << "p:" << v.x << "," << v.y << "," << v.z; +} + +PYBIND11_PACKED(struct NestedStruct { + SimpleStruct a; + PackedStruct b; +}); + +std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { + return os << "n:a=" << v.a << ";b=" << v.b; +} + +struct PartialStruct { + bool x; + uint32_t y; + float z; + uint64_t dummy2; +}; + +struct PartialNestedStruct { + uint64_t dummy1; + PartialStruct a; + uint64_t dummy2; +}; + +struct UnboundStruct { }; + +struct StringStruct { + char a[3]; + std::array b; +}; + +std::ostream& operator<<(std::ostream& os, const StringStruct& v) { + os << "a='"; + for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i]; + os << "',b='"; + for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i]; + return os << "'"; +} + +template +py::array mkarray_via_buffer(size_t n) { + return py::array(py::buffer_info(nullptr, sizeof(T), + py::format_descriptor::format(), + 1, { n }, { sizeof(T) })); +} + +template +py::array_t create_recarray(size_t n) { + auto arr = mkarray_via_buffer(n); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; + } + return arr; +} + +std::string get_format_unbound() { + return py::format_descriptor::format(); +} + +py::array_t create_nested(size_t n) { + auto arr = mkarray_via_buffer(n); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; + } + return arr; +} + +py::array_t create_partial_nested(size_t n) { + auto arr = mkarray_via_buffer(n); + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < n; i++) { + ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + } + return arr; +} + +py::array_t create_string_array(bool non_empty) { + auto arr = mkarray_via_buffer(non_empty ? 4 : 0); + if (non_empty) { + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < req.size * req.itemsize; i++) + static_cast(req.ptr)[i] = 0; + ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a'; + ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a'; + ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a'; + + ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b'; + ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b'; + + ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c'; + } + return arr; +} + +template +void print_recarray(py::array_t arr) { + auto req = arr.request(); + auto ptr = static_cast(req.ptr); + for (size_t i = 0; i < req.size; i++) + std::cout << ptr[i] << std::endl; +} + +void print_format_descriptors() { + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; + std::cout << py::format_descriptor::format() << std::endl; +} + +void print_dtypes() { + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; + std::cout << (std::string) py::dtype::of().str() << std::endl; +} + +py::array_t test_array_ctors(int i) { + using arr_t = py::array_t; + + std::vector data { 1, 2, 3, 4, 5, 6 }; + std::vector shape { 3, 2 }; + std::vector strides { 8, 4 }; + + auto ptr = data.data(); + auto vptr = (void *) ptr; + auto dtype = py::dtype("int32"); + + py::buffer_info buf_ndim1(vptr, 4, "i", 6); + py::buffer_info buf_ndim1_null(nullptr, 4, "i", 6); + py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides); + py::buffer_info buf_ndim2_null(nullptr, 4, "i", 2, shape, strides); + + auto fill = [](py::array arr) { + auto req = arr.request(); + for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1; + return arr; + }; + + switch (i) { + // shape: (3, 2) + case 10: return arr_t(shape, strides, ptr); + case 11: return py::array(shape, strides, ptr); + case 12: return py::array(dtype, shape, strides, vptr); + case 13: return arr_t(shape, ptr); + case 14: return py::array(shape, ptr); + case 15: return py::array(dtype, shape, vptr); + case 16: return arr_t(buf_ndim2); + case 17: return py::array(buf_ndim2); + // shape: (3, 2) - post-fill + case 20: return fill(arr_t(shape, strides)); + case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor + case 22: return fill(py::array(dtype, shape, strides)); + case 23: return fill(arr_t(shape)); + case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor + case 25: return fill(py::array(dtype, shape)); + case 26: return fill(arr_t(buf_ndim2_null)); + case 27: return fill(py::array(buf_ndim2_null)); + // shape: (6, ) + case 30: return arr_t(6, ptr); + case 31: return py::array(6, ptr); + case 32: return py::array(dtype, 6, vptr); + case 33: return arr_t(buf_ndim1); + case 34: return py::array(buf_ndim1); + // shape: (6, ) + case 40: return fill(arr_t(6)); + case 41: return py::array(6, ptr); // can't have nullptr due to templated ctor + case 42: return fill(py::array(dtype, 6)); + case 43: return fill(arr_t(buf_ndim1_null)); + case 44: return fill(py::array(buf_ndim1_null)); + } + return arr_t(); +} + +py::list test_dtype_ctors() { + py::list list; + list.append(py::dtype("int32")); + list.append(py::dtype(std::string("float64"))); + list.append(py::dtype::from_args(py::str("bool"))); + py::list names, offsets, formats; + py::dict dict; + names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names; + offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets; + formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats; + dict["itemsize"] = py::int_(20); + list.append(py::dtype::from_args(dict)); + list.append(py::dtype(names, formats, offsets, 20)); + list.append(py::dtype(py::buffer_info((void *) 0, 1, "I", 1))); + list.append(py::dtype(py::buffer_info((void *) 0, 1, "T{i:a:f:b:}", 1))); + return list; +} + +py::list test_dtype_methods() { + py::list list; + auto dt1 = py::dtype::of(); + auto dt2 = py::dtype::of(); + list.append(dt1); list.append(dt2); + list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields())); + list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize())); + return list; +} + +void init_ex_numpy_dtypes(py::module &m) { + PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); + PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a); + PYBIND11_NUMPY_DTYPE(StringStruct, a, b); + + m.def("create_rec_simple", &create_recarray); + m.def("create_rec_packed", &create_recarray); + m.def("create_rec_nested", &create_nested); + m.def("create_rec_partial", &create_recarray); + m.def("create_rec_partial_nested", &create_partial_nested); + m.def("print_format_descriptors", &print_format_descriptors); + m.def("print_rec_simple", &print_recarray); + m.def("print_rec_packed", &print_recarray); + m.def("print_rec_nested", &print_recarray); + m.def("print_dtypes", &print_dtypes); + m.def("get_format_unbound", &get_format_unbound); + m.def("create_string_array", &create_string_array); + m.def("print_string_array", &print_recarray); + m.def("test_array_ctors", &test_array_ctors); + m.def("test_dtype_ctors", &test_dtype_ctors); + m.def("test_dtype_methods", &test_dtype_methods); +} + +#undef PYBIND11_PACKED diff --git a/example/example-numpy-dtypes.py b/example/example-numpy-dtypes.py new file mode 100644 index 000000000..06185914b --- /dev/null +++ b/example/example-numpy-dtypes.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +from example import ( + create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors, + print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound, + create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array, + test_array_ctors, test_dtype_ctors, test_dtype_methods +) + + +def check_eq(arr, data, dtype): + np.testing.assert_equal(arr, np.array(data, dtype=dtype)) + +try: + get_format_unbound() + raise Exception +except RuntimeError as e: + assert 'unsupported buffer format' in str(e) + +print_format_descriptors() +print_dtypes() + +simple_dtype = np.dtype({'names': ['x', 'y', 'z'], + 'formats': ['?', 'u4', 'f4'], + 'offsets': [0, 4, 8]}) +packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) + +elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)] + +for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]: + arr = func(0) + assert arr.dtype == dtype + check_eq(arr, [], simple_dtype) + check_eq(arr, [], packed_dtype) + + arr = func(3) + assert arr.dtype == dtype + check_eq(arr, elements, simple_dtype) + check_eq(arr, elements, packed_dtype) + + if dtype == simple_dtype: + print_rec_simple(arr) + else: + print_rec_packed(arr) + +arr = create_rec_partial(3) +print(arr.dtype) +partial_dtype = arr.dtype +assert '' not in arr.dtype.fields +assert partial_dtype.itemsize > simple_dtype.itemsize +check_eq(arr, elements, simple_dtype) +check_eq(arr, elements, packed_dtype) + +arr = create_rec_partial_nested(3) +print(arr.dtype) +assert '' not in arr.dtype.fields +assert '' not in arr.dtype.fields['a'][0].fields +assert arr.dtype.itemsize > partial_dtype.itemsize +np.testing.assert_equal(arr['a'], create_rec_partial(3)) + +nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)]) + +arr = create_rec_nested(0) +assert arr.dtype == nested_dtype +check_eq(arr, [], nested_dtype) + +arr = create_rec_nested(3) +assert arr.dtype == nested_dtype +check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)), + ((True, 1, 1.5), (False, 2, 3.0)), + ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype) +print_rec_nested(arr) + +assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[NestedStruct]') + +arr = create_string_array(True) +print(arr.dtype) +print_string_array(arr) +dtype = arr.dtype +assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc'] +assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc'] +arr = create_string_array(False) +assert dtype == arr.dtype + +data = np.arange(1, 7, dtype='int32') +for i in range(8): + np.testing.assert_array_equal(test_array_ctors(10 + i), data.reshape((3, 2))) + np.testing.assert_array_equal(test_array_ctors(20 + i), data.reshape((3, 2))) +for i in range(5): + np.testing.assert_array_equal(test_array_ctors(30 + i), data) + np.testing.assert_array_equal(test_array_ctors(40 + i), data) + +d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'], + 'offsets': [1, 10], 'itemsize': 20}) +d2 = np.dtype([('a', 'i4'), ('b', 'f4')]) +assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'), + np.dtype('bool'), d1, d1, np.dtype('uint32'), d2] + +assert test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True, + np.dtype('int32').itemsize, simple_dtype.itemsize] diff --git a/example/example-numpy-dtypes.ref b/example/example-numpy-dtypes.ref new file mode 100644 index 000000000..4f07ce445 --- /dev/null +++ b/example/example-numpy-dtypes.ref @@ -0,0 +1,28 @@ +T{=?:x:3x=I:y:=f:z:} +T{=?:x:=I:y:=f:z:} +T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:} +T{=?:x:3x=I:y:=f:z:12x} +T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x} +T{=3s:a:=3s:b:} +{'names':['x','y','z'], 'formats':['?',' List[unicode[2]] | Return a C++ array - | + | | ggeett__ddiicctt(...) | Signature : (example.ExamplePythonTypes) -> dict - | + | | Return a Python dictionary - | + | | ggeett__ddiicctt__22(...) - | + | | Signature : (example.ExamplePythonTypes) -> Dict[unicode, unicode] | Return a C++ dictionary - | + | | ggeett__lliisstt(...) | Signature : (example.ExamplePythonTypes) -> list - | + | | Return a Python list - | + | | ggeett__lliisstt__22(...) - | + | | Signature : (example.ExamplePythonTypes) -> List[unicode] | Return a C++ list - | + | | ggeett__sseett(...) | Signature : (example.ExamplePythonTypes) -> set - | + | | Return a Python set - | + | | ggeett__sseett22(...) | Signature : (example.ExamplePythonTypes) -> set - | + | | Return a C++ set - | + | | ppaaiirr__ppaasssstthhrroouugghh(...) - | + | | Signature : (example.ExamplePythonTypes, Tuple[bool, unicode]) -> Tuple[unicode, bool] | Return a pair in reversed order - | + | | pprriinntt__aarrrraayy(...) - | + | | Signature : (example.ExamplePythonTypes, List[unicode[2]]) -> None | Print entries of a C++ array - | + | | pprriinntt__ddiicctt(...) - | + | | Signature : (example.ExamplePythonTypes, dict) -> None | Print entries of a Python dictionary - | + | | pprriinntt__ddiicctt__22(...) - | + | | Signature : (example.ExamplePythonTypes, Dict[unicode, unicode]) -> None | Print entries of a C++ dictionary - | + | | pprriinntt__lliisstt(...) - | + | | Signature : (example.ExamplePythonTypes, list) -> None | Print entries of a Python list - | + | | pprriinntt__lliisstt__22(...) - | + | | Signature : (example.ExamplePythonTypes, List[unicode]) -> None | Print entries of a C++ list - | + | | pprriinntt__sseett(...) - | + | | Signature : (example.ExamplePythonTypes, set) -> None | Print entries of a Python set - | + | | pprriinntt__sseett__22(...) - | + | | Signature : (example.ExamplePythonTypes, Set[unicode]) -> None | Print entries of a C++ set - | + | | tthhrrooww__eexxcceeppttiioonn(...) - | + | | Signature : (example.ExamplePythonTypes) -> None | Throw an exception - | + | | ttuuppllee__ppaasssstthhrroouugghh(...) - | + | | Signature : (example.ExamplePythonTypes, Tuple[bool, unicode, int]) -> Tuple[int, unicode, bool] | Return a triple in reversed order - | + | | ---------------------------------------------------------------------- | Data and other attributes defined here: - | + | | ____nneeww____ = | T.__new__(S, ...) -> a new object with type S, a subtype of T - | + | | nneeww__iinnssttaannccee = | Signature : () -> example.ExamplePythonTypes - | + | | Return an instance __name__(example) = example @@ -135,6 +135,10 @@ __name__(example.ExamplePythonTypes) = ExamplePythonTypes __module__(example.ExamplePythonTypes) = example __name__(example.ExamplePythonTypes.get_set) = get_set __module__(example.ExamplePythonTypes.get_set) = example +foo +bar +baz +boo Instances not destroyed: 1 ### ExamplePythonTypes @ 0x1045b80 destroyed Instances not destroyed: 0 diff --git a/example/example.cpp b/example/example.cpp index b831e3e84..bd6ac9b35 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -29,6 +29,7 @@ void init_ex_inheritance(py::module &); void init_ex_stl_binder_vector(py::module &); void init_ex_eval(py::module &); void init_ex_custom_exceptions(py::module &); +void init_ex_numpy_dtypes(py::module &); void init_issues(py::module &); #if defined(PYBIND11_TEST_EIGEN) @@ -72,6 +73,7 @@ PYBIND11_PLUGIN(example) { init_ex_stl_binder_vector(m); init_ex_eval(m); init_ex_custom_exceptions(m); + init_ex_numpy_dtypes(m); init_issues(m); #if defined(PYBIND11_TEST_EIGEN) diff --git a/example/run_test.py b/example/run_test.py index a11c3fc68..2785c8eaa 100755 --- a/example/run_test.py +++ b/example/run_test.py @@ -46,12 +46,11 @@ name = sys.argv[1] try: output_bytes = subprocess.check_output([sys.executable, "-u", name + ".py"], stderr=subprocess.STDOUT) -except subprocess.CalledProcessError as e: - if e.returncode == 99: - print('Test "%s" could not be run.' % name) - exit(0) - else: - raise +except subprocess.CalledProcessError as exc: + print('Test `{}` failed:\n{}\n'.format(name, '-' * 50)) + print(exc.output.decode()) + print('-' * 50) + sys.exit(1) output = sanitize(output_bytes.decode('utf-8')) reference = sanitize(open(name + '.ref', 'r').read()) diff --git a/include/pybind11/common.h b/include/pybind11/common.h index 0b2092920..9db37adc8 100644 --- a/include/pybind11/common.h +++ b/include/pybind11/common.h @@ -95,6 +95,7 @@ #define PYBIND11_STRING_NAME "str" #define PYBIND11_SLICE_OBJECT PyObject #define PYBIND11_FROM_STRING PyUnicode_FromString +#define PYBIND11_STR_TYPE ::pybind11::str #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type #define PYBIND11_PLUGIN_IMPL(name) \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() @@ -113,6 +114,7 @@ #define PYBIND11_STRING_NAME "unicode" #define PYBIND11_SLICE_OBJECT PySliceObject #define PYBIND11_FROM_STRING PyString_FromString +#define PYBIND11_STR_TYPE ::pybind11::bytes #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type #define PYBIND11_PLUGIN_IMPL(name) \ extern "C" PYBIND11_EXPORT PyObject *init##name() @@ -204,12 +206,13 @@ struct buffer_info { void *ptr; // Pointer to the underlying storage size_t itemsize; // Size of individual items in bytes size_t size; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor::value + std::string format; // For homogeneous buffers, this should be set to format_descriptor::format() size_t ndim; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) std::vector strides; // Number of entries between adjacent entries (for each per dimension) buffer_info() : ptr(nullptr), view(nullptr) {} + buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim, const std::vector &shape, const std::vector &strides) : ptr(ptr), itemsize(itemsize), size(1), format(format), @@ -218,6 +221,10 @@ struct buffer_info { size *= shape[i]; } + buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size) + : buffer_info(ptr, itemsize, format, 1, std::vector { size }, + std::vector { itemsize }) { } + buffer_info(Py_buffer *view) : ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format), ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view) { @@ -231,6 +238,7 @@ struct buffer_info { ~buffer_info() { if (view) { PyBuffer_Release(view); delete view; } } + private: Py_buffer *view = nullptr; }; @@ -323,14 +331,23 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); } /// Format strings for basic number types -#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor { static constexpr const char *value = v; } +#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor \ + { static constexpr const char* value = v; /* for backwards compatibility */ \ + static constexpr const char* format() { return value; } } + template struct format_descriptor { }; + template struct format_descriptor::value>::type> { static constexpr const char value[2] = { "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned::value ? 1 : 0)], '\0' }; + static constexpr const char* format() { return value; } }; + template constexpr const char format_descriptor< T, typename std::enable_if::value>::type>::value[2]; -PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); PYBIND11_DECL_FMT(bool, "?"); + +PYBIND11_DECL_FMT(float, "f"); +PYBIND11_DECL_FMT(double, "d"); +PYBIND11_DECL_FMT(bool, "?"); NAMESPACE_END(pybind11) diff --git a/include/pybind11/descr.h b/include/pybind11/descr.h index 4123cc1b3..e4a504e02 100644 --- a/include/pybind11/descr.h +++ b/include/pybind11/descr.h @@ -181,7 +181,7 @@ PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; } template PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward(args)...); } PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); } -#define PYBIND11_DESCR descr +#define PYBIND11_DESCR ::pybind11::detail::descr #endif NAMESPACE_END(detail) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index bfb934060..2ea7d486b 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -133,7 +133,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value, + format_descriptor::format(), /* Number of dimensions */ 1, /* Buffer dimensions */ @@ -148,7 +148,7 @@ struct type_caster::value && /* Size of one scalar */ sizeof(Scalar), /* Python struct-style format descriptor */ - format_descriptor::value, + format_descriptor::format(), /* Number of dimensions */ isVector ? 1 : 2, /* Buffer dimensions */ @@ -233,7 +233,7 @@ struct type_caster::value>:: try { obj = matrix_type(obj); } catch (const error_already_set &) { - PyErr_Clear(); + PyErr_Clear(); return false; } } @@ -276,7 +276,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(Scalar), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions @@ -291,7 +291,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions @@ -306,7 +306,7 @@ struct type_caster::value>:: // Size of one scalar sizeof(StorageIndex), // Python struct-style format descriptor - format_descriptor::value, + format_descriptor::format(), // Number of dimensions 1, // Buffer dimensions diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 5d355e2fc..c71d9bbca 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -13,6 +13,11 @@ #include "complex.h" #include #include +#include +#include +#include +#include +#include #if defined(_MSC_VER) #pragma warning(push) @@ -20,123 +25,275 @@ #endif NAMESPACE_BEGIN(pybind11) -namespace detail { template struct npy_format_descriptor { }; } +namespace detail { +template struct npy_format_descriptor { }; +template struct is_pod_struct; + +struct npy_api { + enum constants { + NPY_C_CONTIGUOUS_ = 0x0001, + NPY_F_CONTIGUOUS_ = 0x0002, + NPY_ARRAY_FORCECAST_ = 0x0010, + NPY_ENSURE_ARRAY_ = 0x0040, + NPY_BOOL_ = 0, + NPY_BYTE_, NPY_UBYTE_, + NPY_SHORT_, NPY_USHORT_, + NPY_INT_, NPY_UINT_, + NPY_LONG_, NPY_ULONG_, + NPY_LONGLONG_, NPY_ULONGLONG_, + NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, + NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_OBJECT_ = 17, + NPY_STRING_, NPY_UNICODE_, NPY_VOID_ + }; + + static npy_api& get() { + static npy_api api = lookup(); + return api; + } + + bool PyArray_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArray_Type_); + } + bool PyArrayDescr_Check_(PyObject *obj) const { + return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_); + } + + PyObject *(*PyArray_DescrFromType_)(int); + PyObject *(*PyArray_NewFromDescr_) + (PyTypeObject *, PyObject *, int, Py_intptr_t *, + Py_intptr_t *, void *, int, PyObject *); + PyObject *(*PyArray_DescrNewFromType_)(int); + PyObject *(*PyArray_NewCopy_)(PyObject *, int); + PyTypeObject *PyArray_Type_; + PyTypeObject *PyArrayDescr_Type_; + PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_) (PyObject *, PyObject **); + bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *, + Py_ssize_t *, PyObject **, PyObject *); +private: + enum functions { + API_PyArray_Type = 2, + API_PyArrayDescr_Type = 3, + API_PyArray_DescrFromType = 45, + API_PyArray_FromAny = 69, + API_PyArray_NewCopy = 85, + API_PyArray_NewFromDescr = 94, + API_PyArray_DescrNewFromType = 9, + API_PyArray_DescrConverter = 174, + API_PyArray_EquivTypes = 182, + API_PyArray_GetArrayParamsFromObject = 278, + }; + + static npy_api lookup() { + module m = module::import("numpy.core.multiarray"); + object c = (object) m.attr("_ARRAY_API"); +#if PY_MAJOR_VERSION >= 3 + void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); +#else + void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); +#endif + npy_api api; +#define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; + DECL_NPY_API(PyArray_Type); + DECL_NPY_API(PyArrayDescr_Type); + DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_FromAny); + DECL_NPY_API(PyArray_NewCopy); + DECL_NPY_API(PyArray_NewFromDescr); + DECL_NPY_API(PyArray_DescrNewFromType); + DECL_NPY_API(PyArray_DescrConverter); + DECL_NPY_API(PyArray_EquivTypes); + DECL_NPY_API(PyArray_GetArrayParamsFromObject); +#undef DECL_NPY_API + return api; + } +}; +} + +class dtype : public object { +public: + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); + + dtype(const buffer_info &info) { + dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); + m_ptr = descr.strip_padding().release().ptr(); + } + + dtype(std::string format) { + m_ptr = from_args(pybind11::str(format)).release().ptr(); + } + + dtype(const char *format) : dtype(std::string(format)) { } + + dtype(list names, list formats, list offsets, size_t itemsize) { + dict args; + args["names"] = names; + args["formats"] = formats; + args["offsets"] = offsets; + args["itemsize"] = int_(itemsize); + m_ptr = from_args(args).release().ptr(); + } + + static dtype from_args(object args) { + // This is essentially the same as calling np.dtype() constructor in Python + PyObject *ptr = nullptr; + if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr) + pybind11_fail("NumPy: failed to create structured dtype"); + return object(ptr, false); + } + + template static dtype of() { + return detail::npy_format_descriptor::dtype(); + } + + size_t itemsize() const { + return (size_t) attr("itemsize").cast(); + } + + bool has_fields() const { + return attr("fields").cast().ptr() != Py_None; + } + + std::string kind() const { + return (std::string) attr("kind").cast(); + } + +private: + static object _dtype_from_pep3118() { + static PyObject *obj = module::import("numpy.core._internal") + .attr("_dtype_from_pep3118").cast().release().ptr(); + return object(obj, true); + } + + dtype strip_padding() { + // Recursively strip all void fields with empty names that are generated for + // padding fields (as of NumPy v1.11). + auto fields = attr("fields").cast(); + if (fields.ptr() == Py_None) + return *this; + + struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; }; + std::vector field_descriptors; + + auto items = fields.attr("items").cast(); + for (auto field : items()) { + auto spec = object(field, true).cast(); + auto name = spec[0].cast(); + auto format = spec[1].cast()[0].cast(); + auto offset = spec[1].cast()[1].cast(); + if (!len(name) && format.kind() == "V") + continue; + field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset}); + } + + std::sort(field_descriptors.begin(), field_descriptors.end(), + [](const field_descr& a, const field_descr& b) { + return (int) a.offset < (int) b.offset; + }); + + list names, formats, offsets; + for (auto& descr : field_descriptors) { + names.append(descr.name); + formats.append(descr.format); + offsets.append(descr.offset); + } + return dtype(names, formats, offsets, itemsize()); + } +}; class array : public buffer { public: - struct API { - enum Entries { - API_PyArray_Type = 2, - API_PyArray_DescrFromType = 45, - API_PyArray_FromAny = 69, - API_PyArray_NewCopy = 85, - API_PyArray_NewFromDescr = 94, - - NPY_C_CONTIGUOUS_ = 0x0001, - NPY_F_CONTIGUOUS_ = 0x0002, - NPY_ARRAY_FORCECAST_ = 0x0010, - NPY_ENSURE_ARRAY_ = 0x0040, - NPY_BOOL_ = 0, - NPY_BYTE_, NPY_UBYTE_, - NPY_SHORT_, NPY_USHORT_, - NPY_INT_, NPY_UINT_, - NPY_LONG_, NPY_ULONG_, - NPY_LONGLONG_, NPY_ULONGLONG_, - NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_ - }; - - static API lookup() { - module m = module::import("numpy.core.multiarray"); - object c = (object) m.attr("_ARRAY_API"); -#if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr); -#else - void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr); -#endif - API api; - api.PyArray_Type_ = (decltype(api.PyArray_Type_)) api_ptr[API_PyArray_Type]; - api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType]; - api.PyArray_FromAny_ = (decltype(api.PyArray_FromAny_)) api_ptr[API_PyArray_FromAny]; - api.PyArray_NewCopy_ = (decltype(api.PyArray_NewCopy_)) api_ptr[API_PyArray_NewCopy]; - api.PyArray_NewFromDescr_ = (decltype(api.PyArray_NewFromDescr_)) api_ptr[API_PyArray_NewFromDescr]; - return api; - } - - bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); } - - PyObject *(*PyArray_DescrFromType_)(int); - PyObject *(*PyArray_NewFromDescr_) - (PyTypeObject *, PyObject *, int, Py_intptr_t *, - Py_intptr_t *, void *, int, PyObject *); - PyObject *(*PyArray_NewCopy_)(PyObject *, int); - PyTypeObject *PyArray_Type_; - PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); - }; - - PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_) + PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) enum { - c_style = API::NPY_C_CONTIGUOUS_, - f_style = API::NPY_F_CONTIGUOUS_, - forcecast = API::NPY_ARRAY_FORCECAST_ + c_style = detail::npy_api::NPY_C_CONTIGUOUS_, + f_style = detail::npy_api::NPY_F_CONTIGUOUS_, + forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ }; - template array(size_t size, const Type *ptr) { - API& api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format!"); - Py_intptr_t shape = (Py_intptr_t) size; - object tmp = object(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false); - if (ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); + array(const pybind11::dtype& dt, const std::vector& shape, + const std::vector& strides, void *ptr = nullptr) { + auto& api = detail::npy_api::get(); + auto ndim = shape.size(); + if (shape.size() != strides.size()) + pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); + auto descr = dt; + object tmp(api.PyArray_NewFromDescr_( + api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), + (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); + if (ptr) + tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); } - array(const buffer_info &info) { - API& api = lookup_api(); - if ((info.format.size() < 1) || (info.format.size() > 2)) - pybind11_fail("Unsupported buffer format!"); - int fmt = (int) info.format[0]; - if (info.format == "Zd") fmt = API::NPY_CDOUBLE_; - else if (info.format == "Zf") fmt = API::NPY_CFLOAT_; + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) + : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } - PyObject *descr = api.PyArray_DescrFromType_(fmt); - if (descr == nullptr) - pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!"); - object tmp(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0], - (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false); - if (info.ptr && tmp) - tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); - if (!tmp) - pybind11_fail("NumPy: unable to create array!"); - m_ptr = tmp.release().ptr(); + array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr) + : array(dt, std::vector { size }, ptr) { } + + template array(const std::vector& shape, + const std::vector& strides, T* ptr) + : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } + + template array(const std::vector& shape, T* ptr) + : array(shape, default_strides(shape, sizeof(T)), ptr) { } + + template array(size_t size, T* ptr) + : array(std::vector { size }, ptr) { } + + array(const buffer_info &info) + : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } + + pybind11::dtype dtype() { + return attr("dtype").cast(); } protected: - static API &lookup_api() { - static API api = API::lookup(); - return api; + template friend struct detail::npy_format_descriptor; + + static std::vector default_strides(const std::vector& shape, size_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim); + if (ndim) { + std::fill(strides.begin(), strides.end(), itemsize); + for (size_t i = 0; i < ndim - 1; i++) + for (size_t j = 0; j < ndim - 1 - i; j++) + strides[j] *= shape[ndim - 1 - i]; + } + return strides; } }; template class array_t : public array { public: PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr)); + array_t() : array() { } - array_t(const buffer_info& info) : array(info) {} + + array_t(const buffer_info& info) : array(info) { } + + array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr) + : array(shape, strides, ptr) { } + + array_t(const std::vector& shape, T* ptr = nullptr) + : array(shape, ptr) { } + + array_t(size_t size, T* ptr = nullptr) + : array(size, ptr) { } + static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } + static PyObject *ensure(PyObject *ptr) { if (ptr == nullptr) return nullptr; - API &api = lookup_api(); - PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor::value); - PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); + auto& api = detail::npy_api::get(); + PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of().release().ptr(), 0, 0, + detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr); if (!result) PyErr_Clear(); Py_DECREF(ptr); @@ -144,15 +301,47 @@ public: } }; +template +struct format_descriptor::value>::type> { + static const char *format() { return detail::npy_format_descriptor::format(); } +}; + +template struct format_descriptor { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } +}; +template struct format_descriptor> { + static const char *format() { PYBIND11_DESCR s = detail::_() + detail::_("s"); return s.text(); } +}; + NAMESPACE_BEGIN(detail) +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; + +template +struct is_pod_struct { + enum { value = std::is_pod::value && // offsetof only works correctly for POD types + !std::is_array::value && + !is_std_array::value && + !std::is_integral::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same::value && + !std::is_same>::value && + !std::is_same>::value }; +}; template struct npy_format_descriptor::value>::type> { private: constexpr static const int values[8] = { - array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_, array::API::NPY_USHORT_, - array::API::NPY_INT_, array::API::NPY_UINT_, array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ }; + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; public: enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static pybind11::dtype dtype() { + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) + return object(ptr, true); + pybind11_fail("Unsupported buffer format!"); + } template ::value, int>::type = 0> static PYBIND11_DESCR name() { return _("int") + _(); } template ::value, int>::type = 0> @@ -162,12 +351,149 @@ template constexpr const int npy_format_descriptor< T, typename std::enable_if::value>::type>::values[8]; #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = array::API::NumPyName }; \ + enum { value = npy_api::NumPyName }; \ + static pybind11::dtype dtype() { \ + if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ + return object(ptr, true); \ + pybind11_fail("Unsupported buffer format!"); \ + } \ static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); +DECL_FMT(float, NPY_FLOAT_, "float32"); +DECL_FMT(double, NPY_DOUBLE_, "float64"); +DECL_FMT(bool, NPY_BOOL_, "bool"); +DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); +DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); #undef DECL_FMT +#define DECL_CHAR_FMT \ + static PYBIND11_DESCR name() { return _("S") + _(); } \ + static pybind11::dtype dtype() { \ + PYBIND11_DESCR fmt = _("S") + _(); \ + return pybind11::dtype(fmt.text()); \ + } \ + static const char *format() { PYBIND11_DESCR s = _() + _("s"); return s.text(); } +template struct npy_format_descriptor { DECL_CHAR_FMT }; +template struct npy_format_descriptor> { DECL_CHAR_FMT }; +#undef DECL_CHAR_FMT + +struct field_descriptor { + const char *name; + size_t offset; + size_t size; + const char *format; + dtype descr; +}; + +template +struct npy_format_descriptor::value>::type> { + static PYBIND11_DESCR name() { return _("struct"); } + + static pybind11::dtype dtype() { + if (!dtype_()) + pybind11_fail("NumPy: unsupported buffer format!"); + return object(dtype_(), true); + } + + static const char* format() { + if (!dtype_()) + pybind11_fail("NumPy: unsupported buffer format!"); + return format_().c_str(); + } + + static void register_dtype(std::initializer_list fields) { + list names, formats, offsets; + for (auto field : fields) { + if (!field.descr) + pybind11_fail("NumPy: unsupported field dtype"); + names.append(PYBIND11_STR_TYPE(field.name)); + formats.append(field.descr); + offsets.append(int_(field.offset)); + } + dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr(); + + // There is an existing bug in NumPy (as of v1.11): trailing bytes are + // not encoded explicitly into the format string. This will supposedly + // get fixed in v1.12; for further details, see these: + // - https://github.com/numpy/numpy/issues/7797 + // - https://github.com/numpy/numpy/pull/7798 + // Because of this, we won't use numpy's logic to generate buffer format + // strings and will just do it ourselves. + std::vector ordered_fields(fields); + std::sort(ordered_fields.begin(), ordered_fields.end(), + [](const field_descriptor& a, const field_descriptor &b) { + return a.offset < b.offset; + }); + size_t offset = 0; + std::ostringstream oss; + oss << "T{"; + for (auto& field : ordered_fields) { + if (field.offset > offset) + oss << (field.offset - offset) << 'x'; + // note that '=' is required to cover the case of unaligned fields + oss << '=' << field.format << ':' << field.name << ':'; + offset = field.offset + field.size; + } + if (sizeof(T) > offset) + oss << (sizeof(T) - offset) << 'x'; + oss << '}'; + format_() = oss.str(); + + // Sanity check: verify that NumPy properly parses our buffer format string + auto& api = npy_api::get(); + auto arr = array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) })); + if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr())) + pybind11_fail("NumPy: invalid buffer descriptor!"); + } + +private: + static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; } + static inline std::string& format_() { static std::string s; return s; } +}; + +// Extract name, offset and format descriptor for a struct field +#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \ + ::pybind11::detail::field_descriptor { \ + #Field, offsetof(Type, Field), sizeof(decltype(static_cast(0)->Field)), \ + ::pybind11::format_descriptor(0)->Field)>::format(), \ + ::pybind11::detail::npy_format_descriptor(0)->Field)>::dtype() \ + } + +// The main idea of this macro is borrowed from https://github.com/swansontec/map-macro +// (C) William Swanson, Paul Fultz +#define PYBIND11_EVAL0(...) __VA_ARGS__ +#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__))) +#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__))) +#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__))) +#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__))) +#define PYBIND11_EVAL(...) PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__))) +#define PYBIND11_MAP_END(...) +#define PYBIND11_MAP_OUT +#define PYBIND11_MAP_COMMA , +#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END +#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT +#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) +#define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) +#ifdef _MSC_VER // MSVC is not as eager to expand macros, hence this workaround +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) +#else +#define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) +#endif +#define PYBIND11_MAP_LIST_NEXT(test, next) \ + PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) +#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) +#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +// PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ... +#define PYBIND11_MAP_LIST(f, t, ...) \ + PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) + +#define PYBIND11_NUMPY_DTYPE(Type, ...) \ + ::pybind11::detail::npy_format_descriptor::register_dtype \ + ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + template using array_iterator = typename std::add_pointer::type; @@ -348,7 +674,7 @@ struct vectorize_helper { return cast(f(*((Args *) buffers[Index].ptr)...)); array result(buffer_info(nullptr, sizeof(Return), - format_descriptor::value, + format_descriptor::format(), ndim, shape, strides)); buffer_info buf = result.request(); diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0a0f50878..d3895db57 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -339,12 +339,14 @@ inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr()), inline iterator handle::end() const { return iterator(nullptr, false); } inline detail::args_proxy handle::operator*() const { return detail::args_proxy(*this); } +class bytes; + class str : public object { public: PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive) - str(const std::string &s) - : object(PyUnicode_FromStringAndSize(s.c_str(), (ssize_t) s.length()), false) { + str(const char *c, size_t n) + : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), false) { if (!m_ptr) pybind11_fail("Could not allocate string object!"); } @@ -353,6 +355,10 @@ public: if (!m_ptr) pybind11_fail("Could not allocate string object!"); } + str(const std::string &s) : str(s.data(), s.size()) { } + + str(const bytes &b); + operator std::string() const { object temp = *this; if (PyUnicode_Check(m_ptr)) { @@ -362,8 +368,7 @@ public: } char *buffer; ssize_t length; - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length); - if (err == -1) + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) pybind11_fail("Unable to extract string contents! (invalid type)"); return std::string(buffer, (size_t) length); } @@ -382,21 +387,57 @@ class bytes : public object { public: PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK) - bytes(const std::string &s) - : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(s.data(), (ssize_t) s.size()), false) { + bytes(const char *c) + : object(PYBIND11_BYTES_FROM_STRING(c), false) { if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); } + bytes(const char *c, size_t n) + : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) { + if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); + } + + bytes(const std::string &s) : bytes(s.data(), s.size()) { } + + bytes(const pybind11::str &s); + operator std::string() const { char *buffer; ssize_t length; - int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length); - if (err == -1) + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) pybind11_fail("Unable to extract bytes contents!"); return std::string(buffer, (size_t) length); } }; +inline bytes::bytes(const pybind11::str &s) { + object temp = s; + if (PyUnicode_Check(s.ptr())) { + temp = object(PyUnicode_AsUTF8String(s.ptr()), false); + if (!temp) + pybind11_fail("Unable to extract string contents! (encoding issue)"); + } + char *buffer; + ssize_t length; + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) + pybind11_fail("Unable to extract string contents! (invalid type)"); + auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false); + if (!obj) + pybind11_fail("Could not allocate bytes object!"); + m_ptr = obj.release().ptr(); +} + +inline str::str(const bytes& b) { + char *buffer; + ssize_t length; + if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) + pybind11_fail("Unable to extract bytes contents!"); + auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false); + if (!obj) + pybind11_fail("Could not allocate string object!"); + m_ptr = obj.release().ptr(); +} + class none : public object { public: PYBIND11_OBJECT(none, object, detail::PyNone_Check) @@ -570,6 +611,38 @@ public: } }; +class memoryview : public object { +public: + memoryview(const buffer_info& info) { + static Py_buffer buf { }; + // Py_buffer uses signed sizes, strides and shape!.. + static std::vector py_strides { }; + static std::vector py_shape { }; + buf.buf = info.ptr; + buf.itemsize = (Py_ssize_t) info.itemsize; + buf.format = const_cast(info.format.c_str()); + buf.ndim = (int) info.ndim; + buf.len = (Py_ssize_t) info.size; + py_strides.clear(); + py_shape.clear(); + for (size_t i = 0; i < info.ndim; ++i) { + py_strides.push_back((Py_ssize_t) info.strides[i]); + py_shape.push_back((Py_ssize_t) info.shape[i]); + } + buf.strides = py_strides.data(); + buf.shape = py_shape.data(); + buf.suboffsets = nullptr; + buf.readonly = false; + buf.internal = nullptr; + + m_ptr = PyMemoryView_FromBuffer(&buf); + if (!m_ptr) + pybind11_fail("Unable to create memoryview from buffer descriptor"); + } + + PYBIND11_OBJECT_DEFAULT(memoryview, object, PyMemoryView_Check) +}; + inline size_t len(handle h) { ssize_t result = PyObject_Length(h.ptr()); if (result < 0)