diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index 111ff0e3c..acbefd922 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -176,9 +176,10 @@ function overload. Structured types ================ -In order for ``py::array_t`` to work with structured (record) types, we first need -to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE`` -macro which expects the type followed by field names: +In order for ``py::array_t`` to work with structured (record) types, we first +need to register the memory layout of the type. This can be done via +``PYBIND11_NUMPY_DTYPE`` macro, called in the plugin definition code, which +expects the type followed by field names: .. code-block:: cpp @@ -192,10 +193,14 @@ macro which expects the type followed by field names: A a; }; - PYBIND11_NUMPY_DTYPE(A, x, y); - PYBIND11_NUMPY_DTYPE(B, z, a); + // ... + PYBIND11_PLUGIN(test) { + // ... - /* now both A and B can be used as template arguments to py::array_t */ + PYBIND11_NUMPY_DTYPE(A, x, y); + PYBIND11_NUMPY_DTYPE(B, z, a); + /* now both A and B can be used as template arguments to py::array_t */ + } Vectorizing functions ===================== diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index a52d843ba..97a366014 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1009,8 +1009,8 @@ class type_caster::value>> : public pyobject_caste // - if the type is non-copy-constructible, the object must be the sole owner of the type (i.e. it // must have ref_count() == 1)h // If any of the above are not satisfied, we fall back to copying. -template using move_is_plain_type = none_of< - std::is_void, std::is_pointer, std::is_reference, std::is_const +template using move_is_plain_type = satisfies_none_of; template struct move_always : std::false_type {}; template struct move_always using any_of = std::disjunction; #endif template using none_of = negation>; +template class... Predicates> using satisfies_all_of = all_of...>; +template class... Predicates> using satisfies_any_of = any_of...>; +template class... Predicates> using satisfies_none_of = none_of...>; + /// Strip the class from a method type template struct remove_class { }; template struct remove_class { typedef R type(A...); }; @@ -567,21 +571,31 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const char *reason) { throw std::runtime_error(reason); } [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); } -/// Format strings for basic number types -#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor \ - { static constexpr const char* value = v; /* for backwards compatibility */ \ - static std::string format() { return value; } } - template struct format_descriptor { }; -template struct format_descriptor::value>> { - static constexpr const char c = "bBhHiIqQ"[detail::log2(sizeof(T))*2 + std::is_unsigned::value]; +NAMESPACE_BEGIN(detail) +// Returns the index of the given type in the type char array below, and in the list in numpy.h +// The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; +// complex float,double,long double. Note that the long double types only participate when long +// double is actually longer than double (it isn't under MSVC). +// NB: not only the string below but also complex.h and numpy.h rely on this order. +template struct is_fmt_numeric { static constexpr bool value = false; }; +template struct is_fmt_numeric::value>> { + static constexpr bool value = true; + static constexpr int index = std::is_same::value ? 0 : 1 + ( + std::is_integral::value ? detail::log2(sizeof(T))*2 + std::is_unsigned::value : 8 + ( + std::is_same::value ? 1 : std::is_same::value ? 2 : 0)); +}; +NAMESPACE_END(detail) + +template struct format_descriptor::value>> { + static constexpr const char c = "?bBhHiIqQfdgFDG"[detail::is_fmt_numeric::index]; static constexpr const char value[2] = { c, '\0' }; static std::string format() { return std::string(1, c); } }; template constexpr const char format_descriptor< - T, detail::enable_if_t::value>>::value[2]; + T, detail::enable_if_t::value>>::value[2]; /// RAII wrapper that temporarily clears any Python error state struct error_scope { @@ -590,10 +604,6 @@ struct error_scope { ~error_scope() { PyErr_Restore(type, value, trace); } }; -PYBIND11_DECL_FMT(float, "f"); -PYBIND11_DECL_FMT(double, "d"); -PYBIND11_DECL_FMT(bool, "?"); - /// Dummy destructor wrapper that can be used to expose classes with a private destructor struct nodelete { template void operator()(T*) { } }; diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index f767f354c..a6b7b23cd 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -18,11 +18,14 @@ #endif NAMESPACE_BEGIN(pybind11) - -PYBIND11_DECL_FMT(std::complex, "Zf"); -PYBIND11_DECL_FMT(std::complex, "Zd"); - NAMESPACE_BEGIN(detail) + +// The format codes are already in the string in common.h, we just need to provide a specialization +template struct is_fmt_numeric> { + static constexpr bool value = true; + static constexpr int index = is_fmt_numeric::index + 3; +}; + template class type_caster> { public: bool load(handle src, bool) { diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6fecf2853..691b194b2 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -36,8 +36,7 @@ static_assert(sizeof(size_t) == sizeof(Py_intptr_t), "size_t != Py_intptr_t"); NAMESPACE_BEGIN(pybind11) NAMESPACE_BEGIN(detail) -template struct npy_format_descriptor { }; -template struct is_pod_struct; +template struct npy_format_descriptor; struct PyArrayDescr_Proxy { PyObject_HEAD @@ -220,6 +219,16 @@ inline bool check_flags(const void* ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } +template struct is_std_array : std::false_type { }; +template struct is_std_array> : std::true_type { }; +template struct is_complex : std::false_type { }; +template struct is_complex> : std::true_type { }; + +template using is_pod_struct = all_of< + std::is_pod, // since we're accessing directly in memory we need a POD type + satisfies_none_of +>; + NAMESPACE_END(detail) class dtype : public object { @@ -685,65 +694,48 @@ struct pyobject_caster> { PYBIND11_TYPE_CASTER(type, handle_type_name::name()); }; -template struct is_std_array : std::false_type { }; -template struct is_std_array> : std::true_type { }; - -template -struct is_pod_struct { - enum { value = std::is_pod::value && // offsetof only works correctly for POD types - !std::is_reference::value && - !std::is_array::value && - !is_std_array::value && - !std::is_integral::value && - !std::is_enum::value && - !std::is_same::type, float>::value && - !std::is_same::type, double>::value && - !std::is_same::type, bool>::value && - !std::is_same::type, std::complex>::value && - !std::is_same::type, std::complex>::value }; -}; - -template struct npy_format_descriptor::value>> { +template struct npy_format_descriptor::value>> { private: - constexpr static const int values[8] = { - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, - npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ }; + // NB: the order here must match the one in common.h + constexpr static const int values[15] = { + npy_api::NPY_BOOL_, + npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_, npy_api::NPY_USHORT_, + npy_api::NPY_INT_, npy_api::NPY_UINT_, npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_, + npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, + npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ + }; + public: - enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned::value ? 1 : 0)] }; + static constexpr int value = values[detail::is_fmt_numeric::index]; + static pybind11::dtype dtype() { if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) return reinterpret_borrow(ptr); pybind11_fail("Unsupported buffer format!"); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { return _("int") + _(); } - template ::value, int> = 0> - static PYBIND11_DESCR name() { return _("uint") + _(); } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value>(_("bool"), + _::value>("int", "uint") + _()); + } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value || std::is_same::value>( + _("float") + _(), _("longdouble")); + } + template ::value, int> = 0> + static PYBIND11_DESCR name() { + return _::value || std::is_same::value>( + _("complex") + _(), _("longcomplex")); + } }; -template constexpr const int npy_format_descriptor< - T, enable_if_t::value>>::values[8]; -#define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor { \ - enum { value = npy_api::NumPyName }; \ - static pybind11::dtype dtype() { \ - if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \ - return reinterpret_borrow(ptr); \ - pybind11_fail("Unsupported buffer format!"); \ - } \ - static PYBIND11_DESCR name() { return _(Name); } } -DECL_FMT(float, NPY_FLOAT_, "float32"); -DECL_FMT(double, NPY_DOUBLE_, "float64"); -DECL_FMT(bool, NPY_BOOL_, "bool"); -DECL_FMT(std::complex, NPY_CFLOAT_, "complex64"); -DECL_FMT(std::complex, NPY_CDOUBLE_, "complex128"); -#undef DECL_FMT - -#define DECL_CHAR_FMT \ +#define PYBIND11_DECL_CHAR_FMT \ static PYBIND11_DESCR name() { return _("S") + _(); } \ static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } -template struct npy_format_descriptor { DECL_CHAR_FMT }; -template struct npy_format_descriptor> { DECL_CHAR_FMT }; -#undef DECL_CHAR_FMT +template struct npy_format_descriptor { PYBIND11_DECL_CHAR_FMT }; +template struct npy_format_descriptor> { PYBIND11_DECL_CHAR_FMT }; +#undef PYBIND11_DECL_CHAR_FMT template struct npy_format_descriptor::value>> { private: @@ -798,9 +790,9 @@ inline PYBIND11_NOINLINE void register_structured_dtype( for (auto& field : ordered_fields) { if (field.offset > offset) oss << (field.offset - offset) << 'x'; - // mark unaligned fields with '=' + // mark unaligned fields with '^' (unaligned native type) if (field.offset % field.alignment) - oss << '='; + oss << '^'; oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } @@ -820,8 +812,9 @@ inline PYBIND11_NOINLINE void register_structured_dtype( get_internals().direct_conversions[tindex].push_back(direct_converter); } -template -struct npy_format_descriptor::value>> { +template struct npy_format_descriptor { + static_assert(is_pod_struct::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); + static PYBIND11_DESCR name() { return _("struct"); } static pybind11::dtype dtype() { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index bc806de73..0090e6638 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -547,10 +547,10 @@ public: template using is_keyword = std::is_base_of; template using is_s_unpacking = std::is_same; // * unpacking template using is_ds_unpacking = std::is_same; // ** unpacking -template using is_positional = none_of< - is_keyword, is_s_unpacking, is_ds_unpacking +template using is_positional = satisfies_none_of; -template using is_keyword_or_ds = any_of, is_ds_unpacking>; +template using is_keyword_or_ds = satisfies_any_of; // Call argument collector forward declarations template diff --git a/tests/test_numpy_dtypes.cpp b/tests/test_numpy_dtypes.cpp index 3894f6a30..d74ecc59e 100644 --- a/tests/test_numpy_dtypes.cpp +++ b/tests/test_numpy_dtypes.cpp @@ -19,23 +19,25 @@ namespace py = pybind11; struct SimpleStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; + long double ldbl_; }; std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { - return os << "s:" << v.x << "," << v.y << "," << v.z; + return os << "s:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } PYBIND11_PACKED(struct PackedStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; + long double ldbl_; }); std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { - return os << "p:" << v.x << "," << v.y << "," << v.z; + return os << "p:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } PYBIND11_PACKED(struct NestedStruct { @@ -48,10 +50,11 @@ std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { } struct PartialStruct { - bool x; - uint32_t y; - float z; + bool bool_; + uint32_t uint_; + float float_; uint64_t dummy2; + long double ldbl_; }; struct PartialNestedStruct { @@ -99,13 +102,19 @@ py::array mkarray_via_buffer(size_t n) { 1, { n }, { sizeof(T) })); } +#define SET_TEST_VALS(s, i) do { \ + s.bool_ = (i) % 2 != 0; \ + s.uint_ = (uint32_t) (i); \ + s.float_ = (float) (i) * 1.5f; \ + s.ldbl_ = (long double) (i) * -2.5L; } while (0) + template py::array_t create_recarray(size_t n) { auto arr = mkarray_via_buffer(n); auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f; + SET_TEST_VALS(ptr[i], i); } return arr; } @@ -119,8 +128,8 @@ py::array_t create_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; - ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f; + SET_TEST_VALS(ptr[i].a, i); + SET_TEST_VALS(ptr[i].b, i + 1); } return arr; } @@ -130,7 +139,7 @@ py::array_t create_partial_nested(size_t n) { auto req = arr.request(); auto ptr = static_cast(req.ptr); for (size_t i = 0; i < n; i++) { - ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f; + SET_TEST_VALS(ptr[i].a, i); } return arr; } @@ -320,10 +329,10 @@ test_initializer numpy_dtypes([](py::module &m) { // typeinfo may be registered before the dtype descriptor for scalar casts to work... py::class_(m, "SimpleStruct"); - PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); - PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); + PYBIND11_NUMPY_DTYPE(PackedStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(NestedStruct, a, b); - PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z); + PYBIND11_NUMPY_DTYPE(PartialStruct, bool_, uint_, float_, ldbl_); PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a); PYBIND11_NUMPY_DTYPE(StringStruct, a, b); PYBIND11_NUMPY_DTYPE(EnumStruct, e1, e2); @@ -334,6 +343,11 @@ test_initializer numpy_dtypes([](py::module &m) { PYBIND11_NUMPY_DTYPE_EX(StructWithUglyNames, __x__, "x", __y__, "y"); + // If uncommented, this should produce a static_assert failure telling the user that the struct + // is not a POD type +// struct NotPOD { std::string v; NotPOD() : v("hi") {}; }; +// PYBIND11_NUMPY_DTYPE(NotPOD, v); + m.def("create_rec_simple", &create_recarray); m.def("create_rec_packed", &create_recarray); m.def("create_rec_nested", &create_nested); @@ -354,10 +368,10 @@ test_initializer numpy_dtypes([](py::module &m) { m.def("test_dtype_methods", &test_dtype_methods); m.def("trailing_padding_dtype", &trailing_padding_dtype); m.def("buffer_to_dtype", &buffer_to_dtype); - m.def("f_simple", [](SimpleStruct s) { return s.y * 10; }); - m.def("f_packed", [](PackedStruct s) { return s.y * 10; }); - m.def("f_nested", [](NestedStruct s) { return s.a.y * 10; }); - m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z); }); + m.def("f_simple", [](SimpleStruct s) { return s.uint_ * 10; }); + m.def("f_packed", [](PackedStruct s) { return s.uint_ * 10; }); + m.def("f_nested", [](NestedStruct s) { return s.a.uint_ * 10; }); + m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); }); #undef PYBIND11_PACKED diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index 52ebe0ede..b2f184ff8 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -7,14 +7,51 @@ with pytest.suppress(ImportError): @pytest.fixture(scope='module') def simple_dtype(): - return np.dtype({'names': ['x', 'y', 'z'], - 'formats': ['?', 'u4', 'f4'], - 'offsets': [0, 4, 8]}) + ld = np.dtype('longdouble') + return np.dtype({'names': ['bool_', 'uint_', 'float_', 'ldbl_'], + 'formats': ['?', 'u4', 'f4', 'f{}'.format(ld.itemsize)], + 'offsets': [0, 4, 8, (16 if ld.alignment > 4 else 12)]}) @pytest.fixture(scope='module') def packed_dtype(): - return np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')]) + return np.dtype([('bool_', '?'), ('uint_', 'u4'), ('float_', 'f4'), ('ldbl_', 'g')]) + + +def dt_fmt(): + return ("{{'names':['bool_','uint_','float_','ldbl_'], 'formats':['?',' 4) + return dt_fmt().format(ld.itemsize, simple_ld_off, simple_ld_off + ld.itemsize) + + +def packed_dtype_fmt(): + return "[('bool_', '?'), ('uint_', ' 4) + 8 + 8 * ( + np.dtype('longdouble').alignment > 8) + + +def partial_dtype_fmt(): + ld = np.dtype('longdouble') + partial_ld_off = partial_ld_offset() + return dt_fmt().format(ld.itemsize, partial_ld_off, partial_ld_off + ld.itemsize) + + +def partial_nested_fmt(): + ld = np.dtype('longdouble') + partial_nested_off = 8 + 8 * (ld.alignment > 8) + partial_ld_off = partial_ld_offset() + partial_nested_size = partial_nested_off * 2 + partial_ld_off + ld.itemsize + return "{{'names':['a'], 'formats':[{}], 'offsets':[{}], 'itemsize':{}}}".format( + partial_dtype_fmt(), partial_nested_off, partial_nested_size) def assert_equal(actual, expected_data, expected_dtype): @@ -29,12 +66,20 @@ def test_format_descriptors(): get_format_unbound() assert re.match('^NumPy type info missing for .*UnboundStruct.*$', str(excinfo.value)) + ld = np.dtype('longdouble') + ldbl_fmt = ('4x' if ld.alignment > 4 else '') + ld.char + ss_fmt = "T{?:bool_:3xI:uint_:f:float_:" + ldbl_fmt + ":ldbl_:}" + dbl = np.dtype('double') + partial_fmt = ("T{?:bool_:3xI:uint_:f:float_:" + + str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) + + "xg:ldbl_:}") + nested_extra = str(max(8, ld.alignment)) assert print_format_descriptors() == [ - "T{?:x:3xI:y:f:z:}", - "T{?:x:=I:y:=f:z:}", - "T{T{?:x:3xI:y:f:z:}:a:T{?:x:=I:y:=f:z:}:b:}", - "T{?:x:3xI:y:f:z:12x}", - "T{8xT{?:x:3xI:y:f:z:12x}:a:8x}", + ss_fmt, + "T{?:bool_:^I:uint_:^f:float_:^g:ldbl_:}", + "T{" + ss_fmt + ":a:T{?:bool_:^I:uint_:^f:float_:^g:ldbl_:}:b:}", + partial_fmt, + "T{" + nested_extra + "x" + partial_fmt + ":a:" + nested_extra + "x}", "T{3s:a:3s:b:}", 'T{q:e1:B:e2:}' ] @@ -46,13 +91,11 @@ def test_dtype(simple_dtype): trailing_padding_dtype, buffer_to_dtype) assert print_dtypes() == [ - "{'names':['x','y','z'], 'formats':['?',' simple_dtype.itemsize @@ -129,9 +171,7 @@ def test_recarray(simple_dtype, packed_dtype): assert_equal(arr, elements, packed_dtype) arr = create_rec_partial_nested(3) - assert str(arr.dtype) == \ - "{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?',' partial_dtype.itemsize