Merge branch 'master' into smart_holder

docs/compiling.rst

@@ -70,6 +70,19 @@ that is supported via a ``build_ext`` command override; it will only affect
         ext_modules=ext_modules
     )
 
+If you have single-file extension modules that are directly stored in the
+Python source tree (``foo.cpp`` in the same directory as where a ``foo.py``
+would be located), you can also generate ``Pybind11Extensions`` using
+``setup_helpers.intree_extensions``: ``intree_extensions(["path/to/foo.cpp",
+...])`` returns a list of ``Pybind11Extensions`` which can be passed to
+``ext_modules``, possibly after further customizing their attributes
+(``libraries``, ``include_dirs``, etc.).  By doing so, a ``foo.*.so`` extension
+module will be generated and made available upon installation.
+
+``intree_extension`` will automatically detect if you are using a ``src``-style
+layout (as long as no namespace packages are involved), but you can also
+explicitly pass ``package_dir`` to it (as in ``setuptools.setup``).
+
 Since pybind11 does not require NumPy when building, a light-weight replacement
 for NumPy's parallel compilation distutils tool is included. Use it like this:
 

include/pybind11/attr.h

@@ -62,7 +62,8 @@ struct metaclass {
     handle value;
 
     PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.")
-    metaclass() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute
+    // NOLINTNEXTLINE(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute
+    metaclass() {}
 
     /// Override pybind11's default metaclass
     explicit metaclass(handle value) : value(value) { }

include/pybind11/detail/common.h

@@ -801,7 +801,8 @@ struct nodelete { template <typename T> void operator()(T*) { } };
 PYBIND11_NAMESPACE_BEGIN(detail)
 template <typename... Args>
 struct overload_cast_impl {
-    constexpr overload_cast_impl() {}; // NOLINT(modernize-use-equals-default):  MSVC 2015 needs this
+    // NOLINTNEXTLINE(modernize-use-equals-default):  MSVC 2015 needs this
+    constexpr overload_cast_impl() {}
 
     template <typename Return>
     constexpr auto operator()(Return (*pf)(Args...)) const noexcept

include/pybind11/pytypes.h

@@ -1015,7 +1015,7 @@ public:
         if (PyUnicode_Check(m_ptr)) {
             temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(m_ptr));
             if (!temp)
-                pybind11_fail("Unable to extract string contents! (encoding issue)");
+                throw error_already_set();
         }
         char *buffer = nullptr;
         ssize_t length = 0;

pybind11/setup_helpers.py

@@ -303,6 +303,42 @@ class build_ext(_build_ext):  # noqa: N801
         _build_ext.build_extensions(self)
 
 
+def intree_extensions(paths, package_dir=None):
+    """
+    Generate Pybind11Extensions from source files directly located in a Python
+    source tree.
+
+    ``package_dir`` behaves as in ``setuptools.setup``.  If unset, the Python
+    package root parent is determined as the first parent directory that does
+    not contain an ``__init__.py`` file.
+    """
+    exts = []
+    for path in paths:
+        if package_dir is None:
+            parent, _ = os.path.split(path)
+            while os.path.exists(os.path.join(parent, "__init__.py")):
+                parent, _ = os.path.split(parent)
+            relname, _ = os.path.splitext(os.path.relpath(path, parent))
+            qualified_name = relname.replace(os.path.sep, ".")
+            exts.append(Pybind11Extension(qualified_name, [path]))
+        else:
+            found = False
+            for prefix, parent in package_dir.items():
+                if path.startswith(parent):
+                    found = True
+                    relname, _ = os.path.splitext(os.path.relpath(path, parent))
+                    qualified_name = relname.replace(os.path.sep, ".")
+                    if prefix:
+                        qualified_name = prefix + "." + qualified_name
+                    exts.append(Pybind11Extension(qualified_name, [path]))
+            if not found:
+                raise ValueError(
+                    "path {} is not a child of any of the directories listed "
+                    "in 'package_dir' ({})".format(path, package_dir)
+                )
+    return exts
+
+
 def naive_recompile(obj, src):
     """
     This will recompile only if the source file changes. It does not check

pybind11/setup_helpers.pyi

@@ -1,7 +1,7 @@
 # IMPORTANT: Should stay in sync with setup_helpers.py (mostly checked by CI /
 # pre-commit).
 
-from typing import Any, Callable, Iterator, Optional, Type, TypeVar, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Type, TypeVar, Union
 from types import TracebackType
 
 from distutils.command.build_ext import build_ext as _build_ext  # type: ignore
@@ -33,6 +33,9 @@ def auto_cpp_level(compiler: distutils.ccompiler.CCompiler) -> Union[int, str]:
 class build_ext(_build_ext):  # type: ignore
     def build_extensions(self) -> None: ...
 
+def intree_extensions(
+    paths: Iterator[str], package_dir: Optional[Dict[str, str]] = None
+) -> List[Pybind11Extension]: ...
 def no_recompile(obj: str, src: str) -> bool: ...
 def naive_recompile(obj: str, src: str) -> bool: ...
 

tests/extra_setuptools/test_setuphelper.py

@@ -99,3 +99,45 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std):
     subprocess.check_call(
         [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr
     )
+
+
+def test_intree_extensions(monkeypatch, tmpdir):
+    monkeypatch.syspath_prepend(MAIN_DIR)
+
+    from pybind11.setup_helpers import intree_extensions
+
+    monkeypatch.chdir(tmpdir)
+    root = tmpdir
+    root.ensure_dir()
+    subdir = root / "dir"
+    subdir.ensure_dir()
+    src = subdir / "ext.cpp"
+    src.ensure()
+    (ext,) = intree_extensions([src.relto(tmpdir)])
+    assert ext.name == "ext"
+    subdir.ensure("__init__.py")
+    (ext,) = intree_extensions([src.relto(tmpdir)])
+    assert ext.name == "dir.ext"
+
+
+def test_intree_extensions_package_dir(monkeypatch, tmpdir):
+    monkeypatch.syspath_prepend(MAIN_DIR)
+
+    from pybind11.setup_helpers import intree_extensions
+
+    monkeypatch.chdir(tmpdir)
+    root = tmpdir / "src"
+    root.ensure_dir()
+    subdir = root / "dir"
+    subdir.ensure_dir()
+    src = subdir / "ext.cpp"
+    src.ensure()
+    (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"": "src"})
+    assert ext.name == "dir.ext"
+    (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"foo": "src"})
+    assert ext.name == "foo.dir.ext"
+    subdir.ensure("__init__.py")
+    (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"": "src"})
+    assert ext.name == "dir.ext"
+    (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"foo": "src"})
+    assert ext.name == "foo.dir.ext"

tests/pybind11_tests.h

@@ -1,10 +1,15 @@
 #pragma once
+
+// This must be kept first for MSVC 2015.
+// Do not remove the empty line between the #includes.
 #include <pybind11/pybind11.h>
+
 #include <pybind11/eval.h>
 
 #if defined(_MSC_VER) && _MSC_VER < 1910
 // We get some really long type names here which causes MSVC 2015 to emit warnings
-#  pragma warning(disable: 4503) // warning C4503: decorated name length exceeded, name was truncated
+#    pragma warning(                                                                              \
+        disable : 4503) // warning C4503: decorated name length exceeded, name was truncated
 #endif
 
 namespace py = pybind11;

tests/test_buffers.cpp

@@ -154,7 +154,7 @@ TEST_SUBMODULE(buffers, m) {
                                    py::format_descriptor<int32_t>::format(), 1);
         }
 
-        ConstBuffer() : value(new int32_t{0}) { };
+        ConstBuffer() : value(new int32_t{0}) {}
     };
     py::class_<ConstBuffer>(m, "ConstBuffer", py::buffer_protocol())
         .def(py::init<>())

tests/test_callbacks.cpp

@@ -122,7 +122,7 @@ TEST_SUBMODULE(callbacks, m) {
         // [workaround(intel)] = default does not work here
         // Defaulting this destructor results in linking errors with the Intel compiler
         // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827)
-        virtual ~AbstractBase() {};  // NOLINT(modernize-use-equals-default)
+        virtual ~AbstractBase() {} // NOLINT(modernize-use-equals-default)
         virtual unsigned int func() = 0;
     };
     m.def("func_accepting_func_accepting_base",

tests/test_eigen.cpp

@@ -258,7 +258,10 @@ TEST_SUBMODULE(eigen, m) {
     m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; });
     m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; });
     // test_sparse, test_sparse_signature
-    m.def("sparse_r", [mat]() -> SparseMatrixR { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); //NOLINT(clang-analyzer-core.uninitialized.UndefReturn)
+    m.def("sparse_r", [mat]() -> SparseMatrixR {
+        // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn)
+        return Eigen::SparseView<Eigen::MatrixXf>(mat);
+    });
     m.def("sparse_c", [mat]() -> SparseMatrixC { return Eigen::SparseView<Eigen::MatrixXf>(mat); });
     m.def("sparse_copy_r", [](const SparseMatrixR &m) -> SparseMatrixR { return m; });
     m.def("sparse_copy_c", [](const SparseMatrixC &m) -> SparseMatrixC { return m; });

tests/test_methods_and_attributes.cpp

@@ -123,7 +123,7 @@ class NoneCastTester {
 public:
     int answer = -1;
     NoneCastTester() = default;
-    NoneCastTester(int v) : answer(v) {};
+    NoneCastTester(int v) : answer(v) {}
 };
 
 struct StrIssue {
@@ -394,14 +394,14 @@ TEST_SUBMODULE(methods_and_attributes, m) {
         .def("increase_value", &RegisteredDerived::increase_value)
         .def_readwrite("rw_value", &RegisteredDerived::rw_value)
         .def_readonly("ro_value", &RegisteredDerived::ro_value)
-        // These should trigger a static_assert if uncommented
+        // Uncommenting the next line should trigger a static_assert:
-        //.def_readwrite("fails", &UserType::value) // should trigger a static_assert if uncommented
+        // .def_readwrite("fails", &UserType::value)
-        //.def_readonly("fails", &UserType::value) // should trigger a static_assert if uncommented
+        // Uncommenting the next line should trigger a static_assert:
+        // .def_readonly("fails", &UserType::value)
         .def_property("rw_value_prop", &RegisteredDerived::get_int, &RegisteredDerived::set_int)
         .def_property_readonly("ro_value_prop", &RegisteredDerived::get_double)
         // This one is in the registered class:
-        .def("sum", &RegisteredDerived::sum)
+        .def("sum", &RegisteredDerived::sum);
-        ;
 
     using Adapted = decltype(py::method_adaptor<RegisteredDerived>(&RegisteredDerived::do_nothing));
     static_assert(std::is_same<Adapted, void (RegisteredDerived::*)() const>::value, "");

tests/test_pytypes.cpp

@@ -76,6 +76,9 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("str_from_object", [](const py::object& obj) { return py::str(obj); });
     m.def("repr_from_object", [](const py::object& obj) { return py::repr(obj); });
     m.def("str_from_handle", [](py::handle h) { return py::str(h); });
+    m.def("str_from_string_from_str", [](const py::str& obj) {
+        return py::str(static_cast<std::string>(obj));
+    });
 
     m.def("str_format", []() {
         auto s1 = "{} + {} = {}"_s.format(1, 2, 3);

tests/test_pytypes.py

@@ -133,6 +133,14 @@ def test_str(doc):
     else:
         assert m.str_from_handle(malformed_utf8) == "b'\\x80'"
 
+    assert m.str_from_string_from_str("this is a str") == "this is a str"
+    ucs_surrogates_str = u"\udcc3"
+    if env.PY2:
+        assert u"\udcc3" == m.str_from_string_from_str(ucs_surrogates_str)
+    else:
+        with pytest.raises(UnicodeEncodeError):
+            m.str_from_string_from_str(ucs_surrogates_str)
+
 
 def test_bytes(doc):
     assert m.bytes_from_string().decode() == "foo"

tests/test_smart_ptr.cpp

@@ -24,7 +24,7 @@ template <typename T> class huge_unique_ptr {
     std::unique_ptr<T> ptr;
     uint64_t padding[10];
 public:
-    huge_unique_ptr(T *p) : ptr(p) {};
+    huge_unique_ptr(T *p) : ptr(p) {}
     T *get() { return ptr.get(); }
 };