Merge branch 'master' of https://github.com/pybind/pybind11 into lazy-error-string

.gitattributes

@@ -0,0 +1 @@
+docs/*.svg binary

.github/matchers/pylint.json

@@ -0,0 +1,32 @@
+{
+  "problemMatcher": [
+    {
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-warning"
+    },
+    {
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-error"
+    }
+  ]
+}

.github/workflows/ci.yml

@@ -30,8 +30,8 @@ jobs:
         - '3.6'
         - '3.9'
         - '3.10'
-        - 'pypy-3.7-v7.3.7'
+        - 'pypy-3.7'
-        - 'pypy-3.8-v7.3.7'
+        - 'pypy-3.8'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -45,7 +45,7 @@ jobs:
             args: >
               -DPYBIND11_FINDPYTHON=ON
               -DCMAKE_CXX_FLAGS="-D_=1"
-          - runs-on: windows-latest
+          - runs-on: windows-2019
             python: '3.6'
             args: >
               -DPYBIND11_FINDPYTHON=ON
@@ -713,12 +713,12 @@ jobs:
 
         include:
           - python: 3.9
-            args: -DCMAKE_CXX_STANDARD=20 -DDOWNLOAD_EIGEN=OFF
+            args: -DCMAKE_CXX_STANDARD=20
           - python: 3.8
             args: -DCMAKE_CXX_STANDARD=17
 
     name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}"
-    runs-on: windows-latest
+    runs-on: windows-2019
 
     steps:
     - uses: actions/checkout@v2
@@ -836,7 +836,7 @@ jobs:
         cmake -S . -B build
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
-        -DDOWNLOAD_EIGEN=OFF
+        -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=20
 
     - name: Build C++20

.github/workflows/format.yml

@@ -12,6 +12,9 @@ on:
     - stable
     - "v*"
 
+env:
+  FORCE_COLOR: 3
+
 jobs:
   pre-commit:
     name: Format
@@ -19,6 +22,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
+    - name: Add matchers
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json"
     - uses: pre-commit/action@v2.0.3
       with:
         # Slow hooks are marked with manual - slow is okay here, run them too

.pre-commit-config.yaml

@@ -59,7 +59,7 @@ repos:
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: "v1.1.12"
+  rev: "v1.1.13"
   hooks:
   - id: remove-tabs
 
@@ -71,7 +71,7 @@ repos:
 
 # Autoremoves unused imports
 - repo: https://github.com/hadialqattan/pycln
-  rev: "v1.1.0"
+  rev: "v1.2.4"
   hooks:
   - id: pycln
 
@@ -87,23 +87,30 @@ repos:
   - id: rst-directive-colons
   - id: rst-inline-touching-normal
 
-# Flake8 also supports pre-commit natively (same author)
-- repo: https://github.com/PyCQA/flake8
-  rev: "4.0.1"
-  hooks:
-  - id: flake8
-    additional_dependencies: &flake8_dependencies
-      - flake8-bugbear
-      - pep8-naming
-    exclude: ^(docs/.*|tools/.*)$
-
 # Automatically remove noqa that are not used
 - repo: https://github.com/asottile/yesqa
   rev: "v1.3.0"
   hooks:
   - id: yesqa
+    additional_dependencies: &flake8_dependencies
+      - flake8-bugbear
+      - pep8-naming
+
+# Flake8 also supports pre-commit natively (same author)
+- repo: https://github.com/PyCQA/flake8
+  rev: "4.0.1"
+  hooks:
+  - id: flake8
+    exclude: ^(docs/.*|tools/.*)$
     additional_dependencies: *flake8_dependencies
 
+# PyLint has native support - not always usable, but works for us
+- repo: https://github.com/PyCQA/pylint
+  rev: "v2.12.2"
+  hooks:
+  - id: pylint
+    files: ^pybind11
+
 # CMake formatting
 - repo: https://github.com/cheshirekow/cmake-format-precommit
   rev: "v0.6.13"
@@ -156,6 +163,7 @@ repos:
 
 # Clang format the codebase automatically
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v13.0.0"
+  rev: "v13.0.1"
   hooks:
   - id: clang-format
+    types_or: [c++, c, cuda]

include/pybind11/cast.h

@@ -85,11 +85,15 @@ protected:
                                                                                                   \
 public:                                                                                           \
     static constexpr auto name = py_name;                                                         \
-    template <typename T_, enable_if_t<std::is_same<type, remove_cv_t<T_>>::value, int> = 0>      \
+    template <typename T_,                                                                        \
-    static handle cast(T_ *src, return_value_policy policy, handle parent) {                      \
+              ::pybind11::detail::enable_if_t<                                                    \
+                  std::is_same<type, ::pybind11::detail::remove_cv_t<T_>>::value,                 \
+                  int> = 0>                                                                       \
+    static ::pybind11::handle cast(                                                               \
+        T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) {             \
         if (!src)                                                                                 \
-            return none().release();                                                              \
+            return ::pybind11::none().release();                                                  \
-        if (policy == return_value_policy::take_ownership) {                                      \
+        if (policy == ::pybind11::return_value_policy::take_ownership) {                          \
             auto h = cast(std::move(*src), policy, parent);                                       \
             delete src;                                                                           \
             return h;                                                                             \
@@ -100,7 +104,7 @@ public:
     operator type &() { return value; }                /* NOLINT(bugprone-macro-parentheses) */   \
     operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */   \
     template <typename T_>                                                                        \
-    using cast_op_type = pybind11::detail::movable_cast_op_type<T_>
+    using cast_op_type = ::pybind11::detail::movable_cast_op_type<T_>
 
 template <typename CharT>
 using is_std_char_type = any_of<std::is_same<CharT, char>, /* std::string */
@@ -380,7 +384,7 @@ struct string_caster {
             return false;
         }
         if (!PyUnicode_Check(load_src.ptr())) {
-            return load_bytes(load_src);
+            return load_raw(load_src);
         }
 
         // For UTF-8 we avoid the need for a temporary `bytes` object by using
@@ -458,26 +462,37 @@ private:
 #endif
     }
 
-    // When loading into a std::string or char*, accept a bytes object as-is (i.e.
+    // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e.
     // without any encoding/decoding attempt).  For other C++ char sizes this is a no-op.
     // which supports loading a unicode from a str, doesn't take this path.
     template <typename C = CharT>
-    bool load_bytes(enable_if_t<std::is_same<C, char>::value, handle> src) {
+    bool load_raw(enable_if_t<std::is_same<C, char>::value, handle> src) {
         if (PYBIND11_BYTES_CHECK(src.ptr())) {
             // We were passed raw bytes; accept it into a std::string or char*
             // without any encoding attempt.
             const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr());
-            if (bytes) {
+            if (!bytes) {
+                pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure.");
+            }
             value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr()));
             return true;
         }
+        if (PyByteArray_Check(src.ptr())) {
+            // We were passed a bytearray; accept it into a std::string or char*
+            // without any encoding attempt.
+            const char *bytearray = PyByteArray_AsString(src.ptr());
+            if (!bytearray) {
+                pybind11_fail("Unexpected PyByteArray_AsString() failure.");
+            }
+            value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr()));
+            return true;
         }
 
         return false;
     }
 
     template <typename C = CharT>
-    bool load_bytes(enable_if_t<!std::is_same<C, char>::value, handle>) {
+    bool load_raw(enable_if_t<!std::is_same<C, char>::value, handle>) {
         return false;
     }
 };

include/pybind11/detail/class.h

@@ -101,14 +101,14 @@ inline PyTypeObject *make_static_property_type() {
 inline PyTypeObject *make_static_property_type() {
     auto d = dict();
     PyObject *result = PyRun_String(R"(\
-        class pybind11_static_property(property):
+class pybind11_static_property(property):
     def __get__(self, obj, cls):
         return property.__get__(self, cls, cls)
 
     def __set__(self, obj, value):
         cls = obj if isinstance(obj, type) else type(obj)
         property.__set__(self, cls, value)
-        )",
+)",
                                     Py_file_input,
                                     d.ptr(),
                                     d.ptr());

include/pybind11/detail/common.h

@@ -822,7 +822,9 @@ struct is_template_base_of_impl {
 /// Check if a template is the base of a type. For example:
 /// `is_template_base_of<Base, T>` is true if `struct T : Base<U> {}` where U can be anything
 template <template <typename...> class Base, typename T>
-#if defined(PYBIND11_CPP20) || !defined(_MSC_VER) // Sadly, all MSVC versions incl. 2022 need this.
+// Sadly, all MSVC versions incl. 2022 need the workaround, even in C++20 mode.
+// See also: https://github.com/pybind/pybind11/pull/3741
+#if !defined(_MSC_VER)
 using is_template_base_of
     = decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T> *) nullptr));
 #else
@@ -934,9 +936,11 @@ PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybin
 PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally
 
 [[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const char *reason) {
+    assert(!PyErr_Occurred());
     throw std::runtime_error(reason);
 }
 [[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const std::string &reason) {
+    assert(!PyErr_Occurred());
     throw std::runtime_error(reason);
 }
 

include/pybind11/detail/type_caster_base.h

@@ -493,9 +493,10 @@ error_string(PyObject *exc_type, PyObject *exc_value, PyObject *exc_trace) {
         }
 
         PyFrameObject *frame = tb->tb_frame;
+        Py_XINCREF(frame);
         result += "\n\nAt:\n";
         while (frame) {
-#    if PY_VERSION_HEX >= 0x03090000
+#    if PY_VERSION_HEX >= 0x030900B1
             PyCodeObject *f_code = PyFrame_GetCode(frame);
 #    else
             PyCodeObject *f_code = frame->f_code;
@@ -511,6 +512,14 @@ error_string(PyObject *exc_type, PyObject *exc_value, PyObject *exc_trace) {
             result += '\n';
             frame = frame->f_back;
             Py_DECREF(f_code);
+#    if PY_VERSION_HEX >= 0x030900B1
+            auto *b_frame = PyFrame_GetBack(frame);
+#    else
+            auto *b_frame = frame->f_back;
+            Py_XINCREF(b_frame);
+#    endif
+            Py_DECREF(frame);
+            frame = b_frame;
         }
 #endif //! defined(PYPY_VERSION)
     }

include/pybind11/eigen.h

@@ -25,6 +25,8 @@
 #if defined(_MSC_VER)
 #    pragma warning(push)
 #    pragma warning(disable : 4127) // C4127: conditional expression is constant
+#    pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741
+//       C5054: operator '&': deprecated between enumerations of different types
 #endif
 
 #include <Eigen/Core>

include/pybind11/embed.h

@@ -198,6 +198,10 @@ inline void finalize_interpreter() {
     if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
         internals_ptr_ptr = capsule(builtins[id]);
     }
+    // Local internals contains data managed by the current interpreter, so we must clear them to
+    // avoid undefined behaviors when initializing another interpreter
+    detail::get_local_internals().registered_types_cpp.clear();
+    detail::get_local_internals().registered_exception_translators.clear();
 
     Py_Finalize();
 

include/pybind11/pybind11.h

@@ -334,7 +334,7 @@ protected:
                             const std::type_info *const *types,
                             size_t args) {
         // Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr,
-        // we do not want this to destuct the pointer. `initialize` (the caller) still relies on
+        // we do not want this to destruct the pointer. `initialize` (the caller) still relies on
         // the pointee being alive after this call. Only move out if a `capsule` is going to keep
         // it alive.
         auto *rec = unique_rec.get();
@@ -2636,9 +2636,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
 
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION) && PY_VERSION_HEX < 0x030B0000
+#if !defined(PYPY_VERSION)
-    // TODO: Remove PyPy workaround for Python 3.11.
-    // Current API fails on 3.11 since co_varnames can be null.
 #    if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {
@@ -2646,9 +2644,11 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
         // f_code is guaranteed to not be NULL
         if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
             PyObject *locals = PyEval_GetLocals();
-            if (locals != nullptr && f_code->co_varnames != nullptr) {
+            if (locals != nullptr) {
-                PyObject *self_caller
+                PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames");
-                    = dict_getitem(locals, PyTuple_GET_ITEM(f_code->co_varnames, 0));
+                PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0);
+                Py_DECREF(co_varnames);
+                PyObject *self_caller = dict_getitem(locals, self_arg);
                 if (self_caller == self.ptr()) {
                     Py_DECREF(f_code);
                     Py_DECREF(frame);

include/pybind11/pytypes.h

@@ -1542,6 +1542,9 @@ public:
     explicit weakref(handle obj, handle callback = {})
         : object(PyWeakref_NewRef(obj.ptr(), callback.ptr()), stolen_t{}) {
         if (!m_ptr) {
+            if (PyErr_Occurred()) {
+                throw error_already_set();
+            }
             pybind11_fail("Could not allocate weak reference!");
         }
     }

noxfile.py

@@ -1,8 +1,14 @@
+import os
+
 import nox
 
+nox.needs_version = ">=2022.1.7"
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+PYTHON_VERISONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"]
+
+if os.environ.get("CI", None):
+    nox.options.error_on_missing_interpreters = True
 
 
 @nox.session(reuse_venv=True)
@@ -14,7 +20,7 @@ def lint(session: nox.Session) -> None:
     session.run("pre-commit", "run", "-a")
 
 
-@nox.session(python=PYTHON_VERSIONS)
+@nox.session(python=PYTHON_VERISONS)
 def tests(session: nox.Session) -> None:
     """
     Run the tests (requires a compiler).
@@ -24,14 +30,12 @@ def tests(session: nox.Session) -> None:
     session.install("-r", "tests/requirements.txt")
     session.run(
         "cmake",
-        "-S",
+        "-S.",
-        ".",
+        f"-B{tmpdir}",
-        "-B",
-        tmpdir,
         "-DPYBIND11_WERROR=ON",
         "-DDOWNLOAD_CATCH=ON",
         "-DDOWNLOAD_EIGEN=ON",
-        *session.posargs
+        *session.posargs,
     )
     session.run("cmake", "--build", tmpdir)
     session.run("cmake", "--build", tmpdir, "--config=Release", "--target", "check")

pybind11/__main__.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-function-docstring
+
 import argparse
 import sys
 import sysconfig

pybind11/commands.py

@@ -3,16 +3,23 @@ import os
 DIR = os.path.abspath(os.path.dirname(__file__))
 
 
-def get_include(user: bool = False) -> str:
+def get_include(user: bool = False) -> str:  # pylint: disable=unused-argument
+    """
+    Return the path to the pybind11 include directory. The historical "user"
+    argument is unused, and may be removed.
+    """
     installed_path = os.path.join(DIR, "include")
     source_path = os.path.join(os.path.dirname(DIR), "include")
     return installed_path if os.path.exists(installed_path) else source_path
 
 
 def get_cmake_dir() -> str:
+    """
+    Return the path to the pybind11 CMake module directory.
+    """
     cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11")
     if os.path.exists(cmake_installed_path):
         return cmake_installed_path
-    else:
+
     msg = "pybind11 not installed, installation required to access the CMake files"
     raise ImportError(msg)

pybind11/setup_helpers.py

@@ -239,7 +239,7 @@ def has_flag(compiler: Any, flag: str) -> bool:
     with tmp_chdir():
         fname = Path("flagcheck.cpp")
         # Don't trigger -Wunused-parameter.
-        fname.write_text("int main (int, char **) { return 0; }")
+        fname.write_text("int main (int, char **) { return 0; }", encoding="utf-8")
 
         try:
             compiler.compile([str(fname)], extra_postargs=[flag])
@@ -303,15 +303,17 @@ def intree_extensions(
     """
     exts = []
 
-    for path in paths:
     if package_dir is None:
+        for path in paths:
             parent, _ = os.path.split(path)
             while os.path.exists(os.path.join(parent, "__init__.py")):
                 parent, _ = os.path.split(parent)
             relname, _ = os.path.splitext(os.path.relpath(path, parent))
             qualified_name = relname.replace(os.path.sep, ".")
             exts.append(Pybind11Extension(qualified_name, [path]))
-        else:
+        return exts
+
+    for path in paths:
         for prefix, parent in package_dir.items():
             if path.startswith(parent):
                 relname, _ = os.path.splitext(os.path.relpath(path, parent))
@@ -319,8 +321,8 @@ def intree_extensions(
                 if prefix:
                     qualified_name = prefix + "." + qualified_name
                 exts.append(Pybind11Extension(qualified_name, [path]))
-
+                break
-    if not exts:
+        else:
             msg = (
                 f"path {path} is not a child of any of the directories listed "
                 f"in 'package_dir' ({package_dir})"
@@ -339,7 +341,7 @@ def naive_recompile(obj: str, src: str) -> bool:
     return os.stat(obj).st_mtime < os.stat(src).st_mtime
 
 
-def no_recompile(obg: str, src: str) -> bool:
+def no_recompile(obg: str, src: str) -> bool:  # pylint: disable=unused-argument
     """
     This is the safest but slowest choice (and is the default) - will always
     recompile sources.
@@ -412,7 +414,7 @@ class ParallelCompile:
         self,
         envvar: Optional[str] = None,
         default: int = 0,
-        max: int = 0,
+        max: int = 0,  # pylint: disable=redefined-builtin
         needs_recompile: Callable[[str, str], bool] = no_recompile,
     ) -> None:
         self.envvar = envvar
@@ -488,6 +490,9 @@ class ParallelCompile:
         return compile_function
 
     def install(self: S) -> S:
+        """
+        Installs the compile function into distutils.ccompiler.CCompiler.compile.
+        """
         distutils.ccompiler.CCompiler.compile = self.function()  # type: ignore[assignment]
         return self
 

pyproject.toml

@@ -30,3 +30,30 @@ strict = true
 [[tool.mypy.overrides]]
 module = ["ghapi.*", "setuptools.*"]
 ignore_missing_imports = true
+
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+filterwarnings = ["error"]
+log_cli_level = "info"
+testpaths = [
+    "tests",
+]
+timeout=300
+
+
+[tool.pylint]
+master.py-version = "3.6"
+reports.output-format = "colorized"
+messages_control.disable = [
+  "design",
+  "fixme",
+  "imports",
+  "line-too-long",
+  "imports",
+  "invalid-name",
+  "protected-access",
+  "missing-module-docstring",
+]

tests/test_builtin_casters.py

@@ -133,6 +133,15 @@ def test_bytes_to_string():
     assert m.string_length("💩".encode()) == 4
 
 
+def test_bytearray_to_string():
+    """Tests the ability to pass bytearray to C++ string-accepting functions"""
+    assert m.string_length(bytearray(b"Hi")) == 2
+    assert m.strlen(bytearray(b"bytearray")) == 9
+    assert m.string_length(bytearray()) == 0
+    assert m.string_length(bytearray("🦜", "utf-8", "strict")) == 4
+    assert m.string_length(bytearray(b"\x80")) == 1
+
+
 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no <string_view>")
 def test_string_view(capture):
     """Tests support for C++17 string_view arguments and return values"""

tests/test_custom_type_casters.cpp

@@ -20,6 +20,7 @@ public:
     std::string arg = "(default arg inspector 2)";
 };
 class ArgAlwaysConverts {};
+
 namespace pybind11 {
 namespace detail {
 template <>
@@ -105,6 +106,34 @@ struct type_caster<DestructionTester> {
 } // namespace detail
 } // namespace pybind11
 
+// Define type caster outside of `pybind11::detail` and then alias it.
+namespace other_lib {
+struct MyType {};
+// Corrupt `py` shorthand alias for surrounding context.
+namespace py {}
+// Corrupt unqualified relative `pybind11` namespace.
+namespace pybind11 {}
+// Correct alias.
+namespace py_ = ::pybind11;
+// Define caster. This is effectively no-op, we only ensure it compiles and we
+// don't have any symbol collision when using macro mixin.
+struct my_caster {
+    PYBIND11_TYPE_CASTER(MyType, py_::detail::const_name("MyType"));
+    bool load(py_::handle, bool) { return true; }
+
+    static py_::handle cast(const MyType &, py_::return_value_policy, py_::handle) {
+        return py_::bool_(true).release();
+    }
+};
+} // namespace other_lib
+// Effectively "alias" it into correct namespace (via inheritance).
+namespace pybind11 {
+namespace detail {
+template <>
+struct type_caster<other_lib::MyType> : public other_lib::my_caster {};
+} // namespace detail
+} // namespace pybind11
+
 TEST_SUBMODULE(custom_type_casters, m) {
     // test_custom_type_casters
 
@@ -175,4 +204,6 @@ TEST_SUBMODULE(custom_type_casters, m) {
     m.def("destruction_tester_cstats",
           &ConstructorStats::get<DestructionTester>,
           py::return_value_policy::reference);
+
+    m.def("other_lib_type", [](other_lib::MyType x) { return x; });
 }

tests/test_custom_type_casters.py

@@ -114,3 +114,7 @@ def test_custom_caster_destruction():
 
     # Make sure we still only have the original object (from ..._no_destroy()) alive:
     assert cstats.alive() == 1
+
+
+def test_custom_caster_other_lib():
+    assert m.other_lib_type(True)

tests/test_embed/test_interpreter.cpp

@@ -375,3 +375,21 @@ TEST_CASE("sys.argv gets initialized properly") {
     }
     py::initialize_interpreter();
 }
+
+TEST_CASE("make_iterator can be called before then after finalizing an interpreter") {
+    // Reproduction of issue #2101 (https://github.com/pybind/pybind11/issues/2101)
+    py::finalize_interpreter();
+
+    std::vector<int> container;
+    {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }
+
+    REQUIRE_NOTHROW([&]() {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }());
+
+    py::initialize_interpreter();
+}

tests/test_pickling.cpp

@@ -1,4 +1,3 @@
-// clang-format off
 /*
     tests/test_pickling.cpp -- pickle support
 
@@ -11,8 +10,6 @@
 
 #include "pybind11_tests.h"
 
-// clang-format on
-
 #include <memory>
 #include <stdexcept>
 #include <utility>
@@ -63,19 +60,18 @@ void wrap(py::module m) {
 
 } // namespace exercise_trampoline
 
-// clang-format off
-
 TEST_SUBMODULE(pickling, m) {
     // test_roundtrip
     class Pickleable {
     public:
-        explicit Pickleable(const std::string &value) : m_value(value) { }
+        explicit Pickleable(const std::string &value) : m_value(value) {}
         const std::string &value() const { return m_value; }
 
         void setExtra1(int extra1) { m_extra1 = extra1; }
         void setExtra2(int extra2) { m_extra2 = extra2; }
         int extra1() const { return m_extra1; }
         int extra2() const { return m_extra2; }
+
     private:
         std::string m_value;
         int m_extra1 = 0;
@@ -88,8 +84,7 @@ TEST_SUBMODULE(pickling, m) {
     };
 
     py::class_<Pickleable> pyPickleable(m, "Pickleable");
-    pyPickleable
+    pyPickleable.def(py::init<std::string>())
-        .def(py::init<std::string>())
         .def("value", &Pickleable::value)
         .def("extra1", &Pickleable::extra1)
         .def("extra2", &Pickleable::extra2)
@@ -105,7 +100,7 @@ TEST_SUBMODULE(pickling, m) {
         pyPickleable.def("__setstate__", [](Pickleable &p, const py::tuple &t) {
             if (t.size() != 3) {
                 throw std::runtime_error("Invalid state!");
-}
+            }
             /* Invoke the constructor (need to use in-place version) */
             new (&p) Pickleable(t[0].cast<std::string>());
 
@@ -124,7 +119,7 @@ TEST_SUBMODULE(pickling, m) {
             [](const py::tuple &t) {
                 if (t.size() != 3) {
                     throw std::runtime_error("Invalid state!");
-}
+                }
                 auto p = PickleableNew(t[0].cast<std::string>());
 
                 p.setExtra1(t[1].cast<int>());
@@ -136,7 +131,7 @@ TEST_SUBMODULE(pickling, m) {
     // test_roundtrip_with_dict
     class PickleableWithDict {
     public:
-        explicit PickleableWithDict(const std::string &value) : value(value) { }
+        explicit PickleableWithDict(const std::string &value) : value(value) {}
 
         std::string value;
         int extra;
@@ -147,7 +142,8 @@ TEST_SUBMODULE(pickling, m) {
         using PickleableWithDict::PickleableWithDict;
     };
 
-    py::class_<PickleableWithDict> pyPickleableWithDict(m, "PickleableWithDict", py::dynamic_attr());
+    py::class_<PickleableWithDict> pyPickleableWithDict(
+        m, "PickleableWithDict", py::dynamic_attr());
     pyPickleableWithDict.def(py::init<std::string>())
         .def_readwrite("value", &PickleableWithDict::value)
         .def_readwrite("extra", &PickleableWithDict::extra)
@@ -159,7 +155,7 @@ TEST_SUBMODULE(pickling, m) {
         pyPickleableWithDict.def("__setstate__", [](const py::object &self, const py::tuple &t) {
             if (t.size() != 3) {
                 throw std::runtime_error("Invalid state!");
-}
+            }
             /* Cast and construct */
             auto &p = self.cast<PickleableWithDict &>();
             new (&p) PickleableWithDict(t[0].cast<std::string>());
@@ -176,12 +172,13 @@ TEST_SUBMODULE(pickling, m) {
         .def(py::init<std::string>())
         .def(py::pickle(
             [](const py::object &self) {
-                return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__"));
+                return py::make_tuple(
+                    self.attr("value"), self.attr("extra"), self.attr("__dict__"));
             },
             [](const py::tuple &t) {
                 if (t.size() != 3) {
                     throw std::runtime_error("Invalid state!");
-}
+                }
 
                 auto cpp_state = PickleableWithDictNew(t[0].cast<std::string>());
                 cpp_state.extra = t[1].cast<int>();

tests/test_pytypes.py

@@ -1,8 +1,9 @@
+import contextlib
 import sys
 
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import debug_enabled
 from pybind11_tests import pytypes as m
 
@@ -583,6 +584,31 @@ def test_weakref(create_weakref, create_weakref_with_callback):
     assert callback_called
 
 
+@pytest.mark.parametrize(
+    "create_weakref, has_callback",
+    [
+        (m.weakref_from_handle, False),
+        (m.weakref_from_object, False),
+        (m.weakref_from_handle_and_function, True),
+        (m.weakref_from_object_and_function, True),
+    ],
+)
+def test_weakref_err(create_weakref, has_callback):
+    class C:
+        __slots__ = []
+
+    def callback(_):
+        pass
+
+    ob = C()
+    # Should raise TypeError on CPython
+    with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext():
+        if has_callback:
+            _ = create_weakref(ob, callback)
+        else:
+            _ = create_weakref(ob)
+
+
 def test_cpp_iterators():
     assert m.tuple_iterator() == 12
     assert m.dict_iterator() == 305 + 711

tools/FindPythonLibsNew.cmake

@@ -112,11 +112,24 @@ endif()
 # VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows.
 execute_process(
   COMMAND
-    "${PYTHON_EXECUTABLE}" "-c" "from distutils import sysconfig as s;import sys;import struct;
+    "${PYTHON_EXECUTABLE}" "-c" "
+import sys;import struct;
+import sysconfig as s
+USE_SYSCONFIG = sys.version_info >= (3, 10)
+if not USE_SYSCONFIG:
+    from distutils import sysconfig as ds
 print('.'.join(str(v) for v in sys.version_info));
 print(sys.prefix);
-print(s.get_python_inc(plat_specific=True));
+if USE_SYSCONFIG:
-print(s.get_python_lib(plat_specific=True));
+    scheme = s.get_default_scheme()
+    if scheme == 'posix_local':
+        # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
+        scheme = 'posix_prefix'
+    print(s.get_path('platinclude', scheme))
+    print(s.get_path('platlib'))
+else:
+    print(ds.get_python_inc(plat_specific=True));
+    print(ds.get_python_lib(plat_specific=True));
 print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'));
 print(hasattr(sys, 'gettotalrefcount')+0);
 print(struct.calcsize('@P'));