Merge branch 'v2.10' into stable

.clang-tidy

@@ -63,6 +63,8 @@ Checks: |
   -bugprone-unused-raii,
 
 CheckOptions:
+- key:             modernize-use-equals-default.IgnoreMacros
+  value:           false
 - key:             performance-for-range-copy.WarnOnAllAutoCopies
   value:           true
 - key:             performance-inefficient-string-concatenation.StrictMode

.codespell-ignore-lines

@@ -0,0 +1,24 @@
+template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t>
+    template <typename ThisT>
+        auto &this_ = static_cast<ThisT &>(*this);
+                if (load_impl<ThisT>(temp, false)) {
+        ssize_t nd = 0;
+        auto trivial = broadcast(buffers, nd, shape);
+        auto ndim = (size_t) nd;
+    int nd;
+    ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; }
+        using op = op_impl<id, ot, Base, L_type, R_type>;
+template <op_id id, op_type ot, typename L, typename R>
+    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])
+struct IntStruct {
+    explicit IntStruct(int v) : value(v){};
+    ~IntStruct() { value = -value; }
+    IntStruct(const IntStruct &) = default;
+    IntStruct &operator=(const IntStruct &) = default;
+    py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); }));
+    py::implicitly_convertible<int, IntStruct>();
+    m.def("test", [](int expected, const IntStruct &in) {
+        [](int expected, const IntStruct &in) {

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
         - '3.6'
         - '3.9'
         - '3.10'
-        - '3.11-dev'
+        - '3.11'
         - 'pypy-3.7'
         - 'pypy-3.8'
         - 'pypy-3.9'
@@ -80,7 +80,7 @@ jobs:
       run: brew install boost
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Cache wheels
       if: runner.os == 'macOS'
@@ -102,10 +102,12 @@ jobs:
       run: python -m pip install pytest-github-actions-annotate-failures
 
     # First build - C++11 mode and inplace
+    # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here.
     - name: Configure C++11 ${{ matrix.args }}
       run: >
         cmake -S . -B .
         -DPYBIND11_WERROR=ON
+        -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=11
@@ -119,7 +121,7 @@ jobs:
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))"
       run: cmake --build .  --target cpptest -j 2
 
     - name: Interface test C++11
@@ -129,10 +131,12 @@ jobs:
       run: git clean -fdx
 
     # Second build - C++17 mode and in a build directory
+    # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here.
     - name: Configure C++17
       run: >
         cmake -S . -B build2
         -DPYBIND11_WERROR=ON
+        -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
@@ -146,7 +150,7 @@ jobs:
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))"
       run: cmake --build build2 --target cpptest
 
     # Third build - C++17 mode with unstable ABI
@@ -186,7 +190,7 @@ jobs:
         - python-version: "3.9"
           python-debug: true
           valgrind: true
-        - python-version: "3.11-dev"
+        - python-version: "3.11"
           python-debug: false
 
     name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64"
@@ -202,7 +206,7 @@ jobs:
         debug: ${{ matrix.python-debug }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Valgrind cache
       if: matrix.valgrind
@@ -281,6 +285,12 @@ jobs:
             std: 20
           - clang: 10
             std: 17
+          - clang: 11
+            std: 20
+          - clang: 12
+            std: 20
+          - clang: 13
+            std: 20
           - clang: 14
             std: 20
 
@@ -318,8 +328,8 @@ jobs:
   # Testing NVCC; forces sources to behave like .cu files
   cuda:
     runs-on: ubuntu-latest
-    name: "🐍 3.8 • CUDA 11.2 • Ubuntu 20.04"
+    name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04"
-    container: nvidia/cuda:11.2.2-devel-ubuntu20.04
+    container: nvidia/cuda:11.7.0-devel-ubuntu22.04
 
     steps:
     - uses: actions/checkout@v3
@@ -385,7 +395,7 @@ jobs:
   # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds
   centos-nvhpc7:
     runs-on: ubuntu-latest
-    name: "🐍 3 • CentOS7 / PGI 22.3 • x64"
+    name: "🐍 3 • CentOS7 / PGI 22.9 • x64"
     container: centos:7
 
     steps:
@@ -395,7 +405,7 @@ jobs:
       run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils
 
     - name: Install NVidia HPC SDK
-      run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.3
+      run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9
 
     # On CentOS 7, we have to filter a few tests (compiler internal error)
     # and allow deeper template recursion (not needed on CentOS 8 with a newer
@@ -405,12 +415,12 @@ jobs:
       shell: bash
       run: |
         source /etc/profile.d/modules.sh
-        module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.3
+        module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9
         cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \
                             -DCMAKE_CXX_STANDARD=11 \
                             -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \
                             -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \
-                            -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp"
+                            -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp"
 
     # Building before installing Pip should produce a warning but not an error
     - name: Build
@@ -437,14 +447,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        gcc:
-          - 7
-          - latest
-        std:
-          - 11
         include:
-          - gcc: 10
+          - { gcc: 7, std: 11 }
-            std: 20
+          - { gcc: 7, std: 17 }
+          - { gcc: 8, std: 14 }
+          - { gcc: 8, std: 17 }
+          - { gcc: 10, std: 17 }
+          - { gcc: 11, std: 20 }
+          - { gcc: 12, std: 20 }
 
     name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64"
     container: "gcc:${{ matrix.gcc }}"
@@ -459,7 +469,7 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Configure
       shell: bash
@@ -731,6 +741,9 @@ jobs:
             args: -DCMAKE_CXX_STANDARD=20
           - python: 3.8
             args: -DCMAKE_CXX_STANDARD=17
+          - python: 3.7
+            args: -DCMAKE_CXX_STANDARD=14
+
 
     name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}"
     runs-on: windows-2019
@@ -745,10 +758,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.10.0
+      uses: ilammy/msvc-dev-cmd@v1.12.0
       with:
         arch: x86
 
@@ -798,10 +811,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.10.0
+      uses: ilammy/msvc-dev-cmd@v1.12.0
       with:
         arch: x86
 
@@ -849,7 +862,7 @@ jobs:
         python3 -m pip install -r tests/requirements.txt
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Configure C++20
       run: >
@@ -905,7 +918,7 @@ jobs:
     - name: Configure C++11
       # LTO leads to many undefined reference like
       # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&)
-      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DDOWNLOAD_CATCH=ON -S . -B build
+      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build
 
     - name: Build C++11
       run: cmake --build build -j 2
@@ -923,7 +936,7 @@ jobs:
       run: git clean -fdx
 
     - name: Configure C++14
-      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DDOWNLOAD_CATCH=ON -S . -B build2
+      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2
 
     - name: Build C++14
       run: cmake --build build2 -j 2
@@ -941,7 +954,7 @@ jobs:
       run: git clean -fdx
 
     - name: Configure C++17
-      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DDOWNLOAD_CATCH=ON -S . -B build3
+      run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DCMAKE_VERBOSE_MAKEFILE=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3
 
     - name: Build C++17
       run: cmake --build build3 -j 2

.github/workflows/configure.yml

@@ -51,7 +51,7 @@ jobs:
     # An action for adding a specific version of CMake:
     #   https://github.com/jwlawson/actions-setup-cmake
     - name: Setup CMake ${{ matrix.cmake }}
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
       with:
         cmake-version: ${{ matrix.cmake }}
 

.github/workflows/pip.yml

@@ -98,13 +98,13 @@ jobs:
     - uses: actions/download-artifact@v3
 
     - name: Publish standard package
-      uses: pypa/gh-action-pypi-publish@v1.5.0
+      uses: pypa/gh-action-pypi-publish@v1.5.1
       with:
         password: ${{ secrets.pypi_password }}
         packages_dir: standard/
 
     - name: Publish global package
-      uses: pypa/gh-action-pypi-publish@v1.5.0
+      uses: pypa/gh-action-pypi-publish@v1.5.1
       with:
         password: ${{ secrets.pypi_password_global }}
         packages_dir: global/

.github/workflows/upstream.yml

@@ -31,7 +31,7 @@ jobs:
       run: sudo apt-get install libboost-dev
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.12
+      uses: jwlawson/actions-setup-cmake@v1.13
 
     - name: Prepare env
       run: |
@@ -104,7 +104,7 @@ jobs:
       run: cmake --build build3 --target pytest
 
     - name: Interface test
-      run: cmake --build build2 --target test_cmake_build
+      run: cmake --build build3 --target test_cmake_build
 
     # This makes sure the setup_helpers module can build packages using
     # setuptools

.pre-commit-config.yaml

@@ -12,6 +12,15 @@
 #
 # See https://github.com/pre-commit/pre-commit
 
+
+ci:
+  autoupdate_commit_msg: "chore(deps): update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+  autoupdate_schedule: monthly
+
+# third-party content
+exclude: ^tools/JoinPaths.cmake$
+
 repos:
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -32,7 +41,7 @@ repos:
 
 # Upgrade old Python syntax
 - repo: https://github.com/asottile/pyupgrade
-  rev: "v2.37.1"
+  rev: "v2.38.2"
   hooks:
   - id: pyupgrade
     args: [--py36-plus]
@@ -45,7 +54,7 @@ repos:
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black
-  rev: "22.6.0" # Keep in sync with blacken-docs
+  rev: "22.8.0" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
@@ -55,23 +64,23 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==22.6.0 # keep in sync with black hook
+    - black==22.8.0 # keep in sync with black hook
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: "v1.3.0"
+  rev: "v1.3.1"
   hooks:
   - id: remove-tabs
 
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.3.1"
+  rev: "0.4.0"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
 
 # Autoremoves unused imports
 - repo: https://github.com/hadialqattan/pycln
-  rev: "v2.0.1"
+  rev: "v2.1.1"
   hooks:
   - id: pycln
     stages: [manual]
@@ -90,7 +99,7 @@ repos:
 
 # Automatically remove noqa that are not used
 - repo: https://github.com/asottile/yesqa
-  rev: "v1.3.0"
+  rev: "v1.4.0"
   hooks:
   - id: yesqa
     additional_dependencies: &flake8_dependencies
@@ -99,7 +108,7 @@ repos:
 
 # Flake8 also supports pre-commit natively (same author)
 - repo: https://github.com/PyCQA/flake8
-  rev: "4.0.1"
+  rev: "5.0.4"
   hooks:
   - id: flake8
     exclude: ^(docs/.*|tools/.*)$
@@ -107,7 +116,7 @@ repos:
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint
-  rev: "v2.14.4"
+  rev: "v2.15.3"
   hooks:
   - id: pylint
     files: ^pybind11
@@ -123,7 +132,7 @@ repos:
 
 # Check static types with mypy
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v0.961"
+  rev: "v0.981"
   hooks:
   - id: mypy
     args: []
@@ -140,12 +149,14 @@ repos:
     additional_dependencies: [cmake, ninja]
 
 # Check for spelling
+# Use tools/codespell_ignore_lines_from_errors.py
+# to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.1.0"
+  rev: "v2.2.1"
   hooks:
   - id: codespell
     exclude: ".supp$"
-    args: ["-L", "nd,ot,thist"]
+    args: ["-x", ".codespell-ignore-lines"]
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py

CMakeLists.txt

@@ -91,10 +91,16 @@ endif()
 option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT})
 option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT})
 option(PYBIND11_NOPYTHON "Disable search for Python" OFF)
+option(PYBIND11_SIMPLE_GIL_MANAGEMENT
+       "Use simpler GIL management logic that does not support disassociation" OFF)
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
 
+if(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+  add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+endif()
+
 cmake_dependent_option(
   USE_PYTHON_INCLUDE_DIR
   "Install pybind11 headers in Python include directory instead of default installation prefix"
@@ -198,6 +204,9 @@ else()
 endif()
 
 include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake")
+# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files
+# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet
+include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake")
 
 # Relative directory setting
 if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS)
@@ -262,6 +271,16 @@ if(PYBIND11_INSTALL)
     NAMESPACE "pybind11::"
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
+  # pkg-config support
+  if(NOT prefix_for_pc_file)
+    set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}")
+  endif()
+  join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"
+                 "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY)
+  install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"
+          DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/")
+
   # Uninstall target
   if(PYBIND11_MASTER_PROJECT)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in"

docs/advanced/cast/custom.rst

@@ -38,7 +38,7 @@ type is explicitly allowed.
 
 .. code-block:: cpp
 
-    namespace pybind11 { namespace detail {
+    namespace PYBIND11_NAMESPACE { namespace detail {
         template <> struct type_caster<inty> {
         public:
             /**
@@ -78,7 +78,7 @@ type is explicitly allowed.
                 return PyLong_FromLong(src.long_value);
             }
         };
-    }} // namespace pybind11::detail
+    }} // namespace PYBIND11_NAMESPACE::detail
 
 .. note::
 

docs/advanced/cast/stl.rst

@@ -42,7 +42,7 @@ types:
 .. code-block:: cpp
 
     // `boost::optional` as an example -- can be any `std::optional`-like container
-    namespace pybind11 { namespace detail {
+    namespace PYBIND11_NAMESPACE { namespace detail {
         template <typename T>
         struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
     }}
@@ -54,7 +54,7 @@ for custom variant types:
 .. code-block:: cpp
 
     // `boost::variant` as an example -- can be any `std::variant`-like container
-    namespace pybind11 { namespace detail {
+    namespace PYBIND11_NAMESPACE { namespace detail {
         template <typename... Ts>
         struct type_caster<boost::variant<Ts...>> : variant_caster<boost::variant<Ts...>> {};
 
@@ -66,7 +66,7 @@ for custom variant types:
                 return boost::apply_visitor(args...);
             }
         };
-    }} // namespace pybind11::detail
+    }} // namespace PYBIND11_NAMESPACE::detail
 
 The ``visit_helper`` specialization is not required if your ``name::variant`` provides
 a ``name::visit()`` function. For any other function name, the specialization must be

docs/advanced/classes.rst

@@ -1228,7 +1228,7 @@ whether a downcast is safe, you can proceed by specializing the
         std::string bark() const { return sound; }
     };
 
-    namespace pybind11 {
+    namespace PYBIND11_NAMESPACE {
         template<> struct polymorphic_type_hook<Pet> {
             static const void *get(const Pet *src, const std::type_info*& type) {
                 // note that src may be nullptr
@@ -1239,7 +1239,7 @@ whether a downcast is safe, you can proceed by specializing the
                 return src;
             }
         };
-    } // namespace pybind11
+    } // namespace PYBIND11_NAMESPACE
 
 When pybind11 wants to convert a C++ pointer of type ``Base*`` to a
 Python object, it calls ``polymorphic_type_hook<Base>::get()`` to

docs/advanced/misc.rst

@@ -39,15 +39,42 @@ The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds.
 Global Interpreter Lock (GIL)
 =============================
 
-When calling a C++ function from Python, the GIL is always held.
+The Python C API dictates that the Global Interpreter Lock (GIL) must always
+be held by the current thread to safely access Python objects. As a result,
+when Python calls into C++ via pybind11 the GIL must be held, and pybind11
+will never implicitly release the GIL.
+
+.. code-block:: cpp
+
+    void my_function() {
+        /* GIL is held when this function is called from Python */
+    }
+
+    PYBIND11_MODULE(example, m) {
+        m.def("my_function", &my_function);
+    }
+
+pybind11 will ensure that the GIL is held when it knows that it is calling
+Python code. For example, if a Python callback is passed to C++ code via
+``std::function``, when C++ code calls the function the built-in wrapper
+will acquire the GIL before calling the Python callback. Similarly, the
+``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling
+back into Python.
+
+When writing C++ code that is called from other C++ code, if that code accesses
+Python state, it must explicitly acquire and release the GIL.
+
 The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be
 used to acquire and release the global interpreter lock in the body of a C++
 function call. In this way, long-running C++ code can be parallelized using
-multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this
+multiple Python threads, **but great care must be taken** when any
+:class:`gil_scoped_release` appear: if there is any way that the C++ code
+can access Python objects, :class:`gil_scoped_acquire` should be used to
+reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this
 could be realized as follows (important changes highlighted):
 
 .. code-block:: cpp
-    :emphasize-lines: 8,9,31,32
+    :emphasize-lines: 8,30,31
 
     class PyAnimal : public Animal {
     public:
@@ -56,9 +83,7 @@ could be realized as follows (important changes highlighted):
 
         /* Trampoline (need one for each virtual function) */
         std::string go(int n_times) {
-            /* Acquire GIL before calling Python code */
+            /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */
-            py::gil_scoped_acquire acquire;
-
             PYBIND11_OVERRIDE_PURE(
                 std::string, /* Return type */
                 Animal,      /* Parent class */
@@ -78,7 +103,8 @@ could be realized as follows (important changes highlighted):
             .def(py::init<>());
 
         m.def("call_go", [](Animal *animal) -> std::string {
-            /* Release GIL before calling into (potentially long-running) C++ code */
+            // GIL is held when called from Python code. Release GIL before
+            // calling into (potentially long-running) C++ code
             py::gil_scoped_release release;
             return call_go(animal);
         });

docs/advanced/pycpp/numpy.rst

@@ -433,7 +433,7 @@ following:
             { 2, 4 },                                  // shape (rows, cols)
             { sizeof(uint8_t) * 4, sizeof(uint8_t) }   // strides in bytes
         );
-    })
+    });
 
 This approach is meant for providing a ``memoryview`` for a C/C++ buffer not
 managed by Python. The user is responsible for managing the lifetime of the
@@ -449,7 +449,7 @@ We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer:
             buffer,               // buffer pointer
             sizeof(uint8_t) * 8   // buffer size
         );
-    })
+    });
 
 .. versionchanged:: 2.6
     ``memoryview::from_memory`` added.

docs/advanced/smart_ptrs.rst

@@ -157,7 +157,7 @@ specialized:
     PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>);
 
     // Only needed if the type's `.get()` goes by another name
-    namespace pybind11 { namespace detail {
+    namespace PYBIND11_NAMESPACE { namespace detail {
         template <typename T>
         struct holder_helper<SmartPtr<T>> { // <-- specialization
             static const T *get(const SmartPtr<T> &p) { return p.getPointer(); }

docs/changelog.rst

@@ -9,6 +9,148 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning
 Changes will be added here periodically from the "Suggested changelog entry"
 block in pull request descriptions.
 
+
+IN DEVELOPMENT
+--------------
+
+Changes will be summarized here periodically.
+
+Version 2.10.1 (Oct 31, 2022)
+-----------------------------
+
+This is the first version to fully support embedding the newly released Python 3.11.
+
+Changes:
+
+* Allow ``pybind11::capsule`` constructor to take null destructor pointers.
+  `#4221 <https://github.com/pybind/pybind11/pull/4221>`_
+
+* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11
+  (established behavior).
+  `#4119 <https://github.com/pybind/pybind11/pull/4119>`_
+
+* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define),
+  along with many additional tests in ``test_gil_scoped.py``. The option may be
+  useful to try when debugging GIL-related issues, to determine if the more
+  complex default implementation is or is not to blame. See #4216 for
+  background. WARNING: Please be careful to not create ODR violations when
+  using the option: everything that is linked together with mutual symbol
+  visibility needs to be rebuilt.
+  `#4216 <https://github.com/pybind/pybind11/pull/4216>`_
+
+* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes
+  Linux builds safer, and enables the removal of warning suppression pragmas for
+  Windows.
+  `#4298 <https://github.com/pybind/pybind11/pull/4298>`_
+
+Bug fixes:
+
+* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various
+  ``py::str`` ctors when decoding surrogate utf characters.
+  `#4294 <https://github.com/pybind/pybind11/pull/4294>`_
+
+* Revert perfect forwarding for ``make_iterator``. This broke at least one
+  valid use case. May revisit later.
+  `#4234 <https://github.com/pybind/pybind11/pull/4234>`_
+
+* Fix support for safe casts to ``void*`` (regression in 2.10.0).
+  `#4275 <https://github.com/pybind/pybind11/pull/4275>`_
+
+* Fix ``char8_t`` support (regression in 2.9).
+  `#4278 <https://github.com/pybind/pybind11/pull/4278>`_
+
+* Unicode surrogate character in Python exception message leads to process
+  termination in ``error_already_set::what()``.
+  `#4297 <https://github.com/pybind/pybind11/pull/4297>`_
+
+* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``.
+  `#4188 <https://github.com/pybind/pybind11/pull/4188>`_
+
+* Make augmented assignment operators non-const for the object-api. Behavior
+  was previously broken for augmented assignment operators.
+  `#4065 <https://github.com/pybind/pybind11/pull/4065>`_
+
+* Add proper error checking to C++ bindings for Python list append and insert.
+  `#4208 <https://github.com/pybind/pybind11/pull/4208>`_
+
+* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0.
+  `#4220 <https://github.com/pybind/pybind11/pull/4220>`_
+
+* A workaround for PyPy was added in the ``py::error_already_set``
+  implementation, related to PR `#1895 <https://github.com/pybind/pybind11/pull/1895>`_
+  released with v2.10.0.
+  `#4079 <https://github.com/pybind/pybind11/pull/4079>`_
+
+* Fixed compiler errors when C++23 ``std::forward_like`` is available.
+  `#4136 <https://github.com/pybind/pybind11/pull/4136>`_
+
+* Properly raise exceptions in contains methods (like when an object in unhashable).
+  `#4209 <https://github.com/pybind/pybind11/pull/4209>`_
+
+* Further improve another error in exception handling.
+  `#4232 <https://github.com/pybind/pybind11/pull/4232>`_
+
+* ``get_local_internals()`` was made compatible with
+  ``finalize_interpreter()``, fixing potential freezes during interpreter
+  finalization.
+  `#4192 <https://github.com/pybind/pybind11/pull/4192>`_
+
+
+Performance and style:
+
+* Reserve space in set and STL map casters if possible. This will prevent
+  unnecessary rehashing / resizing by knowing the number of keys ahead of time
+  for Python to C++ casting. This improvement will greatly speed up the casting
+  of large unordered maps and sets.
+  `#4194 <https://github.com/pybind/pybind11/pull/4194>`_
+
+* GIL RAII scopes are non-copyable to avoid potential bugs.
+  `#4183 <https://github.com/pybind/pybind11/pull/4183>`_
+
+* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT``
+  macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in
+  macros as well.
+  `#4017 <https://github.com/pybind/pybind11/pull/4017>`_
+
+* Optimize iterator advancement in C++ bindings.
+  `#4237 <https://github.com/pybind/pybind11/pull/4237>`_
+
+* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict``
+  for handling dynamic attribute dictionaries.
+  `#4106 <https://github.com/pybind/pybind11/pull/4106>`_
+
+* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when
+  opening namespaces. Using namespace declarations and namespace qualification
+  remain the same as ``pybind11``. This is done to ensure consistent symbol
+  visibility.
+  `#4098 <https://github.com/pybind/pybind11/pull/4098>`_
+
+* Mark ``detail::forward_like`` as constexpr.
+  `#4147 <https://github.com/pybind/pybind11/pull/4147>`_
+
+* Optimize unpacking_collector when processing ``arg_v`` arguments.
+  `#4219 <https://github.com/pybind/pybind11/pull/4219>`_
+
+* Optimize casting C++ object to ``None``.
+  `#4269 <https://github.com/pybind/pybind11/pull/4269>`_
+
+
+Build system improvements:
+
+* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``.
+  `#4195 <https://github.com/pybind/pybind11/pull/4195>`_
+
+* Include a pkg-config file when installing pybind11, such as in the Python
+  package.
+  `#4077 <https://github.com/pybind/pybind11/pull/4077>`_
+
+* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG``
+  instead of ``Debug``.
+  `#4078 <https://github.com/pybind/pybind11/pull/4078>`_
+
+* Followup to `#3948 <https://github.com/pybind/pybind11/pull/3948>`_, fixing vcpkg again.
+  `#4123 <https://github.com/pybind/pybind11/pull/4123>`_
+
 Version 2.10.0 (Jul 15, 2022)
 -----------------------------
 

include/pybind11/cast.h

@@ -248,7 +248,7 @@ public:
         return false;
     }
     static handle cast(T, return_value_policy /* policy */, handle /* parent */) {
-        return none().inc_ref();
+        return none().release();
     }
     PYBIND11_TYPE_CASTER(T, const_name("None"));
 };
@@ -291,7 +291,7 @@ public:
         if (ptr) {
             return capsule(ptr).release();
         }
-        return none().inc_ref();
+        return none().release();
     }
 
     template <typename T>
@@ -537,7 +537,7 @@ public:
 
     static handle cast(const CharT *src, return_value_policy policy, handle parent) {
         if (src == nullptr) {
-            return pybind11::none().inc_ref();
+            return pybind11::none().release();
         }
         return StringCaster::cast(StringType(src), policy, parent);
     }
@@ -1179,11 +1179,9 @@ enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&)
     pybind11_fail("Internal error: cast_safe fallback invoked");
 }
 template <typename T>
-enable_if_t<std::is_same<void, intrinsic_t<T>>::value, void> cast_safe(object &&) {}
+enable_if_t<std::is_void<T>::value, void> cast_safe(object &&) {}
 template <typename T>
-enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>,
+enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>, std::is_void<T>>::value, T>
-                            std::is_same<void, intrinsic_t<T>>>::value,
-            T>
 cast_safe(object &&o) {
     return pybind11::cast<T>(std::move(o));
 }
@@ -1545,7 +1543,7 @@ private:
             throw cast_error_unable_to_convert_call_arg(a.name, a.type);
 #endif
         }
-        m_kwargs[a.name] = a.value;
+        m_kwargs[a.name] = std::move(a.value);
     }
 
     void process(list & /*args_list*/, detail::kwargs_proxy kp) {

include/pybind11/detail/class.h

@@ -55,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec
     return PyProperty_Type.tp_descr_set(self, cls, value);
 }
 
+// Forward declaration to use in `make_static_property_type()`
+inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type);
+
 /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()`
     methods are modified to always use the object type instead of a concrete instance.
     Return value: New reference. */
@@ -87,6 +90,13 @@ inline PyTypeObject *make_static_property_type() {
         pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");
     }
 
+#    if PY_VERSION_HEX >= 0x030C0000
+    // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE.
+    // Since Python-3.12 property-derived types are required to
+    // have dynamic attributes (to set `__doc__`)
+    enable_dynamic_attributes(heap_type);
+#    endif
+
     setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
     PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
 
@@ -502,31 +512,6 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) {
     return (PyObject *) heap_type;
 }
 
-/// dynamic_attr: Support for `d = instance.__dict__`.
-extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) {
-    PyObject *&dict = *_PyObject_GetDictPtr(self);
-    if (!dict) {
-        dict = PyDict_New();
-    }
-    Py_XINCREF(dict);
-    return dict;
-}
-
-/// dynamic_attr: Support for `instance.__dict__ = dict()`.
-extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) {
-    if (!PyDict_Check(new_dict)) {
-        PyErr_Format(PyExc_TypeError,
-                     "__dict__ must be set to a dictionary, not a '%.200s'",
-                     get_fully_qualified_tp_name(Py_TYPE(new_dict)).c_str());
-        return -1;
-    }
-    PyObject *&dict = *_PyObject_GetDictPtr(self);
-    Py_INCREF(new_dict);
-    Py_CLEAR(dict);
-    dict = new_dict;
-    return 0;
-}
-
 /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`.
 extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) {
     PyObject *&dict = *_PyObject_GetDictPtr(self);
@@ -558,9 +543,17 @@ inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) {
     type->tp_traverse = pybind11_traverse;
     type->tp_clear = pybind11_clear;
 
-    static PyGetSetDef getset[] = {
+    static PyGetSetDef getset[] = {{
-        {const_cast<char *>("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr},
+#if PY_VERSION_HEX < 0x03070000
-        {nullptr, nullptr, nullptr, nullptr, nullptr}};
+                                       const_cast<char *>("__dict__"),
+#else
+                                       "__dict__",
+#endif
+                                       PyObject_GenericGetDict,
+                                       PyObject_GenericSetDict,
+                                       nullptr,
+                                       nullptr},
+                                   {nullptr, nullptr, nullptr, nullptr, nullptr}};
     type->tp_getset = getset;
 }
 

include/pybind11/detail/common.h

@@ -11,11 +11,11 @@
 
 #define PYBIND11_VERSION_MAJOR 2
 #define PYBIND11_VERSION_MINOR 10
-#define PYBIND11_VERSION_PATCH 0
+#define PYBIND11_VERSION_PATCH 1
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x020A0000
+#define PYBIND11_VERSION_HEX 0x020A0100
 
 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name {
 #define PYBIND11_NAMESPACE_END(name) }
@@ -205,11 +205,8 @@
 #    endif
 #endif
 
-#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
-#    define PYBIND11_HAS_U8STRING
-#endif
-
 #include <Python.h>
+// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000
 #if PY_VERSION_HEX < 0x03060000
 #    error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5."
 #endif
@@ -233,6 +230,10 @@
 #    undef copysign
 #endif
 
+#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#    define PYBIND11_SIMPLE_GIL_MANAGEMENT
+#endif
+
 #if defined(_MSC_VER)
 #    if defined(PYBIND11_DEBUG_MARKER)
 #        define _DEBUG
@@ -259,6 +260,11 @@
 #    endif
 #endif
 
+// Must be after including <version> or one of the other headers specified by the standard
+#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
+#    define PYBIND11_HAS_U8STRING
+#endif
+
 // #define PYBIND11_STR_LEGACY_PERMISSIVE
 // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject
 //             (probably surprising and never documented, but this was the
@@ -1033,12 +1039,7 @@ PYBIND11_NAMESPACE_END(detail)
 ///  - regular: static_cast<Return (Class::*)(Arg0, Arg1, Arg2)>(&Class::func)
 ///  - sweet:   overload_cast<Arg0, Arg1, Arg2>(&Class::func)
 template <typename... Args>
-#    if (defined(_MSC_VER) && _MSC_VER < 1920) /* MSVC 2017 */                                    \
+static constexpr detail::overload_cast_impl<Args...> overload_cast{};
-        || (defined(__clang__) && __clang_major__ == 5)
-static constexpr detail::overload_cast_impl<Args...> overload_cast = {};
-#    else
-static constexpr detail::overload_cast_impl<Args...> overload_cast;
-#    endif
 #endif
 
 /// Const member function selector for overload_cast
@@ -1158,6 +1159,24 @@ constexpr inline bool silence_msvc_c4127(bool cond) { return cond; }
 #    define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__
 #endif
 
+#if defined(__clang__)                                                                            \
+    && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point    \
+                                            available. */                                         \
+        || (__clang_major__ >= 7                                                                  \
+            && __clang_major__ <= 12) /* Clang 3, 5, 13, 14, 15 do not generate the warning. */   \
+    )
+#    define PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+// Example:
+// tests/test_kwargs_and_defaults.cpp:46:68: error: local variable 'args' will be copied despite
+// being returned by name [-Werror,-Wreturn-std-move]
+//     m.def("args_function", [](py::args args) -> py::tuple { return args; });
+//                                                                    ^~~~
+// test_kwargs_and_defaults.cpp:46:68: note: call 'std::move' explicitly to avoid copying
+//     m.def("args_function", [](py::args args) -> py::tuple { return args; });
+//                                                                    ^~~~
+//                                                                    std::move(args)
+#endif
+
 // Pybind offers detailed error messages by default for all builts that are debug (through the
 // negation of ndebug). This can also be manually enabled by users, for any builds, through
 // defining PYBIND11_DETAILED_ERROR_MESSAGES.

include/pybind11/detail/internals.h

@@ -9,6 +9,12 @@
 
 #pragma once
 
+#include "common.h"
+
+#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#    include "../gil.h"
+#endif
+
 #include "../pytypes.h"
 
 #include <exception>
@@ -49,7 +55,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass);
 // `Py_LIMITED_API` anyway.
 #    if PYBIND11_INTERNALS_VERSION > 4
 #        define PYBIND11_TLS_KEY_REF Py_tss_t &
-#        ifdef __GNUC__
+#        if defined(__GNUC__) && !defined(__INTEL_COMPILER)
 // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer
 // for every field.
 #            define PYBIND11_TLS_KEY_INIT(var)                                                    \
@@ -169,10 +175,12 @@ struct internals {
     PyTypeObject *default_metaclass;
     PyObject *instance_base;
 #if defined(WITH_THREAD)
+    // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PYBIND11_TLS_KEY_INIT(tstate)
 #    if PYBIND11_INTERNALS_VERSION > 4
     PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
 #    endif // PYBIND11_INTERNALS_VERSION > 4
+    // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
     PyInterpreterState *istate = nullptr;
     ~internals() {
 #    if PYBIND11_INTERNALS_VERSION > 4
@@ -408,13 +416,21 @@ PYBIND11_NOINLINE internals &get_internals() {
         return **internals_pp;
     }
 
+#if defined(WITH_THREAD)
+#    if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+    gil_scoped_acquire gil;
+#    else
     // Ensure that the GIL is held since we will need to make Python calls.
     // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals.
     struct gil_scoped_acquire_local {
         gil_scoped_acquire_local() : state(PyGILState_Ensure()) {}
+        gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete;
+        gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete;
         ~gil_scoped_acquire_local() { PyGILState_Release(state); }
         const PyGILState_STATE state;
     } gil;
+#    endif
+#endif
     error_scope err_scope;
 
     PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID);
@@ -512,8 +528,13 @@ struct local_internals {
 
 /// Works like `get_internals`, but for things which are locally registered.
 inline local_internals &get_local_internals() {
-    static local_internals locals;
+    // Current static can be created in the interpreter finalization routine. If the later will be
-    return locals;
+    // destroyed in another static variable destructor, creation of this static there will cause
+    // static deinitialization fiasco. In order to avoid it we avoid destruction of the
+    // local_internals static. One can read more about the problem and current solution here:
+    // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables
+    static auto *locals = new local_internals();
+    return *locals;
 }
 
 /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its

include/pybind11/eigen.h

@@ -27,6 +27,9 @@
 #    pragma warning(disable : 4127) // C4127: conditional expression is constant
 #    pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741
 //       C5054: operator '&': deprecated between enumerations of different types
+#elif defined(__MINGW32__)
+#    pragma GCC diagnostic push
+#    pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
 #endif
 
 #include <Eigen/Core>
@@ -34,6 +37,8 @@
 
 #if defined(_MSC_VER)
 #    pragma warning(pop)
+#elif defined(__MINGW32__)
+#    pragma GCC diagnostic pop
 #endif
 
 // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit

include/pybind11/embed.h

@@ -150,6 +150,8 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
 #else
     PyConfig config;
     PyConfig_InitIsolatedConfig(&config);
+    config.isolated = 0;
+    config.use_environment = 1;
     config.install_signal_handlers = init_signal_handlers ? 1 : 0;
 
     PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast<char *const *>(argv));

include/pybind11/functional.h

@@ -110,7 +110,7 @@ public:
     template <typename Func>
     static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) {
         if (!f_) {
-            return none().inc_ref();
+            return none().release();
         }
 
         auto result = f_.template target<function_type>();

include/pybind11/gil.h

@@ -10,7 +10,10 @@
 #pragma once
 
 #include "detail/common.h"
-#include "detail/internals.h"
+
+#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#    include "detail/internals.h"
+#endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked();
 
 PYBIND11_NAMESPACE_END(detail)
 
-#if defined(WITH_THREAD) && !defined(PYPY_VERSION)
+#if defined(WITH_THREAD)
+
+#    if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 
 /* The functions below essentially reproduce the PyGILState_* API using a RAII
  * pattern, but there are a few important differences:
@@ -62,11 +67,11 @@ public:
 
         if (!tstate) {
             tstate = PyThreadState_New(internals.istate);
-#    if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+#        if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
             if (!tstate) {
                 pybind11_fail("scoped_acquire: could not create thread state!");
             }
-#    endif
+#        endif
             tstate->gilstate_counter = 0;
             PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate);
         } else {
@@ -80,24 +85,27 @@ public:
         inc_ref();
     }
 
+    gil_scoped_acquire(const gil_scoped_acquire &) = delete;
+    gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete;
+
     void inc_ref() { ++tstate->gilstate_counter; }
 
     PYBIND11_NOINLINE void dec_ref() {
         --tstate->gilstate_counter;
-#    if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+#        if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
         if (detail::get_thread_state_unchecked() != tstate) {
             pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!");
         }
         if (tstate->gilstate_counter < 0) {
             pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!");
         }
-#    endif
+#        endif
         if (tstate->gilstate_counter == 0) {
-#    if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+#        if defined(PYBIND11_DETAILED_ERROR_MESSAGES)
             if (!release) {
                 pybind11_fail("scoped_acquire::dec_ref(): internal error!");
             }
-#    endif
+#        endif
             PyThreadState_Clear(tstate);
             if (active) {
                 PyThreadState_DeleteCurrent();
@@ -144,6 +152,9 @@ public:
         }
     }
 
+    gil_scoped_release(const gil_scoped_acquire &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
+
     /// This method will disable the PyThreadState_DeleteCurrent call and the
     /// GIL won't be acquired. This method should be used if the interpreter
     /// could be shutting down when this is called, as thread deletion is not
@@ -172,12 +183,16 @@ private:
     bool disassoc;
     bool active = true;
 };
-#elif defined(PYPY_VERSION)
+
+#    else // PYBIND11_SIMPLE_GIL_MANAGEMENT
+
 class gil_scoped_acquire {
     PyGILState_STATE state;
 
 public:
-    gil_scoped_acquire() { state = PyGILState_Ensure(); }
+    gil_scoped_acquire() : state{PyGILState_Ensure()} {}
+    gil_scoped_acquire(const gil_scoped_acquire &) = delete;
+    gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete;
     ~gil_scoped_acquire() { PyGILState_Release(state); }
     void disarm() {}
 };
@@ -186,17 +201,39 @@ class gil_scoped_release {
     PyThreadState *state;
 
 public:
-    gil_scoped_release() { state = PyEval_SaveThread(); }
+    gil_scoped_release() : state{PyEval_SaveThread()} {}
+    gil_scoped_release(const gil_scoped_release &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
     ~gil_scoped_release() { PyEval_RestoreThread(state); }
     void disarm() {}
 };
-#else
+
+#    endif // PYBIND11_SIMPLE_GIL_MANAGEMENT
+
+#else // WITH_THREAD
+
 class gil_scoped_acquire {
+public:
+    gil_scoped_acquire() {
+        // Trick to suppress `unused variable` error messages (at call sites).
+        (void) (this != (this + 1));
+    }
+    gil_scoped_acquire(const gil_scoped_acquire &) = delete;
+    gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete;
     void disarm() {}
 };
+
 class gil_scoped_release {
+public:
+    gil_scoped_release() {
+        // Trick to suppress `unused variable` error messages (at call sites).
+        (void) (this != (this + 1));
+    }
+    gil_scoped_release(const gil_scoped_release &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
     void disarm() {}
 };
-#endif
+
+#endif // WITH_THREAD
 
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/numpy.h

@@ -537,7 +537,7 @@ PYBIND11_NAMESPACE_END(detail)
 
 class dtype : public object {
 public:
-    PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_);
+    PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_)
 
     explicit dtype(const buffer_info &info) {
         dtype descr(_dtype_from_pep3118()(pybind11::str(info.format)));
@@ -1401,7 +1401,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor>
     oss << '}';
     auto format_str = oss.str();
 
-    // Sanity check: verify that NumPy properly parses our buffer format string
+    // Smoke test: verify that NumPy properly parses our buffer format string
     auto &api = npy_api::get();
     auto arr = array(buffer_info(nullptr, itemsize, format_str, 1));
     if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) {
@@ -1865,9 +1865,13 @@ private:
         }
 
         auto result = returned_array::create(trivial, shape);
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+#    pragma clang diagnostic push
+#    pragma clang diagnostic ignored "-Wreturn-std-move"
+#endif
 
         if (size == 0) {
-            return std::move(result);
+            return result;
         }
 
         /* Call the function */
@@ -1878,7 +1882,10 @@ private:
             apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq);
         }
 
-        return std::move(result);
+        return result;
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+#    pragma clang diagnostic pop
+#endif
     }
 
     template <size_t... Index, size_t... VIndex, size_t... BIndex>

include/pybind11/operators.h

@@ -84,6 +84,7 @@ struct op_impl {};
 /// Operator implementation generator
 template <op_id id, op_type ot, typename L, typename R>
 struct op_ {
+    static constexpr bool op_enable_if_hook = true;
     template <typename Class, typename... Extra>
     void execute(Class &cl, const Extra &...extra) const {
         using Base = typename Class::type;

include/pybind11/pybind11.h

@@ -1578,14 +1578,14 @@ public:
         return *this;
     }
 
-    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0>
-    class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def(const T &op, const Extra &...extra) {
         op.execute(*this, extra...);
         return *this;
     }
 
-    template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra>
+    template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0>
-    class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) {
+    class_ &def_cast(const T &op, const Extra &...extra) {
         op.execute_cast(*this, extra...);
         return *this;
     }
@@ -2073,7 +2073,7 @@ struct enum_base {
                               + "\" already exists!");
         }
 
-        entries[name] = std::make_pair(value, doc);
+        entries[name] = pybind11::make_tuple(value, doc);
         m_base.attr(std::move(name)) = std::move(value);
     }
 
@@ -2333,7 +2333,7 @@ template <typename Access,
           typename Sentinel,
           typename ValueType,
           typename... Extra>
-iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) {
     using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>;
     // TODO: state captures only the types of Extra, not the values
 
@@ -2359,7 +2359,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra)
                 Policy);
     }
 
-    return cast(state{std::forward<Iterator>(first), std::forward<Sentinel>(last), true});
+    return cast(state{first, last, true});
 }
 
 PYBIND11_NAMESPACE_END(detail)
@@ -2370,15 +2370,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_access<Iterator>::result_type,
           typename... Extra>
-iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       ValueType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a
@@ -2388,15 +2386,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename KeyType = typename detail::iterator_key_access<Iterator>::result_type,
           typename... Extra>
-iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_key_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       KeyType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a
@@ -2406,15 +2402,13 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Sentinel,
           typename ValueType = typename detail::iterator_value_access<Iterator>::result_type,
           typename... Extra>
-iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) {
+iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
     return detail::make_iterator_impl<detail::iterator_value_access<Iterator>,
                                       Policy,
                                       Iterator,
                                       Sentinel,
                                       ValueType,
-                                      Extra...>(std::forward<Iterator>(first),
+                                      Extra...>(first, last, std::forward<Extra>(extra)...);
-                                                std::forward<Sentinel>(last),
-                                                std::forward<Extra>(extra)...);
 }
 
 /// Makes an iterator over values of an stl container or other container supporting

include/pybind11/pytypes.h

@@ -155,23 +155,23 @@ public:
     object operator-() const;
     object operator~() const;
     object operator+(object_api const &other) const;
-    object operator+=(object_api const &other) const;
+    object operator+=(object_api const &other);
     object operator-(object_api const &other) const;
-    object operator-=(object_api const &other) const;
+    object operator-=(object_api const &other);
     object operator*(object_api const &other) const;
-    object operator*=(object_api const &other) const;
+    object operator*=(object_api const &other);
     object operator/(object_api const &other) const;
-    object operator/=(object_api const &other) const;
+    object operator/=(object_api const &other);
     object operator|(object_api const &other) const;
-    object operator|=(object_api const &other) const;
+    object operator|=(object_api const &other);
     object operator&(object_api const &other) const;
-    object operator&=(object_api const &other) const;
+    object operator&=(object_api const &other);
     object operator^(object_api const &other) const;
-    object operator^=(object_api const &other) const;
+    object operator^=(object_api const &other);
     object operator<<(object_api const &other) const;
-    object operator<<=(object_api const &other) const;
+    object operator<<=(object_api const &other);
     object operator>>(object_api const &other) const;
-    object operator>>=(object_api const &other) const;
+    object operator>>=(object_api const &other);
 
     PYBIND11_DEPRECATED("Use py::str(obj) instead")
     pybind11::str str() const;
@@ -334,12 +334,15 @@ public:
     }
 
     object &operator=(const object &other) {
-        other.inc_ref();
+        // Skip inc_ref and dec_ref if both objects are the same
-        // Use temporary variable to ensure `*this` remains valid while
+        if (!this->is(other)) {
-        // `Py_XDECREF` executes, in case `*this` is accessible from Python.
+            other.inc_ref();
-        handle temp(m_ptr);
+            // Use temporary variable to ensure `*this` remains valid while
-        m_ptr = other.m_ptr;
+            // `Py_XDECREF` executes, in case `*this` is accessible from Python.
-        temp.dec_ref();
+            handle temp(m_ptr);
+            m_ptr = other.m_ptr;
+            temp.dec_ref();
+        }
         return *this;
     }
 
@@ -353,6 +356,20 @@ public:
         return *this;
     }
 
+#define PYBIND11_INPLACE_OP(iop)                                                                  \
+    object iop(object_api const &other) { return operator=(handle::iop(other)); }
+
+    PYBIND11_INPLACE_OP(operator+=)
+    PYBIND11_INPLACE_OP(operator-=)
+    PYBIND11_INPLACE_OP(operator*=)
+    PYBIND11_INPLACE_OP(operator/=)
+    PYBIND11_INPLACE_OP(operator|=)
+    PYBIND11_INPLACE_OP(operator&=)
+    PYBIND11_INPLACE_OP(operator^=)
+    PYBIND11_INPLACE_OP(operator<<=)
+    PYBIND11_INPLACE_OP(operator>>=)
+#undef PYBIND11_INPLACE_OP
+
     // Calling cast() on an object lvalue just copies (via handle::cast)
     template <typename T>
     T cast() const &;
@@ -456,6 +473,12 @@ struct error_fetch_and_normalize {
                           + " failed to obtain the name "
                             "of the normalized active exception type.");
         }
+#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00
+        // This behavior runs the risk of masking errors in the error handling, but avoids a
+        // conflict with PyPy, which relies on the normalization here to change OSError to
+        // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075).
+        m_lazy_error_string = exc_type_name_norm;
+#else
         if (exc_type_name_norm != m_lazy_error_string) {
             std::string msg = std::string(called)
                               + ": MISMATCH of original and normalized "
@@ -467,6 +490,7 @@ struct error_fetch_and_normalize {
             msg += ": " + format_value_and_trace();
             pybind11_fail(msg);
         }
+#endif
     }
 
     error_fetch_and_normalize(const error_fetch_and_normalize &) = delete;
@@ -477,11 +501,29 @@ struct error_fetch_and_normalize {
         std::string message_error_string;
         if (m_value) {
             auto value_str = reinterpret_steal<object>(PyObject_Str(m_value.ptr()));
+            constexpr const char *message_unavailable_exc
+                = "<MESSAGE UNAVAILABLE DUE TO ANOTHER EXCEPTION>";
             if (!value_str) {
                 message_error_string = detail::error_string();
-                result = "<MESSAGE UNAVAILABLE DUE TO ANOTHER EXCEPTION>";
+                result = message_unavailable_exc;
             } else {
-                result = value_str.cast<std::string>();
+                // Not using `value_str.cast<std::string>()`, to not potentially throw a secondary
+                // error_already_set that will then result in process termination (#4288).
+                auto value_bytes = reinterpret_steal<object>(
+                    PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace"));
+                if (!value_bytes) {
+                    message_error_string = detail::error_string();
+                    result = message_unavailable_exc;
+                } else {
+                    char *buffer = nullptr;
+                    Py_ssize_t length = 0;
+                    if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) {
+                        message_error_string = detail::error_string();
+                        result = message_unavailable_exc;
+                    } else {
+                        result = std::string(buffer, static_cast<std::size_t>(length));
+                    }
+                }
             }
         } else {
             result = "<MESSAGE UNAVAILABLE>";
@@ -1266,7 +1308,7 @@ public:
 
 #define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun)                           \
     PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun)                                       \
-    Name() : Parent() {}
+    Name() = default;
 
 #define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr)                                                 \
     ::pybind11::type_error("Object of type '"                                                     \
@@ -1289,7 +1331,7 @@ public:
 
 #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun)                                           \
     PYBIND11_OBJECT(Name, Parent, CheckFun)                                                       \
-    Name() : Parent() {}
+    Name() = default;
 
 /// \addtogroup pytypes
 /// @{
@@ -1358,7 +1400,7 @@ public:
 private:
     void advance() {
         value = reinterpret_steal<object>(PyIter_Next(m_ptr));
-        if (PyErr_Occurred()) {
+        if (value.ptr() == nullptr && PyErr_Occurred()) {
             throw error_already_set();
         }
     }
@@ -1408,6 +1450,9 @@ public:
     str(const char *c, const SzType &n)
         : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) {
         if (!m_ptr) {
+            if (PyErr_Occurred()) {
+                throw error_already_set();
+            }
             pybind11_fail("Could not allocate string object!");
         }
     }
@@ -1417,6 +1462,9 @@ public:
     // NOLINTNEXTLINE(google-explicit-constructor)
     str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) {
         if (!m_ptr) {
+            if (PyErr_Occurred()) {
+                throw error_already_set();
+            }
             pybind11_fail("Could not allocate string object!");
         }
     }
@@ -1574,6 +1622,9 @@ inline str::str(const bytes &b) {
     }
     auto obj = reinterpret_steal<object>(PyUnicode_FromStringAndSize(buffer, length));
     if (!obj) {
+        if (PyErr_Occurred()) {
+            throw error_already_set();
+        }
         pybind11_fail("Could not allocate string object!");
     }
     m_ptr = obj.release().ptr();
@@ -1785,16 +1836,16 @@ public:
 
     explicit capsule(const void *value,
                      const char *name = nullptr,
-                     void (*destructor)(PyObject *) = nullptr)
+                     PyCapsule_Destructor destructor = nullptr)
         : object(PyCapsule_New(const_cast<void *>(value), name, destructor), stolen_t{}) {
         if (!m_ptr) {
             throw error_already_set();
         }
     }
 
-    PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input")
+    PYBIND11_DEPRECATED("Please use the ctor with value, name, destructor args")
-    capsule(const void *value, void (*destruct)(PyObject *))
+    capsule(const void *value, PyCapsule_Destructor destructor)
-        : object(PyCapsule_New(const_cast<void *>(value), nullptr, destruct), stolen_t{}) {
+        : object(PyCapsule_New(const_cast<void *>(value), nullptr, destructor), stolen_t{}) {
         if (!m_ptr) {
             throw error_already_set();
         }
@@ -1805,21 +1856,21 @@ public:
             // guard if destructor called while err indicator is set
             error_scope error_guard;
             auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o));
-            if (destructor == nullptr) {
+            if (destructor == nullptr && PyErr_Occurred()) {
-                if (PyErr_Occurred()) {
+                throw error_already_set();
-                    throw error_already_set();
-                }
-                pybind11_fail("Unable to get capsule context");
             }
             const char *name = get_name_in_error_scope(o);
             void *ptr = PyCapsule_GetPointer(o, name);
             if (ptr == nullptr) {
                 throw error_already_set();
             }
-            destructor(ptr);
+
+            if (destructor != nullptr) {
+                destructor(ptr);
+            }
         });
 
-        if (!m_ptr || PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) {
+        if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast<void *>(destructor)) != 0) {
             throw error_already_set();
         }
     }
@@ -1943,7 +1994,11 @@ public:
     void clear() /* py-non-const */ { PyDict_Clear(ptr()); }
     template <typename T>
     bool contains(T &&key) const {
-        return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr()) == 1;
+        auto result = PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr());
+        if (result == -1) {
+            throw error_already_set();
+        }
+        return result == 1;
     }
 
 private:
@@ -1998,14 +2053,20 @@ public:
     detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; }
     template <typename T>
     void append(T &&val) /* py-non-const */ {
-        PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr());
+        if (PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) != 0) {
+            throw error_already_set();
+        }
     }
     template <typename IdxType,
               typename ValType,
               detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0>
     void insert(const IdxType &index, ValType &&val) /* py-non-const */ {
-        PyList_Insert(
+        if (PyList_Insert(m_ptr,
-            m_ptr, ssize_t_cast(index), detail::object_or_cast(std::forward<ValType>(val)).ptr());
+                          ssize_t_cast(index),
+                          detail::object_or_cast(std::forward<ValType>(val)).ptr())
+            != 0) {
+            throw error_already_set();
+        }
     }
 };
 
@@ -2023,7 +2084,11 @@ public:
     bool empty() const { return size() == 0; }
     template <typename T>
     bool contains(T &&val) const {
-        return PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 1;
+        auto result = PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr());
+        if (result == -1) {
+            throw error_already_set();
+        }
+        return result == 1;
     }
 };
 
@@ -2364,29 +2429,39 @@ bool object_api<D>::rich_compare(object_api const &other, int value) const {
         return result;                                                                            \
     }
 
+#define PYBIND11_MATH_OPERATOR_BINARY_INPLACE(iop, fn)                                            \
+    template <typename D>                                                                         \
+    object object_api<D>::iop(object_api const &other) {                                          \
+        object result = reinterpret_steal<object>(fn(derived().ptr(), other.derived().ptr()));    \
+        if (!result.ptr())                                                                        \
+            throw error_already_set();                                                            \
+        return result;                                                                            \
+    }
+
 PYBIND11_MATH_OPERATOR_UNARY(operator~, PyNumber_Invert)
 PYBIND11_MATH_OPERATOR_UNARY(operator-, PyNumber_Negative)
 PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add)
-PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator+=, PyNumber_InPlaceAdd)
 PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract)
-PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator-=, PyNumber_InPlaceSubtract)
 PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply)
-PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator*=, PyNumber_InPlaceMultiply)
 PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide)
-PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator/=, PyNumber_InPlaceTrueDivide)
 PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or)
-PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator|=, PyNumber_InPlaceOr)
 PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And)
-PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator&=, PyNumber_InPlaceAnd)
 PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor)
-PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator^=, PyNumber_InPlaceXor)
 PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift)
-PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator<<=, PyNumber_InPlaceLshift)
 PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift)
-PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift)
+PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift)
 
 #undef PYBIND11_MATH_OPERATOR_UNARY
 #undef PYBIND11_MATH_OPERATOR_BINARY
+#undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE
 
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/stl.h

@@ -45,21 +45,35 @@ using forwarded_type = conditional_t<std::is_lvalue_reference<T>::value,
 /// Forwards a value U as rvalue or lvalue according to whether T is rvalue or lvalue; typically
 /// used for forwarding a container's elements.
 template <typename T, typename U>
-forwarded_type<T, U> forward_like(U &&u) {
+constexpr forwarded_type<T, U> forward_like(U &&u) {
     return std::forward<detail::forwarded_type<T, U>>(std::forward<U>(u));
 }
 
+// Checks if a container has a STL style reserve method.
+// This will only return true for a `reserve()` with a `void` return.
+template <typename C>
+using has_reserve_method = std::is_same<decltype(std::declval<C>().reserve(0)), void>;
+
 template <typename Type, typename Key>
 struct set_caster {
     using type = Type;
     using key_conv = make_caster<Key>;
 
+private:
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
+    void reserve_maybe(const anyset &s, Type *) {
+        value.reserve(s.size());
+    }
+    void reserve_maybe(const anyset &, void *) {}
+
+public:
     bool load(handle src, bool convert) {
         if (!isinstance<anyset>(src)) {
             return false;
         }
         auto s = reinterpret_borrow<anyset>(src);
         value.clear();
+        reserve_maybe(s, &value);
         for (auto entry : s) {
             key_conv conv;
             if (!conv.load(entry, convert)) {
@@ -78,7 +92,7 @@ struct set_caster {
         pybind11::set s;
         for (auto &&value : src) {
             auto value_ = reinterpret_steal<object>(
-                key_conv::cast(forward_like<T>(value), policy, parent));
+                key_conv::cast(detail::forward_like<T>(value), policy, parent));
             if (!value_ || !s.add(std::move(value_))) {
                 return handle();
             }
@@ -94,12 +108,21 @@ struct map_caster {
     using key_conv = make_caster<Key>;
     using value_conv = make_caster<Value>;
 
+private:
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
+    void reserve_maybe(const dict &d, Type *) {
+        value.reserve(d.size());
+    }
+    void reserve_maybe(const dict &, void *) {}
+
+public:
     bool load(handle src, bool convert) {
         if (!isinstance<dict>(src)) {
             return false;
         }
         auto d = reinterpret_borrow<dict>(src);
         value.clear();
+        reserve_maybe(d, &value);
         for (auto it : d) {
             key_conv kconv;
             value_conv vconv;
@@ -122,9 +145,9 @@ struct map_caster {
         }
         for (auto &&kv : src) {
             auto key = reinterpret_steal<object>(
-                key_conv::cast(forward_like<T>(kv.first), policy_key, parent));
+                key_conv::cast(detail::forward_like<T>(kv.first), policy_key, parent));
             auto value = reinterpret_steal<object>(
-                value_conv::cast(forward_like<T>(kv.second), policy_value, parent));
+                value_conv::cast(detail::forward_like<T>(kv.second), policy_value, parent));
             if (!key || !value) {
                 return handle();
             }
@@ -160,9 +183,7 @@ struct list_caster {
     }
 
 private:
-    template <
+    template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0>
-        typename T = Type,
-        enable_if_t<std::is_same<decltype(std::declval<T>().reserve(0)), void>::value, int> = 0>
     void reserve_maybe(const sequence &s, Type *) {
         value.reserve(s.size());
     }
@@ -178,7 +199,7 @@ public:
         ssize_t index = 0;
         for (auto &&value : src) {
             auto value_ = reinterpret_steal<object>(
-                value_conv::cast(forward_like<T>(value), policy, parent));
+                value_conv::cast(detail::forward_like<T>(value), policy, parent));
             if (!value_) {
                 return handle();
             }
@@ -242,7 +263,7 @@ public:
         ssize_t index = 0;
         for (auto &&value : src) {
             auto value_ = reinterpret_steal<object>(
-                value_conv::cast(forward_like<T>(value), policy, parent));
+                value_conv::cast(detail::forward_like<T>(value), policy, parent));
             if (!value_) {
                 return handle();
             }
@@ -290,7 +311,7 @@ struct optional_caster {
     template <typename T>
     static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!src) {
-            return none().inc_ref();
+            return none().release();
         }
         if (!std::is_lvalue_reference<T>::value) {
             policy = return_value_policy_override<Value>::policy(policy);

noxfile.py

@@ -5,7 +5,17 @@ import nox
 nox.needs_version = ">=2022.1.7"
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERISONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"]
+PYTHON_VERSIONS = [
+    "3.6",
+    "3.7",
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+    "pypy3.7",
+    "pypy3.8",
+    "pypy3.9",
+]
 
 if os.environ.get("CI", None):
     nox.options.error_on_missing_interpreters = True
@@ -17,10 +27,10 @@ def lint(session: nox.Session) -> None:
     Lint the codebase (except for clang-format/tidy).
     """
     session.install("pre-commit")
-    session.run("pre-commit", "run", "-a")
+    session.run("pre-commit", "run", "-a", *session.posargs)
 
 
-@nox.session(python=PYTHON_VERISONS)
+@nox.session(python=PYTHON_VERSIONS)
 def tests(session: nox.Session) -> None:
     """
     Run the tests (requires a compiler).
@@ -48,7 +58,7 @@ def tests_packaging(session: nox.Session) -> None:
     """
 
     session.install("-r", "tests/requirements.txt", "--prefer-binary")
-    session.run("pytest", "tests/extra_python_package")
+    session.run("pytest", "tests/extra_python_package", *session.posargs)
 
 
 @nox.session(reuse_venv=True)

pybind11/__init__.py

@@ -6,11 +6,12 @@ if sys.version_info < (3, 6):
 
 
 from ._version import __version__, version_info
-from .commands import get_cmake_dir, get_include
+from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
 
 __all__ = (
     "version_info",
     "__version__",
     "get_include",
     "get_cmake_dir",
+    "get_pkgconfig_dir",
 )

pybind11/__main__.py

@@ -4,7 +4,7 @@ import argparse
 import sys
 import sysconfig
 
-from .commands import get_cmake_dir, get_include
+from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
 
 
 def print_includes() -> None:
@@ -36,6 +36,11 @@ def main() -> None:
         action="store_true",
         help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.",
     )
+    parser.add_argument(
+        "--pkgconfigdir",
+        action="store_true",
+        help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
+    )
     args = parser.parse_args()
     if not sys.argv[1:]:
         parser.print_help()
@@ -43,6 +48,8 @@ def main() -> None:
         print_includes()
     if args.cmakedir:
         print(get_cmake_dir())
+    if args.pkgconfigdir:
+        print(get_pkgconfig_dir())
 
 
 if __name__ == "__main__":

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]:
         return s
 
 
-__version__ = "2.10.0"
+__version__ = "2.10.1"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

pybind11/commands.py

@@ -23,3 +23,15 @@ def get_cmake_dir() -> str:
 
     msg = "pybind11 not installed, installation required to access the CMake files"
     raise ImportError(msg)
+
+
+def get_pkgconfig_dir() -> str:
+    """
+    Return the path to the pybind11 pkgconfig directory.
+    """
+    pkgconfig_installed_path = os.path.join(DIR, "share", "pkgconfig")
+    if os.path.exists(pkgconfig_installed_path):
+        return pkgconfig_installed_path
+
+    msg = "pybind11 not installed, installation required to access the pkgconfig files"
+    raise ImportError(msg)

setup.cfg

@@ -46,5 +46,5 @@ zip_safe = False
 max-line-length = 120
 show_source = True
 exclude = .git, __pycache__, build, dist, docs, tools, venv
-extend-ignore = E203, E722, B950
+extend-ignore = E203, E722, B903, B950
 extend-select = B9

setup.py

@@ -127,6 +127,7 @@ with remove_output("pybind11/include", "pybind11/share"):
             "-DCMAKE_INSTALL_PREFIX=pybind11",
             "-DBUILD_TESTING=OFF",
             "-DPYBIND11_NOPYTHON=ON",
+            "-Dprefix_for_pc_file=${pcfiledir}/../../",
         ]
         if "CMAKE_ARGS" in os.environ:
             fcommand = [

tests/conftest.py

@@ -210,4 +210,5 @@ def pytest_report_header(config):
         f" {pybind11_tests.compiler_info}"
         f" {pybind11_tests.cpp_std}"
         f" {pybind11_tests.PYBIND11_INTERNALS_ID}"
+        f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}"
     )

tests/constructor_stats.h

@@ -115,7 +115,7 @@ public:
 #if defined(PYPY_VERSION)
         PyObject *globals = PyEval_GetGlobals();
         PyObject *result = PyRun_String("import gc\n"
-                                        "for i in range(2):"
+                                        "for i in range(2):\n"
                                         "    gc.collect()\n",
                                         Py_file_input,
                                         globals,

tests/cross_module_gil_utils.cpp

@@ -6,9 +6,15 @@
     All rights reserved. Use of this source code is governed by a
     BSD-style license that can be found in the LICENSE file.
 */
+#if defined(PYBIND11_INTERNALS_VERSION)
+#    undef PYBIND11_INTERNALS_VERSION
+#endif
+#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance.
 #include <pybind11/pybind11.h>
 
 #include <cstdint>
+#include <string>
+#include <thread>
 
 // This file mimics a DSO that makes pybind11 calls but does not define a
 // PYBIND11_MODULE. The purpose is to test that such a DSO can create a
@@ -21,8 +27,54 @@
 namespace {
 
 namespace py = pybind11;
+
 void gil_acquire() { py::gil_scoped_acquire gil; }
 
+std::string gil_multi_acquire_release(unsigned bits) {
+    if ((bits & 0x1u) != 0u) {
+        py::gil_scoped_acquire gil;
+    }
+    if ((bits & 0x2u) != 0u) {
+        py::gil_scoped_release gil;
+    }
+    if ((bits & 0x4u) != 0u) {
+        py::gil_scoped_acquire gil;
+    }
+    if ((bits & 0x8u) != 0u) {
+        py::gil_scoped_release gil;
+    }
+    return PYBIND11_INTERNALS_ID;
+}
+
+struct CustomAutoGIL {
+    CustomAutoGIL() : gstate(PyGILState_Ensure()) {}
+    ~CustomAutoGIL() { PyGILState_Release(gstate); }
+
+    PyGILState_STATE gstate;
+};
+struct CustomAutoNoGIL {
+    CustomAutoNoGIL() : save(PyEval_SaveThread()) {}
+    ~CustomAutoNoGIL() { PyEval_RestoreThread(save); }
+
+    PyThreadState *save;
+};
+
+template <typename Acquire, typename Release>
+void gil_acquire_inner() {
+    Acquire acquire_outer;
+    Acquire acquire_inner;
+    Release release;
+}
+
+template <typename Acquire, typename Release>
+void gil_acquire_nested() {
+    Acquire acquire_outer;
+    Acquire acquire_inner;
+    Release release;
+    auto thread = std::thread(&gil_acquire_inner<Acquire, Release>);
+    thread.join();
+}
+
 constexpr char kModuleName[] = "cross_module_gil_utils";
 
 struct PyModuleDef moduledef = {
@@ -30,6 +82,9 @@ struct PyModuleDef moduledef = {
 
 } // namespace
 
+#define ADD_FUNCTION(Name, ...)                                                                   \
+    PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast<void *>(&__VA_ARGS__)));
+
 extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() {
 
     PyObject *m = PyModule_Create(&moduledef);
@@ -37,8 +92,16 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() {
     if (m != nullptr) {
         static_assert(sizeof(&gil_acquire) == sizeof(void *),
                       "Function pointer must have the same size as void*");
-        PyModule_AddObject(
+        ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire)
-            m, "gil_acquire_funcaddr", PyLong_FromVoidPtr(reinterpret_cast<void *>(&gil_acquire)));
+        ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release)
+        ADD_FUNCTION("gil_acquire_inner_custom_funcaddr",
+                     gil_acquire_inner<CustomAutoGIL, CustomAutoNoGIL>)
+        ADD_FUNCTION("gil_acquire_nested_custom_funcaddr",
+                     gil_acquire_nested<CustomAutoGIL, CustomAutoNoGIL>)
+        ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr",
+                     gil_acquire_inner<py::gil_scoped_acquire, py::gil_scoped_release>)
+        ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr",
+                     gil_acquire_nested<py::gil_scoped_acquire, py::gil_scoped_release>)
     }
 
     return m;

tests/extra_python_package/test_files.py

@@ -12,6 +12,16 @@ import zipfile
 DIR = os.path.abspath(os.path.dirname(__file__))
 MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
 
+PKGCONFIG = """\
+prefix=${{pcfiledir}}/../../
+includedir=${{prefix}}/include
+
+Name: pybind11
+Description: Seamless operability between C++11 and Python
+Version: {VERSION}
+Cflags: -I${{includedir}}
+"""
+
 
 main_headers = {
     "include/pybind11/attr.h",
@@ -59,6 +69,10 @@ cmake_files = {
     "share/cmake/pybind11/pybind11Tools.cmake",
 }
 
+pkgconfig_files = {
+    "share/pkgconfig/pybind11.pc",
+}
+
 py_files = {
     "__init__.py",
     "__main__.py",
@@ -69,7 +83,7 @@ py_files = {
 }
 
 headers = main_headers | detail_headers | stl_headers
-src_files = headers | cmake_files
+src_files = headers | cmake_files | pkgconfig_files
 all_files = src_files | py_files
 
 
@@ -82,6 +96,7 @@ sdist_files = {
     "pybind11/share",
     "pybind11/share/cmake",
     "pybind11/share/cmake/pybind11",
+    "pybind11/share/pkgconfig",
     "pyproject.toml",
     "setup.cfg",
     "setup.py",
@@ -101,22 +116,25 @@ local_sdist_files = {
 }
 
 
+def read_tz_file(tar: tarfile.TarFile, name: str) -> bytes:
+    start = tar.getnames()[0] + "/"
+    inner_file = tar.extractfile(tar.getmember(f"{start}{name}"))
+    assert inner_file
+    with contextlib.closing(inner_file) as f:
+        return f.read()
+
+
+def normalize_line_endings(value: bytes) -> bytes:
+    return value.replace(os.linesep.encode("utf-8"), b"\n")
+
+
 def test_build_sdist(monkeypatch, tmpdir):
 
     monkeypatch.chdir(MAIN_DIR)
 
-    out = subprocess.check_output(
+    subprocess.run(
-        [
+        [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}"], check=True
-            sys.executable,
-            "-m",
-            "build",
-            "--sdist",
-            "--outdir",
-            str(tmpdir),
-        ]
     )
-    if hasattr(out, "decode"):
-        out = out.decode()
 
     (sdist,) = tmpdir.visit("*.tar.gz")
 
@@ -125,25 +143,17 @@ def test_build_sdist(monkeypatch, tmpdir):
         version = start[9:-1]
         simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]}
 
-        with contextlib.closing(
+        setup_py = read_tz_file(tar, "setup.py")
-            tar.extractfile(tar.getmember(start + "setup.py"))
+        pyproject_toml = read_tz_file(tar, "pyproject.toml")
-        ) as f:
+        pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc")
-            setup_py = f.read()
+        cmake_cfg = read_tz_file(
+            tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake"
+        )
 
-        with contextlib.closing(
+    assert (
-            tar.extractfile(tar.getmember(start + "pyproject.toml"))
+        'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")'
-        ) as f:
+        in cmake_cfg.decode("utf-8")
-            pyproject_toml = f.read()
+    )
-
-        with contextlib.closing(
-            tar.extractfile(
-                tar.getmember(
-                    start + "pybind11/share/cmake/pybind11/pybind11Config.cmake"
-                )
-            )
-        ) as f:
-            contents = f.read().decode("utf8")
-        assert 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' in contents
 
     files = {f"pybind11/{n}" for n in all_files}
     files |= sdist_files
@@ -154,9 +164,9 @@ def test_build_sdist(monkeypatch, tmpdir):
 
     with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f:
         contents = (
-            string.Template(f.read().decode())
+            string.Template(f.read().decode("utf-8"))
             .substitute(version=version, extra_cmd="")
-            .encode()
+            .encode("utf-8")
         )
     assert setup_py == contents
 
@@ -164,25 +174,19 @@ def test_build_sdist(monkeypatch, tmpdir):
         contents = f.read()
     assert pyproject_toml == contents
 
+    simple_version = ".".join(version.split(".")[:3])
+    pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8")
+    assert normalize_line_endings(pkgconfig) == pkgconfig_expected
+
 
 def test_build_global_dist(monkeypatch, tmpdir):
 
     monkeypatch.chdir(MAIN_DIR)
     monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1")
-    out = subprocess.check_output(
+    subprocess.run(
-        [
+        [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir)], check=True
-            sys.executable,
-            "-m",
-            "build",
-            "--sdist",
-            "--outdir",
-            str(tmpdir),
-        ]
     )
 
-    if hasattr(out, "decode"):
-        out = out.decode()
-
     (sdist,) = tmpdir.visit("*.tar.gz")
 
     with tarfile.open(str(sdist), "r:gz") as tar:
@@ -190,15 +194,17 @@ def test_build_global_dist(monkeypatch, tmpdir):
         version = start[16:-1]
         simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]}
 
-        with contextlib.closing(
+        setup_py = read_tz_file(tar, "setup.py")
-            tar.extractfile(tar.getmember(start + "setup.py"))
+        pyproject_toml = read_tz_file(tar, "pyproject.toml")
-        ) as f:
+        pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc")
-            setup_py = f.read()
+        cmake_cfg = read_tz_file(
+            tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake"
+        )
 
-        with contextlib.closing(
+    assert (
-            tar.extractfile(tar.getmember(start + "pyproject.toml"))
+        'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")'
-        ) as f:
+        in cmake_cfg.decode("utf-8")
-            pyproject_toml = f.read()
+    )
 
     files = {f"pybind11/{n}" for n in all_files}
     files |= sdist_files
@@ -209,7 +215,7 @@ def test_build_global_dist(monkeypatch, tmpdir):
         contents = (
             string.Template(f.read().decode())
             .substitute(version=version, extra_cmd="")
-            .encode()
+            .encode("utf-8")
         )
         assert setup_py == contents
 
@@ -217,12 +223,16 @@ def test_build_global_dist(monkeypatch, tmpdir):
         contents = f.read()
         assert pyproject_toml == contents
 
+    simple_version = ".".join(version.split(".")[:3])
+    pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8")
+    assert normalize_line_endings(pkgconfig) == pkgconfig_expected
+
 
 def tests_build_wheel(monkeypatch, tmpdir):
     monkeypatch.chdir(MAIN_DIR)
 
-    subprocess.check_output(
+    subprocess.run(
-        [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)]
+        [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True
     )
 
     (wheel,) = tmpdir.visit("*.whl")
@@ -249,8 +259,8 @@ def tests_build_global_wheel(monkeypatch, tmpdir):
     monkeypatch.chdir(MAIN_DIR)
     monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1")
 
-    subprocess.check_output(
+    subprocess.run(
-        [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)]
+        [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True
     )
 
     (wheel,) = tmpdir.visit("*.whl")

tests/pybind11_tests.cpp

@@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m) {
 #endif
     m.attr("cpp_std") = cpp_std();
     m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID;
+    m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") =
+#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+        true;
+#else
+        false;
+#endif
 
     bind_ConstructorStats(m);
 

tests/test_builtin_casters.cpp

@@ -266,9 +266,14 @@ TEST_SUBMODULE(builtin_casters, m) {
     });
     m.def("lvalue_nested", []() -> const decltype(lvnested) & { return lvnested; });
 
-    static std::pair<int, std::string> int_string_pair{2, "items"};
     m.def(
-        "int_string_pair", []() { return &int_string_pair; }, py::return_value_policy::reference);
+        "int_string_pair",
+        []() {
+            // Using no-destructor idiom to side-step warnings from overzealous compilers.
+            static auto *int_string_pair = new std::pair<int, std::string>{2, "items"};
+            return int_string_pair;
+        },
+        py::return_value_policy::reference);
 
     // test_builtins_cast_return_none
     m.def("return_none_string", []() -> std::string * { return nullptr; });

tests/test_class.cpp

@@ -36,6 +36,26 @@ struct NoBraceInitialization {
     std::vector<int> vec;
 };
 
+namespace test_class {
+namespace pr4220_tripped_over_this { // PR #4227
+
+template <int>
+struct SoEmpty {};
+
+template <typename T>
+std::string get_msg(const T &) {
+    return "This is really only meant to exercise successful compilation.";
+}
+
+using Empty0 = SoEmpty<0x0>;
+
+void bind_empty0(py::module_ &m) {
+    py::class_<Empty0>(m, "Empty0").def(py::init<>()).def("get_msg", get_msg<Empty0>);
+}
+
+} // namespace pr4220_tripped_over_this
+} // namespace test_class
+
 TEST_SUBMODULE(class_, m) {
     // test_instance
     struct NoConstructor {
@@ -364,6 +384,8 @@ TEST_SUBMODULE(class_, m) {
 
     protected:
         virtual int foo() const { return value; }
+        virtual void *void_foo() { return static_cast<void *>(&value); }
+        virtual void *get_self() { return static_cast<void *>(this); }
 
     private:
         int value = 42;
@@ -372,6 +394,8 @@ TEST_SUBMODULE(class_, m) {
     class TrampolineB : public ProtectedB {
     public:
         int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); }
+        void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); }
+        void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); }
     };
 
     class PublicistB : public ProtectedB {
@@ -381,11 +405,23 @@ TEST_SUBMODULE(class_, m) {
         // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827)
         ~PublicistB() override{}; // NOLINT(modernize-use-equals-default)
         using ProtectedB::foo;
+        using ProtectedB::get_self;
+        using ProtectedB::void_foo;
     };
 
+    m.def("read_foo", [](const void *original) {
+        const int *ptr = reinterpret_cast<const int *>(original);
+        return *ptr;
+    });
+
+    m.def("pointers_equal",
+          [](const void *original, const void *comparison) { return original == comparison; });
+
     py::class_<ProtectedB, TrampolineB>(m, "ProtectedB")
         .def(py::init<>())
-        .def("foo", &PublicistB::foo);
+        .def("foo", &PublicistB::foo)
+        .def("void_foo", &PublicistB::void_foo)
+        .def("get_self", &PublicistB::get_self);
 
     // test_brace_initialization
     struct BraceInitialization {
@@ -517,6 +553,8 @@ TEST_SUBMODULE(class_, m) {
         py::class_<OtherDuplicateNested>(gt, "OtherDuplicateNested");
         py::class_<OtherDuplicateNested>(gt, "YetAnotherDuplicateNested");
     });
+
+    test_class::pr4220_tripped_over_this::bind_empty0(m);
 }
 
 template <int N>

tests/test_class.py

@@ -313,6 +313,8 @@ def test_bind_protected_functions():
 
     b = m.ProtectedB()
     assert b.foo() == 42
+    assert m.read_foo(b.void_foo()) == 42
+    assert m.pointers_equal(b.get_self(), b)
 
     class C(m.ProtectedB):
         def __init__(self):
@@ -469,3 +471,10 @@ def test_register_duplicate_class():
         m.register_duplicate_nested_class_type(ClassScope)
     expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!'
     assert str(exc_info.value) == expected
+
+
+def test_pr4220_tripped_over_this():
+    assert (
+        m.Empty0().get_msg()
+        == "This is really only meant to exercise successful compilation."
+    )

tests/test_custom_type_casters.cpp

@@ -21,7 +21,7 @@ public:
 };
 class ArgAlwaysConverts {};
 
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <>
 struct type_caster<ArgInspector1> {
@@ -74,7 +74,7 @@ public:
     }
 };
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 // test_custom_caster_destruction
 class DestructionTester {
@@ -92,7 +92,7 @@ public:
         return *this;
     }
 };
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <>
 struct type_caster<DestructionTester> {
@@ -104,7 +104,7 @@ struct type_caster<DestructionTester> {
     }
 };
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 // Define type caster outside of `pybind11::detail` and then alias it.
 namespace other_lib {
@@ -112,7 +112,7 @@ struct MyType {};
 // Corrupt `py` shorthand alias for surrounding context.
 namespace py {}
 // Corrupt unqualified relative `pybind11` namespace.
-namespace pybind11 {}
+namespace PYBIND11_NAMESPACE {}
 // Correct alias.
 namespace py_ = ::pybind11;
 // Define caster. This is effectively no-op, we only ensure it compiles and we
@@ -127,12 +127,12 @@ struct my_caster {
 };
 } // namespace other_lib
 // Effectively "alias" it into correct namespace (via inheritance).
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <>
 struct type_caster<other_lib::MyType> : public other_lib::my_caster {};
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 TEST_SUBMODULE(custom_type_casters, m) {
     // test_custom_type_casters

tests/test_eigen.cpp

@@ -197,11 +197,40 @@ TEST_SUBMODULE(eigen, m) {
 
     // Return a block of a matrix (gives non-standard strides)
     m.def("block",
-          [](const Eigen::Ref<const Eigen::MatrixXd> &x,
+          [m](const py::object &x_obj,
-             int start_row,
+              int start_row,
-             int start_col,
+              int start_col,
-             int block_rows,
+              int block_rows,
-             int block_cols) { return x.block(start_row, start_col, block_rows, block_cols); });
+              int block_cols) {
+              return m.attr("_block")(x_obj, x_obj, start_row, start_col, block_rows, block_cols);
+          });
+
+    m.def(
+        "_block",
+        [](const py::object &x_obj,
+           const Eigen::Ref<const Eigen::MatrixXd> &x,
+           int start_row,
+           int start_col,
+           int block_rows,
+           int block_cols) {
+            // See PR #4217 for background. This test is a bit over the top, but might be useful
+            // as a concrete example to point to when explaining the dangling reference trap.
+            auto i0 = py::make_tuple(0, 0);
+            auto x0_orig = x_obj[*i0].cast<double>();
+            if (x(0, 0) != x0_orig) {
+                throw std::runtime_error(
+                    "Something in the type_caster for Eigen::Ref is terribly wrong.");
+            }
+            double x0_mod = x0_orig + 1;
+            x_obj[*i0] = x0_mod;
+            auto copy_detected = (x(0, 0) != x0_mod);
+            x_obj[*i0] = x0_orig;
+            if (copy_detected) {
+                throw std::runtime_error("type_caster for Eigen::Ref made a copy.");
+            }
+            return x.block(start_row, start_col, block_rows, block_cols);
+        },
+        py::keep_alive<0, 1>());
 
     // test_eigen_return_references, test_eigen_keepalive
     // return value referencing/copying tests:

tests/test_eigen.py

@@ -217,15 +217,24 @@ def test_negative_stride_from_python(msg):
     )
 
 
+def test_block_runtime_error_type_caster_eigen_ref_made_a_copy():
+    with pytest.raises(RuntimeError) as excinfo:
+        m.block(ref, 0, 0, 0, 0)
+    assert str(excinfo.value) == "type_caster for Eigen::Ref made a copy."
+
+
 def test_nonunit_stride_to_python():
     assert np.all(m.diagonal(ref) == ref.diagonal())
     assert np.all(m.diagonal_1(ref) == ref.diagonal(1))
     for i in range(-5, 7):
         assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), f"m.diagonal_n({i})"
 
-    assert np.all(m.block(ref, 2, 1, 3, 3) == ref[2:5, 1:4])
+    # Must be order="F", otherwise the type_caster will make a copy and
-    assert np.all(m.block(ref, 1, 4, 4, 2) == ref[1:, 4:])
+    # m.block() will return a dangling reference (heap-use-after-free).
-    assert np.all(m.block(ref, 1, 4, 3, 2) == ref[1:4, 4:])
+    rof = np.asarray(ref, order="F")
+    assert np.all(m.block(rof, 2, 1, 3, 3) == rof[2:5, 1:4])
+    assert np.all(m.block(rof, 1, 4, 4, 2) == rof[1:, 4:])
+    assert np.all(m.block(rof, 1, 4, 3, 2) == rof[1:4, 4:])
 
 
 def test_eigen_ref_to_python():
@@ -251,14 +260,14 @@ def array_copy_but_one(a, r, c, v):
 def test_eigen_return_references():
     """Tests various ways of returning references and non-referencing copies"""
 
-    master = np.ones((10, 10))
+    primary = np.ones((10, 10))
     a = m.ReturnTester()
     a_get1 = a.get()
     assert not a_get1.flags.owndata and a_get1.flags.writeable
-    assign_both(a_get1, master, 3, 3, 5)
+    assign_both(a_get1, primary, 3, 3, 5)
     a_get2 = a.get_ptr()
     assert not a_get2.flags.owndata and a_get2.flags.writeable
-    assign_both(a_get1, master, 2, 3, 6)
+    assign_both(a_get1, primary, 2, 3, 6)
 
     a_view1 = a.view()
     assert not a_view1.flags.owndata and not a_view1.flags.writeable
@@ -271,25 +280,25 @@ def test_eigen_return_references():
 
     a_copy1 = a.copy_get()
     assert a_copy1.flags.owndata and a_copy1.flags.writeable
-    np.testing.assert_array_equal(a_copy1, master)
+    np.testing.assert_array_equal(a_copy1, primary)
     a_copy1[7, 7] = -44  # Shouldn't affect anything else
-    c1want = array_copy_but_one(master, 7, 7, -44)
+    c1want = array_copy_but_one(primary, 7, 7, -44)
     a_copy2 = a.copy_view()
     assert a_copy2.flags.owndata and a_copy2.flags.writeable
-    np.testing.assert_array_equal(a_copy2, master)
+    np.testing.assert_array_equal(a_copy2, primary)
     a_copy2[4, 4] = -22  # Shouldn't affect anything else
-    c2want = array_copy_but_one(master, 4, 4, -22)
+    c2want = array_copy_but_one(primary, 4, 4, -22)
 
     a_ref1 = a.ref()
     assert not a_ref1.flags.owndata and a_ref1.flags.writeable
-    assign_both(a_ref1, master, 1, 1, 15)
+    assign_both(a_ref1, primary, 1, 1, 15)
     a_ref2 = a.ref_const()
     assert not a_ref2.flags.owndata and not a_ref2.flags.writeable
     with pytest.raises(ValueError):
         a_ref2[5, 5] = 33
     a_ref3 = a.ref_safe()
     assert not a_ref3.flags.owndata and a_ref3.flags.writeable
-    assign_both(a_ref3, master, 0, 7, 99)
+    assign_both(a_ref3, primary, 0, 7, 99)
     a_ref4 = a.ref_const_safe()
     assert not a_ref4.flags.owndata and not a_ref4.flags.writeable
     with pytest.raises(ValueError):
@@ -297,23 +306,23 @@ def test_eigen_return_references():
 
     a_copy3 = a.copy_ref()
     assert a_copy3.flags.owndata and a_copy3.flags.writeable
-    np.testing.assert_array_equal(a_copy3, master)
+    np.testing.assert_array_equal(a_copy3, primary)
     a_copy3[8, 1] = 11
-    c3want = array_copy_but_one(master, 8, 1, 11)
+    c3want = array_copy_but_one(primary, 8, 1, 11)
     a_copy4 = a.copy_ref_const()
     assert a_copy4.flags.owndata and a_copy4.flags.writeable
-    np.testing.assert_array_equal(a_copy4, master)
+    np.testing.assert_array_equal(a_copy4, primary)
     a_copy4[8, 4] = 88
-    c4want = array_copy_but_one(master, 8, 4, 88)
+    c4want = array_copy_but_one(primary, 8, 4, 88)
 
     a_block1 = a.block(3, 3, 2, 2)
     assert not a_block1.flags.owndata and a_block1.flags.writeable
     a_block1[0, 0] = 55
-    master[3, 3] = 55
+    primary[3, 3] = 55
     a_block2 = a.block_safe(2, 2, 3, 2)
     assert not a_block2.flags.owndata and a_block2.flags.writeable
     a_block2[2, 1] = -123
-    master[4, 3] = -123
+    primary[4, 3] = -123
     a_block3 = a.block_const(6, 7, 4, 3)
     assert not a_block3.flags.owndata and not a_block3.flags.writeable
     with pytest.raises(ValueError):
@@ -321,18 +330,18 @@ def test_eigen_return_references():
 
     a_copy5 = a.copy_block(2, 2, 2, 3)
     assert a_copy5.flags.owndata and a_copy5.flags.writeable
-    np.testing.assert_array_equal(a_copy5, master[2:4, 2:5])
+    np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5])
     a_copy5[1, 1] = 777
-    c5want = array_copy_but_one(master[2:4, 2:5], 1, 1, 777)
+    c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777)
 
     a_corn1 = a.corners()
     assert not a_corn1.flags.owndata and a_corn1.flags.writeable
     a_corn1 *= 50
     a_corn1[1, 1] = 999
-    master[0, 0] = 50
+    primary[0, 0] = 50
-    master[0, 9] = 50
+    primary[0, 9] = 50
-    master[9, 0] = 50
+    primary[9, 0] = 50
-    master[9, 9] = 999
+    primary[9, 9] = 999
     a_corn2 = a.corners_const()
     assert not a_corn2.flags.owndata and not a_corn2.flags.writeable
     with pytest.raises(ValueError):
@@ -340,22 +349,22 @@ def test_eigen_return_references():
 
     # All of the changes made all the way along should be visible everywhere
     # now (except for the copies, of course)
-    np.testing.assert_array_equal(a_get1, master)
+    np.testing.assert_array_equal(a_get1, primary)
-    np.testing.assert_array_equal(a_get2, master)
+    np.testing.assert_array_equal(a_get2, primary)
-    np.testing.assert_array_equal(a_view1, master)
+    np.testing.assert_array_equal(a_view1, primary)
-    np.testing.assert_array_equal(a_view2, master)
+    np.testing.assert_array_equal(a_view2, primary)
-    np.testing.assert_array_equal(a_ref1, master)
+    np.testing.assert_array_equal(a_ref1, primary)
-    np.testing.assert_array_equal(a_ref2, master)
+    np.testing.assert_array_equal(a_ref2, primary)
-    np.testing.assert_array_equal(a_ref3, master)
+    np.testing.assert_array_equal(a_ref3, primary)
-    np.testing.assert_array_equal(a_ref4, master)
+    np.testing.assert_array_equal(a_ref4, primary)
-    np.testing.assert_array_equal(a_block1, master[3:5, 3:5])
+    np.testing.assert_array_equal(a_block1, primary[3:5, 3:5])
-    np.testing.assert_array_equal(a_block2, master[2:5, 2:4])
+    np.testing.assert_array_equal(a_block2, primary[2:5, 2:4])
-    np.testing.assert_array_equal(a_block3, master[6:10, 7:10])
+    np.testing.assert_array_equal(a_block3, primary[6:10, 7:10])
     np.testing.assert_array_equal(
-        a_corn1, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1]
+        a_corn1, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1]
     )
     np.testing.assert_array_equal(
-        a_corn2, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1]
+        a_corn2, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1]
     )
 
     np.testing.assert_array_equal(a_copy1, c1want)

tests/test_embed/catch.cpp

@@ -20,7 +20,25 @@
 namespace py = pybind11;
 
 int main(int argc, char *argv[]) {
+    // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number:
+    std::string updated_pythonpath("pybind11_test_embed_PYTHONPATH_2099743835476552");
+    const char *preexisting_pythonpath = getenv("PYTHONPATH");
+    if (preexisting_pythonpath != nullptr) {
+#if defined(_WIN32)
+        updated_pythonpath += ';';
+#else
+        updated_pythonpath += ':';
+#endif
+        updated_pythonpath += preexisting_pythonpath;
+    }
+#if defined(_WIN32)
+    _putenv_s("PYTHONPATH", updated_pythonpath.c_str());
+#else
+    setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1);
+#endif
+
     py::scoped_interpreter guard{};
+
     auto result = Catch::Session().run(argc, argv);
 
     return result < 0xff ? result : 0xff;

tests/test_embed/test_interpreter.cpp

@@ -75,6 +75,13 @@ PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) {
     d["missing"].cast<py::object>();
 }
 
+TEST_CASE("PYTHONPATH is used to update sys.path") {
+    // The setup for this TEST_CASE is in catch.cpp!
+    auto sys_path = py::str(py::module_::import("sys").attr("path")).cast<std::string>();
+    REQUIRE_THAT(sys_path,
+                 Catch::Matchers::Contains("pybind11_test_embed_PYTHONPATH_2099743835476552"));
+}
+
 TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
     auto module_ = py::module_::import("test_interpreter");
     REQUIRE(py::hasattr(module_, "DerivedWidget"));
@@ -286,7 +293,6 @@ TEST_CASE("Threads") {
 
     {
         py::gil_scoped_release gil_release{};
-        REQUIRE(has_pybind11_internals_static());
 
         auto threads = std::vector<std::thread>();
         for (auto i = 0; i < num_threads; ++i) {

tests/test_exceptions.cpp

@@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor {
     py::str s;
 };
 
-std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) {
-    PyErr_SetObject(exc_type.ptr(), exc_value.ptr());
-    return py::error_already_set().what();
-}
-
 TEST_SUBMODULE(exceptions, m) {
     m.def("throw_std_exception",
           []() { throw std::runtime_error("This exception was intentionally thrown."); });
@@ -334,4 +329,14 @@ TEST_SUBMODULE(exceptions, m) {
             e.restore();
         }
     });
+
+    // https://github.com/pybind/pybind11/issues/4075
+    m.def("test_pypy_oserror_normalization", []() {
+        try {
+            py::module_::import("io").attr("open")("this_filename_must_not_exist", "r");
+        } catch (const py::error_already_set &e) {
+            return py::str(e.what()); // str must be built before e goes out of scope.
+        }
+        return py::str("UNEXPECTED");
+    });
 }

tests/test_exceptions.py

@@ -185,8 +185,8 @@ def test_custom(msg):
     with pytest.raises(m.MyException5) as excinfo:
         try:
             m.throws5()
-        except m.MyException5_1:
+        except m.MyException5_1 as err:
-            raise RuntimeError("Exception error: caught child from parent")
+            raise RuntimeError("Exception error: caught child from parent") from err
     assert msg(excinfo.value) == "this is a helper-defined translated exception"
 
 
@@ -275,6 +275,20 @@ def test_local_translator(msg):
     assert msg(excinfo.value) == "this mod"
 
 
+def test_error_already_set_message_with_unicode_surrogate():  # Issue #4288
+    assert m.error_already_set_what(RuntimeError, "\ud927") == (
+        "RuntimeError: \\ud927",
+        False,
+    )
+
+
+def test_error_already_set_message_with_malformed_utf8():
+    assert m.error_already_set_what(RuntimeError, b"\x80") == (
+        "RuntimeError: b'\\x80'",
+        False,
+    )
+
+
 class FlakyException(Exception):
     def __init__(self, failure_point):
         if failure_point == "failure_point_init":
@@ -360,3 +374,9 @@ def test_error_already_set_double_restore():
         "Internal error: pybind11::detail::error_fetch_and_normalize::restore()"
         " called a second time. ORIGINAL ERROR: ValueError: Random error."
     )
+
+
+def test_pypy_oserror_normalization():
+    # https://github.com/pybind/pybind11/issues/4075
+    what = m.test_pypy_oserror_normalization()
+    assert "this_filename_must_not_exist" in what

tests/test_gil_scoped.cpp

@@ -11,6 +11,13 @@
 
 #include "pybind11_tests.h"
 
+#include <string>
+#include <thread>
+
+#define CROSS_MODULE(Function)                                                                    \
+    auto cm = py::module_::import("cross_module_gil_utils");                                      \
+    auto target = reinterpret_cast<void (*)()>(PyLong_AsVoidPtr(cm.attr(Function).ptr()));
+
 class VirtClass {
 public:
     virtual ~VirtClass() = default;
@@ -28,6 +35,16 @@ class PyVirtClass : public VirtClass {
 };
 
 TEST_SUBMODULE(gil_scoped, m) {
+    m.attr("defined_THREAD_SANITIZER") =
+#if defined(THREAD_SANITIZER)
+        true;
+#else
+        false;
+#endif
+
+    m.def("intentional_deadlock",
+          []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); });
+
     py::class_<VirtClass, PyVirtClass>(m, "VirtClass")
         .def(py::init<>())
         .def("virtual_func", &VirtClass::virtual_func)
@@ -37,11 +54,91 @@ TEST_SUBMODULE(gil_scoped, m) {
     m.def("test_callback_std_func", [](const std::function<void()> &func) { func(); });
     m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); });
     m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); });
-    m.def("test_cross_module_gil", []() {
+    m.def("test_cross_module_gil_released", []() {
-        auto cm = py::module_::import("cross_module_gil_utils");
+        CROSS_MODULE("gil_acquire_funcaddr")
-        auto gil_acquire = reinterpret_cast<void (*)()>(
-            PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr()));
         py::gil_scoped_release gil_release;
-        gil_acquire();
+        target();
+    });
+    m.def("test_cross_module_gil_acquired", []() {
+        CROSS_MODULE("gil_acquire_funcaddr")
+        py::gil_scoped_acquire gil_acquire;
+        target();
+    });
+    m.def("test_cross_module_gil_inner_custom_released", []() {
+        CROSS_MODULE("gil_acquire_inner_custom_funcaddr")
+        py::gil_scoped_release gil_release;
+        target();
+    });
+    m.def("test_cross_module_gil_inner_custom_acquired", []() {
+        CROSS_MODULE("gil_acquire_inner_custom_funcaddr")
+        py::gil_scoped_acquire gil_acquire;
+        target();
+    });
+    m.def("test_cross_module_gil_inner_pybind11_released", []() {
+        CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr")
+        py::gil_scoped_release gil_release;
+        target();
+    });
+    m.def("test_cross_module_gil_inner_pybind11_acquired", []() {
+        CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr")
+        py::gil_scoped_acquire gil_acquire;
+        target();
+    });
+    m.def("test_cross_module_gil_nested_custom_released", []() {
+        CROSS_MODULE("gil_acquire_nested_custom_funcaddr")
+        py::gil_scoped_release gil_release;
+        target();
+    });
+    m.def("test_cross_module_gil_nested_custom_acquired", []() {
+        CROSS_MODULE("gil_acquire_nested_custom_funcaddr")
+        py::gil_scoped_acquire gil_acquire;
+        target();
+    });
+    m.def("test_cross_module_gil_nested_pybind11_released", []() {
+        CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr")
+        py::gil_scoped_release gil_release;
+        target();
+    });
+    m.def("test_cross_module_gil_nested_pybind11_acquired", []() {
+        CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr")
+        py::gil_scoped_acquire gil_acquire;
+        target();
+    });
+    m.def("test_release_acquire", [](const py::object &obj) {
+        py::gil_scoped_release gil_released;
+        py::gil_scoped_acquire gil_acquired;
+        return py::str(obj);
+    });
+    m.def("test_nested_acquire", [](const py::object &obj) {
+        py::gil_scoped_release gil_released;
+        py::gil_scoped_acquire gil_acquired_outer;
+        py::gil_scoped_acquire gil_acquired_inner;
+        return py::str(obj);
+    });
+    m.def("test_multi_acquire_release_cross_module", [](unsigned bits) {
+        py::set internals_ids;
+        internals_ids.add(PYBIND11_INTERNALS_ID);
+        {
+            py::gil_scoped_release gil_released;
+            auto thread_f = [bits, &internals_ids]() {
+                py::gil_scoped_acquire gil_acquired;
+                auto cm = py::module_::import("cross_module_gil_utils");
+                auto target = reinterpret_cast<std::string (*)(unsigned)>(
+                    PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr()));
+                std::string cm_internals_id = target(bits >> 3);
+                internals_ids.add(cm_internals_id);
+            };
+            if ((bits & 0x1u) != 0u) {
+                thread_f();
+            }
+            if ((bits & 0x2u) != 0u) {
+                std::thread non_python_thread(thread_f);
+                non_python_thread.join();
+            }
+            if ((bits & 0x4u) != 0u) {
+                thread_f();
+            }
+        }
+        return internals_ids;
     });
 }

tests/test_gil_scoped.py

@@ -1,45 +1,199 @@
 import multiprocessing
+import sys
 import threading
+import time
+
+import pytest
 
 from pybind11_tests import gil_scoped as m
 
 
+class ExtendedVirtClass(m.VirtClass):
+    def virtual_func(self):
+        pass
+
+    def pure_virtual_func(self):
+        pass
+
+
+def test_callback_py_obj():
+    m.test_callback_py_obj(lambda: None)
+
+
+def test_callback_std_func():
+    m.test_callback_std_func(lambda: None)
+
+
+def test_callback_virtual_func():
+    extended = ExtendedVirtClass()
+    m.test_callback_virtual_func(extended)
+
+
+def test_callback_pure_virtual_func():
+    extended = ExtendedVirtClass()
+    m.test_callback_pure_virtual_func(extended)
+
+
+def test_cross_module_gil_released():
+    """Makes sure that the GIL can be acquired by another module from a GIL-released state."""
+    m.test_cross_module_gil_released()  # Should not raise a SIGSEGV
+
+
+def test_cross_module_gil_acquired():
+    """Makes sure that the GIL can be acquired by another module from a GIL-acquired state."""
+    m.test_cross_module_gil_acquired()  # Should not raise a SIGSEGV
+
+
+def test_cross_module_gil_inner_custom_released():
+    """Makes sure that the GIL can be acquired/released by another module
+    from a GIL-released state using custom locking logic."""
+    m.test_cross_module_gil_inner_custom_released()
+
+
+def test_cross_module_gil_inner_custom_acquired():
+    """Makes sure that the GIL can be acquired/acquired by another module
+    from a GIL-acquired state using custom locking logic."""
+    m.test_cross_module_gil_inner_custom_acquired()
+
+
+def test_cross_module_gil_inner_pybind11_released():
+    """Makes sure that the GIL can be acquired/released by another module
+    from a GIL-released state using pybind11 locking logic."""
+    m.test_cross_module_gil_inner_pybind11_released()
+
+
+def test_cross_module_gil_inner_pybind11_acquired():
+    """Makes sure that the GIL can be acquired/acquired by another module
+    from a GIL-acquired state using pybind11 locking logic."""
+    m.test_cross_module_gil_inner_pybind11_acquired()
+
+
+def test_cross_module_gil_nested_custom_released():
+    """Makes sure that the GIL can be nested acquired/released by another module
+    from a GIL-released state using custom locking logic."""
+    m.test_cross_module_gil_nested_custom_released()
+
+
+def test_cross_module_gil_nested_custom_acquired():
+    """Makes sure that the GIL can be nested acquired/acquired by another module
+    from a GIL-acquired state using custom locking logic."""
+    m.test_cross_module_gil_nested_custom_acquired()
+
+
+def test_cross_module_gil_nested_pybind11_released():
+    """Makes sure that the GIL can be nested acquired/released by another module
+    from a GIL-released state using pybind11 locking logic."""
+    m.test_cross_module_gil_nested_pybind11_released()
+
+
+def test_cross_module_gil_nested_pybind11_acquired():
+    """Makes sure that the GIL can be nested acquired/acquired by another module
+    from a GIL-acquired state using pybind11 locking logic."""
+    m.test_cross_module_gil_nested_pybind11_acquired()
+
+
+def test_release_acquire():
+    assert m.test_release_acquire(0xAB) == "171"
+
+
+def test_nested_acquire():
+    assert m.test_nested_acquire(0xAB) == "171"
+
+
+def test_multi_acquire_release_cross_module():
+    for bits in range(16 * 8):
+        internals_ids = m.test_multi_acquire_release_cross_module(bits)
+        assert len(internals_ids) == 2 if bits % 8 else 1
+
+
+# Intentionally putting human review in the loop here, to guard against accidents.
+VARS_BEFORE_ALL_BASIC_TESTS = dict(vars())  # Make a copy of the dict (critical).
+ALL_BASIC_TESTS = (
+    test_callback_py_obj,
+    test_callback_std_func,
+    test_callback_virtual_func,
+    test_callback_pure_virtual_func,
+    test_cross_module_gil_released,
+    test_cross_module_gil_acquired,
+    test_cross_module_gil_inner_custom_released,
+    test_cross_module_gil_inner_custom_acquired,
+    test_cross_module_gil_inner_pybind11_released,
+    test_cross_module_gil_inner_pybind11_acquired,
+    test_cross_module_gil_nested_custom_released,
+    test_cross_module_gil_nested_custom_acquired,
+    test_cross_module_gil_nested_pybind11_released,
+    test_cross_module_gil_nested_pybind11_acquired,
+    test_release_acquire,
+    test_nested_acquire,
+    test_multi_acquire_release_cross_module,
+)
+
+
+def test_all_basic_tests_completeness():
+    num_found = 0
+    for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items():
+        if not key.startswith("test_"):
+            continue
+        assert value in ALL_BASIC_TESTS
+        num_found += 1
+    assert len(ALL_BASIC_TESTS) == num_found
+
+
+def _intentional_deadlock():
+    m.intentional_deadlock()
+
+
+ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,)
+SKIP_IF_DEADLOCK = True  # See PR #4216
+
+
 def _run_in_process(target, *args, **kwargs):
-    """Runs target in process and returns its exitcode after 10s (None if still alive)."""
+    if len(args) == 0:
+        test_fn = target
+    else:
+        test_fn = args[0]
+    # Do not need to wait much, 10s should be more than enough.
+    timeout = 0.1 if test_fn is _intentional_deadlock else 10
     process = multiprocessing.Process(target=target, args=args, kwargs=kwargs)
     process.daemon = True
     try:
+        t_start = time.time()
         process.start()
-        # Do not need to wait much, 10s should be more than enough.
+        if timeout >= 100:  # For debugging.
-        process.join(timeout=10)
+            print(
+                "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs)
+            )
+            print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True)
+        process.join(timeout=timeout)
+        if timeout >= 100:
+            print("\nprocess.pid JOINED", process.pid, flush=True)
+        t_delta = time.time() - t_start
+        if process.exitcode == 66 and m.defined_THREAD_SANITIZER:  # Issue #2754
+            # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output.
+            # Maybe this could work:
+            # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e
+            pytest.skip(
+                "ThreadSanitizer: starting new threads after multi-threaded fork is not supported."
+            )
+        elif test_fn is _intentional_deadlock:
+            assert process.exitcode is None
+            return 0
+        elif process.exitcode is None:
+            assert t_delta > 0.9 * timeout
+            msg = "DEADLOCK, most likely, exactly what this test is meant to detect."
+            if SKIP_IF_DEADLOCK:
+                pytest.skip(msg)
+            raise RuntimeError(msg)
         return process.exitcode
     finally:
         if process.is_alive():
             process.terminate()
 
 
-def _python_to_cpp_to_python():
+def _run_in_threads(test_fn, num_threads, parallel):
-    """Calls different C++ functions that come back to Python."""
-
-    class ExtendedVirtClass(m.VirtClass):
-        def virtual_func(self):
-            pass
-
-        def pure_virtual_func(self):
-            pass
-
-    extended = ExtendedVirtClass()
-    m.test_callback_py_obj(lambda: None)
-    m.test_callback_std_func(lambda: None)
-    m.test_callback_virtual_func(extended)
-    m.test_callback_pure_virtual_func(extended)
-
-
-def _python_to_cpp_to_python_from_threads(num_threads, parallel=False):
-    """Calls different C++ functions that come back to Python, from Python threads."""
     threads = []
     for _ in range(num_threads):
-        thread = threading.Thread(target=_python_to_cpp_to_python)
+        thread = threading.Thread(target=test_fn)
         thread.daemon = True
         thread.start()
         if parallel:
@@ -51,43 +205,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False):
 
 
 # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9
-def test_python_to_cpp_to_python_from_thread():
+@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
+def test_run_in_process_one_thread(test_fn):
     """Makes sure there is no GIL deadlock when running in a thread.
 
     It runs in a separate process to be able to stop and assert if it deadlocks.
     """
-    assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0
+    assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0
 
 
 # TODO: FIXME on macOS Python 3.9
-def test_python_to_cpp_to_python_from_thread_multiple_parallel():
+@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
+def test_run_in_process_multiple_threads_parallel(test_fn):
     """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel.
 
     It runs in a separate process to be able to stop and assert if it deadlocks.
     """
-    assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0
+    assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0
 
 
 # TODO: FIXME on macOS Python 3.9
-def test_python_to_cpp_to_python_from_thread_multiple_sequential():
+@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
+def test_run_in_process_multiple_threads_sequential(test_fn):
     """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially.
 
     It runs in a separate process to be able to stop and assert if it deadlocks.
     """
-    assert (
+    assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0
-        _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0
-    )
 
 
 # TODO: FIXME on macOS Python 3.9
-def test_python_to_cpp_to_python_from_process():
+@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
+def test_run_in_process_direct(test_fn):
     """Makes sure there is no GIL deadlock when using processes.
 
     This test is for completion, but it was never an issue.
     """
-    assert _run_in_process(_python_to_cpp_to_python) == 0
+    assert _run_in_process(test_fn) == 0
-
-
-def test_cross_module_gil():
-    """Makes sure that the GIL can be acquired by another module from a GIL-released state."""
-    m.test_cross_module_gil()  # Should not raise a SIGSEGV

tests/test_kwargs_and_defaults.cpp

@@ -43,7 +43,16 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
     m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a = 0);
 
     // test_args_and_kwargs
-    m.def("args_function", [](py::args args) -> py::tuple { return std::move(args); });
+    m.def("args_function", [](py::args args) -> py::tuple {
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+#    pragma clang diagnostic push
+#    pragma clang diagnostic ignored "-Wreturn-std-move"
+#endif
+        return args;
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+#    pragma clang diagnostic pop
+#endif
+    });
     m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) {
         return py::make_tuple(args, kwargs);
     });

tests/test_pytypes.cpp

@@ -109,6 +109,11 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("get_iterator", [] { return py::iterator(); });
     // test_iterable
     m.def("get_iterable", [] { return py::iterable(); });
+    m.def("get_frozenset_from_iterable",
+          [](const py::iterable &iter) { return py::frozenset(iter); });
+    m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); });
+    m.def("get_set_from_iterable", [](const py::iterable &iter) { return py::set(iter); });
+    m.def("get_tuple_from_iterable", [](const py::iterable &iter) { return py::tuple(iter); });
     // test_float
     m.def("get_float", [] { return py::float_(0.0f); });
     // test_list
@@ -178,7 +183,7 @@ TEST_SUBMODULE(pytypes, m) {
         return d2;
     });
     m.def("dict_contains",
-          [](const py::dict &dict, py::object val) { return dict.contains(val); });
+          [](const py::dict &dict, const py::object &val) { return dict.contains(val); });
     m.def("dict_contains",
           [](const py::dict &dict, const char *val) { return dict.contains(val); });
 
@@ -201,7 +206,12 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; });
     m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; });
     m.def("str_from_string", []() { return py::str(std::string("baz")); });
+    m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); });
+    m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); });
     m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); });
+    m.def("str_from_bytes_input",
+          [](const py::bytes &encoded_str) { return py::str(encoded_str); });
+
     m.def("str_from_object", [](const py::object &obj) { return py::str(obj); });
     m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); });
     m.def("str_from_handle", [](py::handle h) { return py::str(h); });
@@ -284,6 +294,12 @@ TEST_SUBMODULE(pytypes, m) {
         return capsule;
     });
 
+    m.def("return_capsule_with_explicit_nullptr_dtor", []() {
+        py::print("creating capsule with explicit nullptr dtor");
+        return py::capsule(reinterpret_cast<void *>(1234),
+                           static_cast<void (*)(void *)>(nullptr)); // PR #4221
+    });
+
     // test_accessors
     m.def("accessor_api", [](const py::object &o) {
         auto d = py::dict();
@@ -527,6 +543,9 @@ TEST_SUBMODULE(pytypes, m) {
 
     m.def("hash_function", [](py::object obj) { return py::hash(std::move(obj)); });
 
+    m.def("obj_contains",
+          [](py::object &obj, const py::object &key) { return obj.contains(key); });
+
     m.def("test_number_protocol", [](const py::object &a, const py::object &b) {
         py::list l;
         l.append(a.equal(b));
@@ -756,4 +775,38 @@ TEST_SUBMODULE(pytypes, m) {
         }
         return o;
     });
+
+    // testing immutable object augmented assignment: #issue 3812
+    m.def("inplace_append", [](py::object &a, const py::object &b) {
+        a += b;
+        return a;
+    });
+    m.def("inplace_subtract", [](py::object &a, const py::object &b) {
+        a -= b;
+        return a;
+    });
+    m.def("inplace_multiply", [](py::object &a, const py::object &b) {
+        a *= b;
+        return a;
+    });
+    m.def("inplace_divide", [](py::object &a, const py::object &b) {
+        a /= b;
+        return a;
+    });
+    m.def("inplace_or", [](py::object &a, const py::object &b) {
+        a |= b;
+        return a;
+    });
+    m.def("inplace_and", [](py::object &a, const py::object &b) {
+        a &= b;
+        return a;
+    });
+    m.def("inplace_lshift", [](py::object &a, const py::object &b) {
+        a <<= b;
+        return a;
+    });
+    m.def("inplace_rshift", [](py::object &a, const py::object &b) {
+        a >>= b;
+        return a;
+    });
 }

tests/test_pytypes.py

@@ -26,6 +26,22 @@ def test_iterator(doc):
     assert doc(m.get_iterator) == "get_iterator() -> Iterator"
 
 
+@pytest.mark.parametrize(
+    "pytype, from_iter_func",
+    [
+        (frozenset, m.get_frozenset_from_iterable),
+        (list, m.get_list_from_iterable),
+        (set, m.get_set_from_iterable),
+        (tuple, m.get_tuple_from_iterable),
+    ],
+)
+def test_from_iterable(pytype, from_iter_func):
+    my_iter = iter(range(10))
+    s = from_iter_func(my_iter)
+    assert type(s) == pytype
+    assert s == pytype(range(10))
+
+
 def test_iterable(doc):
     assert doc(m.get_iterable) == "get_iterable() -> Iterable"
 
@@ -152,6 +168,31 @@ def test_dict(capture, doc):
     assert m.dict_keyword_constructor() == {"x": 1, "y": 2, "z": 3}
 
 
+class CustomContains:
+    d = {"key": None}
+
+    def __contains__(self, m):
+        return m in self.d
+
+
+@pytest.mark.parametrize(
+    "arg,func",
+    [
+        (set(), m.anyset_contains),
+        (dict(), m.dict_contains),
+        (CustomContains(), m.obj_contains),
+    ],
+)
+@pytest.mark.xfail("env.PYPY and sys.pypy_version_info < (7, 3, 10)", strict=False)
+def test_unhashable_exceptions(arg, func):
+    class Unhashable:
+        __hash__ = None
+
+    with pytest.raises(TypeError) as exc_info:
+        func(arg, Unhashable())
+    assert "unhashable type:" in str(exc_info.value)
+
+
 def test_tuple():
     assert m.tuple_no_args() == ()
     assert m.tuple_ssize_t() == ()
@@ -203,6 +244,20 @@ def test_str(doc):
         m.str_from_string_from_str(ucs_surrogates_str)
 
 
+@pytest.mark.parametrize(
+    "func",
+    [
+        m.str_from_bytes_input,
+        m.str_from_cstr_input,
+        m.str_from_std_string_input,
+    ],
+)
+def test_surrogate_pairs_unicode_error(func):
+    input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass")
+    with pytest.raises(UnicodeDecodeError):
+        func(input_str)
+
+
 def test_bytes(doc):
     assert m.bytes_from_char_ssize_t().decode() == "green"
     assert m.bytes_from_char_size_t().decode() == "purple"
@@ -283,6 +338,17 @@ def test_capsule(capture):
     """
     )
 
+    with capture:
+        a = m.return_capsule_with_explicit_nullptr_dtor()
+        del a
+        pytest.gc_collect()
+    assert (
+        capture.unordered
+        == """
+        creating capsule with explicit nullptr dtor
+    """
+    )
+
 
 def test_accessors():
     class SubTestObject:
@@ -739,3 +805,75 @@ def test_populate_obj_str_attrs():
     new_attrs = {k: v for k, v in new_o.__dict__.items() if not k.startswith("_")}
     assert all(isinstance(v, str) for v in new_attrs.values())
     assert len(new_attrs) == pop
+
+
+@pytest.mark.parametrize(
+    "a,b", [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))]
+)
+def test_inplace_append(a, b):
+    expected = a + b
+    assert m.inplace_append(a, b) == expected
+
+
+@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))])
+def test_inplace_subtract(a, b):
+    expected = a - b
+    assert m.inplace_subtract(a, b) == expected
+
+
+@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), ([1], 3)])
+def test_inplace_multiply(a, b):
+    expected = a * b
+    assert m.inplace_multiply(a, b) == expected
+
+
+@pytest.mark.parametrize("a,b", [(6, 3), (6.0, 3.0)])
+def test_inplace_divide(a, b):
+    expected = a / b
+    assert m.inplace_divide(a, b) == expected
+
+
+@pytest.mark.parametrize(
+    "a,b",
+    [
+        (False, True),
+        (
+            set(),
+            {
+                1,
+            },
+        ),
+    ],
+)
+def test_inplace_or(a, b):
+    expected = a | b
+    assert m.inplace_or(a, b) == expected
+
+
+@pytest.mark.parametrize(
+    "a,b",
+    [
+        (True, False),
+        (
+            {1, 2, 3},
+            {
+                1,
+            },
+        ),
+    ],
+)
+def test_inplace_and(a, b):
+    expected = a & b
+    assert m.inplace_and(a, b) == expected
+
+
+@pytest.mark.parametrize("a,b", [(8, 1), (-3, 2)])
+def test_inplace_lshift(a, b):
+    expected = a << b
+    assert m.inplace_lshift(a, b) == expected
+
+
+@pytest.mark.parametrize("a,b", [(8, 1), (-2, 2)])
+def test_inplace_rshift(a, b):
+    expected = a >> b
+    assert m.inplace_rshift(a, b) == expected

tests/test_sequences_and_iterators.cpp

@@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
           []() { return py::make_iterator<py::return_value_policy::copy>(list); });
     m.def("make_iterator_2",
           []() { return py::make_iterator<py::return_value_policy::automatic>(list); });
+
+    // test_iterator on c arrays
+    // #4100: ensure lvalue required as increment operand
+    class CArrayHolder {
+    public:
+        CArrayHolder(double x, double y, double z) {
+            values[0] = x;
+            values[1] = y;
+            values[2] = z;
+        };
+        double values[3];
+    };
+
+    py::class_<CArrayHolder>(m, "CArrayHolder")
+        .def(py::init<double, double, double>())
+        .def(
+            "__iter__",
+            [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); },
+            py::keep_alive<0, 1>());
 }

tests/test_sequences_and_iterators.py

@@ -241,3 +241,11 @@ def test_iterator_rvp():
     assert list(m.make_iterator_1()) == [1, 2, 3]
     assert list(m.make_iterator_2()) == [1, 2, 3]
     assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2()))
+
+
+def test_carray_iterator():
+    """#4100: Check for proper iterator overload with C-Arrays"""
+    args_gt = list(float(i) for i in range(3))
+    arr_h = m.CArrayHolder(*args_gt)
+    args = list(arr_h)
+    assert args_gt == args

tests/test_smart_ptr.cpp

@@ -266,14 +266,14 @@ struct ElementList {
 // It is always possible to construct a ref<T> from an Object* pointer without
 // possible inconsistencies, hence the 'true' argument at the end.
 // Make pybind11 aware of the non-standard getter member function
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <typename T>
 struct holder_helper<ref<T>> {
     static const T *get(const ref<T> &p) { return p.get_ptr(); }
 };
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 // Make pybind aware of the ref-counted wrapper type (s):
 PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true);

tests/test_stl.cpp

@@ -23,7 +23,7 @@
 #if defined(PYBIND11_TEST_BOOST)
 #    include <boost/optional.hpp>
 
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <typename T>
 struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
@@ -31,7 +31,7 @@ struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
 template <>
 struct type_caster<boost::none_t> : void_caster<boost::none_t> {};
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 #endif
 
 // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14
@@ -43,7 +43,7 @@ using std::variant;
 #    define PYBIND11_TEST_VARIANT 1
 using boost::variant;
 
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <typename... Ts>
 struct type_caster<boost::variant<Ts...>> : variant_caster<boost::variant<Ts...>> {};
@@ -56,7 +56,7 @@ struct visit_helper<boost::variant> {
     }
 };
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 #endif
 
 PYBIND11_MAKE_OPAQUE(std::vector<std::string, std::allocator<std::string>>);
@@ -159,13 +159,13 @@ private:
     std::vector<T> storage;
 };
 
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 namespace detail {
 template <typename T>
 struct type_caster<ReferenceSensitiveOptional<T>>
     : optional_caster<ReferenceSensitiveOptional<T>> {};
 } // namespace detail
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 TEST_SUBMODULE(stl, m) {
     // test_vector
@@ -176,9 +176,14 @@ TEST_SUBMODULE(stl, m) {
     m.def("load_bool_vector",
           [](const std::vector<bool> &v) { return v.at(0) == true && v.at(1) == false; });
     // Unnumbered regression (caused by #936): pointers to stl containers aren't castable
-    static std::vector<RValueCaster> lvv{2};
     m.def(
-        "cast_ptr_vector", []() { return &lvv; }, py::return_value_policy::reference);
+        "cast_ptr_vector",
+        []() {
+            // Using no-destructor idiom to side-step warnings from overzealous compilers.
+            static auto *v = new std::vector<RValueCaster>{2};
+            return v;
+        },
+        py::return_value_policy::reference);
 
     // test_deque
     m.def("cast_deque", []() { return std::deque<int>{1}; });
@@ -237,6 +242,7 @@ TEST_SUBMODULE(stl, m) {
     lvn["b"].emplace_back();        // add a list
     lvn["b"].back().emplace_back(); // add an array
     lvn["b"].back().emplace_back(); // add another array
+    static std::vector<RValueCaster> lvv{2};
     m.def("cast_lv_vector", []() -> const decltype(lvv) & { return lvv; });
     m.def("cast_lv_array", []() -> const decltype(lva) & { return lva; });
     m.def("cast_lv_map", []() -> const decltype(lvm) & { return lvm; });

tests/test_tagbased_polymorphic.cpp

@@ -117,7 +117,7 @@ std::string Animal::name_of_kind(Kind kind) {
     return raw_name;
 }
 
-namespace pybind11 {
+namespace PYBIND11_NAMESPACE {
 template <typename itype>
 struct polymorphic_type_hook<itype, detail::enable_if_t<std::is_base_of<Animal, itype>::value>> {
     static const void *get(const itype *src, const std::type_info *&type) {
@@ -125,7 +125,7 @@ struct polymorphic_type_hook<itype, detail::enable_if_t<std::is_base_of<Animal,
         return src;
     }
 };
-} // namespace pybind11
+} // namespace PYBIND11_NAMESPACE
 
 TEST_SUBMODULE(tagbased_polymorphic, m) {
     py::class_<Animal>(m, "Animal").def_readonly("name", &Animal::name);

tools/FindPythonLibsNew.cmake

@@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0)
   return()
 endif()
 
+option(
+  PYBIND11_PYTHONLIBS_OVERWRITE
+  "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values."
+  ON)
 # Can manually set values when cross-compiling
 macro(_PYBIND11_GET_IF_UNDEF lst index name)
-  if(NOT DEFINED "${name}")
+  if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}")
     list(GET "${lst}" "${index}" "${name}")
   endif()
 endmacro()

tools/JoinPaths.cmake

@@ -0,0 +1,23 @@
+# This module provides function for joining paths
+# known from most languages
+#
+# SPDX-License-Identifier: (MIT OR CC0-1.0)
+# Copyright 2020 Jan Tojnar
+# https://github.com/jtojnar/cmake-snips
+#
+# Modelled after Python’s os.path.join
+# https://docs.python.org/3.7/library/os.path.html#os.path.join
+# Windows not supported
+function(join_paths joined_path first_path_segment)
+    set(temp_path "${first_path_segment}")
+    foreach(current_segment IN LISTS ARGN)
+        if(NOT ("${current_segment}" STREQUAL ""))
+            if(IS_ABSOLUTE "${current_segment}")
+                set(temp_path "${current_segment}")
+            else()
+                set(temp_path "${temp_path}/${current_segment}")
+            endif()
+        endif()
+    endforeach()
+    set(${joined_path} "${temp_path}" PARENT_SCOPE)
+endfunction()

tools/codespell_ignore_lines_from_errors.py

@@ -0,0 +1,35 @@
+"""Simple script for rebuilding .codespell-ignore-lines
+
+Usage:
+
+cat < /dev/null > .codespell-ignore-lines
+pre-commit run --all-files codespell >& /tmp/codespell_errors.txt
+python3 tools/codespell_ignore_lines_from_errors.py /tmp/codespell_errors.txt > .codespell-ignore-lines
+
+git diff to review changes, then commit, push.
+"""
+
+import sys
+from typing import List
+
+
+def run(args: List[str]) -> None:
+    assert len(args) == 1, "codespell_errors.txt"
+    cache = {}
+    done = set()
+    for line in sorted(open(args[0]).read().splitlines()):
+        i = line.find(" ==> ")
+        if i > 0:
+            flds = line[:i].split(":")
+            if len(flds) >= 2:
+                filename, line_num = flds[:2]
+                if filename not in cache:
+                    cache[filename] = open(filename).read().splitlines()
+                supp = cache[filename][int(line_num) - 1]
+                if supp not in done:
+                    print(supp)
+                    done.add(supp)
+
+
+if __name__ == "__main__":
+    run(args=sys.argv[1:])

tools/pybind11.pc.in

@@ -0,0 +1,7 @@
+prefix=@prefix_for_pc_file@
+includedir=@includedir_for_pc_file@
+
+Name: @PROJECT_NAME@
+Description: Seamless operability between C++11 and Python
+Version: @PROJECT_VERSION@
+Cflags: -I${includedir}

tools/pybind11NewTools.cmake

@@ -233,7 +233,9 @@ function(pybind11_add_module target_name)
     endif()
   endif()
 
-  if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
+  # Use case-insensitive comparison to match the result of $<CONFIG:cfgs>
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+  if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO)
     # Strip unnecessary sections of the binary on Linux/macOS
     pybind11_strip(${target_name})
   endif()

tools/pybind11Tools.cmake

@@ -115,6 +115,7 @@ if(PYTHON_IS_DEBUG)
     PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG)
 endif()
 
+# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg
 if(CMAKE_VERSION VERSION_LESS 3.11)
   set_property(
     TARGET pybind11::module
@@ -130,16 +131,19 @@ if(CMAKE_VERSION VERSION_LESS 3.11)
     APPEND
     PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $<BUILD_INTERFACE:${PYTHON_LIBRARIES}>)
 else()
+  # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside
+  # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are
+  # target_link_library keywords rather than real libraries.
+  add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE)
+  target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES})
   target_link_libraries(
     pybind11::module
     INTERFACE
       pybind11::python_link_helper
-      "$<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:Cygwin>>:$<BUILD_INTERFACE:${PYTHON_LIBRARIES}>>"
+      "$<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:Cygwin>>:pybind11::_ClassicPythonLibraries>")
-  )
 
   target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11
-                                                  $<BUILD_INTERFACE:${PYTHON_LIBRARIES}>)
+                                                  pybind11::_ClassicPythonLibraries)
-
 endif()
 
 function(pybind11_extension name)
@@ -208,7 +212,9 @@ function(pybind11_add_module target_name)
     endif()
   endif()
 
-  if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
+  # Use case-insensitive comparison to match the result of $<CONFIG:cfgs>
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+  if(NOT MSVC AND NOT ${uppercase_CMAKE_BUILD_TYPE} MATCHES DEBUG|RELWITHDEBINFO)
     pybind11_strip(${target_name})
   endif()
 

tools/setup_global.py.in

@@ -29,6 +29,7 @@ main_headers = glob.glob("pybind11/include/pybind11/*.h")
 detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h")
 stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h")
 cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake")
+pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc")
 headers = main_headers + detail_headers + stl_headers
 
 cmdclass = {"install_headers": InstallHeadersNested}
@@ -51,6 +52,7 @@ setup(
     headers=headers,
     data_files=[
         (base + "share/cmake/pybind11", cmake_files),
+        (base + "share/pkgconfig", pkgconfig_files),
         (base + "include/pybind11", main_headers),
         (base + "include/pybind11/detail", detail_headers),
         (base + "include/pybind11/stl", stl_headers),

tools/setup_main.py.in

@@ -17,6 +17,7 @@ setup(
         "pybind11.include.pybind11.detail",
         "pybind11.include.pybind11.stl",
         "pybind11.share.cmake.pybind11",
+        "pybind11.share.pkgconfig",
     ],
     package_data={
         "pybind11": ["py.typed"],
@@ -24,6 +25,7 @@ setup(
         "pybind11.include.pybind11.detail": ["*.h"],
         "pybind11.include.pybind11.stl": ["*.h"],
         "pybind11.share.cmake.pybind11": ["*.cmake"],
+        "pybind11.share.pkgconfig": ["*.pc"],
     },
     extras_require={
         "global": ["pybind11_global==$version"]