Merge pull request #1 from zgojcic/zg/sync_pybind11

Zg/sync pybind11

.github/workflows/ci.yml

@@ -9,11 +9,14 @@ on:
       - stable
       - v*
 
+permissions: read-all
+
 concurrency:
   group: test-${{ github.ref }}
   cancel-in-progress: true
 
 env:
+  PIP_BREAK_SYSTEM_PACKAGES: 1
   PIP_ONLY_BINARY: numpy
   FORCE_COLOR: 3
   PYTEST_TIMEOUT: 300
@@ -33,6 +36,7 @@ jobs:
         - '3.9'
         - '3.10'
         - '3.11'
+        - '3.12'
         - 'pypy-3.7'
         - 'pypy-3.8'
         - 'pypy-3.9'
@@ -71,6 +75,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
+        allow-prereleases: true
 
     - name: Setup Boost (Linux)
       # Can't use boost + define _
@@ -82,7 +87,7 @@ jobs:
       run: brew install boost
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Cache wheels
       if: runner.os == 'macOS'
@@ -164,7 +169,6 @@ jobs:
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
         -DPYBIND11_INTERNALS_VERSION=10000000
-        "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
         ${{ matrix.args }}
 
     - name: Build (unstable ABI)
@@ -179,7 +183,9 @@ jobs:
     # This makes sure the setup_helpers module can build packages using
     # setuptools
     - name: Setuptools helpers test
-      run: pytest tests/extra_setuptools
+      run: |
+        pip install setuptools
+        pytest tests/extra_setuptools
       if: "!(matrix.runs-on == 'windows-2022')"
 
 
@@ -202,13 +208,13 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Setup Python ${{ matrix.python-version }} (deadsnakes)
-      uses: deadsnakes/action@v3.0.0
+      uses: deadsnakes/action@v3.0.1
       with:
         python-version: ${{ matrix.python-version }}
         debug: ${{ matrix.python-debug }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Valgrind cache
       if: matrix.valgrind
@@ -241,8 +247,6 @@ jobs:
         python -m pip install -r tests/requirements.txt
 
     - name: Configure
-      env:
-        SETUPTOOLS_USE_DISTUTILS: stdlib
       run: >
         cmake -S . -B build
         -DCMAKE_BUILD_TYPE=Debug
@@ -280,6 +284,8 @@ jobs:
           - dev
         std:
           - 11
+        container_suffix:
+          - ""
         include:
           - clang: 5
             std: 14
@@ -293,9 +299,12 @@ jobs:
             std: 20
           - clang: 14
             std: 20
+          - clang: 15
+            std: 20
+            container_suffix: "-bullseye"
 
     name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64"
-    container: "silkeh/clang:${{ matrix.clang }}"
+    container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}"
 
     steps:
     - uses: actions/checkout@v3
@@ -394,6 +403,7 @@ jobs:
 
   # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds
   centos-nvhpc7:
+    if: ${{ false }}  # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690
     runs-on: ubuntu-latest
     name: "🐍 3 • CentOS7 / PGI 22.9 • x64"
     container: centos:7
@@ -469,7 +479,7 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Configure
       shell: bash
@@ -492,6 +502,24 @@ jobs:
     - name: Interface test
       run: cmake --build build --target test_cmake_build
 
+    - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      if: matrix.gcc == '12'
+      shell: bash
+      run: >
+        cmake -S . -B build_partial
+        -DPYBIND11_WERROR=ON
+        -DDOWNLOAD_CATCH=ON
+        -DCMAKE_CXX_STANDARD=${{ matrix.std }}
+        -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
+        "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
+
+    - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      if: matrix.gcc == '12'
+      run: cmake --build build_partial -j 2
+
+    - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      if: matrix.gcc == '12'
+      run: cmake --build build_partial --target pytest
 
   # Testing on ICC using the oneAPI apt repo
   icc:
@@ -758,10 +786,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.12.0
+      uses: ilammy/msvc-dev-cmd@v1.12.1
       with:
         arch: x86
 
@@ -811,10 +839,10 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Prepare MSVC
-      uses: ilammy/msvc-dev-cmd@v1.12.0
+      uses: ilammy/msvc-dev-cmd@v1.12.1
       with:
         arch: x86
 
@@ -862,7 +890,7 @@ jobs:
         python3 -m pip install -r tests/requirements.txt
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
     - name: Configure C++20
       run: >
@@ -884,6 +912,21 @@ jobs:
     - name: Interface test C++20
       run: cmake --build build --target test_cmake_build
 
+    - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      run: >
+        cmake -S . -B build_partial
+        -DPYBIND11_WERROR=ON
+        -DDOWNLOAD_CATCH=ON
+        -DDOWNLOAD_EIGEN=ON
+        -DCMAKE_CXX_STANDARD=20
+        "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
+
+    - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      run: cmake --build build_partial -j 2
+
+    - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+      run: cmake --build build_partial --target pytest
+
   mingw:
     name: "🐍 3 • windows-latest • ${{ matrix.sys }}"
     runs-on: windows-latest
@@ -995,7 +1038,7 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Update CMake
-        uses: jwlawson/actions-setup-cmake@v1.13
+        uses: jwlawson/actions-setup-cmake@v1.14
 
       - name: Install ninja-build tool
         uses: seanmiddleditch/gha-setup-ninja@v3
@@ -1065,7 +1108,7 @@ jobs:
         run: clang++ --version
 
       - name: Update CMake
-        uses: jwlawson/actions-setup-cmake@v1.13
+        uses: jwlawson/actions-setup-cmake@v1.14
 
       - name: Run pip installs
         run: |
@@ -1100,5 +1143,23 @@ jobs:
       - name: Interface test
         run: cmake --build . --target test_cmake_build -j 2
 
+      - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+        run: >
+          cmake -S . -B build_partial
+          -DPYBIND11_WERROR=ON
+          -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF
+          -DDOWNLOAD_CATCH=ON
+          -DDOWNLOAD_EIGEN=ON
+          -DCMAKE_CXX_COMPILER=clang++
+          -DCMAKE_CXX_STANDARD=17
+          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
+          "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
+
+      - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+        run: cmake --build build_partial -j 2
+
+      - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE
+        run: cmake --build build_partial --target pytest -j 2
+
       - name: Clean directory
         run: git clean -fdx

.github/workflows/configure.yml

@@ -9,7 +9,11 @@ on:
       - stable
       - v*
 
+permissions:
+  contents: read
+
 env:
+  PIP_BREAK_SYSTEM_PACKAGES: 1
   # For cmake:
   VERBOSE: 1
 
@@ -55,7 +59,7 @@ jobs:
     # An action for adding a specific version of CMake:
     #   https://github.com/jwlawson/actions-setup-cmake
     - name: Setup CMake ${{ matrix.cmake }}
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
       with:
         cmake-version: ${{ matrix.cmake }}
 

.github/workflows/format.yml

@@ -12,6 +12,9 @@ on:
     - stable
     - "v*"
 
+permissions:
+  contents: read
+
 env:
   FORCE_COLOR: 3
   # For cmake:

.github/workflows/labeler.yml

@@ -3,10 +3,15 @@ on:
   pull_request_target:
     types: [closed]
 
+permissions: {}
+
 jobs:
   label:
     name: Labeler
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
     steps:
 
     - uses: actions/labeler@main

.github/workflows/pip.yml

@@ -12,7 +12,11 @@ on:
     types:
     - published
 
+permissions:
+  contents: read
+
 env:
+  PIP_BREAK_SYSTEM_PACKAGES: 1
   PIP_ONLY_BINARY: numpy
 
 jobs:
@@ -98,13 +102,13 @@ jobs:
     - uses: actions/download-artifact@v3
 
     - name: Publish standard package
-      uses: pypa/gh-action-pypi-publish@v1.6.4
+      uses: pypa/gh-action-pypi-publish@v1.8.6
       with:
         password: ${{ secrets.pypi_password }}
-        packages_dir: standard/
+        packages-dir: standard/
 
     - name: Publish global package
-      uses: pypa/gh-action-pypi-publish@v1.6.4
+      uses: pypa/gh-action-pypi-publish@v1.8.6
       with:
         password: ${{ secrets.pypi_password_global }}
-        packages_dir: global/
+        packages-dir: global/

.github/workflows/upstream.yml

@@ -1,114 +1,116 @@
-
 name: Upstream
 
 on:
   workflow_dispatch:
   pull_request:
 
+permissions:
+  contents: read
+
 concurrency:
   group: upstream-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  PIP_ONLY_BINARY: numpy
+  PIP_BREAK_SYSTEM_PACKAGES: 1
+  PIP_ONLY_BINARY: ":all:"
   # For cmake:
   VERBOSE: 1
 
 jobs:
   standard:
-    name: "🐍 3.11 latest internals • ubuntu-latest • x64"
+    name: "🐍 3.12 latest • ubuntu-latest • x64"
     runs-on: ubuntu-latest
+    # Only runs when the  'python dev' label is selected
     if: "contains(github.event.pull_request.labels.*.name, 'python dev')"
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python 3.11
+    - name: Setup Python 3.12
       uses: actions/setup-python@v4
       with:
-        python-version: "3.11-dev"
+        python-version: "3.12-dev"
 
-    - name: Setup Boost (Linux)
-      if: runner.os == 'Linux'
+    - name: Setup Boost
       run: sudo apt-get install libboost-dev
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v1.14
 
-    - name: Prepare env
+    - name: Run pip installs
       run: |
+        python -m pip install --upgrade pip
         python -m pip install -r tests/requirements.txt
 
-    - name: Setup annotations on Linux
-      if: runner.os == 'Linux'
-      run: python -m pip install pytest-github-actions-annotate-failures
+    - name: Show platform info
+      run: |
+        python -m platform
+        cmake --version
+        pip list
 
     # First build - C++11 mode and inplace
     - name: Configure C++11
       run: >
-        cmake -S . -B .
+        cmake -S . -B build11
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=11
+        -DCMAKE_BUILD_TYPE=Debug
 
     - name: Build C++11
-      run: cmake --build . -j 2
+      run: cmake --build build11 -j 2
 
     - name: Python tests C++11
-      run: cmake --build . --target pytest -j 2
+      run: cmake --build build11 --target pytest -j 2
 
     - name: C++11 tests
-      run: cmake --build .  --target cpptest -j 2
+      run: cmake --build build11  --target cpptest -j 2
 
     - name: Interface test C++11
-      run: cmake --build . --target test_cmake_build
-
-    - name: Clean directory
-      run: git clean -fdx
+      run: cmake --build build11 --target test_cmake_build
 
     # Second build - C++17 mode and in a build directory
     - name: Configure C++17
       run: >
-        cmake -S . -B build2
+        cmake -S . -B build17
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
-        ${{ matrix.args }}
-        ${{ matrix.args2 }}
 
-    - name: Build
-      run: cmake --build build2 -j 2
+    - name: Build C++17
+      run: cmake --build build17 -j 2
 
-    - name: Python tests
-      run: cmake --build build2 --target pytest
+    - name: Python tests C++17
+      run: cmake --build build17 --target pytest
 
-    - name: C++ tests
-      run: cmake --build build2 --target cpptest
+    - name: C++17 tests
+      run: cmake --build build17 --target cpptest
 
     # Third build - C++17 mode with unstable ABI
     - name: Configure (unstable ABI)
       run: >
-        cmake -S . -B build3
+        cmake -S . -B build17max
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
         -DPYBIND11_INTERNALS_VERSION=10000000
-        "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
-        ${{ matrix.args }}
 
     - name: Build (unstable ABI)
-      run: cmake --build build3 -j 2
+      run: cmake --build build17max -j 2
 
     - name: Python tests (unstable ABI)
-      run: cmake --build build3 --target pytest
+      run: cmake --build build17max --target pytest
 
-    - name: Interface test
-      run: cmake --build build3 --target test_cmake_build
+    - name: Interface test (unstable ABI)
+      run: cmake --build build17max --target test_cmake_build
 
     # This makes sure the setup_helpers module can build packages using
     # setuptools
     - name: Setuptools helpers test
-      run: pytest tests/extra_setuptools
+      run: |
+        pip install setuptools
+        pytest tests/extra_setuptools

.pre-commit-config.yaml

@@ -22,6 +22,49 @@ ci:
 exclude: ^tools/JoinPaths.cmake$
 
 repos:
+
+# Clang format the codebase automatically
+- repo: https://github.com/pre-commit/mirrors-clang-format
+  rev: "v16.0.4"
+  hooks:
+  - id: clang-format
+    types_or: [c++, c, cuda]
+
+# Black, the code formatter, natively supports pre-commit
+- repo: https://github.com/psf/black
+  rev: "23.3.0" # Keep in sync with blacken-docs
+  hooks:
+  - id: black
+
+# Ruff, the Python auto-correcting linter written in Rust
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  rev: v0.0.270
+  hooks:
+  - id: ruff
+    args: ["--fix", "--show-fixes"]
+
+# Check static types with mypy
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: "v1.3.0"
+  hooks:
+  - id: mypy
+    args: []
+    exclude: ^(tests|docs)/
+    additional_dependencies:
+    - markdown-it-py<3 # Drop this together with dropping Python 3.7 support.
+    - nox
+    - rich
+    - types-setuptools
+
+# CMake formatting
+- repo: https://github.com/cheshirekow/cmake-format-precommit
+  rev: "v0.6.13"
+  hooks:
+  - id: cmake-format
+    additional_dependencies: [pyyaml]
+    types: [file]
+    files: (\.cmake|CMakeLists.txt)(.in)?$
+
 # Standard hooks
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: "v4.4.0"
@@ -39,106 +82,35 @@ repos:
   - id: requirements-txt-fixer
   - id: trailing-whitespace
 
-# Upgrade old Python syntax
-- repo: https://github.com/asottile/pyupgrade
-  rev: "v3.3.1"
-  hooks:
-  - id: pyupgrade
-    args: [--py36-plus]
-
-# Nicely sort includes
-- repo: https://github.com/PyCQA/isort
-  rev: "5.11.4"
-  hooks:
-  - id: isort
-
-# Black, the code formatter, natively supports pre-commit
-- repo: https://github.com/psf/black
-  rev: "22.12.0" # Keep in sync with blacken-docs
-  hooks:
-  - id: black
-
 # Also code format the docs
 - repo: https://github.com/asottile/blacken-docs
-  rev: "v1.12.1"
+  rev: "1.13.0"
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==22.10.0 # keep in sync with black hook
+    - black==23.3.0 # keep in sync with black hook
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: "v1.3.1"
+  rev: "v1.5.1"
   hooks:
   - id: remove-tabs
 
+# Avoid directional quotes
 - repo: https://github.com/sirosen/texthooks
-  rev: "0.4.0"
+  rev: "0.5.0"
   hooks:
   - id: fix-ligatures
   - id: fix-smartquotes
 
-# Autoremoves unused imports
-- repo: https://github.com/hadialqattan/pycln
-  rev: "v2.1.2"
-  hooks:
-  - id: pycln
-    stages: [manual]
-
 # Checking for common mistakes
 - repo: https://github.com/pre-commit/pygrep-hooks
-  rev: "v1.9.0"
+  rev: "v1.10.0"
   hooks:
-  - id: python-check-blanket-noqa
-  - id: python-check-blanket-type-ignore
-  - id: python-no-log-warn
-  - id: python-use-type-annotations
   - id: rst-backticks
   - id: rst-directive-colons
   - id: rst-inline-touching-normal
 
-# Automatically remove noqa that are not used
-- repo: https://github.com/asottile/yesqa
-  rev: "v1.4.0"
-  hooks:
-  - id: yesqa
-    additional_dependencies: &flake8_dependencies
-      - flake8-bugbear
-      - pep8-naming
-
-# Flake8 also supports pre-commit natively (same author)
-- repo: https://github.com/PyCQA/flake8
-  rev: "6.0.0"
-  hooks:
-  - id: flake8
-    exclude: ^(docs/.*|tools/.*)$
-    additional_dependencies: *flake8_dependencies
-
-# PyLint has native support - not always usable, but works for us
-- repo: https://github.com/PyCQA/pylint
-  rev: "v2.15.9"
-  hooks:
-  - id: pylint
-    files: ^pybind11
-
-# CMake formatting
-- repo: https://github.com/cheshirekow/cmake-format-precommit
-  rev: "v0.6.13"
-  hooks:
-  - id: cmake-format
-    additional_dependencies: [pyyaml]
-    types: [file]
-    files: (\.cmake|CMakeLists.txt)(.in)?$
-
-# Check static types with mypy
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: "v0.991"
-  hooks:
-  - id: mypy
-    args: []
-    exclude: ^(tests|docs)/
-    additional_dependencies: [nox, rich]
-
 # Checks the manifest for missing files (native support)
 - repo: https://github.com/mgedmin/check-manifest
   rev: "0.49"
@@ -152,15 +124,15 @@ repos:
 # Use tools/codespell_ignore_lines_from_errors.py
 # to rebuild .codespell-ignore-lines
 - repo: https://github.com/codespell-project/codespell
-  rev: "v2.2.2"
+  rev: "v2.2.4"
   hooks:
   - id: codespell
     exclude: ".supp$"
-    args: ["-x", ".codespell-ignore-lines"]
+    args: ["-x.codespell-ignore-lines", "-Lccompiler"]
 
 # Check for common shell mistakes
 - repo: https://github.com/shellcheck-py/shellcheck-py
-  rev: "v0.9.0.2"
+  rev: "v0.9.0.5"
   hooks:
   - id: shellcheck
 
@@ -173,9 +145,9 @@ repos:
     entry: PyBind|Numpy|Cmake|CCache|PyTest
     exclude: ^\.pre-commit-config.yaml$
 
-# Clang format the codebase automatically
-- repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: "v15.0.6"
+# PyLint has native support - not always usable, but works for us
+- repo: https://github.com/PyCQA/pylint
+  rev: "v3.0.0a6"
   hooks:
-  - id: clang-format
-    types_or: [c++, c, cuda]
+  - id: pylint
+    files: ^pybind11

CMakeLists.txt

@@ -126,6 +126,7 @@ set(PYBIND11_HEADERS
     include/pybind11/complex.h
     include/pybind11/options.h
     include/pybind11/eigen.h
+    include/pybind11/eigen/common.h
     include/pybind11/eigen/matrix.h
     include/pybind11/eigen/tensor.h
     include/pybind11/embed.h
@@ -139,7 +140,8 @@ set(PYBIND11_HEADERS
     include/pybind11/pytypes.h
     include/pybind11/stl.h
     include/pybind11/stl_bind.h
-    include/pybind11/stl/filesystem.h)
+    include/pybind11/stl/filesystem.h
+    include/pybind11/type_caster_pyobject_ptr.h)
 
 # Compare with grep and warn if mismatched
 if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)

MANIFEST.in

@@ -1,5 +1,6 @@
+prune tests
 recursive-include pybind11/include/pybind11 *.h
 recursive-include pybind11 *.py
 recursive-include pybind11 py.typed
 include pybind11/share/cmake/pybind11/*.cmake
-include LICENSE README.rst pyproject.toml setup.py setup.cfg
+include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg

SECURITY.md

@@ -0,0 +1,13 @@
+# Security Policy
+
+## Supported Versions
+
+Security updates are applied only to the latest release.
+
+## Reporting a Vulnerability
+
+If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
+
+Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new).
+
+This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure.

docs/advanced/cast/strings.rst

@@ -101,8 +101,11 @@ conversion has the same overhead as implicit conversion.
     m.def("str_output",
         []() {
             std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1
-            py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length());
-            return py_s;
+            py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr);
+            if (!py_s) {
+                throw py::error_already_set();
+            }
+            return py::reinterpret_steal<py::str>(py_s);
         }
     );
 
@@ -113,7 +116,8 @@ conversion has the same overhead as implicit conversion.
 
 The `Python C API
 <https://docs.python.org/3/c-api/unicode.html#built-in-codecs>`_ provides
-several built-in codecs.
+several built-in codecs. Note that these all return *new* references, so
+use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`.
 
 
 One could also use a third party encoding library such as libiconv to transcode

docs/changelog.rst

@@ -10,10 +10,25 @@ Changes will be added here periodically from the "Suggested changelog entry"
 block in pull request descriptions.
 
 
-IN DEVELOPMENT
---------------
+Version 2.11.0 (June 2, 2023)
+-----------------------------
 
-Changes will be summarized here periodically.
+New features:
+
+* ``pybind11::detail::is_move_constructible`` can now be specialized for cases
+   in which ``std::is_move_constructible`` does not work as needed. This is
+   very similar to the long-established
+   ``pybind11::detail::is_copy_constructible``.
+  `#4631 <https://github.com/pybind/pybind11/pull/4631>`_
+
+* Introduce ``recursive_container_traits``.
+  `#4623 <https://github.com/pybind/pybind11/pull/4623>`_
+
+* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic
+  wrapping of APIs that make use of ``PyObject *``. This header needs to
+  included explicitly (i.e. it is not included implicitly
+  with ``pybind/pybind11.h``).
+  `#4601 <https://github.com/pybind/pybind11/pull/4601>`_
 
 Changes:
 
@@ -21,11 +36,78 @@ Changes:
   ``dec_ref()`` are now enabled by default again.
   `#4246 <https://github.com/pybind/pybind11/pull/4246>`_
 
+* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()``
+  instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete
+  ``sys.path``.
+  `#4473 <https://github.com/pybind/pybind11/pull/4473>`_
+
+* Cast errors now always include Python type information, even if
+  ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary
+  sizes slightly (~1.5%) but the error messages are much more informative.
+  `#4463 <https://github.com/pybind/pybind11/pull/4463>`_
+
+* Setter return values (which are inaccessible for all practical purposes) are
+  no longer converted to Python (only to be discarded).
+  `#4621 <https://github.com/pybind/pybind11/pull/4621>`_
+
+* Allow lambda specified to function definition to be ``noexcept(true)``
+  in C++17.
+  `#4593 <https://github.com/pybind/pybind11/pull/4593>`_
+
+* Get rid of recursive template instantiations for concatenating type
+  signatures on C++17 and higher.
+  `#4587 <https://github.com/pybind/pybind11/pull/4587>`_
+
+* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11
+  ABI version for Python 3.12 is version 5. (The default ABI version
+  for Python versions up to and including 3.11 is still version 4.).
+  `#4570 <https://github.com/pybind/pybind11/pull/4570>`_
+
+* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds
+  use ``std::hash<std::type_index>`` and ``std::equal_to<std::type_index>``
+  instead of string-based type comparisons. This resolves issues when binding
+  types defined in the unnamed namespace.
+  `#4319 <https://github.com/pybind/pybind11/pull/4319>`_
+
 Build system improvements:
 
 * Update clang-tidy to 15 in CI.
   `#4387 <https://github.com/pybind/pybind11/pull/4387>`_
 
+* Moved the linting framework over to Ruff.
+  `#4483 <https://github.com/pybind/pybind11/pull/4483>`_
+
+* Skip lto checks and target generation when
+  ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined.
+  `#4643 <https://github.com/pybind/pybind11/pull/4643>`_
+
+* No longer inject ``-stdlib=libc++``, not needed for modern Pythons
+  (macOS 10.9+).
+  `#4639 <https://github.com/pybind/pybind11/pull/4639>`_
+
+
+Version 2.10.4 (Mar 16, 2023)
+-----------------------------
+
+Changes:
+
+* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and
+  exits).
+  `#4526 <https://github.com/pybind/pybind11/pull/4526>`_
+
+Bug Fixes:
+
+* Fix a warning when pydebug is enabled on Python 3.11.
+  `#4461 <https://github.com/pybind/pybind11/pull/4461>`_
+
+* Ensure ``gil_scoped_release`` RAII is non-copyable.
+  `#4490 <https://github.com/pybind/pybind11/pull/4490>`_
+
+* Ensure the tests dir does not show up with new versions of setuptools.
+  `#4510 <https://github.com/pybind/pybind11/pull/4510>`_
+
+* Better stacklevel for a warning in setuptools helpers.
+  `#4516 <https://github.com/pybind/pybind11/pull/4516>`_
 
 Version 2.10.3 (Jan 3, 2023)
 ----------------------------

docs/classes.rst

@@ -58,6 +58,16 @@ interactive Python session demonstrating this example is shown below:
     Static member functions can be bound in the same way using
     :func:`class_::def_static`.
 
+.. note::
+
+    Binding C++ types in unnamed namespaces (also known as anonymous namespaces)
+    works reliably on many platforms, but not all. The `XFAIL_CONDITION` in
+    tests/test_unnamed_namespace_a.py encodes the currently known conditions.
+    For background see `#4319 <https://github.com/pybind/pybind11/pull/4319>`_.
+    If portability is a concern, it is therefore not recommended to bind C++
+    types in unnamed namespaces. It will be safest to manually pick unique
+    namespace names.
+
 Keyword and default arguments
 =============================
 It is possible to specify keyword and default arguments using the syntax

docs/conf.py

@@ -353,12 +353,11 @@ def prepare(app):
         f.write(contents)
 
 
-def clean_up(app, exception):
+def clean_up(app, exception):  # noqa: ARG001
     (DIR / "readme.rst").unlink()
 
 
 def setup(app):
-
     # Add hook for building doxygen xml when needed
     app.connect("builder-inited", generate_doxygen_xml)
 

include/pybind11/attr.h

@@ -26,6 +26,9 @@ struct is_method {
     explicit is_method(const handle &c) : class_(c) {}
 };
 
+/// Annotation for setters
+struct is_setter {};
+
 /// Annotation for operators
 struct is_operator {};
 
@@ -188,8 +191,8 @@ struct argument_record {
 struct function_record {
     function_record()
         : is_constructor(false), is_new_style_constructor(false), is_stateless(false),
-          is_operator(false), is_method(false), has_args(false), has_kwargs(false),
-          prepend(false) {}
+          is_operator(false), is_method(false), is_setter(false), has_args(false),
+          has_kwargs(false), prepend(false) {}
 
     /// Function name
     char *name = nullptr; /* why no C++ strings? They generate heavier code.. */
@@ -230,6 +233,9 @@ struct function_record {
     /// True if this is a method
     bool is_method : 1;
 
+    /// True if this is a setter
+    bool is_setter : 1;
+
     /// True if the function has a '*args' argument
     bool has_args : 1;
 
@@ -426,6 +432,12 @@ struct process_attribute<is_method> : process_attribute_default<is_method> {
     }
 };
 
+/// Process an attribute which indicates that this function is a setter
+template <>
+struct process_attribute<is_setter> : process_attribute_default<is_setter> {
+    static void init(const is_setter &, function_record *r) { r->is_setter = true; }
+};
+
 /// Process an attribute which indicates the parent scope of a method
 template <>
 struct process_attribute<scope> : process_attribute_default<scope> {

include/pybind11/buffer_info.h

@@ -37,6 +37,9 @@ inline std::vector<ssize_t> f_strides(const std::vector<ssize_t> &shape, ssize_t
     return strides;
 }
 
+template <typename T, typename SFINAE = void>
+struct compare_buffer_info;
+
 PYBIND11_NAMESPACE_END(detail)
 
 /// Information record describing a Python buffer object
@@ -150,6 +153,17 @@ struct buffer_info {
     Py_buffer *view() const { return m_view; }
     Py_buffer *&view() { return m_view; }
 
+    /* True if the buffer item type is equivalent to `T`. */
+    // To define "equivalent" by example:
+    // `buffer_info::item_type_is_equivalent_to<int>(b)` and
+    // `buffer_info::item_type_is_equivalent_to<long>(b)` may both be true
+    // on some platforms, but `int` and `unsigned` will never be equivalent.
+    // For the ground truth, please inspect `detail::compare_buffer_info<>`.
+    template <typename T>
+    bool item_type_is_equivalent_to() const {
+        return detail::compare_buffer_info<T>::compare(*this);
+    }
+
 private:
     struct private_ctr_tag {};
 
@@ -170,9 +184,10 @@ private:
 
 PYBIND11_NAMESPACE_BEGIN(detail)
 
-template <typename T, typename SFINAE = void>
+template <typename T, typename SFINAE>
 struct compare_buffer_info {
     static bool compare(const buffer_info &b) {
+        // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *`
         return b.format == format_descriptor<T>::format() && b.itemsize == (ssize_t) sizeof(T);
     }
 };

include/pybind11/cast.h

@@ -964,7 +964,7 @@ struct move_always<
     enable_if_t<
         all_of<move_is_plain_type<T>,
                negation<is_copy_constructible<T>>,
-               std::is_move_constructible<T>,
+               is_move_constructible<T>,
                std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>>
     : std::true_type {};
 template <typename T, typename SFINAE = void>
@@ -975,7 +975,7 @@ struct move_if_unreferenced<
     enable_if_t<
         all_of<move_is_plain_type<T>,
                negation<move_always<T>>,
-               std::is_move_constructible<T>,
+               is_move_constructible<T>,
                std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>>
     : std::true_type {};
 template <typename T>
@@ -1017,11 +1017,14 @@ type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &ha
                   "Internal error: type_caster should only be used for C++ types");
     if (!conv.load(handle, true)) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
-        throw cast_error("Unable to cast Python instance to C++ type (#define "
+        throw cast_error(
+            "Unable to cast Python instance of type "
+            + str(type::handle_of(handle)).cast<std::string>()
+            + " to C++ type '?' (#define "
               "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)");
 #else
         throw cast_error("Unable to cast Python instance of type "
-                         + (std::string) str(type::handle_of(handle)) + " to C++ type '"
+                         + str(type::handle_of(handle)).cast<std::string>() + " to C++ type '"
                          + type_id<T>() + "'");
 #endif
     }
@@ -1038,7 +1041,11 @@ make_caster<T> load_type(const handle &handle) {
 PYBIND11_NAMESPACE_END(detail)
 
 // pytype -> C++ type
-template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
+template <typename T,
+          detail::enable_if_t<!detail::is_pyobject<T>::value
+                                  && !detail::is_same_ignoring_cvref<T, PyObject *>::value,
+                              int>
+          = 0>
 T cast(const handle &handle) {
     using namespace detail;
     static_assert(!cast_is_temporary_value_reference<T>::value,
@@ -1052,6 +1059,34 @@ T cast(const handle &handle) {
     return T(reinterpret_borrow<object>(handle));
 }
 
+// Note that `cast<PyObject *>(obj)` increments the reference count of `obj`.
+// This is necessary for the case that `obj` is a temporary, and could
+// not possibly be different, given
+// 1. the established convention that the passed `handle` is borrowed, and
+// 2. we don't want to force all generic code using `cast<T>()` to special-case
+//    handling of `T` = `PyObject *` (to increment the reference count there).
+// It is the responsibility of the caller to ensure that the reference count
+// is decremented.
+template <typename T,
+          typename Handle,
+          detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value
+                                  && detail::is_same_ignoring_cvref<Handle, handle>::value,
+                              int>
+          = 0>
+T cast(Handle &&handle) {
+    return handle.inc_ref().ptr();
+}
+// To optimize way an inc_ref/dec_ref cycle:
+template <typename T,
+          typename Object,
+          detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value
+                                  && detail::is_same_ignoring_cvref<Object, object>::value,
+                              int>
+          = 0>
+T cast(Object &&obj) {
+    return obj.release().ptr();
+}
+
 // C++ type -> py::object
 template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
 object cast(T &&value,
@@ -1085,12 +1120,13 @@ detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) {
     if (obj.ref_count() > 1) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
         throw cast_error(
-            "Unable to cast Python instance to C++ rvalue: instance has multiple references"
+            "Unable to cast Python " + str(type::handle_of(obj)).cast<std::string>()
+            + " instance to C++ rvalue: instance has multiple references"
               " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)");
 #else
-        throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj))
-                         + " instance to C++ " + type_id<T>()
-                         + " instance: instance has multiple references");
+        throw cast_error("Unable to move from Python "
+                         + str(type::handle_of(obj)).cast<std::string>() + " instance to C++ "
+                         + type_id<T>() + " instance: instance has multiple references");
 #endif
     }
 
@@ -1195,8 +1231,9 @@ PYBIND11_NAMESPACE_END(detail)
 // The overloads could coexist, i.e. the #if is not strictly speaking needed,
 // but it is an easy minor optimization.
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
-inline cast_error cast_error_unable_to_convert_call_arg() {
-    return cast_error("Unable to convert call argument to Python object (#define "
+inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) {
+    return cast_error("Unable to convert call argument '" + name
+                      + "' to Python object (#define "
                         "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)");
 }
 #else
@@ -1220,7 +1257,7 @@ tuple make_tuple(Args &&...args_) {
     for (size_t i = 0; i < args.size(); i++) {
         if (!args[i]) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
-            throw cast_error_unable_to_convert_call_arg();
+            throw cast_error_unable_to_convert_call_arg(std::to_string(i));
 #else
             std::array<std::string, size> argtypes{{type_id<Args>()...}};
             throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]);
@@ -1510,7 +1547,7 @@ private:
             detail::make_caster<T>::cast(std::forward<T>(x), policy, {}));
         if (!o) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
-            throw cast_error_unable_to_convert_call_arg();
+            throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()));
 #else
             throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()),
                                                         type_id<T>());
@@ -1542,7 +1579,7 @@ private:
         }
         if (!a.value) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
-            throw cast_error_unable_to_convert_call_arg();
+            throw cast_error_unable_to_convert_call_arg(a.name);
 #else
             throw cast_error_unable_to_convert_call_arg(a.name, a.type);
 #endif

include/pybind11/detail/class.h

@@ -445,9 +445,17 @@ inline void clear_instance(PyObject *self) {
 /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc`
 /// to destroy the C++ object itself, while the rest is Python bookkeeping.
 extern "C" inline void pybind11_object_dealloc(PyObject *self) {
+    auto *type = Py_TYPE(self);
+
+    // If this is a GC tracked object, untrack it first
+    // Note that the track call is implicitly done by the
+    // default tp_alloc, which we never override.
+    if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) {
+        PyObject_GC_UnTrack(self);
+    }
+
     clear_instance(self);
 
-    auto *type = Py_TYPE(self);
     type->tp_free(self);
 
 #if PY_VERSION_HEX < 0x03080000

include/pybind11/detail/common.h

@@ -661,6 +661,10 @@ template <class T>
 using remove_cvref_t = typename remove_cvref<T>::type;
 #endif
 
+/// Example usage: is_same_ignoring_cvref<T, PyObject *>::value
+template <typename T, typename U>
+using is_same_ignoring_cvref = std::is_same<detail::remove_cvref_t<T>, U>;
+
 /// Index sequences
 #if defined(PYBIND11_CPP14)
 using std::index_sequence;
@@ -754,7 +758,16 @@ template <typename C, typename R, typename... A>
 struct remove_class<R (C::*)(A...) const> {
     using type = R(A...);
 };
-
+#ifdef __cpp_noexcept_function_type
+template <typename C, typename R, typename... A>
+struct remove_class<R (C::*)(A...) noexcept> {
+    using type = R(A...);
+};
+template <typename C, typename R, typename... A>
+struct remove_class<R (C::*)(A...) const noexcept> {
+    using type = R(A...);
+};
+#endif
 /// Helper template to strip away type modifiers
 template <typename T>
 struct intrinsic_type {
@@ -1012,6 +1025,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in
 template <typename T, typename SFINAE = void>
 struct format_descriptor {};
 
+template <typename T>
+struct format_descriptor<
+    T,
+    detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value>> {
+    static constexpr const char c = 'O';
+    static constexpr const char value[2] = {c, '\0'};
+    static std::string format() { return std::string(1, c); }
+};
+
 PYBIND11_NAMESPACE_BEGIN(detail)
 // Returns the index of the given type in the type char array below, and in the list in numpy.h
 // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double;
@@ -1225,8 +1247,9 @@ constexpr
 #endif
 
 // Pybind offers detailed error messages by default for all builts that are debug (through the
-// negation of ndebug). This can also be manually enabled by users, for any builds, through
-// defining PYBIND11_DETAILED_ERROR_MESSAGES.
+// negation of NDEBUG). This can also be manually enabled by users, for any builds, through
+// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those
+// who are writing (as opposed to merely using) libraries that use pybind11.
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG)
 #    define PYBIND11_DETAILED_ERROR_MESSAGES
 #endif

include/pybind11/detail/descr.h

@@ -143,11 +143,24 @@ constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) {
     return descr;
 }
 
+#ifdef __cpp_fold_expressions
+template <size_t N1, size_t N2, typename... Ts1, typename... Ts2>
+constexpr descr<N1 + N2 + 2, Ts1..., Ts2...> operator,(const descr<N1, Ts1...> &a,
+                                                       const descr<N2, Ts2...> &b) {
+    return a + const_name(", ") + b;
+}
+
+template <size_t N, typename... Ts, typename... Args>
+constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) {
+    return (d, ..., args);
+}
+#else
 template <size_t N, typename... Ts, typename... Args>
 constexpr auto concat(const descr<N, Ts...> &d, const Args &...args)
     -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) {
     return d + const_name(", ") + concat(args...);
 }
+#endif
 
 template <size_t N, typename... Ts>
 constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) {

include/pybind11/detail/init.h

@@ -175,7 +175,7 @@ void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) {
 template <typename Class>
 void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) {
     PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias);
-    static_assert(std::is_move_constructible<Cpp<Class>>::value,
+    static_assert(is_move_constructible<Cpp<Class>>::value,
                   "pybind11::init() return-by-value factory function requires a movable class");
     if (Class::has_alias && need_alias) {
         construct_alias_from_cpp<Class>(is_alias_constructible<Class>{}, v_h, std::move(result));
@@ -190,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) {
 template <typename Class>
 void construct(value_and_holder &v_h, Alias<Class> &&result, bool) {
     static_assert(
-        std::is_move_constructible<Alias<Class>>::value,
+        is_move_constructible<Alias<Class>>::value,
         "pybind11::init() return-by-alias-value factory function requires a movable alias class");
     v_h.value_ptr() = new Alias<Class>(std::move(result));
 }

include/pybind11/detail/internals.h

@@ -34,8 +34,17 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
+#    if PY_VERSION_HEX >= 0x030C0000
+// Version bump for Python 3.12+, before first 3.12 beta release.
+#        define PYBIND11_INTERNALS_VERSION 5
+#    else
 #        define PYBIND11_INTERNALS_VERSION 4
 #    endif
+#endif
+
+// This requirement is mainly to reduce the support burden (see PR #4570).
+static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5,
+              "pybind11 ABI version 5 is the minimum for Python 3.12+");
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
@@ -114,7 +123,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) {
 // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name,
 // which works.  If not under a known-good stl, provide our own name-based hash and equality
 // functions that use the type name.
-#if defined(__GLIBCXX__)
+#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__))                                     \
+    || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION))
 inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; }
 using type_hash = std::hash<std::type_index>;
 using type_equal_to = std::equal_to<std::type_index>;
@@ -421,6 +431,38 @@ inline void translate_local_exception(std::exception_ptr p) {
 }
 #endif
 
+inline object get_python_state_dict() {
+    object state_dict;
+#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION)
+    state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
+#else
+#    if PY_VERSION_HEX < 0x03090000
+    PyInterpreterState *istate = _PyInterpreterState_Get();
+#    else
+    PyInterpreterState *istate = PyInterpreterState_Get();
+#    endif
+    if (istate) {
+        state_dict = reinterpret_borrow<object>(PyInterpreterState_GetDict(istate));
+    }
+#endif
+    if (!state_dict) {
+        raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED");
+    }
+    return state_dict;
+}
+
+inline object get_internals_obj_from_state_dict(handle state_dict) {
+    return reinterpret_borrow<object>(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID));
+}
+
+inline internals **get_internals_pp_from_capsule(handle obj) {
+    void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr);
+    if (raw_ptr == nullptr) {
+        raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED");
+    }
+    return static_cast<internals **>(raw_ptr);
+}
+
 /// Return a reference to the current `internals` data
 PYBIND11_NOINLINE internals &get_internals() {
     auto **&internals_pp = get_internals_pp();
@@ -445,12 +487,12 @@ PYBIND11_NOINLINE internals &get_internals() {
 #endif
     error_scope err_scope;
 
-    PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID);
-    auto builtins = handle(PyEval_GetBuiltins());
-    if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
-        internals_pp = static_cast<internals **>(capsule(builtins[id]));
-
-        // We loaded builtins through python's builtins, which means that our `error_already_set`
+    dict state_dict = get_python_state_dict();
+    if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) {
+        internals_pp = get_internals_pp_from_capsule(internals_obj);
+    }
+    if (internals_pp && *internals_pp) {
+        // We loaded the internals through `state_dict`, which means that our `error_already_set`
         // and `builtin_exception` may be different local classes than the ones set up in the
         // initial exception translator, below, so add another for our local exception classes.
         //
@@ -484,7 +526,7 @@ PYBIND11_NOINLINE internals &get_internals() {
 #    endif
         internals_ptr->istate = tstate->interp;
 #endif
-        builtins[id] = capsule(internals_pp);
+        state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp);
         internals_ptr->registered_exception_translators.push_front(&translate_exception);
         internals_ptr->static_property_type = make_static_property_type();
         internals_ptr->default_metaclass = make_default_metaclass();

include/pybind11/detail/type_caster_base.h

@@ -258,8 +258,8 @@ struct value_and_holder {
 
     // Main constructor for a found value/holder:
     value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index)
-        : inst{i}, index{index}, type{type}, vh{inst->simple_layout
-                                                    ? inst->simple_value_holder
+        : inst{i}, index{index}, type{type},
+          vh{inst->simple_layout ? inst->simple_value_holder
                                  : &inst->nonsimple.values_and_holders[vpos]} {}
 
     // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
@@ -822,23 +822,179 @@ using movable_cast_op_type
                                   typename std::add_rvalue_reference<intrinsic_t<T>>::type,
                                   typename std::add_lvalue_reference<intrinsic_t<T>>::type>>;
 
+// Does the container have a mapped type and is it recursive?
+// Implemented by specializations below.
+template <typename Container, typename SFINAE = void>
+struct container_mapped_type_traits {
+    static constexpr bool has_mapped_type = false;
+    static constexpr bool has_recursive_mapped_type = false;
+};
+
+template <typename Container>
+struct container_mapped_type_traits<
+    Container,
+    typename std::enable_if<
+        std::is_same<typename Container::mapped_type, Container>::value>::type> {
+    static constexpr bool has_mapped_type = true;
+    static constexpr bool has_recursive_mapped_type = true;
+};
+
+template <typename Container>
+struct container_mapped_type_traits<
+    Container,
+    typename std::enable_if<
+        negation<std::is_same<typename Container::mapped_type, Container>>::value>::type> {
+    static constexpr bool has_mapped_type = true;
+    static constexpr bool has_recursive_mapped_type = false;
+};
+
+// Does the container have a value type and is it recursive?
+// Implemented by specializations below.
+template <typename Container, typename SFINAE = void>
+struct container_value_type_traits : std::false_type {
+    static constexpr bool has_value_type = false;
+    static constexpr bool has_recursive_value_type = false;
+};
+
+template <typename Container>
+struct container_value_type_traits<
+    Container,
+    typename std::enable_if<
+        std::is_same<typename Container::value_type, Container>::value>::type> {
+    static constexpr bool has_value_type = true;
+    static constexpr bool has_recursive_value_type = true;
+};
+
+template <typename Container>
+struct container_value_type_traits<
+    Container,
+    typename std::enable_if<
+        negation<std::is_same<typename Container::value_type, Container>>::value>::type> {
+    static constexpr bool has_value_type = true;
+    static constexpr bool has_recursive_value_type = false;
+};
+
+/*
+ * Tag to be used for representing the bottom of recursively defined types.
+ * Define this tag so we don't have to use void.
+ */
+struct recursive_bottom {};
+
+/*
+ * Implementation detail of `recursive_container_traits` below.
+ * `T` is the `value_type` of the container, which might need to be modified to
+ * avoid recursive types and const types.
+ */
+template <typename T, bool is_this_a_map>
+struct impl_type_to_check_recursively {
+    /*
+     * If the container is recursive, then no further recursion should be done.
+     */
+    using if_recursive = recursive_bottom;
+    /*
+     * Otherwise yield `T` unchanged.
+     */
+    using if_not_recursive = T;
+};
+
+/*
+ * For pairs - only as value type of a map -, the first type should remove the `const`.
+ * Also, if the map is recursive, then the recursive checking should consider
+ * the first type only.
+ */
+template <typename A, typename B>
+struct impl_type_to_check_recursively<std::pair<A, B>, /* is_this_a_map = */ true> {
+    using if_recursive = typename std::remove_const<A>::type;
+    using if_not_recursive = std::pair<typename std::remove_const<A>::type, B>;
+};
+
+/*
+ * Implementation of `recursive_container_traits` below.
+ */
+template <typename Container, typename SFINAE = void>
+struct impl_recursive_container_traits {
+    using type_to_check_recursively = recursive_bottom;
+};
+
+template <typename Container>
+struct impl_recursive_container_traits<
+    Container,
+    typename std::enable_if<container_value_type_traits<Container>::has_value_type>::type> {
+    static constexpr bool is_recursive
+        = container_mapped_type_traits<Container>::has_recursive_mapped_type
+          || container_value_type_traits<Container>::has_recursive_value_type;
+    /*
+     * This member dictates which type Pybind11 should check recursively in traits
+     * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ...
+     * Direct access to `value_type` should be avoided:
+     * 1. `value_type` might recursively contain the type again
+     * 2. `value_type` of STL map types is `std::pair<A const, B>`, the `const`
+     *    should be removed.
+     *
+     */
+    using type_to_check_recursively = typename std::conditional<
+        is_recursive,
+        typename impl_type_to_check_recursively<
+            typename Container::value_type,
+            container_mapped_type_traits<Container>::has_mapped_type>::if_recursive,
+        typename impl_type_to_check_recursively<
+            typename Container::value_type,
+            container_mapped_type_traits<Container>::has_mapped_type>::if_not_recursive>::type;
+};
+
+/*
+ * This trait defines the `type_to_check_recursively` which is needed to properly
+ * handle recursively defined traits such as `is_move_constructible` without going
+ * into an infinite recursion.
+ * Should be used instead of directly accessing the `value_type`.
+ * It cancels the recursion by returning the `recursive_bottom` tag.
+ *
+ * The default definition of `type_to_check_recursively` is as follows:
+ *
+ * 1. By default, it is `recursive_bottom`, so that the recursion is canceled.
+ * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used.
+ *    If the `value_type` is a pair and a `mapped_type` is defined,
+ *    then the `const` is removed from the first type.
+ * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned.
+ * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined,
+ *    then `const` is removed from the first type and the first type is returned.
+ *
+ * This behavior can be extended by the user as seen in test_stl_binders.cpp.
+ *
+ * This struct is exactly the same as impl_recursive_container_traits.
+ * The duplication achieves that user-defined specializations don't compete
+ * with internal specializations, but take precedence.
+ */
+template <typename Container, typename SFINAE = void>
+struct recursive_container_traits : impl_recursive_container_traits<Container> {};
+
+template <typename T>
+struct is_move_constructible
+    : all_of<std::is_move_constructible<T>,
+             is_move_constructible<
+                 typename recursive_container_traits<T>::type_to_check_recursively>> {};
+
+template <>
+struct is_move_constructible<recursive_bottom> : std::true_type {};
+
+// Likewise for std::pair
+// (after C++17 it is mandatory that the move constructor not exist when the two types aren't
+// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves
+// containers).
+template <typename T1, typename T2>
+struct is_move_constructible<std::pair<T1, T2>>
+    : all_of<is_move_constructible<T1>, is_move_constructible<T2>> {};
+
 // std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when
 // T is non-copyable, but code containing such a copy constructor fails to actually compile.
-template <typename T, typename SFINAE = void>
-struct is_copy_constructible : std::is_copy_constructible<T> {};
+template <typename T>
+struct is_copy_constructible
+    : all_of<std::is_copy_constructible<T>,
+             is_copy_constructible<
+                 typename recursive_container_traits<T>::type_to_check_recursively>> {};
 
-// Specialization for types that appear to be copy constructible but also look like stl containers
-// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if
-// so, copy constructability depends on whether the value_type is copy constructible.
-template <typename Container>
-struct is_copy_constructible<
-    Container,
-    enable_if_t<
-        all_of<std::is_copy_constructible<Container>,
-               std::is_same<typename Container::value_type &, typename Container::reference>,
-               // Avoid infinite recursion
-               negation<std::is_same<Container, typename Container::value_type>>>::value>>
-    : is_copy_constructible<typename Container::value_type> {};
+template <>
+struct is_copy_constructible<recursive_bottom> : std::true_type {};
 
 // Likewise for std::pair
 // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't
@@ -849,14 +1005,16 @@ struct is_copy_constructible<std::pair<T1, T2>>
     : all_of<is_copy_constructible<T1>, is_copy_constructible<T2>> {};
 
 // The same problems arise with std::is_copy_assignable, so we use the same workaround.
-template <typename T, typename SFINAE = void>
-struct is_copy_assignable : std::is_copy_assignable<T> {};
-template <typename Container>
-struct is_copy_assignable<Container,
-                          enable_if_t<all_of<std::is_copy_assignable<Container>,
-                                             std::is_same<typename Container::value_type &,
-                                                          typename Container::reference>>::value>>
-    : is_copy_assignable<typename Container::value_type> {};
+template <typename T>
+struct is_copy_assignable
+    : all_of<
+          std::is_copy_assignable<T>,
+          is_copy_assignable<typename recursive_container_traits<T>::type_to_check_recursively>> {
+};
+
+template <>
+struct is_copy_assignable<recursive_bottom> : std::true_type {};
+
 template <typename T1, typename T2>
 struct is_copy_assignable<std::pair<T1, T2>>
     : all_of<is_copy_assignable<T1>, is_copy_assignable<T2>> {};
@@ -994,7 +1152,7 @@ protected:
         return [](const void *arg) -> void * { return new T(*reinterpret_cast<const T *>(arg)); };
     }
 
-    template <typename T, typename = enable_if_t<std::is_move_constructible<T>::value>>
+    template <typename T, typename = enable_if_t<is_move_constructible<T>::value>>
     static auto make_move_constructor(const T *)
         -> decltype(new T(std::declval<T &&>()), Constructor{}) {
         return [](const void *arg) -> void * {

include/pybind11/eigen/common.h

@@ -0,0 +1,9 @@
+// Copyright (c) 2023 The pybind Community.
+
+#pragma once
+
+// Common message for `static_assert()`s, which are useful to easily
+// preempt much less obvious errors.
+#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED                                    \
+    "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen "     \
+    "types."

include/pybind11/eigen/matrix.h

@@ -10,6 +10,7 @@
 #pragma once
 
 #include "../numpy.h"
+#include "common.h"
 
 /* HINT: To suppress warnings originating from the Eigen headers, use -isystem.
    See also:
@@ -19,7 +20,9 @@
 PYBIND11_WARNING_PUSH
 PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741
 //       C5054: operator '&': deprecated between enumerations of different types
+#if defined(__MINGW32__)
 PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized")
+#endif
 
 #include <Eigen/Core>
 #include <Eigen/SparseCore>
@@ -285,6 +288,8 @@ handle eigen_encapsulate(Type *src) {
 template <typename Type>
 struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> {
     using Scalar = typename Type::Scalar;
+    static_assert(!std::is_pointer<Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
     using props = EigenProps<Type>;
 
     bool load(handle src, bool convert) {
@@ -403,6 +408,9 @@ private:
 // Base class for casting reference/map/block/etc. objects back to python.
 template <typename MapType>
 struct eigen_map_caster {
+    static_assert(!std::is_pointer<typename MapType::Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
+
 private:
     using props = EigenProps<MapType>;
 
@@ -455,6 +463,8 @@ private:
     using Type = Eigen::Ref<PlainObjectType, 0, StrideType>;
     using props = EigenProps<Type>;
     using Scalar = typename props::Scalar;
+    static_assert(!std::is_pointer<Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
     using MapType = Eigen::Map<PlainObjectType, 0, StrideType>;
     using Array
         = array_t<Scalar,
@@ -602,6 +612,9 @@ private:
 // regular Eigen::Matrix, then casting that.
 template <typename Type>
 struct type_caster<Type, enable_if_t<is_eigen_other<Type>::value>> {
+    static_assert(!std::is_pointer<typename Type::Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
+
 protected:
     using Matrix
         = Eigen::Matrix<typename Type::Scalar, Type::RowsAtCompileTime, Type::ColsAtCompileTime>;
@@ -630,6 +643,8 @@ public:
 template <typename Type>
 struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> {
     using Scalar = typename Type::Scalar;
+    static_assert(!std::is_pointer<Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
     using StorageIndex = remove_reference_t<decltype(*std::declval<Type>().outerIndexPtr())>;
     using Index = typename Type::Index;
     static constexpr bool rowMajor = Type::IsRowMajor;

include/pybind11/eigen/tensor.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "../numpy.h"
+#include "common.h"
 
 #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
 static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0");
@@ -17,7 +18,9 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0
 PYBIND11_WARNING_PUSH
 PYBIND11_WARNING_DISABLE_MSVC(4554)
 PYBIND11_WARNING_DISABLE_MSVC(4127)
+#if defined(__MINGW32__)
 PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized")
+#endif
 
 #include <unsupported/Eigen/CXX11/Tensor>
 
@@ -162,6 +165,8 @@ PYBIND11_WARNING_POP
 
 template <typename Type>
 struct type_caster<Type, typename eigen_tensor_helper<Type>::ValidType> {
+    static_assert(!std::is_pointer<typename Type::Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
     using Helper = eigen_tensor_helper<Type>;
     static constexpr auto temp_name = get_tensor_descriptor<Type, false>::value;
     PYBIND11_TYPE_CASTER(Type, temp_name);
@@ -357,6 +362,8 @@ struct get_storage_pointer_type<MapType, void_t<typename MapType::PointerArgType
 template <typename Type, int Options>
 struct type_caster<Eigen::TensorMap<Type, Options>,
                    typename eigen_tensor_helper<remove_cv_t<Type>>::ValidType> {
+    static_assert(!std::is_pointer<typename Type::Scalar>::value,
+                  PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED);
     using MapType = Eigen::TensorMap<Type, Options>;
     using Helper = eigen_tensor_helper<remove_cv_t<Type>>;
 

include/pybind11/embed.h

@@ -198,9 +198,10 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
         init_signal_handlers, argc, argv, add_program_dir_to_path);
 #else
     PyConfig config;
-    PyConfig_InitIsolatedConfig(&config);
-    config.isolated = 0;
-    config.use_environment = 1;
+    PyConfig_InitPythonConfig(&config);
+    // See PR #4473 for background
+    config.parse_argv = 0;
+
     config.install_signal_handlers = init_signal_handlers ? 1 : 0;
     initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
 #endif
@@ -242,16 +243,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
 
  \endrst */
 inline void finalize_interpreter() {
-    handle builtins(PyEval_GetBuiltins());
-    const char *id = PYBIND11_INTERNALS_ID;
-
     // Get the internals pointer (without creating it if it doesn't exist).  It's possible for the
     // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()`
     // during destruction), so we get the pointer-pointer here and check it after Py_Finalize().
     detail::internals **internals_ptr_ptr = detail::get_internals_pp();
-    // It could also be stashed in builtins, so look there too:
-    if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
-        internals_ptr_ptr = capsule(builtins[id]);
+    // It could also be stashed in state_dict, so look there too:
+    if (object internals_obj
+        = get_internals_obj_from_state_dict(detail::get_python_state_dict())) {
+        internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj);
     }
     // Local internals contains data managed by the current interpreter, so we must clear them to
     // avoid undefined behaviors when initializing another interpreter

include/pybind11/gil.h

@@ -152,8 +152,8 @@ public:
         }
     }
 
-    gil_scoped_release(const gil_scoped_acquire &) = delete;
-    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
+    gil_scoped_release(const gil_scoped_release &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_release &) = delete;
 
     /// This method will disable the PyThreadState_DeleteCurrent call and the
     /// GIL won't be acquired. This method should be used if the interpreter
@@ -203,7 +203,7 @@ class gil_scoped_release {
 public:
     gil_scoped_release() : state{PyEval_SaveThread()} {}
     gil_scoped_release(const gil_scoped_release &) = delete;
-    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_release &) = delete;
     ~gil_scoped_release() { PyEval_RestoreThread(state); }
     void disarm() {}
 };
@@ -230,7 +230,7 @@ public:
         (void) (this != (this + 1));
     }
     gil_scoped_release(const gil_scoped_release &) = delete;
-    gil_scoped_release &operator=(const gil_scoped_acquire &) = delete;
+    gil_scoped_release &operator=(const gil_scoped_release &) = delete;
     void disarm() {}
 };
 

include/pybind11/numpy.h

@@ -564,6 +564,8 @@ public:
         m_ptr = from_args(args).release().ptr();
     }
 
+    /// Return dtype for the given typenum (one of the NPY_TYPES).
+    /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType
     explicit dtype(int typenum)
         : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) {
         if (m_ptr == nullptr) {
@@ -1133,10 +1135,10 @@ public:
 
     /**
      * Returns a proxy object that provides const access to the array's data without bounds or
-     * dimensionality checking.  Unlike `unchecked()`, this does not require that the underlying
-     * array have the `writable` flag.  Use with care: the array must not be destroyed or reshaped
-     * for the duration of the returned object, and the caller must take care not to access invalid
-     * dimensions or dimension indices.
+     * dimensionality checking.  Unlike `mutable_unchecked()`, this does not require that the
+     * underlying array have the `writable` flag.  Use with care: the array must not be destroyed
+     * or reshaped for the duration of the returned object, and the caller must take care not to
+     * access invalid dimensions or dimension indices.
      */
     template <ssize_t Dims = -1>
     detail::unchecked_reference<T, Dims> unchecked() const & {
@@ -1295,12 +1297,16 @@ private:
 public:
     static constexpr int value = values[detail::is_fmt_numeric<T>::index];
 
-    static pybind11::dtype dtype() {
-        if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) {
-            return reinterpret_steal<pybind11::dtype>(ptr);
-        }
-        pybind11_fail("Unsupported buffer format!");
-    }
+    static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); }
+};
+
+template <typename T>
+struct npy_format_descriptor<T, enable_if_t<is_same_ignoring_cvref<T, PyObject *>::value>> {
+    static constexpr auto name = const_name("object");
+
+    static constexpr int value = npy_api::NPY_OBJECT_;
+
+    static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); }
 };
 
 #define PYBIND11_DECL_CHAR_FMT                                                                    \

include/pybind11/pybind11.h

@@ -84,6 +84,7 @@ public:
     cpp_function() = default;
     // NOLINTNEXTLINE(google-explicit-constructor)
     cpp_function(std::nullptr_t) {}
+    cpp_function(std::nullptr_t, const is_setter &) {}
 
     /// Construct a cpp_function from a vanilla function pointer
     template <typename Return, typename... Args, typename... Extra>
@@ -244,10 +245,16 @@ protected:
             using Guard = extract_guard_t<Extra...>;
 
             /* Perform the function call */
-            handle result
-                = cast_out::cast(std::move(args_converter).template call<Return, Guard>(cap->f),
+            handle result;
+            if (call.func.is_setter) {
+                (void) std::move(args_converter).template call<Return, Guard>(cap->f);
+                result = none().release();
+            } else {
+                result = cast_out::cast(
+                    std::move(args_converter).template call<Return, Guard>(cap->f),
                     policy,
                     call.parent);
+            }
 
             /* Invoke call policy post-call hook */
             process_attributes<Extra...>::postcall(call, result);
@@ -501,8 +508,8 @@ protected:
             rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS;
 
             capsule rec_capsule(unique_rec.release(),
+                                detail::get_function_record_capsule_name(),
                                 [](void *ptr) { destruct((detail::function_record *) ptr); });
-            rec_capsule.set_name(detail::get_function_record_capsule_name());
             guarded_strdup.release();
 
             object scope_module;
@@ -1729,7 +1736,8 @@ public:
     template <typename Getter, typename Setter, typename... Extra>
     class_ &
     def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) {
-        return def_property(name, fget, cpp_function(method_adaptor<type>(fset)), extra...);
+        return def_property(
+            name, fget, cpp_function(method_adaptor<type>(fset), is_setter()), extra...);
     }
     template <typename Getter, typename... Extra>
     class_ &def_property(const char *name,

include/pybind11/pytypes.h

@@ -471,13 +471,24 @@ inline const char *obj_class_name(PyObject *obj) {
 
 std::string error_string();
 
+// The code in this struct is very unusual, to minimize the chances of
+// masking bugs (elsewhere) by errors during the error handling (here).
+// This is meant to be a lifeline for troubleshooting long-running processes
+// that crash under conditions that are virtually impossible to reproduce.
+// Low-level implementation alternatives are preferred to higher-level ones
+// that might raise cascading exceptions. Last-ditch-kind-of attempts are made
+// to report as much of the original error as possible, even if there are
+// secondary issues obtaining some of the details.
 struct error_fetch_and_normalize {
+    // This comment only applies to Python <= 3.11:
     //     Immediate normalization is long-established behavior (starting with
     //     https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011
     //     from Sep 2016) and safest. Normalization could be deferred, but this could mask
     //     errors elsewhere, the performance gain is very minor in typical situations
     //     (usually the dominant bottleneck is EH unwinding), and the implementation here
     //     would be more complex.
+    // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately.
+    // Any errors during normalization are tracked under __notes__.
     explicit error_fetch_and_normalize(const char *called) {
         PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr());
         if (!m_type) {
@@ -492,6 +503,14 @@ struct error_fetch_and_normalize {
                             "of the original active exception type.");
         }
         m_lazy_error_string = exc_type_name_orig;
+#if PY_VERSION_HEX >= 0x030C0000
+        // The presence of __notes__ is likely due to exception normalization
+        // errors, although that is not necessarily true, therefore insert a
+        // hint only:
+        if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) {
+            m_lazy_error_string += "[WITH __notes__]";
+        }
+#else
         // PyErr_NormalizeException() may change the exception type if there are cascading
         // failures. This can potentially be extremely confusing.
         PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr());
@@ -523,6 +542,7 @@ struct error_fetch_and_normalize {
             msg += ": " + format_value_and_trace();
             pybind11_fail(msg);
         }
+#    endif
 #endif
     }
 
@@ -558,6 +578,40 @@ struct error_fetch_and_normalize {
                     }
                 }
             }
+#if PY_VERSION_HEX >= 0x030B0000
+            auto notes
+                = reinterpret_steal<object>(PyObject_GetAttrString(m_value.ptr(), "__notes__"));
+            if (!notes) {
+                PyErr_Clear(); // No notes is good news.
+            } else {
+                auto len_notes = PyList_Size(notes.ptr());
+                if (len_notes < 0) {
+                    result += "\nFAILURE obtaining len(__notes__): " + detail::error_string();
+                } else {
+                    result += "\n__notes__ (len=" + std::to_string(len_notes) + "):";
+                    for (ssize_t i = 0; i < len_notes; i++) {
+                        PyObject *note = PyList_GET_ITEM(notes.ptr(), i);
+                        auto note_bytes = reinterpret_steal<object>(
+                            PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace"));
+                        if (!note_bytes) {
+                            result += "\nFAILURE obtaining __notes__[" + std::to_string(i)
+                                      + "]: " + detail::error_string();
+                        } else {
+                            char *buffer = nullptr;
+                            Py_ssize_t length = 0;
+                            if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length)
+                                == -1) {
+                                result += "\nFAILURE formatting __notes__[" + std::to_string(i)
+                                          + "]: " + detail::error_string();
+                            } else {
+                                result += '\n';
+                                result += std::string(buffer, static_cast<std::size_t>(length));
+                            }
+                        }
+                    }
+                }
+            }
+#endif
         } else {
             result = "<MESSAGE UNAVAILABLE>";
         }
@@ -1871,28 +1925,13 @@ public:
         }
     }
 
+    /// Capsule name is nullptr.
     capsule(const void *value, void (*destructor)(void *)) {
-        m_ptr = PyCapsule_New(const_cast<void *>(value), nullptr, [](PyObject *o) {
-            // guard if destructor called while err indicator is set
-            error_scope error_guard;
-            auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o));
-            if (destructor == nullptr && PyErr_Occurred()) {
-                throw error_already_set();
-            }
-            const char *name = get_name_in_error_scope(o);
-            void *ptr = PyCapsule_GetPointer(o, name);
-            if (ptr == nullptr) {
-                throw error_already_set();
+        initialize_with_void_ptr_destructor(value, nullptr, destructor);
     }
 
-            if (destructor != nullptr) {
-                destructor(ptr);
-            }
-        });
-
-        if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast<void *>(destructor)) != 0) {
-            throw error_already_set();
-        }
+    capsule(const void *value, const char *name, void (*destructor)(void *)) {
+        initialize_with_void_ptr_destructor(value, name, destructor);
     }
 
     explicit capsule(void (*destructor)()) {
@@ -1960,6 +1999,32 @@ private:
 
         return name;
     }
+
+    void initialize_with_void_ptr_destructor(const void *value,
+                                             const char *name,
+                                             void (*destructor)(void *)) {
+        m_ptr = PyCapsule_New(const_cast<void *>(value), name, [](PyObject *o) {
+            // guard if destructor called while err indicator is set
+            error_scope error_guard;
+            auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o));
+            if (destructor == nullptr && PyErr_Occurred()) {
+                throw error_already_set();
+            }
+            const char *name = get_name_in_error_scope(o);
+            void *ptr = PyCapsule_GetPointer(o, name);
+            if (ptr == nullptr) {
+                throw error_already_set();
+            }
+
+            if (destructor != nullptr) {
+                destructor(ptr);
+            }
+        });
+
+        if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast<void *>(destructor)) != 0) {
+            throw error_already_set();
+        }
+    }
 };
 
 class tuple : public object {

include/pybind11/stl.h

@@ -273,11 +273,11 @@ public:
     }
 
     PYBIND11_TYPE_CASTER(ArrayType,
-                         const_name("List[") + value_conv::name
+                         const_name<Resizable>(const_name(""), const_name("Annotated["))
+                             + const_name("List[") + value_conv::name + const_name("]")
                              + const_name<Resizable>(const_name(""),
-                                                     const_name("[") + const_name<Size>()
-                                                         + const_name("]"))
-                             + const_name("]"));
+                                                     const_name(", FixedSize(")
+                                                         + const_name<Size>() + const_name(")]")));
 };
 
 template <typename Type, size_t Size>

include/pybind11/stl_bind.h

@@ -61,9 +61,11 @@ struct is_comparable<
 /* For a vector/map data structure, recursively check the value type
    (which is std::pair for maps) */
 template <typename T>
-struct is_comparable<T, enable_if_t<container_traits<T>::is_vector>> {
-    static constexpr const bool value = is_comparable<typename T::value_type>::value;
-};
+struct is_comparable<T, enable_if_t<container_traits<T>::is_vector>>
+    : is_comparable<typename recursive_container_traits<T>::type_to_check_recursively> {};
+
+template <>
+struct is_comparable<recursive_bottom> : std::true_type {};
 
 /* For pairs, recursively check the two data types */
 template <typename T>

include/pybind11/type_caster_pyobject_ptr.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2023 The pybind Community.
+
+#pragma once
+
+#include "detail/common.h"
+#include "detail/descr.h"
+#include "cast.h"
+#include "pytypes.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <>
+class type_caster<PyObject> {
+public:
+    static constexpr auto name = const_name("object"); // See discussion under PR #4601.
+
+    // This overload is purely to guard against accidents.
+    template <typename T,
+              detail::enable_if_t<!is_same_ignoring_cvref<T, PyObject *>::value, int> = 0>
+    static handle cast(T &&, return_value_policy, handle /*parent*/) {
+        static_assert(is_same_ignoring_cvref<T, PyObject *>::value,
+                      "Invalid C++ type T for to-Python conversion (type_caster<PyObject>).");
+        return nullptr; // Unreachable.
+    }
+
+    static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) {
+        if (src == nullptr) {
+            throw error_already_set();
+        }
+        if (PyErr_Occurred()) {
+            raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()");
+            throw error_already_set();
+        }
+        if (policy == return_value_policy::take_ownership) {
+            return src;
+        }
+        if (policy == return_value_policy::reference
+            || policy == return_value_policy::automatic_reference) {
+            return handle(src).inc_ref();
+        }
+        pybind11_fail("type_caster<PyObject>::cast(): unsupported return_value_policy: "
+                      + std::to_string(static_cast<int>(policy)));
+    }
+
+    bool load(handle src, bool) {
+        value = reinterpret_borrow<object>(src);
+        return true;
+    }
+
+    template <typename T>
+    using cast_op_type = PyObject *;
+
+    explicit operator PyObject *() { return value.ptr(); }
+
+private:
+    object value;
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

pybind11/__init__.py

@@ -1,6 +1,6 @@
 import sys
 
-if sys.version_info < (3, 6):
+if sys.version_info < (3, 6):  # noqa: UP036
     msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5."
     raise ImportError(msg)
 

pybind11/__main__.py

@@ -4,6 +4,7 @@ import argparse
 import sys
 import sysconfig
 
+from ._version import __version__
 from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
 
 
@@ -24,8 +25,13 @@ def print_includes() -> None:
 
 
 def main() -> None:
-
     parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=__version__,
+        help="Print the version and exit.",
+    )
     parser.add_argument(
         "--includes",
         action="store_true",

pybind11/commands.py

@@ -3,7 +3,7 @@ import os
 DIR = os.path.abspath(os.path.dirname(__file__))
 
 
-def get_include(user: bool = False) -> str:  # pylint: disable=unused-argument
+def get_include(user: bool = False) -> str:  # noqa: ARG001
     """
     Return the path to the pybind11 include directory. The historical "user"
     argument is unused, and may be removed.

pybind11/setup_helpers.py

@@ -66,8 +66,8 @@ try:
     from setuptools import Extension as _Extension
     from setuptools.command.build_ext import build_ext as _build_ext
 except ImportError:
-    from distutils.command.build_ext import build_ext as _build_ext
-    from distutils.extension import Extension as _Extension
+    from distutils.command.build_ext import build_ext as _build_ext  # type: ignore[assignment]
+    from distutils.extension import Extension as _Extension  # type: ignore[assignment]
 
 import distutils.ccompiler
 import distutils.errors
@@ -84,7 +84,7 @@ STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}"
 # directory into your path if it sits beside your setup.py.
 
 
-class Pybind11Extension(_Extension):  # type: ignore[misc]
+class Pybind11Extension(_Extension):
     """
     Build a C++11+ Extension module with pybind11. This automatically adds the
     recommended flags when you init the extension and assumes C++ sources - you
@@ -118,7 +118,6 @@ class Pybind11Extension(_Extension):  # type: ignore[misc]
         self.extra_link_args[:0] = flags
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
-
         self._cxx_level = 0
         cxx_std = kwargs.pop("cxx_std", 0)
 
@@ -145,7 +144,6 @@ class Pybind11Extension(_Extension):  # type: ignore[misc]
         self.cxx_std = cxx_std
 
         cflags = []
-        ldflags = []
         if WIN:
             cflags += ["/EHsc", "/bigobj"]
         else:
@@ -155,11 +153,7 @@ class Pybind11Extension(_Extension):  # type: ignore[misc]
             c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags)
             if not any(opt.startswith("-g") for opt in c_cpp_flags):
                 cflags += ["-g0"]
-            if MACOS:
-                cflags += ["-stdlib=libc++"]
-                ldflags += ["-stdlib=libc++"]
         self._add_cflags(cflags)
-        self._add_ldflags(ldflags)
 
     @property
     def cxx_std(self) -> int:
@@ -174,9 +168,10 @@ class Pybind11Extension(_Extension):  # type: ignore[misc]
 
     @cxx_std.setter
     def cxx_std(self, level: int) -> None:
-
         if self._cxx_level:
-            warnings.warn("You cannot safely change the cxx_level after setting it!")
+            warnings.warn(
+                "You cannot safely change the cxx_level after setting it!", stacklevel=2
+            )
 
         # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so
         # force a valid flag here.
@@ -271,7 +266,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]:
     raise RuntimeError(msg)
 
 
-class build_ext(_build_ext):  # type: ignore[misc] # noqa: N801
+class build_ext(_build_ext):  # noqa: N801
     """
     Customized build_ext that allows an auto-search for the highest supported
     C++ level for Pybind11Extension. This is only needed for the auto-search
@@ -341,7 +336,7 @@ def naive_recompile(obj: str, src: str) -> bool:
     return os.stat(obj).st_mtime < os.stat(src).st_mtime
 
 
-def no_recompile(obg: str, src: str) -> bool:  # pylint: disable=unused-argument
+def no_recompile(obg: str, src: str) -> bool:  # noqa: ARG001
     """
     This is the safest but slowest choice (and is the default) - will always
     recompile sources.
@@ -439,7 +434,6 @@ class ParallelCompile:
             extra_postargs: Optional[List[str]] = None,
             depends: Optional[List[str]] = None,
         ) -> Any:
-
             # These lines are directly from distutils.ccompiler.CCompiler
             macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile(  # type: ignore[attr-defined]
                 output_dir, macros, include_dirs, sources, depends, extra_postargs

pyproject.toml

@@ -2,6 +2,7 @@
 requires = ["setuptools>=42", "cmake>=3.18", "ninja"]
 build-backend = "setuptools.build_meta"
 
+
 [tool.check-manifest]
 ignore = [
     "tests/**",
@@ -15,11 +16,6 @@ ignore = [
     "noxfile.py",
 ]
 
-[tool.isort]
-# Needs the compiled .so modules and env.py from tests
-known_first_party = "env,pybind11_cross_module_tests,pybind11_tests,"
-# For black compatibility
-profile = "black"
 
 [tool.mypy]
 files = ["pybind11"]
@@ -30,7 +26,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 warn_unreachable = true
 
 [[tool.mypy.overrides]]
-module = ["ghapi.*", "setuptools.*"]
+module = ["ghapi.*"]
 ignore_missing_imports = true
 
 
@@ -58,4 +54,45 @@ messages_control.disable = [
   "invalid-name",
   "protected-access",
   "missing-module-docstring",
+  "unused-argument",  # covered by Ruff ARG
 ]
+
+
+[tool.ruff]
+select = [
+  "E", "F", "W", # flake8
+  "B",           # flake8-bugbear
+  "I",           # isort
+  "N",           # pep8-naming
+  "ARG",         # flake8-unused-arguments
+  "C4",          # flake8-comprehensions
+  "EM",          # flake8-errmsg
+  "ICN",         # flake8-import-conventions
+  "ISC",         # flake8-implicit-str-concat
+  "PGH",         # pygrep-hooks
+  "PIE",         # flake8-pie
+  "PL",          # pylint
+  "PT",          # flake8-pytest-style
+  "RET",         # flake8-return
+  "RUF100",      # Ruff-specific
+  "SIM",         # flake8-simplify
+  "UP",          # pyupgrade
+  "YTT",         # flake8-2020
+]
+ignore = [
+  "PLR",    # Design related pylint
+  "E501",   # Line too long (Black is enough)
+  "PT011",  # Too broad with raises in pytest
+  "PT004",  # Fixture that doesn't return needs underscore (no, it is fine)
+  "SIM118", # iter(x) is not always the same as iter(x.keys())
+]
+target-version = "py37"
+src = ["src"]
+unfixable = ["T20"]
+exclude = []
+line-length = 120
+isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"]
+
+[tool.ruff.per-file-ignores]
+"tests/**" = ["EM", "N"]
+"tests/test_call_policies.py" = ["PLC1901"]

setup.cfg

@@ -20,6 +20,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     License :: OSI Approved :: BSD License
     Programming Language :: Python :: Implementation :: PyPy
     Programming Language :: Python :: Implementation :: CPython
@@ -40,11 +41,3 @@ project_urls =
 [options]
 python_requires = >=3.6
 zip_safe = False
-
-
-[flake8]
-max-line-length = 120
-show_source = True
-exclude = .git, __pycache__, build, dist, docs, tools, venv
-extend-ignore = E203, E722
-extend-select = B902, B904

setup.py

@@ -96,7 +96,7 @@ def get_and_replace(
 
 # Use our input files instead when making the SDist (and anything that depends
 # on it, like a wheel)
-class SDist(setuptools.command.sdist.sdist):  # type: ignore[misc]
+class SDist(setuptools.command.sdist.sdist):
     def make_release_tree(self, base_dir: str, files: List[str]) -> None:
         super().make_release_tree(base_dir, files)
 

tests/CMakeLists.txt

@@ -154,7 +154,11 @@ set(PYBIND11_TEST_FILES
     test_stl_binders
     test_tagbased_polymorphic
     test_thread
+    test_type_caster_pyobject_ptr
     test_union
+    test_unnamed_namespace_a
+    test_unnamed_namespace_b
+    test_vector_unique_ptr_member
     test_virtual_functions)
 
 # Invoking cmake with something like:
@@ -169,7 +173,7 @@ if(PYBIND11_TEST_OVERRIDE)
   # This allows the override to be done with extensions, preserving backwards compatibility.
   foreach(test_name ${TEST_FILES_NO_EXT})
     if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT
-    )# If not in the whitelist, add to be filtered out.
+    )# If not in the allowlist, add to be filtered out.
       list(APPEND PYBIND11_TEST_FILTER ${test_name})
     endif()
   endforeach()

tests/conftest.py

@@ -8,19 +8,26 @@ import contextlib
 import difflib
 import gc
 import multiprocessing
-import os
 import re
+import sys
 import textwrap
+import traceback
 
 import pytest
 
 # Early diagnostic for failed imports
+try:
     import pybind11_tests
+except Exception:
+    # pytest does not show the traceback without this.
+    traceback.print_exc()
+    raise
 
 
 @pytest.fixture(scope="session", autouse=True)
-def always_forkserver_on_unix():
-    if os.name == "nt":
+def use_multiprocessing_forkserver_on_linux():
+    if sys.platform != "linux":
+        # The default on Windows and macOS is "spawn": If it's not broken, don't fix it.
         return
 
     # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592
@@ -28,8 +35,6 @@ def always_forkserver_on_unix():
     # It is actually a well-known pitfall, unfortunately without guard rails.
     # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py,
     # visit the issuecomment link above for details).
-    # Windows does not have fork() and the associated pitfall, therefore it is best left
-    # running with defaults.
     multiprocessing.set_start_method("forkserver")
 
 
@@ -77,7 +82,6 @@ class Output:
         b = _strip_and_dedent(other).splitlines()
         if a == b:
             return True
-        else:
         self.explanation = _make_explanation(a, b)
         return False
 
@@ -90,7 +94,6 @@ class Unordered(Output):
         b = _split_and_sort(other)
         if a == b:
             return True
-        else:
         self.explanation = _make_explanation(a, b)
         return False
 
@@ -113,7 +116,6 @@ class Capture:
         b = other
         if a == b:
             return True
-        else:
         self.explanation = a.explanation
         return False
 
@@ -132,7 +134,7 @@ class Capture:
         return Output(self.err)
 
 
-@pytest.fixture
+@pytest.fixture()
 def capture(capsys):
     """Extended `capsys` with context manager and custom equality operators"""
     return Capture(capsys)
@@ -153,7 +155,6 @@ class SanitizedString:
         b = _strip_and_dedent(other)
         if a == b:
             return True
-        else:
         self.explanation = _make_explanation(a.splitlines(), b.splitlines())
         return False
 
@@ -161,17 +162,15 @@ class SanitizedString:
 def _sanitize_general(s):
     s = s.strip()
     s = s.replace("pybind11_tests.", "m.")
-    s = _long_marker.sub(r"\1", s)
-    return s
+    return _long_marker.sub(r"\1", s)
 
 
 def _sanitize_docstring(thing):
     s = thing.__doc__
-    s = _sanitize_general(s)
-    return s
+    return _sanitize_general(s)
 
 
-@pytest.fixture
+@pytest.fixture()
 def doc():
     """Sanitize docstrings and add custom failure explanation"""
     return SanitizedString(_sanitize_docstring)
@@ -180,30 +179,20 @@ def doc():
 def _sanitize_message(thing):
     s = str(thing)
     s = _sanitize_general(s)
-    s = _hexadecimal.sub("0", s)
-    return s
+    return _hexadecimal.sub("0", s)
 
 
-@pytest.fixture
+@pytest.fixture()
 def msg():
     """Sanitize messages and add custom failure explanation"""
     return SanitizedString(_sanitize_message)
 
 
-# noinspection PyUnusedLocal
-def pytest_assertrepr_compare(op, left, right):
+def pytest_assertrepr_compare(op, left, right):  # noqa: ARG001
     """Hook to insert custom failure explanation"""
     if hasattr(left, "explanation"):
         return left.explanation
-
-
-@contextlib.contextmanager
-def suppress(exception):
-    """Suppress the desired exception"""
-    try:
-        yield
-    except exception:
-        pass
+    return None
 
 
 def gc_collect():
@@ -214,7 +203,7 @@ def gc_collect():
 
 
 def pytest_configure():
-    pytest.suppress = suppress
+    pytest.suppress = contextlib.suppress
     pytest.gc_collect = gc_collect
 
 

tests/env.py

@@ -24,5 +24,4 @@ def deprecated_call():
     pytest_major_minor = (int(pieces[0]), int(pieces[1]))
     if pytest_major_minor < (3, 9):
         return pytest.warns((DeprecationWarning, PendingDeprecationWarning))
-    else:
     return pytest.deprecated_call()

tests/extra_python_package/test_files.py

@@ -43,6 +43,7 @@ main_headers = {
     "include/pybind11/pytypes.h",
     "include/pybind11/stl.h",
     "include/pybind11/stl_bind.h",
+    "include/pybind11/type_caster_pyobject_ptr.h",
 }
 
 detail_headers = {
@@ -56,6 +57,7 @@ detail_headers = {
 }
 
 eigen_headers = {
+    "include/pybind11/eigen/common.h",
     "include/pybind11/eigen/matrix.h",
     "include/pybind11/eigen/tensor.h",
 }
@@ -110,6 +112,7 @@ sdist_files = {
     "MANIFEST.in",
     "README.rst",
     "PKG-INFO",
+    "SECURITY.md",
 }
 
 local_sdist_files = {
@@ -135,7 +138,6 @@ def normalize_line_endings(value: bytes) -> bytes:
 
 
 def test_build_sdist(monkeypatch, tmpdir):
-
     monkeypatch.chdir(MAIN_DIR)
 
     subprocess.run(
@@ -186,7 +188,6 @@ def test_build_sdist(monkeypatch, tmpdir):
 
 
 def test_build_global_dist(monkeypatch, tmpdir):
-
     monkeypatch.chdir(MAIN_DIR)
     monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1")
     subprocess.run(

tests/test_async.py

@@ -4,7 +4,7 @@ asyncio = pytest.importorskip("asyncio")
 m = pytest.importorskip("pybind11_tests.async_module")
 
 
-@pytest.fixture
+@pytest.fixture()
 def event_loop():
     loop = asyncio.new_event_loop()
     yield loop
@@ -16,7 +16,7 @@ async def get_await_result(x):
 
 
 def test_await(event_loop):
-    assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync()))
+    assert event_loop.run_until_complete(get_await_result(m.SupportsAsync())) == 5
 
 
 def test_await_missing(event_loop):

tests/test_buffers.cpp

@@ -7,12 +7,47 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+#include <pybind11/complex.h>
 #include <pybind11/stl.h>
 
 #include "constructor_stats.h"
 #include "pybind11_tests.h"
 
 TEST_SUBMODULE(buffers, m) {
+    m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double));
+
+    m.def("format_descriptor_format_buffer_info_equiv",
+          [](const std::string &cpp_name, const py::buffer &buffer) {
+              // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables
+              static auto *format_table = new std::map<std::string, std::string>;
+              static auto *equiv_table
+                  = new std::map<std::string, bool (py::buffer_info::*)() const>;
+              if (format_table->empty()) {
+#define PYBIND11_ASSIGN_HELPER(...)                                                               \
+    (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format();                 \
+    (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>;
+                  PYBIND11_ASSIGN_HELPER(PyObject *)
+                  PYBIND11_ASSIGN_HELPER(bool)
+                  PYBIND11_ASSIGN_HELPER(std::int8_t)
+                  PYBIND11_ASSIGN_HELPER(std::uint8_t)
+                  PYBIND11_ASSIGN_HELPER(std::int16_t)
+                  PYBIND11_ASSIGN_HELPER(std::uint16_t)
+                  PYBIND11_ASSIGN_HELPER(std::int32_t)
+                  PYBIND11_ASSIGN_HELPER(std::uint32_t)
+                  PYBIND11_ASSIGN_HELPER(std::int64_t)
+                  PYBIND11_ASSIGN_HELPER(std::uint64_t)
+                  PYBIND11_ASSIGN_HELPER(float)
+                  PYBIND11_ASSIGN_HELPER(double)
+                  PYBIND11_ASSIGN_HELPER(long double)
+                  PYBIND11_ASSIGN_HELPER(std::complex<float>)
+                  PYBIND11_ASSIGN_HELPER(std::complex<double>)
+                  PYBIND11_ASSIGN_HELPER(std::complex<long double>)
+#undef PYBIND11_ASSIGN_HELPER
+              }
+              return std::pair<std::string, bool>(
+                  (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))());
+          });
+
     // test_from_python / test_to_python:
     class Matrix {
     public:

tests/test_buffers.py

@@ -10,6 +10,63 @@ from pybind11_tests import buffers as m
 
 np = pytest.importorskip("numpy")
 
+if m.long_double_and_double_have_same_size:
+    # Determined by the compiler used to build the pybind11 tests
+    # (e.g. MSVC gets here, but MinGW might not).
+    np_float128 = None
+    np_complex256 = None
+else:
+    # Determined by the compiler used to build numpy (e.g. MinGW).
+    np_float128 = getattr(np, *["float128"] * 2)
+    np_complex256 = getattr(np, *["complex256"] * 2)
+
+CPP_NAME_FORMAT_NP_DTYPE_TABLE = [
+    ("PyObject *", "O", object),
+    ("bool", "?", np.bool_),
+    ("std::int8_t", "b", np.int8),
+    ("std::uint8_t", "B", np.uint8),
+    ("std::int16_t", "h", np.int16),
+    ("std::uint16_t", "H", np.uint16),
+    ("std::int32_t", "i", np.int32),
+    ("std::uint32_t", "I", np.uint32),
+    ("std::int64_t", "q", np.int64),
+    ("std::uint64_t", "Q", np.uint64),
+    ("float", "f", np.float32),
+    ("double", "d", np.float64),
+    ("long double", "g", np_float128),
+    ("std::complex<float>", "Zf", np.complex64),
+    ("std::complex<double>", "Zd", np.complex128),
+    ("std::complex<long double>", "Zg", np_complex256),
+]
+CPP_NAME_FORMAT_TABLE = [
+    (cpp_name, format)
+    for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE
+    if np_dtype is not None
+]
+CPP_NAME_NP_DTYPE_TABLE = [
+    (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE
+]
+
+
+@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE)
+def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype):
+    if np_dtype is None:
+        pytest.skip(
+            f"cpp_name=`{cpp_name}`: `long double` and `double` have same size."
+        )
+    if isinstance(np_dtype, str):
+        pytest.skip(f"np.{np_dtype} does not exist.")
+    np_array = np.array([], dtype=np_dtype)
+    for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE:
+        format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv(
+            other_cpp_name, np_array
+        )
+        assert format == expected_format
+        if other_cpp_name == cpp_name:
+            assert np_array_is_matching
+        else:
+            assert not np_array_is_matching
+
 
 def test_from_python():
     with pytest.raises(RuntimeError) as excinfo:
@@ -54,7 +111,8 @@ def test_to_python():
     mat2 = np.array(mat, copy=False)
     assert mat2.shape == (5, 4)
     assert abs(mat2).sum() == 11
-    assert mat2[2, 3] == 4 and mat2[3, 2] == 7
+    assert mat2[2, 3] == 4
+    assert mat2[3, 2] == 7
     mat2[2, 3] = 5
     assert mat2[2, 3] == 5
 

tests/test_builtin_casters.py

@@ -126,8 +126,8 @@ def test_bytes_to_string():
 
     assert m.strlen(b"hi") == 2
     assert m.string_length(b"world") == 5
-    assert m.string_length("a\x00b".encode()) == 3
-    assert m.strlen("a\x00b".encode()) == 1  # C-string limitation
+    assert m.string_length(b"a\x00b") == 3
+    assert m.strlen(b"a\x00b") == 1  # C-string limitation
 
     # passing in a utf8 encoded string should work
     assert m.string_length("💩".encode()) == 4
@@ -421,13 +421,15 @@ def test_reference_wrapper():
     a2 = m.refwrap_list(copy=True)
     assert [x.value for x in a1] == [2, 3]
     assert [x.value for x in a2] == [2, 3]
-    assert not a1[0] is a2[0] and not a1[1] is a2[1]
+    assert a1[0] is not a2[0]
+    assert a1[1] is not a2[1]
 
     b1 = m.refwrap_list(copy=False)
     b2 = m.refwrap_list(copy=False)
     assert [x.value for x in b1] == [1, 2]
     assert [x.value for x in b2] == [1, 2]
-    assert b1[0] is b2[0] and b1[1] is b2[1]
+    assert b1[0] is b2[0]
+    assert b1[1] is b2[1]
 
     assert m.refwrap_iiw(IncType(5)) == 5
     assert m.refwrap_call_iiw(IncType(10), m.refwrap_iiw) == [10, 10, 10, 10]

tests/test_callbacks.py

@@ -5,6 +5,7 @@ import pytest
 
 import env  # noqa: F401
 from pybind11_tests import callbacks as m
+from pybind11_tests import detailed_error_messages_enabled
 
 
 def test_callbacks():
@@ -70,11 +71,20 @@ def test_keyword_args_and_generalized_unpacking():
 
     with pytest.raises(RuntimeError) as excinfo:
         m.test_arg_conversion_error1(f)
-    assert "Unable to convert call argument" in str(excinfo.value)
+    assert str(excinfo.value) == "Unable to convert call argument " + (
+        "'1' of type 'UnregisteredType' to Python object"
+        if detailed_error_messages_enabled
+        else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"
+    )
 
     with pytest.raises(RuntimeError) as excinfo:
         m.test_arg_conversion_error2(f)
-    assert "Unable to convert call argument" in str(excinfo.value)
+    assert str(excinfo.value) == "Unable to convert call argument " + (
+        "'expected_name' of type 'UnregisteredType' to Python object"
+        if detailed_error_messages_enabled
+        else "'expected_name' to Python object "
+        "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"
+    )
 
 
 def test_lambda_closure_cleanup():

tests/test_chrono.py

@@ -7,7 +7,6 @@ from pybind11_tests import chrono as m
 
 
 def test_chrono_system_clock():
-
     # Get the time from both c++ and datetime
     date0 = datetime.datetime.today()
     date1 = m.test_chrono1()
@@ -122,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch):
 
 
 def test_chrono_duration_roundtrip():
-
     # Get the difference between two times (a timedelta)
     date1 = datetime.datetime.today()
     date2 = datetime.datetime.today()
@@ -143,7 +141,6 @@ def test_chrono_duration_roundtrip():
 
 
 def test_chrono_duration_subtraction_equivalence():
-
     date1 = datetime.datetime.today()
     date2 = datetime.datetime.today()
 
@@ -154,7 +151,6 @@ def test_chrono_duration_subtraction_equivalence():
 
 
 def test_chrono_duration_subtraction_equivalence_date():
-
     date1 = datetime.date.today()
     date2 = datetime.date.today()
 

tests/test_class.py

@@ -6,10 +6,7 @@ from pybind11_tests import class_ as m
 
 
 def test_obj_class_name():
-    if env.PYPY:
-        expected_name = "UserType"
-    else:
-        expected_name = "pybind11_tests.UserType"
+    expected_name = "UserType" if env.PYPY else "pybind11_tests.UserType"
     assert m.obj_class_name(UserType(1)) == expected_name
     assert m.obj_class_name(UserType) == expected_name
 
@@ -32,7 +29,7 @@ def test_instance(msg):
     assert cstats.alive() == 0
 
 
-def test_instance_new(msg):
+def test_instance_new():
     instance = m.NoConstructorNew()  # .__new__(m.NoConstructor.__class__)
     cstats = ConstructorStats.get(m.NoConstructorNew)
     assert cstats.alive() == 1
@@ -185,7 +182,6 @@ def test_inheritance(msg):
 
 
 def test_inheritance_init(msg):
-
     # Single base
     class Python(m.Pet):
         def __init__(self):
@@ -222,7 +218,7 @@ def test_automatic_upcasting():
 
 
 def test_isinstance():
-    objects = [tuple(), dict(), m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4
+    objects = [(), {}, m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4
     expected = (True, True, True, True, True, False, False)
     assert m.check_instances(objects) == expected
 
@@ -428,7 +424,7 @@ def test_exception_rvalue_abort():
 
 
 # https://github.com/pybind/pybind11/issues/1568
-def test_multiple_instances_with_same_pointer(capture):
+def test_multiple_instances_with_same_pointer():
     n = 100
     instances = [m.SamePointer() for _ in range(n)]
     for i in range(n):

tests/test_const_name.py

@@ -3,9 +3,9 @@ import pytest
 from pybind11_tests import const_name as m
 
 
-@pytest.mark.parametrize("func", (m.const_name_tests, m.underscore_tests))
+@pytest.mark.parametrize("func", [m.const_name_tests, m.underscore_tests])
 @pytest.mark.parametrize(
-    "selector, expected",
+    ("selector", "expected"),
     enumerate(
         (
             "",

tests/test_constants_and_functions.cpp

@@ -148,4 +148,7 @@ TEST_SUBMODULE(constants_and_functions, m) {
             py::arg_v("y", 42, "<the answer>"),
             py::arg_v("z", default_value));
     });
+
+    // test noexcept(true) lambda (#4565)
+    m.def("l1", []() noexcept(true) { return 0; });
 }

tests/test_constants_and_functions.py

@@ -50,3 +50,7 @@ def test_function_record_leaks():
         m.register_large_capture_with_invalid_arguments(m)
     with pytest.raises(RuntimeError):
         m.register_with_raising_repr(m, RaisingRepr())
+
+
+def test_noexcept_lambda():
+    assert m.l1() == 0

tests/test_copy_move.cpp

@@ -13,6 +13,8 @@
 #include "constructor_stats.h"
 #include "pybind11_tests.h"
 
+#include <type_traits>
+
 template <typename derived>
 struct empty {
     static const derived &get_one() { return instance_; }
@@ -293,3 +295,239 @@ TEST_SUBMODULE(copy_move_policies, m) {
     // Make sure that cast from pytype rvalue to other pytype works
     m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast<py::int_>(); });
 }
+
+/*
+ * Rest of the file:
+ * static_assert based tests for pybind11 adaptations of
+ * std::is_move_constructible, std::is_copy_constructible and
+ * std::is_copy_assignable (no adaptation of std::is_move_assignable).
+ * Difference between pybind11 and std traits: pybind11 traits will also check
+ * the contained value_types.
+ */
+
+struct NotMovable {
+    NotMovable() = default;
+    NotMovable(NotMovable const &) = default;
+    NotMovable(NotMovable &&) = delete;
+    NotMovable &operator=(NotMovable const &) = default;
+    NotMovable &operator=(NotMovable &&) = delete;
+};
+static_assert(!std::is_move_constructible<NotMovable>::value,
+              "!std::is_move_constructible<NotMovable>::value");
+static_assert(std::is_copy_constructible<NotMovable>::value,
+              "std::is_copy_constructible<NotMovable>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotMovable>::value,
+              "!pybind11::detail::is_move_constructible<NotMovable>::value");
+static_assert(pybind11::detail::is_copy_constructible<NotMovable>::value,
+              "pybind11::detail::is_copy_constructible<NotMovable>::value");
+static_assert(!std::is_move_assignable<NotMovable>::value,
+              "!std::is_move_assignable<NotMovable>::value");
+static_assert(std::is_copy_assignable<NotMovable>::value,
+              "std::is_copy_assignable<NotMovable>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotMovable>::value,
+//               "!pybind11::detail::is_move_assignable<NotMovable>::value");
+static_assert(pybind11::detail::is_copy_assignable<NotMovable>::value,
+              "pybind11::detail::is_copy_assignable<NotMovable>::value");
+
+struct NotCopyable {
+    NotCopyable() = default;
+    NotCopyable(NotCopyable const &) = delete;
+    NotCopyable(NotCopyable &&) = default;
+    NotCopyable &operator=(NotCopyable const &) = delete;
+    NotCopyable &operator=(NotCopyable &&) = default;
+};
+static_assert(std::is_move_constructible<NotCopyable>::value,
+              "std::is_move_constructible<NotCopyable>::value");
+static_assert(!std::is_copy_constructible<NotCopyable>::value,
+              "!std::is_copy_constructible<NotCopyable>::value");
+static_assert(pybind11::detail::is_move_constructible<NotCopyable>::value,
+              "pybind11::detail::is_move_constructible<NotCopyable>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyable>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyable>::value");
+static_assert(std::is_move_assignable<NotCopyable>::value,
+              "std::is_move_assignable<NotCopyable>::value");
+static_assert(!std::is_copy_assignable<NotCopyable>::value,
+              "!std::is_copy_assignable<NotCopyable>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyable>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyable>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyable>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyable>::value");
+
+struct NotCopyableNotMovable {
+    NotCopyableNotMovable() = default;
+    NotCopyableNotMovable(NotCopyableNotMovable const &) = delete;
+    NotCopyableNotMovable(NotCopyableNotMovable &&) = delete;
+    NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete;
+    NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete;
+};
+static_assert(!std::is_move_constructible<NotCopyableNotMovable>::value,
+              "!std::is_move_constructible<NotCopyableNotMovable>::value");
+static_assert(!std::is_copy_constructible<NotCopyableNotMovable>::value,
+              "!std::is_copy_constructible<NotCopyableNotMovable>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovable>::value,
+              "!pybind11::detail::is_move_constructible<NotCopyableNotMovable>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovable>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyableNotMovable>::value");
+static_assert(!std::is_move_assignable<NotCopyableNotMovable>::value,
+              "!std::is_move_assignable<NotCopyableNotMovable>::value");
+static_assert(!std::is_copy_assignable<NotCopyableNotMovable>::value,
+              "!std::is_copy_assignable<NotCopyableNotMovable>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovable>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyableNotMovable>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovable>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyableNotMovable>::value");
+
+struct NotMovableVector : std::vector<NotMovable> {};
+static_assert(std::is_move_constructible<NotMovableVector>::value,
+              "std::is_move_constructible<NotMovableVector>::value");
+static_assert(std::is_copy_constructible<NotMovableVector>::value,
+              "std::is_copy_constructible<NotMovableVector>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotMovableVector>::value,
+              "!pybind11::detail::is_move_constructible<NotMovableVector>::value");
+static_assert(pybind11::detail::is_copy_constructible<NotMovableVector>::value,
+              "pybind11::detail::is_copy_constructible<NotMovableVector>::value");
+static_assert(std::is_move_assignable<NotMovableVector>::value,
+              "std::is_move_assignable<NotMovableVector>::value");
+static_assert(std::is_copy_assignable<NotMovableVector>::value,
+              "std::is_copy_assignable<NotMovableVector>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotMovableVector>::value,
+//               "!pybind11::detail::is_move_assignable<NotMovableVector>::value");
+static_assert(pybind11::detail::is_copy_assignable<NotMovableVector>::value,
+              "pybind11::detail::is_copy_assignable<NotMovableVector>::value");
+
+struct NotCopyableVector : std::vector<NotCopyable> {};
+static_assert(std::is_move_constructible<NotCopyableVector>::value,
+              "std::is_move_constructible<NotCopyableVector>::value");
+static_assert(std::is_copy_constructible<NotCopyableVector>::value,
+              "std::is_copy_constructible<NotCopyableVector>::value");
+static_assert(pybind11::detail::is_move_constructible<NotCopyableVector>::value,
+              "pybind11::detail::is_move_constructible<NotCopyableVector>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyableVector>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyableVector>::value");
+static_assert(std::is_move_assignable<NotCopyableVector>::value,
+              "std::is_move_assignable<NotCopyableVector>::value");
+static_assert(std::is_copy_assignable<NotCopyableVector>::value,
+              "std::is_copy_assignable<NotCopyableVector>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyableVector>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyableVector>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyableVector>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyableVector>::value");
+
+struct NotCopyableNotMovableVector : std::vector<NotCopyableNotMovable> {};
+static_assert(std::is_move_constructible<NotCopyableNotMovableVector>::value,
+              "std::is_move_constructible<NotCopyableNotMovableVector>::value");
+static_assert(std::is_copy_constructible<NotCopyableNotMovableVector>::value,
+              "std::is_copy_constructible<NotCopyableNotMovableVector>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovableVector>::value,
+              "!pybind11::detail::is_move_constructible<NotCopyableNotMovableVector>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovableVector>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyableNotMovableVector>::value");
+static_assert(std::is_move_assignable<NotCopyableNotMovableVector>::value,
+              "std::is_move_assignable<NotCopyableNotMovableVector>::value");
+static_assert(std::is_copy_assignable<NotCopyableNotMovableVector>::value,
+              "std::is_copy_assignable<NotCopyableNotMovableVector>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovableVector>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyableNotMovableVector>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovableVector>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyableNotMovableVector>::value");
+
+struct NotMovableMap : std::map<int, NotMovable> {};
+static_assert(std::is_move_constructible<NotMovableMap>::value,
+              "std::is_move_constructible<NotMovableMap>::value");
+static_assert(std::is_copy_constructible<NotMovableMap>::value,
+              "std::is_copy_constructible<NotMovableMap>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotMovableMap>::value,
+              "!pybind11::detail::is_move_constructible<NotMovableMap>::value");
+static_assert(pybind11::detail::is_copy_constructible<NotMovableMap>::value,
+              "pybind11::detail::is_copy_constructible<NotMovableMap>::value");
+static_assert(std::is_move_assignable<NotMovableMap>::value,
+              "std::is_move_assignable<NotMovableMap>::value");
+static_assert(std::is_copy_assignable<NotMovableMap>::value,
+              "std::is_copy_assignable<NotMovableMap>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotMovableMap>::value,
+//               "!pybind11::detail::is_move_assignable<NotMovableMap>::value");
+static_assert(pybind11::detail::is_copy_assignable<NotMovableMap>::value,
+              "pybind11::detail::is_copy_assignable<NotMovableMap>::value");
+
+struct NotCopyableMap : std::map<int, NotCopyable> {};
+static_assert(std::is_move_constructible<NotCopyableMap>::value,
+              "std::is_move_constructible<NotCopyableMap>::value");
+static_assert(std::is_copy_constructible<NotCopyableMap>::value,
+              "std::is_copy_constructible<NotCopyableMap>::value");
+static_assert(pybind11::detail::is_move_constructible<NotCopyableMap>::value,
+              "pybind11::detail::is_move_constructible<NotCopyableMap>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyableMap>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyableMap>::value");
+static_assert(std::is_move_assignable<NotCopyableMap>::value,
+              "std::is_move_assignable<NotCopyableMap>::value");
+static_assert(std::is_copy_assignable<NotCopyableMap>::value,
+              "std::is_copy_assignable<NotCopyableMap>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyableMap>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyableMap>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyableMap>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyableMap>::value");
+
+struct NotCopyableNotMovableMap : std::map<int, NotCopyableNotMovable> {};
+static_assert(std::is_move_constructible<NotCopyableNotMovableMap>::value,
+              "std::is_move_constructible<NotCopyableNotMovableMap>::value");
+static_assert(std::is_copy_constructible<NotCopyableNotMovableMap>::value,
+              "std::is_copy_constructible<NotCopyableNotMovableMap>::value");
+static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovableMap>::value,
+              "!pybind11::detail::is_move_constructible<NotCopyableNotMovableMap>::value");
+static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovableMap>::value,
+              "!pybind11::detail::is_copy_constructible<NotCopyableNotMovableMap>::value");
+static_assert(std::is_move_assignable<NotCopyableNotMovableMap>::value,
+              "std::is_move_assignable<NotCopyableNotMovableMap>::value");
+static_assert(std::is_copy_assignable<NotCopyableNotMovableMap>::value,
+              "std::is_copy_assignable<NotCopyableNotMovableMap>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovableMap>::value,
+//               "!pybind11::detail::is_move_assignable<NotCopyableNotMovableMap>::value");
+static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovableMap>::value,
+              "!pybind11::detail::is_copy_assignable<NotCopyableNotMovableMap>::value");
+
+struct RecursiveVector : std::vector<RecursiveVector> {};
+static_assert(std::is_move_constructible<RecursiveVector>::value,
+              "std::is_move_constructible<RecursiveVector>::value");
+static_assert(std::is_copy_constructible<RecursiveVector>::value,
+              "std::is_copy_constructible<RecursiveVector>::value");
+static_assert(pybind11::detail::is_move_constructible<RecursiveVector>::value,
+              "pybind11::detail::is_move_constructible<RecursiveVector>::value");
+static_assert(pybind11::detail::is_copy_constructible<RecursiveVector>::value,
+              "pybind11::detail::is_copy_constructible<RecursiveVector>::value");
+static_assert(std::is_move_assignable<RecursiveVector>::value,
+              "std::is_move_assignable<RecursiveVector>::value");
+static_assert(std::is_copy_assignable<RecursiveVector>::value,
+              "std::is_copy_assignable<RecursiveVector>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<RecursiveVector>::value,
+//               "!pybind11::detail::is_move_assignable<RecursiveVector>::value");
+static_assert(pybind11::detail::is_copy_assignable<RecursiveVector>::value,
+              "pybind11::detail::is_copy_assignable<RecursiveVector>::value");
+
+struct RecursiveMap : std::map<int, RecursiveMap> {};
+static_assert(std::is_move_constructible<RecursiveMap>::value,
+              "std::is_move_constructible<RecursiveMap>::value");
+static_assert(std::is_copy_constructible<RecursiveMap>::value,
+              "std::is_copy_constructible<RecursiveMap>::value");
+static_assert(pybind11::detail::is_move_constructible<RecursiveMap>::value,
+              "pybind11::detail::is_move_constructible<RecursiveMap>::value");
+static_assert(pybind11::detail::is_copy_constructible<RecursiveMap>::value,
+              "pybind11::detail::is_copy_constructible<RecursiveMap>::value");
+static_assert(std::is_move_assignable<RecursiveMap>::value,
+              "std::is_move_assignable<RecursiveMap>::value");
+static_assert(std::is_copy_assignable<RecursiveMap>::value,
+              "std::is_copy_assignable<RecursiveMap>::value");
+// pybind11 does not have this
+// static_assert(!pybind11::detail::is_move_assignable<RecursiveMap>::value,
+//               "!pybind11::detail::is_move_assignable<RecursiveMap>::value");
+static_assert(pybind11::detail::is_copy_assignable<RecursiveMap>::value,
+              "pybind11::detail::is_copy_assignable<RecursiveMap>::value");

tests/test_custom_type_casters.py

@@ -94,12 +94,14 @@ def test_noconvert_args(msg):
 
 def test_custom_caster_destruction():
     """Tests that returning a pointer to a type that gets converted with a custom type caster gets
-    destroyed when the function has py::return_value_policy::take_ownership policy applied."""
+    destroyed when the function has py::return_value_policy::take_ownership policy applied.
+    """
 
     cstats = m.destruction_tester_cstats()
     # This one *doesn't* have take_ownership: the pointer should be used but not destroyed:
     z = m.custom_caster_no_destroy()
-    assert cstats.alive() == 1 and cstats.default_constructions == 1
+    assert cstats.alive() == 1
+    assert cstats.default_constructions == 1
     assert z
 
     # take_ownership applied: this constructs a new object, casts it, then destroys it:

tests/test_custom_type_setup.py

@@ -7,7 +7,7 @@ import env  # noqa: F401
 from pybind11_tests import custom_type_setup as m
 
 
-@pytest.fixture
+@pytest.fixture()
 def gc_tester():
     """Tests that an object is garbage collected.
 

tests/test_eigen_matrix.py

@@ -263,79 +263,96 @@ def test_eigen_return_references():
     primary = np.ones((10, 10))
     a = m.ReturnTester()
     a_get1 = a.get()
-    assert not a_get1.flags.owndata and a_get1.flags.writeable
+    assert not a_get1.flags.owndata
+    assert a_get1.flags.writeable
     assign_both(a_get1, primary, 3, 3, 5)
     a_get2 = a.get_ptr()
-    assert not a_get2.flags.owndata and a_get2.flags.writeable
+    assert not a_get2.flags.owndata
+    assert a_get2.flags.writeable
     assign_both(a_get1, primary, 2, 3, 6)
 
     a_view1 = a.view()
-    assert not a_view1.flags.owndata and not a_view1.flags.writeable
+    assert not a_view1.flags.owndata
+    assert not a_view1.flags.writeable
     with pytest.raises(ValueError):
         a_view1[2, 3] = 4
     a_view2 = a.view_ptr()
-    assert not a_view2.flags.owndata and not a_view2.flags.writeable
+    assert not a_view2.flags.owndata
+    assert not a_view2.flags.writeable
     with pytest.raises(ValueError):
         a_view2[2, 3] = 4
 
     a_copy1 = a.copy_get()
-    assert a_copy1.flags.owndata and a_copy1.flags.writeable
+    assert a_copy1.flags.owndata
+    assert a_copy1.flags.writeable
     np.testing.assert_array_equal(a_copy1, primary)
     a_copy1[7, 7] = -44  # Shouldn't affect anything else
     c1want = array_copy_but_one(primary, 7, 7, -44)
     a_copy2 = a.copy_view()
-    assert a_copy2.flags.owndata and a_copy2.flags.writeable
+    assert a_copy2.flags.owndata
+    assert a_copy2.flags.writeable
     np.testing.assert_array_equal(a_copy2, primary)
     a_copy2[4, 4] = -22  # Shouldn't affect anything else
     c2want = array_copy_but_one(primary, 4, 4, -22)
 
     a_ref1 = a.ref()
-    assert not a_ref1.flags.owndata and a_ref1.flags.writeable
+    assert not a_ref1.flags.owndata
+    assert a_ref1.flags.writeable
     assign_both(a_ref1, primary, 1, 1, 15)
     a_ref2 = a.ref_const()
-    assert not a_ref2.flags.owndata and not a_ref2.flags.writeable
+    assert not a_ref2.flags.owndata
+    assert not a_ref2.flags.writeable
     with pytest.raises(ValueError):
         a_ref2[5, 5] = 33
     a_ref3 = a.ref_safe()
-    assert not a_ref3.flags.owndata and a_ref3.flags.writeable
+    assert not a_ref3.flags.owndata
+    assert a_ref3.flags.writeable
     assign_both(a_ref3, primary, 0, 7, 99)
     a_ref4 = a.ref_const_safe()
-    assert not a_ref4.flags.owndata and not a_ref4.flags.writeable
+    assert not a_ref4.flags.owndata
+    assert not a_ref4.flags.writeable
     with pytest.raises(ValueError):
         a_ref4[7, 0] = 987654321
 
     a_copy3 = a.copy_ref()
-    assert a_copy3.flags.owndata and a_copy3.flags.writeable
+    assert a_copy3.flags.owndata
+    assert a_copy3.flags.writeable
     np.testing.assert_array_equal(a_copy3, primary)
     a_copy3[8, 1] = 11
     c3want = array_copy_but_one(primary, 8, 1, 11)
     a_copy4 = a.copy_ref_const()
-    assert a_copy4.flags.owndata and a_copy4.flags.writeable
+    assert a_copy4.flags.owndata
+    assert a_copy4.flags.writeable
     np.testing.assert_array_equal(a_copy4, primary)
     a_copy4[8, 4] = 88
     c4want = array_copy_but_one(primary, 8, 4, 88)
 
     a_block1 = a.block(3, 3, 2, 2)
-    assert not a_block1.flags.owndata and a_block1.flags.writeable
+    assert not a_block1.flags.owndata
+    assert a_block1.flags.writeable
     a_block1[0, 0] = 55
     primary[3, 3] = 55
     a_block2 = a.block_safe(2, 2, 3, 2)
-    assert not a_block2.flags.owndata and a_block2.flags.writeable
+    assert not a_block2.flags.owndata
+    assert a_block2.flags.writeable
     a_block2[2, 1] = -123
     primary[4, 3] = -123
     a_block3 = a.block_const(6, 7, 4, 3)
-    assert not a_block3.flags.owndata and not a_block3.flags.writeable
+    assert not a_block3.flags.owndata
+    assert not a_block3.flags.writeable
     with pytest.raises(ValueError):
         a_block3[2, 2] = -44444
 
     a_copy5 = a.copy_block(2, 2, 2, 3)
-    assert a_copy5.flags.owndata and a_copy5.flags.writeable
+    assert a_copy5.flags.owndata
+    assert a_copy5.flags.writeable
     np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5])
     a_copy5[1, 1] = 777
     c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777)
 
     a_corn1 = a.corners()
-    assert not a_corn1.flags.owndata and a_corn1.flags.writeable
+    assert not a_corn1.flags.owndata
+    assert a_corn1.flags.writeable
     a_corn1 *= 50
     a_corn1[1, 1] = 999
     primary[0, 0] = 50
@@ -343,7 +360,8 @@ def test_eigen_return_references():
     primary[9, 0] = 50
     primary[9, 9] = 999
     a_corn2 = a.corners_const()
-    assert not a_corn2.flags.owndata and not a_corn2.flags.writeable
+    assert not a_corn2.flags.owndata
+    assert not a_corn2.flags.writeable
     with pytest.raises(ValueError):
         a_corn2[1, 0] = 51
 
@@ -503,10 +521,14 @@ def test_numpy_ref_mutators():
 
     assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4
 
-    assert not zc.flags.owndata and zc.flags.writeable
-    assert not zr.flags.owndata and zr.flags.writeable
-    assert not zcro.flags.owndata and not zcro.flags.writeable
-    assert not zrro.flags.owndata and not zrro.flags.writeable
+    assert not zc.flags.owndata
+    assert zc.flags.writeable
+    assert not zr.flags.owndata
+    assert zr.flags.writeable
+    assert not zcro.flags.owndata
+    assert not zcro.flags.writeable
+    assert not zrro.flags.owndata
+    assert not zrro.flags.writeable
 
     zc[1, 2] = 99
     expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]])
@@ -530,7 +552,8 @@ def test_numpy_ref_mutators():
     # the const should drop away)
     y1 = np.array(m.get_cm_const_ref())
 
-    assert y1.flags.owndata and y1.flags.writeable
+    assert y1.flags.owndata
+    assert y1.flags.writeable
     # We should get copies of the eigen data, which was modified above:
     assert y1[1, 2] == 99
     y1[1, 2] += 12
@@ -603,38 +626,38 @@ def test_nocopy_wrapper():
     # All but the second should fail with m.get_elem_nocopy:
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(int_matrix_colmajor)
-    assert "get_elem_nocopy(): incompatible function arguments." in str(
-        excinfo.value
-    ) and ", flags.f_contiguous" in str(excinfo.value)
+    assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
+    assert ", flags.f_contiguous" in str(excinfo.value)
     assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(int_matrix_rowmajor)
-    assert "get_elem_nocopy(): incompatible function arguments." in str(
-        excinfo.value
-    ) and ", flags.f_contiguous" in str(excinfo.value)
+    assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
+    assert ", flags.f_contiguous" in str(excinfo.value)
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_nocopy(dbl_matrix_rowmajor)
-    assert "get_elem_nocopy(): incompatible function arguments." in str(
-        excinfo.value
-    ) and ", flags.f_contiguous" in str(excinfo.value)
+    assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value)
+    assert ", flags.f_contiguous" in str(excinfo.value)
 
     # For the row-major test, we take a long matrix in row-major, so only the third is allowed:
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_rm_nocopy(int_matrix_colmajor)
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
-    ) and ", flags.c_contiguous" in str(excinfo.value)
+    )
+    assert ", flags.c_contiguous" in str(excinfo.value)
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_rm_nocopy(dbl_matrix_colmajor)
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
-    ) and ", flags.c_contiguous" in str(excinfo.value)
+    )
+    assert ", flags.c_contiguous" in str(excinfo.value)
     assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8
     with pytest.raises(TypeError) as excinfo:
         m.get_elem_rm_nocopy(dbl_matrix_rowmajor)
     assert "get_elem_rm_nocopy(): incompatible function arguments." in str(
         excinfo.value
-    ) and ", flags.c_contiguous" in str(excinfo.value)
+    )
+    assert ", flags.c_contiguous" in str(excinfo.value)
 
 
 def test_eigen_ref_life_support():

tests/test_eigen_tensor.py

@@ -11,14 +11,15 @@ try:
     submodules += [avoid.c_style, avoid.f_style]
 except ImportError as e:
     # Ensure config, build, toolchain, etc. issues are not masked here:
-    raise RuntimeError(
+    msg = (
         "import eigen_tensor_avoid_stl_array FAILED, while "
         "import pybind11_tests.eigen_tensor succeeded. "
         "Please ensure that "
         "test_eigen_tensor.cpp & "
         "eigen_tensor_avoid_stl_array.cpp "
         "are built together (or both are not built if Eigen is not available)."
-    ) from e
+    )
+    raise RuntimeError(msg) from e
 
 tensor_ref = np.empty((3, 5, 2), dtype=np.int64)
 
@@ -59,7 +60,6 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None):
 @pytest.mark.parametrize("m", submodules)
 @pytest.mark.parametrize("member_name", ["member", "member_view"])
 def test_reference_internal(m, member_name):
-
     if not hasattr(sys, "getrefcount"):
         pytest.skip("No reference counting")
     foo = m.CustomExample()
@@ -108,7 +108,6 @@ def test_convert_tensor_to_py(m, func_name):
 
 @pytest.mark.parametrize("m", submodules)
 def test_bad_cpp_to_python_casts(m):
-
     with pytest.raises(
         RuntimeError, match="Cannot use reference internal when there is no parent"
     ):
@@ -131,7 +130,6 @@ def test_bad_cpp_to_python_casts(m):
 
 @pytest.mark.parametrize("m", submodules)
 def test_bad_python_to_cpp_casts(m):
-
     with pytest.raises(
         TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments"
     ):
@@ -150,10 +148,7 @@ def test_bad_python_to_cpp_casts(m):
         m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64))
     )
 
-    if m.needed_options == "F":
-        bad_options = "C"
-    else:
-        bad_options = "F"
+    bad_options = "C" if m.needed_options == "F" else "F"
     # Shape, dtype and the order need to be correct for a TensorMap cast
     with pytest.raises(
         TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments"
@@ -176,25 +171,24 @@ def test_bad_python_to_cpp_casts(m):
             np.zeros((3, 5), dtype=np.float64, order=m.needed_options)
         )
 
+    temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options)
     with pytest.raises(
         TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments"
     ):
-        temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options)
         m.round_trip_view_tensor(
             temp[:, ::-1, :],
         )
 
+    temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options)
+    temp.setflags(write=False)
     with pytest.raises(
         TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments"
     ):
-        temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options)
-        temp.setflags(write=False)
         m.round_trip_view_tensor(temp)
 
 
 @pytest.mark.parametrize("m", submodules)
 def test_references_actually_refer(m):
-
     a = m.reference_tensor()
     temp = a[indices]
     a[indices] = 100
@@ -211,7 +205,6 @@ def test_references_actually_refer(m):
 
 @pytest.mark.parametrize("m", submodules)
 def test_round_trip(m):
-
     assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref))
 
     with pytest.raises(TypeError, match="^Cannot cast array data from"):
@@ -260,7 +253,6 @@ def test_round_trip(m):
 
 @pytest.mark.parametrize("m", submodules)
 def test_round_trip_references_actually_refer(m):
-
     # Need to create a copy that matches the type on the C side
     copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options)
     a = m.round_trip_view_tensor(copy)
@@ -288,9 +280,9 @@ def test_doc_string(m, doc):
     order_flag = f"flags.{m.needed_options.lower()}_contiguous"
     assert doc(m.round_trip_view_tensor) == (
         f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])"
-        + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]"
+        f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]"
     )
     assert doc(m.round_trip_const_view_tensor) == (
         f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])"
-        + " -> numpy.ndarray[numpy.float64[?, ?, ?]]"
+        " -> numpy.ndarray[numpy.float64[?, ?, ?]]"
     )

tests/test_embed/test_interpreter.cpp

@@ -184,7 +184,7 @@ TEST_CASE("Custom PyConfig") {
     py::initialize_interpreter();
 }
 
-TEST_CASE("Custom PyConfig with argv") {
+TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") {
     py::finalize_interpreter();
     {
         PyConfig config;
@@ -199,6 +199,26 @@ TEST_CASE("Custom PyConfig with argv") {
     }
     py::initialize_interpreter();
 }
+
+TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") {
+    py::finalize_interpreter();
+    {
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+
+        // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by
+        // changing it to `0`. This test exercises `scoped_interpreter` with the default config.
+        char *argv[] = {strdup("a.out"), strdup("arg1")};
+        py::scoped_interpreter argv_scope(&config, 2, argv);
+        std::free(argv[0]);
+        std::free(argv[1]);
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0() == "arg1");
+    }
+    py::initialize_interpreter();
+}
 #endif
 
 TEST_CASE("Add program dir to path pre-PyConfig") {
@@ -235,10 +255,10 @@ TEST_CASE("Add program dir to path using PyConfig") {
 }
 #endif
 
-bool has_pybind11_internals_builtin() {
-    auto builtins = py::handle(PyEval_GetBuiltins());
-    return builtins.contains(PYBIND11_INTERNALS_ID);
-};
+bool has_state_dict_internals_obj() {
+    return bool(
+        py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict()));
+}
 
 bool has_pybind11_internals_static() {
     auto **&ipp = py::detail::get_internals_pp();
@@ -248,7 +268,7 @@ bool has_pybind11_internals_static() {
 TEST_CASE("Restart the interpreter") {
     // Verify pre-restart state.
     REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
     REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>()
             == 123);
@@ -265,10 +285,10 @@ TEST_CASE("Restart the interpreter") {
     REQUIRE(Py_IsInitialized() == 1);
 
     // Internals are deleted after a restart.
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
     pybind11::detail::get_internals();
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
     REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp())
             == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
@@ -283,13 +303,13 @@ TEST_CASE("Restart the interpreter") {
               py::detail::get_internals();
               *static_cast<bool *>(ran) = true;
           });
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
     REQUIRE_FALSE(ran);
     py::finalize_interpreter();
     REQUIRE(ran);
     py::initialize_interpreter();
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE_FALSE(has_pybind11_internals_static());
 
     // C++ modules can be reloaded.
@@ -311,7 +331,7 @@ TEST_CASE("Subinterpreter") {
 
         REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
     }
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
 
     /// Create and switch to a subinterpreter.
@@ -321,7 +341,7 @@ TEST_CASE("Subinterpreter") {
     // Subinterpreters get their own copy of builtins. detail::get_internals() still
     // works by returning from the static variable, i.e. all interpreters share a single
     // global pybind11::internals;
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_state_dict_internals_obj());
     REQUIRE(has_pybind11_internals_static());
 
     // Modules tags should be gone.

tests/test_enum.py

@@ -1,3 +1,5 @@
+# ruff: noqa: SIM201 SIM300 SIM202
+
 import pytest
 
 from pybind11_tests import enums as m

tests/test_exceptions.cpp

@@ -339,4 +339,9 @@ TEST_SUBMODULE(exceptions, m) {
         }
         return py::str("UNEXPECTED");
     });
+
+    m.def("test_fn_cast_int", [](const py::function &fn) {
+        // function returns None instead of int, should give a useful error message
+        fn().cast<int>();
+    });
 }

tests/test_exceptions.py

@@ -94,7 +94,6 @@ def ignore_pytest_unraisable_warning(f):
     if hasattr(pytest, unraisable):  # Python >= 3.8 and pytest >= 6
         dec = pytest.mark.filterwarnings(f"ignore::pytest.{unraisable}")
         return dec(f)
-    else:
     return f
 
 
@@ -183,7 +182,7 @@ def test_custom(msg):
         m.throws5_1()
     assert msg(excinfo.value) == "MyException5 subclass"
 
-    with pytest.raises(m.MyException5) as excinfo:
+    with pytest.raises(m.MyException5) as excinfo:  # noqa: PT012
         try:
             m.throws5()
         except m.MyException5_1 as err:
@@ -212,7 +211,7 @@ def test_nested_throws(capture):
         m.try_catch(m.MyException5, throw_myex)
     assert str(excinfo.value) == "nested error"
 
-    def pycatch(exctype, f, *args):
+    def pycatch(exctype, f, *args):  # noqa: ARG001
         try:
             f(*args)
         except m.MyException as e:
@@ -303,12 +302,12 @@ class FlakyException(Exception):
 
 
 @pytest.mark.parametrize(
-    "exc_type, exc_value, expected_what",
-    (
+    ("exc_type", "exc_value", "expected_what"),
+    [
         (ValueError, "plain_str", "ValueError: plain_str"),
         (ValueError, ("tuple_elem",), "ValueError: tuple_elem"),
         (FlakyException, ("happy",), "FlakyException: FlakyException.__str__"),
-    ),
+    ],
 )
 def test_error_already_set_what_with_happy_exceptions(
     exc_type, exc_value, expected_what
@@ -318,8 +317,7 @@ def test_error_already_set_what_with_happy_exceptions(
     assert what == expected_what
 
 
-@pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault")
-def test_flaky_exception_failure_point_init():
+def _test_flaky_exception_failure_point_init_before_py_3_12():
     with pytest.raises(RuntimeError) as excinfo:
         m.error_already_set_what(FlakyException, ("failure_point_init",))
     lines = str(excinfo.value).splitlines()
@@ -333,7 +331,33 @@ def test_flaky_exception_failure_point_init():
     # Checking the first two lines of the traceback as formatted in error_string():
     assert "test_exceptions.py(" in lines[3]
     assert lines[3].endswith("): __init__")
-    assert lines[4].endswith("): test_flaky_exception_failure_point_init")
+    assert lines[4].endswith(
+        "): _test_flaky_exception_failure_point_init_before_py_3_12"
+    )
+
+
+def _test_flaky_exception_failure_point_init_py_3_12():
+    # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594
+    what, py_err_set_after_what = m.error_already_set_what(
+        FlakyException, ("failure_point_init",)
+    )
+    assert not py_err_set_after_what
+    lines = what.splitlines()
+    assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init")
+    assert lines[1] == "__notes__ (len=1):"
+    assert "Normalization failed:" in lines[2]
+    assert "FlakyException" in lines[2]
+
+
+@pytest.mark.skipif(
+    "env.PYPY and sys.version_info[:2] < (3, 12)",
+    reason="PyErr_NormalizeException Segmentation fault",
+)
+def test_flaky_exception_failure_point_init():
+    if sys.version_info[:2] < (3, 12):
+        _test_flaky_exception_failure_point_init_before_py_3_12()
+    else:
+        _test_flaky_exception_failure_point_init_py_3_12()
 
 
 def test_flaky_exception_failure_point_str():
@@ -342,10 +366,7 @@ def test_flaky_exception_failure_point_str():
     )
     assert not py_err_set_after_what
     lines = what.splitlines()
-    if env.PYPY and len(lines) == 3:
-        n = 3  # Traceback is missing.
-    else:
-        n = 5
+    n = 3 if env.PYPY and len(lines) == 3 else 5
     assert (
         lines[:n]
         == [
@@ -381,3 +402,12 @@ def test_pypy_oserror_normalization():
     # https://github.com/pybind/pybind11/issues/4075
     what = m.test_pypy_oserror_normalization()
     assert "this_filename_must_not_exist" in what
+
+
+def test_fn_cast_int_exception():
+    with pytest.raises(RuntimeError) as excinfo:
+        m.test_fn_cast_int(lambda: None)
+
+    assert str(excinfo.value).startswith(
+        "Unable to cast Python instance of type <class 'NoneType'> to C++ type"
+    )

tests/test_factory_constructors.py

@@ -96,7 +96,7 @@ def test_init_factory_signature(msg):
         3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None
 
         4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None
-    """  # noqa: E501 line too long
+    """
     )
 
 

tests/test_gil_scoped.py

@@ -148,10 +148,7 @@ ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_dead
 
 
 def _run_in_process(target, *args, **kwargs):
-    if len(args) == 0:
-        test_fn = target
-    else:
-        test_fn = args[0]
+    test_fn = target if len(args) == 0 else args[0]
     # Do not need to wait much, 10s should be more than enough.
     timeout = 0.1 if test_fn is _intentional_deadlock else 10
     process = multiprocessing.Process(target=target, args=args, kwargs=kwargs)
@@ -178,7 +175,8 @@ def _run_in_process(target, *args, **kwargs):
         elif test_fn is _intentional_deadlock:
             assert process.exitcode is None
             return 0
-        elif process.exitcode is None:
+
+        if process.exitcode is None:
             assert t_delta > 0.9 * timeout
             msg = "DEADLOCK, most likely, exactly what this test is meant to detect."
             if env.PYPY and env.WIN:

tests/test_iostream.py

@@ -9,16 +9,16 @@ def test_captured(capsys):
     m.captured_output(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
     m.captured_err(msg)
     stdout, stderr = capsys.readouterr()
-    assert stdout == ""
+    assert not stdout
     assert stderr == msg
 
 
@@ -30,7 +30,7 @@ def test_captured_large_string(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_2byte_offset0(capsys):
@@ -40,7 +40,7 @@ def test_captured_utf8_2byte_offset0(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_2byte_offset1(capsys):
@@ -50,7 +50,7 @@ def test_captured_utf8_2byte_offset1(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_3byte_offset0(capsys):
@@ -60,7 +60,7 @@ def test_captured_utf8_3byte_offset0(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_3byte_offset1(capsys):
@@ -70,7 +70,7 @@ def test_captured_utf8_3byte_offset1(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_3byte_offset2(capsys):
@@ -80,7 +80,7 @@ def test_captured_utf8_3byte_offset2(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_4byte_offset0(capsys):
@@ -90,7 +90,7 @@ def test_captured_utf8_4byte_offset0(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_4byte_offset1(capsys):
@@ -100,7 +100,7 @@ def test_captured_utf8_4byte_offset1(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_4byte_offset2(capsys):
@@ -110,7 +110,7 @@ def test_captured_utf8_4byte_offset2(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_captured_utf8_4byte_offset3(capsys):
@@ -120,7 +120,7 @@ def test_captured_utf8_4byte_offset3(capsys):
     m.captured_output_default(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_guard_capture(capsys):
@@ -128,7 +128,7 @@ def test_guard_capture(capsys):
     m.guard_output(msg)
     stdout, stderr = capsys.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
 
 
 def test_series_captured(capture):
@@ -145,7 +145,7 @@ def test_flush(capfd):
     with m.ostream_redirect():
         m.noisy_function(msg, flush=False)
         stdout, stderr = capfd.readouterr()
-        assert stdout == ""
+        assert not stdout
 
         m.noisy_function(msg2, flush=True)
         stdout, stderr = capfd.readouterr()
@@ -164,15 +164,15 @@ def test_not_captured(capfd):
         m.raw_output(msg)
     stdout, stderr = capfd.readouterr()
     assert stdout == msg
-    assert stderr == ""
-    assert stream.getvalue() == ""
+    assert not stderr
+    assert not stream.getvalue()
 
     stream = StringIO()
     with redirect_stdout(stream):
         m.captured_output(msg)
     stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
+    assert not stdout
+    assert not stderr
     assert stream.getvalue() == msg
 
 
@@ -182,16 +182,16 @@ def test_err(capfd):
     with redirect_stderr(stream):
         m.raw_err(msg)
     stdout, stderr = capfd.readouterr()
-    assert stdout == ""
+    assert not stdout
     assert stderr == msg
-    assert stream.getvalue() == ""
+    assert not stream.getvalue()
 
     stream = StringIO()
     with redirect_stderr(stream):
         m.captured_err(msg)
     stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
+    assert not stdout
+    assert not stderr
     assert stream.getvalue() == msg
 
 
@@ -221,14 +221,13 @@ def test_redirect(capfd):
         m.raw_output(msg)
     stdout, stderr = capfd.readouterr()
     assert stdout == msg
-    assert stream.getvalue() == ""
+    assert not stream.getvalue()
 
     stream = StringIO()
-    with redirect_stdout(stream):
-        with m.ostream_redirect():
+    with redirect_stdout(stream), m.ostream_redirect():
         m.raw_output(msg)
     stdout, stderr = capfd.readouterr()
-    assert stdout == ""
+    assert not stdout
     assert stream.getvalue() == msg
 
     stream = StringIO()
@@ -236,7 +235,7 @@ def test_redirect(capfd):
         m.raw_output(msg)
     stdout, stderr = capfd.readouterr()
     assert stdout == msg
-    assert stream.getvalue() == ""
+    assert not stream.getvalue()
 
 
 def test_redirect_err(capfd):
@@ -244,13 +243,12 @@ def test_redirect_err(capfd):
     msg2 = "StdErr"
 
     stream = StringIO()
-    with redirect_stderr(stream):
-        with m.ostream_redirect(stdout=False):
+    with redirect_stderr(stream), m.ostream_redirect(stdout=False):
         m.raw_output(msg)
         m.raw_err(msg2)
     stdout, stderr = capfd.readouterr()
     assert stdout == msg
-    assert stderr == ""
+    assert not stderr
     assert stream.getvalue() == msg2
 
 
@@ -260,14 +258,12 @@ def test_redirect_both(capfd):
 
     stream = StringIO()
     stream2 = StringIO()
-    with redirect_stdout(stream):
-        with redirect_stderr(stream2):
-            with m.ostream_redirect():
+    with redirect_stdout(stream), redirect_stderr(stream2), m.ostream_redirect():
         m.raw_output(msg)
         m.raw_err(msg2)
     stdout, stderr = capfd.readouterr()
-    assert stdout == ""
-    assert stderr == ""
+    assert not stdout
+    assert not stderr
     assert stream.getvalue() == msg
     assert stream2.getvalue() == msg2
 

tests/test_kwargs_and_defaults.py

@@ -25,7 +25,7 @@ def test_function_signatures(doc):
     )
 
 
-def test_named_arguments(msg):
+def test_named_arguments():
     assert m.kw_func0(5, 10) == "x=5, y=10"
 
     assert m.kw_func1(5, 10) == "x=5, y=10"
@@ -43,8 +43,7 @@ def test_named_arguments(msg):
         # noinspection PyArgumentList
         m.kw_func2(x=5, y=10, z=12)
     assert excinfo.match(
-        r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))"
-        + "{3}$"
+        r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$)){3}$"
     )
 
     assert m.kw_func4() == "{13 17}"
@@ -59,7 +58,7 @@ def test_arg_and_kwargs():
     assert m.args_function(*args) == args
 
     args = "a1", "a2"
-    kwargs = dict(arg3="a3", arg4=4)
+    kwargs = {"arg3": "a3", "arg4": 4}
     assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs)
 
 
@@ -177,7 +176,7 @@ def test_mixed_args_and_kwargs(msg):
 
     assert (
         m.args_kwonly_kwargs_defaults.__doc__
-        == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n"  # noqa: E501 line too long
+        == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n"
     )
     assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {})
     assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {})
@@ -233,15 +232,15 @@ def test_keyword_only_args(msg):
     x.method(i=1, j=2)
     assert (
         m.first_arg_kw_only.__init__.__doc__
-        == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n"  # noqa: E501 line too long
+        == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n"
     )
     assert (
         m.first_arg_kw_only.method.__doc__
-        == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n"  # noqa: E501 line too long
+        == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n"
     )
 
 
-def test_positional_only_args(msg):
+def test_positional_only_args():
     assert m.pos_only_all(1, 2) == (1, 2)
     assert m.pos_only_all(2, 1) == (2, 1)
 
@@ -283,7 +282,7 @@ def test_positional_only_args(msg):
     # Mix it with args and kwargs:
     assert (
         m.args_kwonly_full_monty.__doc__
-        == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n"  # noqa: E501 line too long
+        == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n"
     )
     assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {})
     assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {})
@@ -326,18 +325,18 @@ def test_positional_only_args(msg):
     # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987
     assert (
         m.first_arg_kw_only.pos_only.__doc__
-        == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n"  # noqa: E501 line too long
+        == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n"
     )
 
 
 def test_signatures():
-    assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__
-    assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__
-    assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__
-    assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__
+    assert m.kw_only_all.__doc__ == "kw_only_all(*, i: int, j: int) -> tuple\n"
+    assert m.kw_only_mixed.__doc__ == "kw_only_mixed(i: int, *, j: int) -> tuple\n"
+    assert m.pos_only_all.__doc__ == "pos_only_all(i: int, j: int, /) -> tuple\n"
+    assert m.pos_only_mix.__doc__ == "pos_only_mix(i: int, /, j: int) -> tuple\n"
     assert (
-        "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n"
-        == m.pos_kw_only_mix.__doc__
+        m.pos_kw_only_mix.__doc__
+        == "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n"
     )
 
 

tests/test_local_bindings.py

@@ -130,7 +130,8 @@ def test_stl_bind_global():
 def test_mixed_local_global():
     """Local types take precedence over globally registered types: a module with a `module_local`
     type can be registered even if the type is already registered globally.  With the module,
-    casting will go to the local type; outside the module casting goes to the global type."""
+    casting will go to the local type; outside the module casting goes to the global type.
+    """
     import pybind11_cross_module_tests as cm
 
     m.register_mixed_global()

tests/test_methods_and_attributes.cpp

@@ -177,6 +177,38 @@ struct RValueRefParam {
     std::size_t func4(std::string &&s) const & { return s.size(); }
 };
 
+namespace pybind11_tests {
+namespace exercise_is_setter {
+
+struct FieldBase {
+    int int_value() const { return int_value_; }
+
+    FieldBase &SetIntValue(int int_value) {
+        int_value_ = int_value;
+        return *this;
+    }
+
+private:
+    int int_value_ = -99;
+};
+
+struct Field : FieldBase {};
+
+void add_bindings(py::module &m) {
+    py::module sm = m.def_submodule("exercise_is_setter");
+    // NOTE: FieldBase is not wrapped, therefore ...
+    py::class_<Field>(sm, "Field")
+        .def(py::init<>())
+        .def_property(
+            "int_value",
+            &Field::int_value,
+            &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted.
+        );
+}
+
+} // namespace exercise_is_setter
+} // namespace pybind11_tests
+
 TEST_SUBMODULE(methods_and_attributes, m) {
     // test_methods_and_attributes
     py::class_<ExampleMandA> emna(m, "ExampleMandA");
@@ -456,4 +488,6 @@ TEST_SUBMODULE(methods_and_attributes, m) {
         .def("func2", &RValueRefParam::func2)
         .def("func3", &RValueRefParam::func3)
         .def("func4", &RValueRefParam::func4);
+
+    pybind11_tests::exercise_is_setter::add_bindings(m);
 }

tests/test_methods_and_attributes.py

@@ -183,9 +183,9 @@ def test_static_properties():
 
     # Only static attributes can be deleted
     del m.TestPropertiesOverride.def_readonly_static
+    assert hasattr(m.TestPropertiesOverride, "def_readonly_static")
     assert (
-        hasattr(m.TestPropertiesOverride, "def_readonly_static")
-        and m.TestPropertiesOverride.def_readonly_static
+        m.TestPropertiesOverride.def_readonly_static
         is m.TestProperties.def_readonly_static
     )
     assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__
@@ -256,10 +256,7 @@ def test_no_mixed_overloads():
 
 @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])
 def test_property_return_value_policies(access):
-    if not access.startswith("static"):
-        obj = m.TestPropRVP()
-    else:
-        obj = m.TestPropRVP
+    obj = m.TestPropRVP() if not access.startswith("static") else m.TestPropRVP
 
     ref = getattr(obj, access + "_ref")
     assert ref.value == 1
@@ -525,3 +522,12 @@ def test_rvalue_ref_param():
     assert r.func2("1234") == 4
     assert r.func3("12345") == 5
     assert r.func4("123456") == 6
+
+
+def test_is_setter():
+    fld = m.exercise_is_setter.Field()
+    assert fld.int_value == -99
+    setter_return = fld.int_value = 100
+    assert isinstance(setter_return, int)
+    assert setter_return == 100
+    assert fld.int_value == 100

tests/test_modules.py

@@ -1,3 +1,5 @@
+import builtins
+
 import pytest
 
 import env
@@ -61,7 +63,6 @@ def test_importing():
     from pybind11_tests.modules import OD
 
     assert OD is OrderedDict
-    assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])"
 
 
 def test_pydoc():
@@ -86,12 +87,7 @@ def test_builtin_key_type():
 
     Previous versions of pybind11 would add a unicode key in python 2.
     """
-    if hasattr(__builtins__, "keys"):
-        keys = __builtins__.keys()
-    else:  # this is to make pypy happy since builtins is different there.
-        keys = __builtins__.__dict__.keys()
-
-    assert {type(k) for k in keys} == {str}
+    assert all(type(k) == str for k in dir(builtins))
 
 
 @pytest.mark.xfail("env.PYPY", reason="PyModule_GetName()")
@@ -107,11 +103,10 @@ def test_def_submodule_failures():
         sm_name_orig = sm.__name__
         sm.__name__ = malformed_utf8
         try:
-            with pytest.raises(Exception):
-                # Seen with Python 3.9: SystemError: nameless module
-                # But we do not want to exercise the internals of PyModule_GetName(), which could
-                # change in future versions of Python, but a bad __name__ is very likely to cause
-                # some kind of failure indefinitely.
+            # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise
+            # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that
+            # changes in future Python versions, simply add the new expected exception types here.
+            with pytest.raises(SystemError):
                 m.def_submodule(sm, b"SubSubModuleName")
         finally:
             # Clean up to ensure nothing gets upset by a module with an invalid __name__.

tests/test_numpy_array.cpp

@@ -523,4 +523,30 @@ TEST_SUBMODULE(numpy_array, sm) {
     sm.def("test_fmt_desc_const_double", [](const py::array_t<const double> &) {});
 
     sm.def("round_trip_float", [](double d) { return d; });
+
+    sm.def("pass_array_pyobject_ptr_return_sum_str_values",
+           [](const py::array_t<PyObject *> &objs) {
+               std::string sum_str_values;
+               for (const auto &obj : objs) {
+                   sum_str_values += py::str(obj.attr("value"));
+               }
+               return sum_str_values;
+           });
+
+    sm.def("pass_array_pyobject_ptr_return_as_list",
+           [](const py::array_t<PyObject *> &objs) -> py::list { return objs; });
+
+    sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) {
+        py::size_t arr_size = py::len(objs);
+        py::array_t<PyObject *> arr_from_list(static_cast<py::ssize_t>(arr_size));
+        PyObject **data = arr_from_list.mutable_data();
+        for (py::size_t i = 0; i < arr_size; i++) {
+            assert(data[i] == nullptr);
+            data[i] = py::cast<PyObject *>(objs[i].attr("value"));
+        }
+        return arr_from_list;
+    });
+
+    sm.def("return_array_pyobject_ptr_from_list",
+           [](const py::list &objs) -> py::array_t<PyObject *> { return objs; });
 }

tests/test_numpy_array.py

@@ -22,7 +22,7 @@ def test_dtypes():
             )
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture()
 def arr():
     return np.array([[1, 2, 3], [4, 5, 6]], "=u2")
 
@@ -67,7 +67,7 @@ def test_array_attributes():
 
 
 @pytest.mark.parametrize(
-    "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)]
+    ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)]
 )
 def test_index_offset(arr, args, ret):
     assert m.index_at(arr, *args) == ret
@@ -93,7 +93,7 @@ def test_dim_check_fail(arr):
 
 
 @pytest.mark.parametrize(
-    "args, ret",
+    ("args", "ret"),
     [
         ([], [1, 2, 3, 4, 5, 6]),
         ([1], [4, 5, 6]),
@@ -211,12 +211,14 @@ def test_wrap():
             assert b[0, 0] == 1234
 
     a1 = np.array([1, 2], dtype=np.int16)
-    assert a1.flags.owndata and a1.base is None
+    assert a1.flags.owndata
+    assert a1.base is None
     a2 = m.wrap(a1)
     assert_references(a1, a2)
 
     a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F")
-    assert a1.flags.owndata and a1.base is None
+    assert a1.flags.owndata
+    assert a1.base is None
     a2 = m.wrap(a1)
     assert_references(a1, a2)
 
@@ -451,13 +453,15 @@ def test_array_resize():
     try:
         m.array_resize3(a, 3, True)
     except ValueError as e:
-        assert str(e).startswith("cannot resize an array")
+        assert str(e).startswith("cannot resize an array")  # noqa: PT017
     # transposed array doesn't own data
     b = a.transpose()
     try:
         m.array_resize3(b, 3, False)
     except ValueError as e:
-        assert str(e).startswith("cannot resize this array: it does not own its data")
+        assert str(e).startswith(  # noqa: PT017
+            "cannot resize this array: it does not own its data"
+        )
     # ... but reshape should be fine
     m.array_reshape2(b)
     assert b.shape == (8, 8)
@@ -591,3 +595,74 @@ def test_round_trip_float():
     arr = np.zeros((), np.float64)
     arr[()] = 37.2
     assert m.round_trip_float(arr) == 37.2
+
+
+# HINT: An easy and robust way (although only manual unfortunately) to check for
+#       ref-count leaks in the test_.*pyobject_ptr.* functions below is to
+#           * temporarily insert `while True:` (one-by-one),
+#           * run this test, and
+#           * run the Linux `top` command in another shell to visually monitor
+#             `RES` for a minute or two.
+#       If there is a leak, it is usually evident in seconds because the `RES`
+#       value increases without bounds. (Don't forget to Ctrl-C the test!)
+
+
+# For use as a temporary user-defined object, to maximize sensitivity of the tests below:
+#     * Ref-count leaks will be immediately evident.
+#     * Sanitizers are much more likely to detect heap-use-after-free due to
+#       other ref-count bugs.
+class PyValueHolder:
+    def __init__(self, value):
+        self.value = value
+
+
+def WrapWithPyValueHolder(*values):
+    return [PyValueHolder(v) for v in values]
+
+
+def UnwrapPyValueHolder(vhs):
+    return [vh.value for vh in vhs]
+
+
+def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray():
+    # Intentionally all temporaries, do not change.
+    assert (
+        m.pass_array_pyobject_ptr_return_sum_str_values(
+            np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object)
+        )
+        == "-3four5.0"
+    )
+
+
+def test_pass_array_pyobject_ptr_return_sum_str_values_list():
+    # Intentionally all temporaries, do not change.
+    assert (
+        m.pass_array_pyobject_ptr_return_sum_str_values(
+            WrapWithPyValueHolder(2, "three", -4.0)
+        )
+        == "2three-4.0"
+    )
+
+
+def test_pass_array_pyobject_ptr_return_as_list():
+    # Intentionally all temporaries, do not change.
+    assert UnwrapPyValueHolder(
+        m.pass_array_pyobject_ptr_return_as_list(
+            np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object)
+        )
+    ) == [-1, "two", 3.0]
+
+
+@pytest.mark.parametrize(
+    ("return_array_pyobject_ptr", "unwrap"),
+    [
+        (m.return_array_pyobject_ptr_cpp_loop, list),
+        (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder),
+    ],
+)
+def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap):
+    # Intentionally all temporaries, do not change.
+    arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0))
+    assert isinstance(arr_from_list, np.ndarray)
+    assert arr_from_list.dtype == np.dtype("O")
+    assert unwrap(arr_from_list) == [6, "seven", -8.0]

tests/test_numpy_dtypes.py

@@ -130,14 +130,10 @@ def test_dtype(simple_dtype):
         partial_nested_fmt(),
         "[('a','S3'),('b','S3')]",
         (
-            "{{'names':['a','b','c','d'],"
-            + "'formats':[('S4',(3,)),('"
-            + e
-            + "i4',(2,)),('u1',(3,)),('"
-            + e
-            + "f4',(4,2))],"
-            + "'offsets':[0,12,20,24],'itemsize':56}}"
-        ).format(e=e),
+            "{'names':['a','b','c','d'],"
+            f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))],"
+            "'offsets':[0,12,20,24],'itemsize':56}"
+        ),
         "[('e1','" + e + "i8'),('e2','u1')]",
         "[('x','i1'),('y','" + e + "u8')]",
         "[('cflt','" + e + "c8'),('cdbl','" + e + "c16')]",
@@ -291,19 +287,17 @@ def test_array_array():
 
     arr = m.create_array_array(3)
     assert str(arr.dtype).replace(" ", "") == (
-        "{{'names':['a','b','c','d'],"
-        + "'formats':[('S4',(3,)),('"
-        + e
-        + "i4',(2,)),('u1',(3,)),('{e}f4',(4,2))],"
-        + "'offsets':[0,12,20,24],'itemsize':56}}"
-    ).format(e=e)
+        "{'names':['a','b','c','d'],"
+        f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))],"
+        "'offsets':[0,12,20,24],'itemsize':56}"
+    )
     assert m.print_array_array(arr) == [
         "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1},"
-        + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}",
+        "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}",
         "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001},"
-        + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}",
+        "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}",
         "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001},"
-        + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}",
+        "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}",
     ]
     assert arr["a"].tolist() == [
         [b"ABCD", b"KLMN", b"UVWX"],

tests/test_numpy_vectorize.py

@@ -149,7 +149,7 @@ def test_docs(doc):
         doc(m.vectorized_func)
         == """
         vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object
-    """  # noqa: E501 line too long
+    """
     )
 
 

tests/test_operator_overloading.py

@@ -130,7 +130,6 @@ def test_nested():
 
 
 def test_overriding_eq_reset_hash():
-
     assert m.Comparable(15) is not m.Comparable(15)
     assert m.Comparable(15) == m.Comparable(15)
 

tests/test_pytypes.cpp

@@ -260,6 +260,15 @@ TEST_SUBMODULE(pytypes, m) {
         });
     });
 
+    m.def("return_capsule_with_destructor_3", []() {
+        py::print("creating capsule");
+        auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) {
+            py::print("destructing capsule: {}"_s.format((size_t) ptr));
+        });
+        py::print("original name: {}"_s.format(cap.name()));
+        return cap;
+    });
+
     m.def("return_renamed_capsule_with_destructor_2", []() {
         py::print("creating capsule");
         auto cap = py::capsule((void *) 1234, [](void *ptr) {

tests/test_pytypes.py

@@ -15,7 +15,7 @@ def test_obj_class_name():
     assert m.obj_class_name([]) == "list"
 
 
-def test_handle_from_move_only_type_with_operator_PyObject():  # noqa: N802
+def test_handle_from_move_only_type_with_operator_PyObject():
     assert m.handle_from_move_only_type_with_operator_PyObject_ncnst()
     assert m.handle_from_move_only_type_with_operator_PyObject_const()
 
@@ -33,7 +33,7 @@ def test_iterator(doc):
 
 
 @pytest.mark.parametrize(
-    "pytype, from_iter_func",
+    ("pytype", "from_iter_func"),
     [
         (frozenset, m.get_frozenset_from_iterable),
         (list, m.get_list_from_iterable),
@@ -87,7 +87,7 @@ def test_list(capture, doc):
     assert doc(m.print_list) == "print_list(arg0: list) -> None"
 
 
-def test_none(capture, doc):
+def test_none(doc):
     assert doc(m.get_none) == "get_none() -> None"
     assert doc(m.print_none) == "print_none(arg0: None) -> None"
 
@@ -182,10 +182,10 @@ class CustomContains:
 
 
 @pytest.mark.parametrize(
-    "arg,func",
+    ("arg", "func"),
     [
         (set(), m.anyset_contains),
-        (dict(), m.dict_contains),
+        ({}, m.dict_contains),
         (CustomContains(), m.obj_contains),
     ],
 )
@@ -273,7 +273,7 @@ def test_bytes(doc):
     assert doc(m.bytes_from_str) == "bytes_from_str() -> bytes"
 
 
-def test_bytearray(doc):
+def test_bytearray():
     assert m.bytearray_from_char_ssize_t().decode() == "$%"
     assert m.bytearray_from_char_size_t().decode() == "@$!"
     assert m.bytearray_from_string().decode() == "foo"
@@ -319,6 +319,19 @@ def test_capsule(capture):
     """
     )
 
+    with capture:
+        a = m.return_capsule_with_destructor_3()
+        del a
+        pytest.gc_collect()
+    assert (
+        capture.unordered
+        == """
+        creating capsule
+        destructing capsule: 1233
+        original name: oname
+    """
+    )
+
     with capture:
         a = m.return_renamed_capsule_with_destructor_2()
         del a
@@ -385,7 +398,7 @@ def test_accessors():
     assert d["implicit_list"] == [1, 2, 3]
     assert all(x in TestObject.__dict__ for x in d["implicit_dict"])
 
-    assert m.tuple_accessor(tuple()) == (0, 1, 2)
+    assert m.tuple_accessor(()) == (0, 1, 2)
 
     d = m.accessor_assignment()
     assert d["get"] == 0
@@ -475,7 +488,7 @@ def test_pybind11_str_raw_str():
     assert cvt({}) == "{}"
     assert cvt({3: 4}) == "{3: 4}"
     assert cvt(set()) == "set()"
-    assert cvt({3, 3}) == "{3}"
+    assert cvt({3}) == "{3}"
 
     valid_orig = "DZ"
     valid_utf8 = valid_orig.encode("utf-8")
@@ -536,7 +549,7 @@ def test_print(capture):
     assert str(excinfo.value) == "Unable to convert call argument " + (
         "'1' of type 'UnregisteredType' to Python object"
         if detailed_error_messages_enabled
-        else "to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"
+        else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"
     )
 
 
@@ -593,7 +606,7 @@ def test_issue2361():
 
 
 @pytest.mark.parametrize(
-    "method, args, fmt, expected_view",
+    ("method", "args", "fmt", "expected_view"),
     [
         (m.test_memoryview_object, (b"red",), "B", b"red"),
         (m.test_memoryview_buffer_info, (b"green",), "B", b"green"),
@@ -651,7 +664,7 @@ def test_memoryview_from_memory():
 
 
 def test_builtin_functions():
-    assert m.get_len([i for i in range(42)]) == 42
+    assert m.get_len(list(range(42))) == 42
     with pytest.raises(TypeError) as exc_info:
         m.get_len(i for i in range(42))
     assert str(exc_info.value) in [
@@ -695,7 +708,7 @@ def test_pass_bytes_or_unicode_to_string_types():
 
 
 @pytest.mark.parametrize(
-    "create_weakref, create_weakref_with_callback",
+    ("create_weakref", "create_weakref_with_callback"),
     [
         (m.weakref_from_handle, m.weakref_from_handle_and_function),
         (m.weakref_from_object, m.weakref_from_object_and_function),
@@ -710,7 +723,7 @@ def test_weakref(create_weakref, create_weakref_with_callback):
 
     callback_called = False
 
-    def callback(wr):
+    def callback(_):
         nonlocal callback_called
         callback_called = True
 
@@ -730,7 +743,7 @@ def test_weakref(create_weakref, create_weakref_with_callback):
 
 
 @pytest.mark.parametrize(
-    "create_weakref, has_callback",
+    ("create_weakref", "has_callback"),
     [
         (m.weakref_from_handle, False),
         (m.weakref_from_object, False),
@@ -748,10 +761,7 @@ def test_weakref_err(create_weakref, has_callback):
     ob = C()
     # Should raise TypeError on CPython
     with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext():
-        if has_callback:
-            _ = create_weakref(ob, callback)
-        else:
-            _ = create_weakref(ob)
+        _ = create_weakref(ob, callback) if has_callback else create_weakref(ob)
 
 
 def test_cpp_iterators():
@@ -814,33 +824,36 @@ def test_populate_obj_str_attrs():
 
 
 @pytest.mark.parametrize(
-    "a,b", [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))]
+    ("a", "b"),
+    [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))],
 )
 def test_inplace_append(a, b):
     expected = a + b
     assert m.inplace_append(a, b) == expected
 
 
-@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))])
+@pytest.mark.parametrize(
+    ("a", "b"), [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))]
+)
 def test_inplace_subtract(a, b):
     expected = a - b
     assert m.inplace_subtract(a, b) == expected
 
 
-@pytest.mark.parametrize("a,b", [(3, 2), (3.0, 2.0), ([1], 3)])
+@pytest.mark.parametrize(("a", "b"), [(3, 2), (3.0, 2.0), ([1], 3)])
 def test_inplace_multiply(a, b):
     expected = a * b
     assert m.inplace_multiply(a, b) == expected
 
 
-@pytest.mark.parametrize("a,b", [(6, 3), (6.0, 3.0)])
+@pytest.mark.parametrize(("a", "b"), [(6, 3), (6.0, 3.0)])
 def test_inplace_divide(a, b):
     expected = a / b
     assert m.inplace_divide(a, b) == expected
 
 
 @pytest.mark.parametrize(
-    "a,b",
+    ("a", "b"),
     [
         (False, True),
         (
@@ -857,7 +870,7 @@ def test_inplace_or(a, b):
 
 
 @pytest.mark.parametrize(
-    "a,b",
+    ("a", "b"),
     [
         (True, False),
         (
@@ -873,13 +886,13 @@ def test_inplace_and(a, b):
     assert m.inplace_and(a, b) == expected
 
 
-@pytest.mark.parametrize("a,b", [(8, 1), (-3, 2)])
+@pytest.mark.parametrize(("a", "b"), [(8, 1), (-3, 2)])
 def test_inplace_lshift(a, b):
     expected = a << b
     assert m.inplace_lshift(a, b) == expected
 
 
-@pytest.mark.parametrize("a,b", [(8, 1), (-2, 2)])
+@pytest.mark.parametrize(("a", "b"), [(8, 1), (-2, 2)])
 def test_inplace_rshift(a, b):
     expected = a >> b
     assert m.inplace_rshift(a, b) == expected

tests/test_sequences_and_iterators.py

@@ -1,5 +1,5 @@
 import pytest
-from pytest import approx
+from pytest import approx  # noqa: PT013
 
 from pybind11_tests import ConstructorStats
 from pybind11_tests import sequences_and_iterators as m
@@ -103,7 +103,8 @@ def test_sequence():
 
     assert "Sequence" in repr(s)
     assert len(s) == 5
-    assert s[0] == 0 and s[3] == 0
+    assert s[0] == 0
+    assert s[3] == 0
     assert 12.34 not in s
     s[0], s[3] = 12.34, 56.78
     assert 12.34 in s
@@ -245,7 +246,7 @@ def test_iterator_rvp():
 
 def test_carray_iterator():
     """#4100: Check for proper iterator overload with C-Arrays"""
-    args_gt = list(float(i) for i in range(3))
+    args_gt = [float(i) for i in range(3)]
     arr_h = m.CArrayHolder(*args_gt)
     args = list(arr_h)
     assert args_gt == args

tests/test_stl.py

@@ -14,7 +14,7 @@ def test_vector(doc):
 
     assert m.cast_bool_vector() == [True, False]
     assert m.load_bool_vector([True, False])
-    assert m.load_bool_vector(tuple([True, False]))
+    assert m.load_bool_vector((True, False))
 
     assert doc(m.cast_vector) == "cast_vector() -> List[int]"
     assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool"
@@ -23,7 +23,7 @@ def test_vector(doc):
     assert m.cast_ptr_vector() == ["lvalue", "lvalue"]
 
 
-def test_deque(doc):
+def test_deque():
     """std::deque <-> list"""
     lst = m.cast_deque()
     assert lst == [1]
@@ -39,8 +39,11 @@ def test_array(doc):
     assert m.load_array(lst)
     assert m.load_array(tuple(lst))
 
-    assert doc(m.cast_array) == "cast_array() -> List[int[2]]"
-    assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool"
+    assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]"
+    assert (
+        doc(m.load_array)
+        == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool"
+    )
 
 
 def test_valarray(doc):
@@ -95,7 +98,8 @@ def test_recursive_casting():
 
     # Issue #853 test case:
     z = m.cast_unique_ptr_vector()
-    assert z[0].value == 7 and z[1].value == 42
+    assert z[0].value == 7
+    assert z[1].value == 42
 
 
 def test_move_out_container():
@@ -366,7 +370,7 @@ def test_issue_1561():
     """check fix for issue #1561"""
     bar = m.Issue1561Outer()
     bar.list = [m.Issue1561Inner("bar")]
-    bar.list
+    assert bar.list
     assert bar.list[0].data == "bar"
 
 

tests/test_stl_binders.cpp

@@ -70,6 +70,44 @@ NestMap *times_hundred(int n) {
     return m;
 }
 
+/*
+ * Recursive data structures as test for issue #4623
+ */
+struct RecursiveVector : std::vector<RecursiveVector> {
+    using Parent = std::vector<RecursiveVector>;
+    using Parent::Parent;
+};
+
+struct RecursiveMap : std::map<int, RecursiveMap> {
+    using Parent = std::map<int, RecursiveMap>;
+    using Parent::Parent;
+};
+
+/*
+ * Pybind11 does not catch more complicated recursion schemes, such as mutual
+ * recursion.
+ * In that case custom recursive_container_traits specializations need to be added,
+ * thus manually telling pybind11 about the recursion.
+ */
+struct MutuallyRecursiveContainerPairMV;
+struct MutuallyRecursiveContainerPairVM;
+
+struct MutuallyRecursiveContainerPairMV : std::map<int, MutuallyRecursiveContainerPairVM> {};
+struct MutuallyRecursiveContainerPairVM : std::vector<MutuallyRecursiveContainerPairMV> {};
+
+namespace pybind11 {
+namespace detail {
+template <typename SFINAE>
+struct recursive_container_traits<MutuallyRecursiveContainerPairMV, SFINAE> {
+    using type_to_check_recursively = recursive_bottom;
+};
+template <typename SFINAE>
+struct recursive_container_traits<MutuallyRecursiveContainerPairVM, SFINAE> {
+    using type_to_check_recursively = recursive_bottom;
+};
+} // namespace detail
+} // namespace pybind11
+
 TEST_SUBMODULE(stl_binders, m) {
     // test_vector_int
     py::bind_vector<std::vector<unsigned int>>(m, "VectorInt", py::buffer_protocol());
@@ -129,6 +167,12 @@ TEST_SUBMODULE(stl_binders, m) {
             m, "VectorUndeclStruct", py::buffer_protocol());
     });
 
+    // Bind recursive container types
+    py::bind_vector<RecursiveVector>(m, "RecursiveVector");
+    py::bind_map<RecursiveMap>(m, "RecursiveMap");
+    py::bind_map<MutuallyRecursiveContainerPairMV>(m, "MutuallyRecursiveContainerPairMV");
+    py::bind_vector<MutuallyRecursiveContainerPairVM>(m, "MutuallyRecursiveContainerPairVM");
+
     // The rest depends on numpy:
     try {
         py::module_::import("numpy");

tests/test_stl_binders.py

@@ -186,9 +186,9 @@ def test_map_string_double():
     um["ua"] = 1.1
     um["ub"] = 2.6
 
-    assert sorted(list(um)) == ["ua", "ub"]
+    assert sorted(um) == ["ua", "ub"]
     assert list(um.keys()) == list(um)
-    assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)]
+    assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)]
     assert list(zip(um.keys(), um.values())) == list(um.items())
     assert "UnorderedMapStringDouble" in str(um)
 
@@ -304,11 +304,11 @@ def test_map_delitem():
     um["ua"] = 1.1
     um["ub"] = 2.6
 
-    assert sorted(list(um)) == ["ua", "ub"]
-    assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)]
+    assert sorted(um) == ["ua", "ub"]
+    assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)]
     del um["ua"]
-    assert sorted(list(um)) == ["ub"]
-    assert sorted(list(um.items())) == [("ub", 2.6)]
+    assert sorted(um) == ["ub"]
+    assert sorted(um.items()) == [("ub", 2.6)]
 
 
 def test_map_view_types():
@@ -335,3 +335,21 @@ def test_map_view_types():
     assert type(unordered_map_string_double.items()) is items_type
     assert type(map_string_double_const.items()) is items_type
     assert type(unordered_map_string_double_const.items()) is items_type
+
+
+def test_recursive_vector():
+    recursive_vector = m.RecursiveVector()
+    recursive_vector.append(m.RecursiveVector())
+    recursive_vector[0].append(m.RecursiveVector())
+    recursive_vector[0].append(m.RecursiveVector())
+    # Can't use len() since test_stl_binders.cpp does not include stl.h,
+    # so the necessary conversion is missing
+    assert recursive_vector[0].count(m.RecursiveVector()) == 2
+
+
+def test_recursive_map():
+    recursive_map = m.RecursiveMap()
+    recursive_map[100] = m.RecursiveMap()
+    recursive_map[100][101] = m.RecursiveMap()
+    recursive_map[100][102] = m.RecursiveMap()
+    assert list(recursive_map[100].keys()) == [101, 102]

tests/test_type_caster_pyobject_ptr.cpp

@@ -0,0 +1,130 @@
+#include <pybind11/functional.h>
+#include <pybind11/stl.h>
+#include <pybind11/type_caster_pyobject_ptr.h>
+
+#include "pybind11_tests.h"
+
+#include <cstddef>
+#include <vector>
+
+namespace {
+
+std::vector<PyObject *> make_vector_pyobject_ptr(const py::object &ValueHolder) {
+    std::vector<PyObject *> vec_obj;
+    for (int i = 1; i < 3; i++) {
+        vec_obj.push_back(ValueHolder(i * 93).release().ptr());
+    }
+    // This vector now owns the refcounts.
+    return vec_obj;
+}
+
+} // namespace
+
+TEST_SUBMODULE(type_caster_pyobject_ptr, m) {
+    m.def("cast_from_pyobject_ptr", []() {
+        PyObject *ptr = PyLong_FromLongLong(6758L);
+        return py::cast(ptr, py::return_value_policy::take_ownership);
+    });
+    m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) {
+        auto rc1 = obj.ref_count();
+        auto *ptr = py::cast<PyObject *>(obj);
+        auto rc2 = obj.ref_count();
+        if (rc2 != rc1 + 1) {
+            return -1;
+        }
+        return 100 - py::reinterpret_steal<py::object>(ptr).attr("value").cast<int>();
+    });
+    m.def("cast_object_to_pyobject_ptr", [](py::object obj) {
+        py::handle hdl = obj;
+        auto rc1 = hdl.ref_count();
+        auto *ptr = py::cast<PyObject *>(std::move(obj));
+        auto rc2 = hdl.ref_count();
+        if (rc2 != rc1) {
+            return -1;
+        }
+        return 300 - py::reinterpret_steal<py::object>(ptr).attr("value").cast<int>();
+    });
+    m.def("cast_list_to_pyobject_ptr", [](py::list lst) {
+        // This is to cover types implicitly convertible to object.
+        py::handle hdl = lst;
+        auto rc1 = hdl.ref_count();
+        auto *ptr = py::cast<PyObject *>(std::move(lst));
+        auto rc2 = hdl.ref_count();
+        if (rc2 != rc1) {
+            return -1;
+        }
+        return 400 - static_cast<int>(py::len(py::reinterpret_steal<py::list>(ptr)));
+    });
+
+    m.def(
+        "return_pyobject_ptr",
+        []() { return PyLong_FromLongLong(2314L); },
+        py::return_value_policy::take_ownership);
+    m.def("pass_pyobject_ptr", [](PyObject *ptr) {
+        return 200 - py::reinterpret_borrow<py::object>(ptr).attr("value").cast<int>();
+    });
+
+    m.def("call_callback_with_object_return",
+          [](const std::function<py::object(int)> &cb, int value) { return cb(value); });
+    m.def(
+        "call_callback_with_pyobject_ptr_return",
+        [](const std::function<PyObject *(int)> &cb, int value) { return cb(value); },
+        py::return_value_policy::take_ownership);
+    m.def(
+        "call_callback_with_pyobject_ptr_arg",
+        [](const std::function<int(PyObject *)> &cb, py::handle obj) { return cb(obj.ptr()); },
+        py::arg("cb"), // This triggers return_value_policy::automatic_reference
+        py::arg("obj"));
+
+    m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) {
+        if (set_error) {
+            PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling.");
+        }
+        PyObject *ptr = nullptr;
+        py::cast(ptr);
+    });
+
+    m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() {
+        PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling.");
+        py::cast(Py_None);
+    });
+
+    m.def("pass_list_pyobject_ptr", [](const std::vector<PyObject *> &vec_obj) {
+        int acc = 0;
+        for (const auto &ptr : vec_obj) {
+            acc = acc * 1000 + py::reinterpret_borrow<py::object>(ptr).attr("value").cast<int>();
+        }
+        return acc;
+    });
+
+    m.def("return_list_pyobject_ptr_take_ownership",
+          make_vector_pyobject_ptr,
+          // Ownership is transferred one-by-one when the vector is converted to a Python list.
+          py::return_value_policy::take_ownership);
+
+    m.def("return_list_pyobject_ptr_reference",
+          make_vector_pyobject_ptr,
+          // Ownership is not transferred.
+          py::return_value_policy::reference);
+
+    m.def("dec_ref_each_pyobject_ptr", [](const std::vector<PyObject *> &vec_obj) {
+        std::size_t i = 0;
+        for (; i < vec_obj.size(); i++) {
+            py::handle h(vec_obj[i]);
+            if (static_cast<std::size_t>(h.ref_count()) < 2) {
+                break; // Something is badly wrong.
+            }
+            h.dec_ref();
+        }
+        return i;
+    });
+
+    m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {});
+
+#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing.
+    {
+        PyObject *ptr = nullptr;
+        (void) py::cast(*ptr);
+    }
+#endif
+}

tests/test_type_caster_pyobject_ptr.py

@@ -0,0 +1,104 @@
+import pytest
+
+from pybind11_tests import type_caster_pyobject_ptr as m
+
+
+# For use as a temporary user-defined object, to maximize sensitivity of the tests below.
+class ValueHolder:
+    def __init__(self, value):
+        self.value = value
+
+
+def test_cast_from_pyobject_ptr():
+    assert m.cast_from_pyobject_ptr() == 6758
+
+
+def test_cast_handle_to_pyobject_ptr():
+    assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76
+
+
+def test_cast_object_to_pyobject_ptr():
+    assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257
+
+
+def test_cast_list_to_pyobject_ptr():
+    assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395
+
+
+def test_return_pyobject_ptr():
+    assert m.return_pyobject_ptr() == 2314
+
+
+def test_pass_pyobject_ptr():
+    assert m.pass_pyobject_ptr(ValueHolder(82)) == 118
+
+
+@pytest.mark.parametrize(
+    "call_callback",
+    [
+        m.call_callback_with_object_return,
+        m.call_callback_with_pyobject_ptr_return,
+    ],
+)
+def test_call_callback_with_object_return(call_callback):
+    def cb(value):
+        if value < 0:
+            raise ValueError("Raised from cb")
+        return ValueHolder(1000 - value)
+
+    assert call_callback(cb, 287).value == 713
+
+    with pytest.raises(ValueError, match="^Raised from cb$"):
+        call_callback(cb, -1)
+
+
+def test_call_callback_with_pyobject_ptr_arg():
+    def cb(obj):
+        return 300 - obj.value
+
+    assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261
+
+
+@pytest.mark.parametrize("set_error", [True, False])
+def test_cast_to_python_nullptr(set_error):
+    expected = {
+        True: r"^Reflective of healthy error handling\.$",
+        False: (
+            r"^Internal error: pybind11::error_already_set called "
+            r"while Python error indicator not set\.$"
+        ),
+    }[set_error]
+    with pytest.raises(RuntimeError, match=expected):
+        m.cast_to_pyobject_ptr_nullptr(set_error)
+
+
+def test_cast_to_python_non_nullptr_with_error_set():
+    with pytest.raises(SystemError) as excinfo:
+        m.cast_to_pyobject_ptr_non_nullptr_with_error_set()
+    assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()"
+    assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling."
+
+
+def test_pass_list_pyobject_ptr():
+    acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)])
+    assert acc == 842452
+
+
+def test_return_list_pyobject_ptr_take_ownership():
+    vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder)
+    assert [e.value for e in vec_obj] == [93, 186]
+
+
+def test_return_list_pyobject_ptr_reference():
+    vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder)
+    assert [e.value for e in vec_obj] == [93, 186]
+    # Commenting out the next `assert` will leak the Python references.
+    # An easy way to see evidence of the leaks:
+    # Insert `while True:` as the first line of this function and monitor the
+    # process RES (Resident Memory Size) with the Unix top command.
+    assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2
+
+
+def test_type_caster_name_via_incompatible_function_arguments_type_error():
+    with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"):
+        m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202))

tests/test_unnamed_namespace_a.cpp

@@ -0,0 +1,38 @@
+#include "pybind11_tests.h"
+
+namespace {
+struct any_struct {};
+} // namespace
+
+TEST_SUBMODULE(unnamed_namespace_a, m) {
+    if (py::detail::get_type_info(typeid(any_struct)) == nullptr) {
+        py::class_<any_struct>(m, "unnamed_namespace_a_any_struct");
+    } else {
+        m.attr("unnamed_namespace_a_any_struct") = py::none();
+    }
+    m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION;
+    m.attr("defined_WIN32_or__WIN32") =
+#if defined(WIN32) || defined(_WIN32)
+        true;
+#else
+        false;
+#endif
+    m.attr("defined___clang__") =
+#if defined(__clang__)
+        true;
+#else
+        false;
+#endif
+    m.attr("defined__LIBCPP_VERSION") =
+#if defined(_LIBCPP_VERSION)
+        true;
+#else
+        false;
+#endif
+    m.attr("defined___GLIBCXX__") =
+#if defined(__GLIBCXX__)
+        true;
+#else
+        false;
+#endif
+}

tests/test_unnamed_namespace_a.py

@@ -0,0 +1,34 @@
+import pytest
+
+from pybind11_tests import unnamed_namespace_a as m
+from pybind11_tests import unnamed_namespace_b as mb
+
+XFAIL_CONDITION = (
+    "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))"
+    " or "
+    "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32"
+    " and "
+    "(m.defined___clang__ or m.defined__LIBCPP_VERSION))"
+)
+XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319"
+
+
+@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False)
+@pytest.mark.parametrize(
+    "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct]
+)
+def test_have_class_any_struct(any_struct):
+    assert any_struct is not None
+
+
+def test_have_at_least_one_class_any_struct():
+    assert (
+        m.unnamed_namespace_a_any_struct is not None
+        or mb.unnamed_namespace_b_any_struct is not None
+    )
+
+
+@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True)
+def test_have_both_class_any_struct():
+    assert m.unnamed_namespace_a_any_struct is not None
+    assert mb.unnamed_namespace_b_any_struct is not None

tests/test_unnamed_namespace_b.cpp

@@ -0,0 +1,13 @@
+#include "pybind11_tests.h"
+
+namespace {
+struct any_struct {};
+} // namespace
+
+TEST_SUBMODULE(unnamed_namespace_b, m) {
+    if (py::detail::get_type_info(typeid(any_struct)) == nullptr) {
+        py::class_<any_struct>(m, "unnamed_namespace_b_any_struct");
+    } else {
+        m.attr("unnamed_namespace_b_any_struct") = py::none();
+    }
+}

tests/test_unnamed_namespace_b.py

@@ -0,0 +1,5 @@
+from pybind11_tests import unnamed_namespace_b as m
+
+
+def test_have_attr_any_struct():
+    assert hasattr(m, "unnamed_namespace_b_any_struct")

tests/test_vector_unique_ptr_member.cpp

@@ -0,0 +1,54 @@
+#include "pybind11_tests.h"
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+namespace pybind11_tests {
+namespace vector_unique_ptr_member {
+
+struct DataType {};
+
+// Reduced from a use case in the wild.
+struct VectorOwner {
+    static std::unique_ptr<VectorOwner> Create(std::size_t num_elems) {
+        return std::unique_ptr<VectorOwner>(
+            new VectorOwner(std::vector<std::unique_ptr<DataType>>(num_elems)));
+    }
+
+    std::size_t data_size() const { return data_.size(); }
+
+private:
+    explicit VectorOwner(std::vector<std::unique_ptr<DataType>> data) : data_(std::move(data)) {}
+
+    const std::vector<std::unique_ptr<DataType>> data_;
+};
+
+} // namespace vector_unique_ptr_member
+} // namespace pybind11_tests
+
+namespace pybind11 {
+namespace detail {
+
+template <>
+struct is_copy_constructible<pybind11_tests::vector_unique_ptr_member::VectorOwner>
+    : std::false_type {};
+
+template <>
+struct is_move_constructible<pybind11_tests::vector_unique_ptr_member::VectorOwner>
+    : std::false_type {};
+
+} // namespace detail
+} // namespace pybind11
+
+using namespace pybind11_tests::vector_unique_ptr_member;
+
+py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); }
+
+TEST_SUBMODULE(vector_unique_ptr_member, m) {
+    py::class_<VectorOwner>(m, "VectorOwner")
+        .def_static("Create", &VectorOwner::Create)
+        .def("data_size", &VectorOwner::data_size);
+
+    m.def("py_cast_VectorOwner_ptr", py_cast_VectorOwner_ptr);
+}

tests/test_vector_unique_ptr_member.py

@@ -0,0 +1,14 @@
+import pytest
+
+from pybind11_tests import vector_unique_ptr_member as m
+
+
+@pytest.mark.parametrize("num_elems", range(3))
+def test_create(num_elems):
+    vo = m.VectorOwner.Create(num_elems)
+    assert vo.data_size() == num_elems
+
+
+def test_cast():
+    vo = m.VectorOwner.Create(0)
+    assert m.py_cast_VectorOwner_ptr(vo) is vo

tests/test_virtual_functions.py

@@ -192,8 +192,7 @@ def test_move_support():
     class NCVirtExt(m.NCVirt):
         def get_noncopyable(self, a, b):
             # Constructs and returns a new instance:
-            nc = m.NonCopyable(a * a, b * b)
-            return nc
+            return m.NonCopyable(a * a, b * b)
 
         def get_movable(self, a, b):
             # Return a referenced copy
@@ -256,7 +255,7 @@ def test_dispatch_issue(msg):
     assert m.dispatch_issue_go(b) == "Yay.."
 
 
-def test_recursive_dispatch_issue(msg):
+def test_recursive_dispatch_issue():
     """#3357: Recursive dispatch fails to find python function override"""
 
     class Data(m.Data):
@@ -269,7 +268,7 @@ def test_recursive_dispatch_issue(msg):
             # lambda is a workaround, which adds extra frame to the
             # current CPython thread. Removing lambda reveals the bug
             # [https://github.com/pybind/pybind11/issues/3357]
-            (lambda: visitor(Data(first.value + second.value)))()
+            (lambda: visitor(Data(first.value + second.value)))()  # noqa: PLC3002
 
     class StoreResultVisitor:
         def __init__(self):

tools/FindCatch.cmake

@@ -36,10 +36,14 @@ endfunction()
 function(_download_catch version destination_dir)
   message(STATUS "Downloading catch v${version}...")
   set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp)
-  file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status)
+  file(
+    DOWNLOAD ${url} "${destination_dir}/catch.hpp"
+    STATUS status
+    LOG log)
   list(GET status 0 error)
   if(error)
-    message(FATAL_ERROR "Could not download ${url}")
+    string(REPLACE "\n" "\n  " log "  ${log}")
+    message(FATAL_ERROR "Could not download URL:\n" "  ${url}\n" "Log:\n" "${log}")
   endif()
   set(CATCH_INCLUDE_DIR
       "${destination_dir}"

tools/codespell_ignore_lines_from_errors.py

@@ -17,14 +17,18 @@ def run(args: List[str]) -> None:
     assert len(args) == 1, "codespell_errors.txt"
     cache = {}
     done = set()
-    for line in sorted(open(args[0]).read().splitlines()):
+    with open(args[0]) as f:
+        lines = f.read().splitlines()
+
+    for line in sorted(lines):
         i = line.find(" ==> ")
         if i > 0:
             flds = line[:i].split(":")
             if len(flds) >= 2:
                 filename, line_num = flds[:2]
                 if filename not in cache:
-                    cache[filename] = open(filename).read().splitlines()
+                    with open(filename) as f:
+                        cache[filename] = f.read().splitlines()
                 supp = cache[filename][int(line_num) - 1]
                 if supp not in done:
                     print(supp)

tools/pybind11Common.cmake

@@ -5,8 +5,8 @@ Adds the following targets::
     pybind11::pybind11 - link to headers and pybind11
     pybind11::module - Adds module links
     pybind11::embed - Adds embed links
-    pybind11::lto - Link time optimizations (manual selection)
-    pybind11::thin_lto - Link time optimizations (manual selection)
+    pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set)
+    pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set)
     pybind11::python_link_helper - Adds link to Python libraries
     pybind11::windows_extras - MSVC bigobj and mp for building multithreaded
     pybind11::opt_size - avoid optimizations that increase code size
@@ -20,7 +20,7 @@ Adds the following functions::
 
 # CMake 3.10 has an include_guard command, but we can't use that yet
 # include_guard(global) (pre-CMake 3.10)
-if(TARGET pybind11::lto)
+if(TARGET pybind11::pybind11)
   return()
 endif()
 
@@ -372,11 +372,13 @@ function(_pybind11_generate_lto target prefer_thin_lto)
   endif()
 endfunction()
 
+if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
   add_library(pybind11::lto IMPORTED INTERFACE ${optional_global})
   _pybind11_generate_lto(pybind11::lto FALSE)
 
   add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global})
   _pybind11_generate_lto(pybind11::thin_lto TRUE)
+endif()
 
 # ---------------------- pybind11_strip -----------------------------