fix: improve support for Python 3.11-dev (#3368)

* ci: support Python 3.11-dev

Also update 3.10 to final, better PyPy usage

* fix: use PyFrame_GetCode on Python 3.9+

* ci: some bitiness of pypy not supported on win

* chore: update CMake support to 3.22rc1 to quiet warning

* fix: use dev version of py to fix Py 3.11

* tests: print proper Eigen version

* ci: include pypy2, not sure why

* ci: avoid running on Python 3.11 for now

* ci: fix runs

* ci: simpler PyPy usage, drop unmaintained scipy + pypy index

* ci: only binary numpy, wait on pypy 3.8

* refactor: address review

.github/workflows/ci.yml

@@ -13,6 +13,9 @@ concurrency:
   group: test-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  PIP_ONLY_BINARY: numpy
+
 jobs:
   # This is the "main" test suite, which tests a large number of different
   # versions of default compilers and Python versions in GitHub Actions.
@@ -22,13 +25,14 @@ jobs:
       matrix:
         runs-on: [ubuntu-latest, windows-latest, macos-latest]
         python:
-        - 2.7
-        - 3.5
-        - 3.6
-        - 3.9
-        - 3.10-dev
-        - pypy2
-        - pypy3
+        - '2.7'
+        - '3.5'
+        - '3.6'
+        - '3.9'
+        - '3.10'
+        # - '3.11-dev'
+        - 'pypy-3.7-v7.3.5'
+        # - 'pypy-3.8'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -46,18 +50,8 @@ jobs:
             python: 3.6
             args: >
               -DPYBIND11_FINDPYTHON=ON
-
-        # These items will be removed from the build matrix, keys must match.
-        exclude:
-            # Currently 32bit only, and we build 64bit
-          - runs-on: windows-latest
-            python: pypy2
-          - runs-on: windows-latest
-            python: pypy3
-
-          # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine.
-          - runs-on: ubuntu-latest
-            python: pypy2
+          - runs-on: macos-latest
+            python: pypy-2.7
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}"
     runs-on: ${{ matrix.runs-on }}
@@ -93,7 +87,8 @@ jobs:
         key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }}
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Setup annotations on Linux
       if: runner.os == 'Linux'
@@ -117,7 +112,7 @@ jobs:
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
       run: cmake --build .  --target cpptest -j 2
 
     - name: Interface test C++11
@@ -145,7 +140,7 @@ jobs:
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
       run: cmake --build build2 --target cpptest
 
     # Third build - C++17 mode with unstable ABI
@@ -196,11 +191,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.9
+        # TODO: Fails on 3.10, investigate
+        - python-version: "3.9"
           python-debug: true
           valgrind: true
-        - python-version: 3.10-dev
-          python-debug: false
+      # - python-version: "3.11-dev"
+      #   python-debug: false
 
     name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64"
     runs-on: ubuntu-latest
@@ -244,7 +240,8 @@ jobs:
         sudo apt-get install libc6-dbg  # Needed by Valgrind
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Configure
       run: >
@@ -521,7 +518,7 @@ jobs:
     - name: Install dependencies
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
-        python3 -m pip install -r tests/requirements.txt --prefer-binary
+        python3 -m pip install -r tests/requirements.txt
 
     - name: Configure C++11
       run: |
@@ -612,7 +609,8 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Install dependencies
-      run: python3 -m pip install cmake -r tests/requirements.txt --prefer-binary
+      run: |
+        python3 -m pip install cmake -r tests/requirements.txt
 
     - name: VAR_BUILD_TYPE 7
       if: matrix.centos == 7
@@ -739,8 +737,7 @@ jobs:
         - 3.7
         - 3.8
         - 3.9
-        - pypy3
-        # TODO: fix hang on pypy2
+        - pypy-3.6
 
         include:
           - python: 3.9
@@ -769,7 +766,8 @@ jobs:
         arch: x86
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure ${{ matrix.args }}
@@ -815,7 +813,8 @@ jobs:
         toolset: 14.0
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure
@@ -868,7 +867,8 @@ jobs:
       uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure

.github/workflows/pip.yml

@@ -12,6 +12,9 @@ on:
     types:
     - published
 
+env:
+  PIP_ONLY_BINARY: numpy
+
 jobs:
   # This builds the sdists and wheels and makes sure the files are exactly as
   # expected. Using Windows and Python 2.7, since that is often the most
@@ -29,7 +32,8 @@ jobs:
         python-version: 2.7
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Python Packaging tests
       run: pytest tests/extra_python_package/
@@ -50,7 +54,8 @@ jobs:
         python-version: 3.8
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt build twine --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt build twine
 
     - name: Python Packaging tests
       run: pytest tests/extra_python_package/

CMakeLists.txt

@@ -7,13 +7,13 @@
 
 cmake_minimum_required(VERSION 3.4)
 
-# The `cmake_minimum_required(VERSION 3.4...3.21)` syntax does not work with
+# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with
 # some versions of VS that have a patched CMake 3.11. This forces us to emulate
 # the behavior using the following workaround:
-if(${CMAKE_VERSION} VERSION_LESS 3.21)
+if(${CMAKE_VERSION} VERSION_LESS 3.22)
   cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
 else()
-  cmake_policy(VERSION 3.21)
+  cmake_policy(VERSION 3.22)
 endif()
 
 # Extract project version from source

include/pybind11/detail/type_caster_base.h

@@ -468,12 +468,19 @@ PYBIND11_NOINLINE std::string error_string() {
         PyFrameObject *frame = trace->tb_frame;
         errorString += "\n\nAt:\n";
         while (frame) {
+#if PY_VERSION_HEX >= 0x03090000
+            PyCodeObject *f_code = PyFrame_GetCode(frame);
+#else
+            PyCodeObject *f_code = frame->f_code;
+            Py_INCREF(f_code);
+#endif
             int lineno = PyFrame_GetLineNumber(frame);
             errorString +=
-                "  " + handle(frame->f_code->co_filename).cast<std::string>() +
+                "  " + handle(f_code->co_filename).cast<std::string>() +
                 "(" + std::to_string(lineno) + "): " +
-                handle(frame->f_code->co_name).cast<std::string>() + "\n";
+                handle(f_code->co_name).cast<std::string>() + "\n";
             frame = frame->f_back;
+            Py_DECREF(f_code);
         }
     }
 #endif

include/pybind11/pybind11.h

@@ -2335,6 +2335,29 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
 #if !defined(PYPY_VERSION)
+
+#if PY_VERSION_HEX >= 0x03090000
+    PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
+    if (frame != nullptr) {
+        PyCodeObject *f_code = PyFrame_GetCode(frame);
+        // f_code is guaranteed to not be NULL
+        if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
+            PyObject* locals = PyEval_GetLocals();
+            if (locals != nullptr) {
+                PyObject *self_caller = dict_getitem(
+                    locals, PyTuple_GET_ITEM(f_code->co_varnames, 0)
+                );
+                if (self_caller == self.ptr()) {
+                    Py_DECREF(f_code);
+                    Py_DECREF(frame);
+                    return function();
+                }
+            }
+        }
+        Py_DECREF(f_code);
+        Py_DECREF(frame);
+    }
+#else
     PyFrameObject *frame = PyThreadState_Get()->frame;
     if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name
         && frame->f_code->co_argcount > 0) {
@@ -2344,6 +2367,8 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
         if (self_caller == self.ptr())
             return function();
     }
+#endif
+
 #else
     /* PyPy currently doesn't provide a detailed cpyext emulation of
        frame objects, so we have to emulate this using Python. This

noxfile.py

@@ -2,7 +2,7 @@ import nox
 
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
+PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
 @nox.session(reuse_venv=True)
@@ -20,7 +20,8 @@ def tests(session: nox.Session) -> None:
     Run the tests (requires a compiler).
     """
     tmpdir = session.create_tmp()
-    session.install("pytest", "cmake")
+    session.install("cmake")
+    session.install("-r", "tests/requirements.txt")
     session.run(
         "cmake",
         "-S",

tests/CMakeLists.txt

@@ -174,10 +174,14 @@ set(PYBIND11_CROSS_MODULE_GIL_TESTS test_gil_scoped.py)
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"
     CACHE STRING "Eigen repository to use for tests")
-# This hash is for 3.4.0, using a hash for security reasons
-set(PYBIND11_EIGEN_VERSION
-    "929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec"
-    CACHE STRING "Eigen version to use for tests")
+# Always use a hash for reconfigure speed and security reasons
+# Include the version number for pretty printing (keep in sync)
+set(PYBIND11_EIGEN_VERSION_AND_HASH
+    "3.4.0;929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec"
+    CACHE STRING "Eigen version to use for tests, format: VERSION;HASH")
+
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 0 PYBIND11_EIGEN_VERSION_STRING)
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH)
 
 # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but
 # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed"
@@ -196,16 +200,22 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
     FetchContent_Declare(
       eigen
       GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}"
-      GIT_TAG "${PYBIND11_EIGEN_VERSION}")
+      GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}")
 
     FetchContent_GetProperties(eigen)
     if(NOT eigen_POPULATED)
-      message(STATUS "Downloading Eigen")
+      message(
+        STATUS
+          "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}"
+      )
       FetchContent_Populate(eigen)
     endif()
 
     set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR})
     set(EIGEN3_FOUND TRUE)
+    # When getting locally, the version is not visible from a superprojet,
+    # so just force it.
+    set(EIGEN3_VERSION "${PYBIND11_EIGEN_VERSION_STRING}")
 
   else()
     find_package(Eigen3 3.2.7 QUIET CONFIG)

tests/requirements.txt

@@ -1,12 +1,12 @@
---extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/
-numpy==1.16.6; python_version<"3.6" and sys_platform!="win32"
-numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6"
-numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6"
-numpy==1.21.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10"
-numpy==1.21.2; platform_python_implementation!="PyPy" and sys_platform=="linux" and python_version=="3.10"
+numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" and platform_python_implementation!="PyPy"
+numpy==1.19.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.6"
+numpy==1.20.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7"
+numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
+numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7"
+py @ git+https://github.com/pytest-dev/py; python_version>="3.11"
 pytest==4.6.9; python_version<"3.5"
 pytest==6.1.2; python_version=="3.5"
 pytest==6.2.4; python_version>="3.6"
 pytest-timeout
-scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6"
-scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10"
+scipy==1.2.3; platform_python_implementation!="PyPy" and python_version<"3.6"
+scipy==1.5.4; platform_python_implementation!="PyPy" and python_version>="3.6" and python_version<"3.10"

tools/pybind11Tools.cmake

@@ -38,7 +38,7 @@ endif()
 
 # A user can set versions manually too
 set(Python_ADDITIONAL_VERSIONS
-    "3.10;3.9;3.8;3.7;3.6;3.5;3.4"
+    "3.11;3.10;3.9;3.8;3.7;3.6;3.5;3.4"
     CACHE INTERNAL "")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")