Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -197,6 +197,35 @@ jobs:
         pytest tests/extra_setuptools
       if: "!(matrix.runs-on == 'windows-2022')"
 
+  manylinux:
+    name: Manylinux on 🐍 3.13t • GIL
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    container: quay.io/pypa/musllinux_1_2_x86_64:latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare venv
+        run: python3.13t -m venv .venv
+
+      - name: Install Python deps
+        run: .venv/bin/pip install -r tests/requirements.txt
+
+      - name: Configure C++11
+        run: >
+          cmake -S. -Bbuild
+          -DPYBIND11_WERROR=ON
+          -DDOWNLOAD_CATCH=ON
+          -DDOWNLOAD_EIGEN=ON
+          -DPython_ROOT_DIR=.venv
+
+      - name: Build C++11
+        run: cmake --build build -j2
+
+      - name: Python tests C++11
+        run: cmake --build build --target pytest -j2
 
   deadsnakes:
     strategy:

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.5
+  rev: v0.4.7
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]

CMakeLists.txt

@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
 
 if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
   add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
@@ -306,6 +307,7 @@ if(PYBIND11_INSTALL)
           tools/pybind11Common.cmake
           tools/pybind11Tools.cmake
           tools/pybind11NewTools.cmake
+          tools/pybind11GuessPythonExtSuffix.cmake
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
   if(NOT PYBIND11_EXPORT_NAME)

include/pybind11/detail/class.h

@@ -388,7 +388,11 @@ inline void clear_patients(PyObject *self) {
     auto *instance = reinterpret_cast<detail::instance *>(self);
     auto &internals = get_internals();
     auto pos = internals.patients.find(self);
-    assert(pos != internals.patients.end());
+
+    if (pos == internals.patients.end()) {
+        pybind11_fail("FATAL: Internal consistency check failed: Invalid clear_patients() call.");
+    }
+
     // Clearing the patients can cause more Python code to run, which
     // can invalidate the iterator. Extract the vector of patients
     // from the unordered_map first.

include/pybind11/pybind11.h

@@ -3040,7 +3040,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
             PyObject *locals = PyEval_GetFrameLocals();
 #        else
             PyObject *locals = PyEval_GetLocals();
-            Py_INCREF(locals);
+            Py_XINCREF(locals);
 #        endif
             if (locals != nullptr) {
 #        if PY_VERSION_HEX >= 0x030b0000

include/pybind11/pytypes.h

@@ -183,7 +183,15 @@ public:
     str_attr_accessor doc() const;
 
     /// Return the object's current reference count
-    int ref_count() const { return static_cast<int>(Py_REFCNT(derived().ptr())); }
+    ssize_t ref_count() const {
+#ifdef PYPY_VERSION
+        // PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT
+        // Following pybind11 2.12.1 and older behavior and removing this part
+        return static_cast<ssize_t>(static_cast<int>(Py_REFCNT(derived().ptr())));
+#else
+        return Py_REFCNT(derived().ptr());
+#endif
+    }
 
     // TODO PYBIND11_DEPRECATED(
     //     "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()")
@@ -2175,6 +2183,11 @@ public:
             throw error_already_set();
         }
     }
+    void clear() /* py-non-const */ {
+        if (PyList_SetSlice(m_ptr, 0, PyList_Size(m_ptr), nullptr) == -1) {
+            throw error_already_set();
+        }
+    }
 };
 
 class args : public tuple {

tests/CMakeLists.txt

@@ -358,7 +358,7 @@ if(Boost_FOUND)
     add_library(Boost::headers IMPORTED INTERFACE)
     if(TARGET Boost::boost)
       # Classic FindBoost
-      set_property(TARGET Boost::boost PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
+      set_property(TARGET Boost::headers PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
     else()
       # Very old FindBoost, or newer Boost than CMake in older CMakes
       set_property(TARGET Boost::headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES

tests/extra_python_package/test_files.py

@@ -80,6 +80,7 @@ cmake_files = {
     "share/cmake/pybind11/pybind11Common.cmake",
     "share/cmake/pybind11/pybind11Config.cmake",
     "share/cmake/pybind11/pybind11ConfigVersion.cmake",
+    "share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
     "share/cmake/pybind11/pybind11NewTools.cmake",
     "share/cmake/pybind11/pybind11Targets.cmake",
     "share/cmake/pybind11/pybind11Tools.cmake",

tests/pybind11_tests.cpp

@@ -89,6 +89,8 @@ PYBIND11_MODULE(pybind11_tests, m) {
 #endif
     m.attr("cpp_std") = cpp_std();
     m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID;
+    // Free threaded Python uses UINT32_MAX for immortal objects.
+    m.attr("PYBIND11_REFCNT_IMMORTAL") = UINT32_MAX;
     m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") =
 #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
         true;

tests/test_class.py

@@ -3,7 +3,7 @@ from unittest import mock
 import pytest
 
 import env
-from pybind11_tests import ConstructorStats, UserType
+from pybind11_tests import PYBIND11_REFCNT_IMMORTAL, ConstructorStats, UserType
 from pybind11_tests import class_ as m
 
 
@@ -377,7 +377,9 @@ def test_class_refcount():
         refcount_3 = getrefcount(cls)
 
         assert refcount_1 == refcount_3
-        assert refcount_2 > refcount_1
+        assert (refcount_2 > refcount_1) or (
+            refcount_2 == refcount_1 == PYBIND11_REFCNT_IMMORTAL
+        )
 
 
 def test_reentrant_implicit_conversion_failure(msg):

tests/test_embed/CMakeLists.txt

@@ -7,6 +7,13 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STR
   return()
 endif()
 
+if(TARGET Python::Module AND NOT TARGET Python::Python)
+  message(STATUS "Skipping embed test since no embed libs found")
+  add_custom_target(cpptest) # Dummy target since embedding is not supported.
+  set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}")
+  return()
+endif()
+
 find_package(Catch 2.13.9)
 
 if(CATCH_FOUND)

tests/test_kwargs_and_defaults.cpp

@@ -150,10 +150,13 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
 
 // test_args_refcount
 // PyPy needs a garbage collection to get the reference count values to match CPython's behaviour
+// PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT, so truncate
 #ifdef PYPY_VERSION
 #    define GC_IF_NEEDED ConstructorStats::gc()
+#    define REFCNT(x) (int) Py_REFCNT(x)
 #else
 #    define GC_IF_NEEDED
+#    define REFCNT(x) Py_REFCNT(x)
 #endif
     m.def("arg_refcount_h", [](py::handle h) {
         GC_IF_NEEDED;
@@ -172,7 +175,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         py::tuple t(a.size());
         for (size_t i = 0; i < a.size(); i++) {
             // Use raw Python API here to avoid an extra, intermediate incref on the tuple item:
-            t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
+            t[i] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
         }
         return t;
     });
@@ -182,7 +185,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) {
         t[0] = o.ref_count();
         for (size_t i = 0; i < a.size(); i++) {
             // Use raw Python API here to avoid an extra, intermediate incref on the tuple item:
-            t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
+            t[i + 1] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i)));
         }
         return t;
     });

tests/test_kwargs_and_defaults.py

@@ -381,7 +381,7 @@ def test_args_refcount():
     arguments"""
     refcount = m.arg_refcount_h
 
-    myval = 54321
+    myval = object()
     expected = refcount(myval)
     assert m.arg_refcount_h(myval) == expected
     assert m.arg_refcount_o(myval) == expected + 1
@@ -420,6 +420,7 @@ def test_args_refcount():
     # for the `py::args`; in the previous case, we could simply inc_ref and pass on Python's input
     # tuple without having to inc_ref the individual elements, but here we can't, hence the extra
     # refs.
-    assert m.mixed_args_refcount(myval, myval, myval) == (exp3 + 3, exp3 + 3, exp3 + 3)
+    exp3_3 = exp3 + 3
+    assert m.mixed_args_refcount(myval, myval, myval) == (exp3_3, exp3_3, exp3_3)
 
     assert m.class_default_argument() == "<class 'decimal.Decimal'>"

tests/test_pytypes.cpp

@@ -135,6 +135,7 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("list_size_t", []() { return py::list{(py::size_t) 0}; });
     m.def("list_insert_ssize_t", [](py::list *l) { return l->insert((py::ssize_t) 1, 83); });
     m.def("list_insert_size_t", [](py::list *l) { return l->insert((py::size_t) 3, 57); });
+    m.def("list_clear", [](py::list *l) { l->clear(); });
     m.def("get_list", []() {
         py::list list;
         list.append("value");

tests/test_pytypes.py

@@ -65,6 +65,8 @@ def test_list(capture, doc):
     assert lins == [1, 83, 2]
     m.list_insert_size_t(lins)
     assert lins == [1, 83, 2, 57]
+    m.list_clear(lins)
+    assert lins == []
 
     with capture:
         lst = m.get_list()
@@ -631,7 +633,8 @@ def test_memoryview(method, args, fmt, expected_view):
     ],
 )
 def test_memoryview_refcount(method):
-    buf = b"\x0a\x0b\x0c\x0d"
+    # Avoiding a literal to avoid an immortal object in free-threaded builds
+    buf = "\x0a\x0b\x0c\x0d".encode("ascii")
     ref_before = sys.getrefcount(buf)
     view = method(buf)
     ref_after = sys.getrefcount(buf)

tools/FindPythonLibsNew.cmake

@@ -205,7 +205,7 @@ endif()
 # Make sure the Python has the same pointer-size as the chosen compiler
 # Skip if CMAKE_SIZEOF_VOID_P is not defined
 # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
-if(NOT CMAKE_CROSSCOMPILING
+if(NOT _PYBIND11_CROSSCOMPILING
    AND CMAKE_SIZEOF_VOID_P
    AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
   if(PythonLibsNew_FIND_REQUIRED)

tools/pybind11Common.cmake

@@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
     "${pybind11_INCLUDE_DIR}"
     CACHE INTERNAL "Include directory for pybind11 (Python not requested)")
 
+if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
+  set(_PYBIND11_CROSSCOMPILING
+      ON
+      CACHE INTERNAL "")
+else()
+  set(_PYBIND11_CROSSCOMPILING
+      OFF
+      CACHE INTERNAL "")
+endif()
+
 # --------------------- Shared targets ----------------------------
 
 # Build an interface library target:
@@ -195,7 +205,7 @@ endif()
 
 # --------------------- pybind11_find_import -------------------------------
 
-if(NOT _pybind11_nopython)
+if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
   # Check to see if modules are importable. Use REQUIRED to force an error if
   # one of the modules is not found. <package_name>_FOUND will be set if the
   # package was found (underscores replace dashes if present). QUIET will hide

tools/pybind11GuessPythonExtSuffix.cmake

@@ -0,0 +1,86 @@
+cmake_minimum_required(VERSION 3.5)
+
+function(pybind11_guess_python_module_extension python)
+
+  # The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
+    message(
+      STATUS
+        "Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
+    )
+    set(PYTHON_MODULE_EXT_SUFFIX
+        "$ENV{SETUPTOOLS_EXT_SUFFIX}"
+        CACHE
+          STRING
+          "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
+  endif()
+  # If that didn't work, use the Python_SOABI variable:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
+    message(
+      STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
+    # The final extension depends on the system
+    set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+      set(_PY_BUILD_EXTENSION ".pyd")
+    endif()
+    # If the SOABI already has an extension, use it as the full suffix
+    # (used for debug versions of Python on Windows)
+    if(${python}_SOABI MATCHES "\\.")
+      set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
+      # If the SOABI is empty, this is usually a bug, but we generate a
+      # correct extension anyway, which is the best we can do
+    elseif("${${python}_SOABI}" STREQUAL "")
+      message(
+        WARNING
+          "${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
+      )
+      set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
+      # Otherwise, add the system-dependent extension to it
+    else()
+      set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
+    endif()
+  endif()
+
+  # If we could not deduce the extension suffix, unset the results:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
+    unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
+    unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
+    unset(PYTHON_IS_DEBUG PARENT_SCOPE)
+    return()
+  endif()
+
+  # Sanity checks:
+  if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
+                              OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
+    message(
+      WARNING
+        "Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
+    )
+  endif()
+
+  # Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
+  get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+  get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)
+
+  # Try to deduce the debug ABI from the extension suffix:
+  if(NOT DEFINED _PYTHON_IS_DEBUG)
+    if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
+       OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
+      set(_PYTHON_IS_DEBUG On)
+    else()
+      set(_PYTHON_IS_DEBUG Off)
+    endif()
+  endif()
+
+  # Return results
+  set(PYTHON_MODULE_DEBUG_POSTFIX
+      "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+      PARENT_SCOPE)
+  set(PYTHON_MODULE_EXTENSION
+      "${_PYTHON_MODULE_EXTENSION}"
+      PARENT_SCOPE)
+  set(PYTHON_IS_DEBUG
+      "${_PYTHON_IS_DEBUG}"
+      PARENT_SCOPE)
+
+endfunction()

tools/pybind11NewTools.cmake

@@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     set(Python_ROOT_DIR "$ENV{pythonLocation}")
   endif()
 
+  # Interpreter should not be found when cross-compiling
+  if(_PYBIND11_CROSSCOMPILING)
+    set(_pybind11_interp_component "")
+  else()
+    set(_pybind11_interp_component Interpreter)
+  endif()
+
   # Development.Module support (required for manylinux) started in 3.18
   if(CMAKE_VERSION VERSION_LESS 3.18)
     set(_pybind11_dev_component Development)
@@ -48,8 +55,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     endif()
   endif()
 
-  find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
+  find_package(
-                                              ${_pybind11_quiet} ${_pybind11_global_keyword})
+    Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component}
+                                   ${_pybind11_quiet} ${_pybind11_global_keyword})
 
   # If we are in submodule mode, export the Python targets to global targets.
   # If this behavior is not desired, FindPython _before_ pybind11.
@@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     if(TARGET Python::Python)
       set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
-    set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
+    if(TARGET Python::Interpreter)
+      set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
+    endif()
     if(TARGET Python::Module)
       set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
@@ -100,69 +110,89 @@ if(PYBIND11_MASTER_PROJECT)
   endif()
 endif()
 
-# If a user finds Python, they may forget to include the Interpreter component
+if(NOT _PYBIND11_CROSSCOMPILING)
-# and the following two steps require it. It is highly recommended by CMake
+  # If a user finds Python, they may forget to include the Interpreter component
-# when finding development libraries anyway, so we will require it.
+  # and the following two steps require it. It is highly recommended by CMake
-if(NOT DEFINED ${_Python}_EXECUTABLE)
+  # when finding development libraries anyway, so we will require it.
-  message(
+  if(NOT DEFINED ${_Python}_EXECUTABLE)
-    FATAL_ERROR
-      "${_Python} was found without the Interpreter component. Pybind11 requires this component.")
-
-endif()
-
-if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
-                                               PYBIND11_PYTHON_EXECUTABLE_LAST)
-  # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
-  unset(PYTHON_IS_DEBUG CACHE)
-  unset(PYTHON_MODULE_EXTENSION CACHE)
-endif()
-
-set(PYBIND11_PYTHON_EXECUTABLE_LAST
-    "${${_Python}_EXECUTABLE}"
-    CACHE INTERNAL "Python executable during the last CMake run")
-
-if(NOT DEFINED PYTHON_IS_DEBUG)
-  # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
-  execute_process(
-    COMMAND "${${_Python}_EXECUTABLE}" "-c"
-            "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
-    RESULT_VARIABLE _PYTHON_IS_DEBUG)
-  set(PYTHON_IS_DEBUG
-      "${_PYTHON_IS_DEBUG}"
-      CACHE INTERNAL "Python debug status")
-endif()
-
-# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
-# required for PyPy3 (as of 7.3.1)
-if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
-  execute_process(
-    COMMAND
-      "${${_Python}_EXECUTABLE}" "-c"
-      "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
-    OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
-    ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-  if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
     message(
-      FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
+      FATAL_ERROR
-                  "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
+        "${_Python} was found without the Interpreter component. Pybind11 requires this component."
     )
+
   endif()
 
-  # This needs to be available for the pybind11_extension function
+  if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
-  if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+                                                 PYBIND11_PYTHON_EXECUTABLE_LAST)
-    get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+    # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
-    set(PYTHON_MODULE_DEBUG_POSTFIX
+    unset(PYTHON_IS_DEBUG CACHE)
-        "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+    unset(PYTHON_MODULE_EXTENSION CACHE)
-        CACHE INTERNAL "")
   endif()
 
-  if(NOT DEFINED PYTHON_MODULE_EXTENSION)
+  set(PYBIND11_PYTHON_EXECUTABLE_LAST
-    get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+      "${${_Python}_EXECUTABLE}"
-    set(PYTHON_MODULE_EXTENSION
+      CACHE INTERNAL "Python executable during the last CMake run")
-        "${_PYTHON_MODULE_EXTENSION}"
+
-        CACHE INTERNAL "")
+  if(NOT DEFINED PYTHON_IS_DEBUG)
+    # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
+    execute_process(
+      COMMAND "${${_Python}_EXECUTABLE}" "-c"
+              "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
+      RESULT_VARIABLE _PYTHON_IS_DEBUG)
+    set(PYTHON_IS_DEBUG
+        "${_PYTHON_IS_DEBUG}"
+        CACHE INTERNAL "Python debug status")
+  endif()
+
+  # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
+  # required for PyPy3 (as of 7.3.1)
+  if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    execute_process(
+      COMMAND
+        "${${_Python}_EXECUTABLE}" "-c"
+        "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
+      OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
+      ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
+      message(
+        FATAL_ERROR
+          "pybind11 could not query the module file extension, likely the 'distutils'"
+          "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}")
+    endif()
+
+    # This needs to be available for the pybind11_extension function
+    if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+      get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+      set(PYTHON_MODULE_DEBUG_POSTFIX
+          "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+          CACHE INTERNAL "")
+    endif()
+
+    if(NOT DEFINED PYTHON_MODULE_EXTENSION)
+      get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+      set(PYTHON_MODULE_EXTENSION
+          "${_PYTHON_MODULE_EXTENSION}"
+          CACHE INTERNAL "")
+    endif()
+  endif()
+else()
+  if(NOT DEFINED PYTHON_IS_DEBUG
+     OR NOT DEFINED PYTHON_MODULE_EXTENSION
+     OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
+    pybind11_guess_python_module_extension("${_Python}")
+  endif()
+  # When cross-compiling, we cannot query the Python interpreter, so we require
+  # the user to set these variables explicitly.
+  if(NOT DEFINED PYTHON_IS_DEBUG
+     OR NOT DEFINED PYTHON_MODULE_EXTENSION
+     OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    message(
+      FATAL_ERROR
+        "When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \
+        variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)")
   endif()
 endif()
 

tools/test-pybind11GuessPythonExtSuffix.cmake

@@ -0,0 +1,161 @@
+cmake_minimum_required(VERSION 3.5)
+
+# Tests for pybind11_guess_python_module_extension
+# Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake`
+
+include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
+
+macro(expect_streq actual expected)
+  if(NOT "${actual}" STREQUAL "${expected}")
+    message(SEND_ERROR "Fail\n *** actual:   '${actual}'\n *** expected: '${expected}'")
+  endif()
+endmacro()
+
+macro(expect_false actual)
+  if("${actual}")
+    message(SEND_ERROR "Fail\n *** actual:   '${actual}'\n *** expected: false")
+  endif()
+endmacro()
+
+macro(expect_true actual)
+  if(NOT "${actual}")
+    message(SEND_ERROR "Fail\n *** actual:   '${actual}'\n *** expected: true")
+  endif()
+endmacro()
+
+# Windows
+set(CMAKE_SYSTEM_NAME "Windows")
+set(CMAKE_SHARED_MODULE_SUFFIX ".dll")
+
+set(Python3_SOABI "")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cp311-win_arm64")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cp311d-win_arm64")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311d-win_arm64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_true("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "pypy310-pp73-win_amd64")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-win_amd64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "_d.cp311-win_amd64.pyd") # This is a quirk of FindPython3
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_amd64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "_d")
+expect_true("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+unset(Python3_SOABI)
+set(ENV{SETUPTOOLS_EXT_SUFFIX} ".cp39-win_arm64.pyd") # Set by cibuildwheel
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp39-win_arm64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+unset(ENV{SETUPTOOLS_EXT_SUFFIX})
+
+set(Python3_SOABI "cp311-win_arm64")
+set(ENV{SETUPTOOLS_EXT_SUFFIX} "") # Should not be used
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+unset(ENV{SETUPTOOLS_EXT_SUFFIX})
+
+# macOS
+set(CMAKE_SYSTEM_NAME "Darwin")
+set(CMAKE_SHARED_MODULE_SUFFIX ".so")
+
+set(Python3_SOABI "")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cpython-312-darwin")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-darwin.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cpython-312d-darwin")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-darwin.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_true("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+# Linux
+set(CMAKE_SYSTEM_NAME "Linux")
+set(CMAKE_SHARED_MODULE_SUFFIX ".so")
+
+set(Python3_SOABI "")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cpython-312-arm-linux-gnueabihf")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-arm-linux-gnueabihf.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "cpython-312d-arm-linux-gnueabihf")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-arm-linux-gnueabihf.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_true("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "pypy310-pp73-x86_64-linux-gnu")
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-x86_64-linux-gnu.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_false("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)
+
+set(Python3_SOABI "pypy310d-pp73-x86_64-linux-gnu")
+# TODO: I'm not sure if this is the right SOABI for PyPy debug builds
+pybind11_guess_python_module_extension("Python3")
+expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310d-pp73-x86_64-linux-gnu.so")
+expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "")
+expect_true("${PYTHON_IS_DEBUG}")
+unset(PYTHON_MODULE_EXT_SUFFIX)
+unset(PYTHON_MODULE_EXT_SUFFIX CACHE)