Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -10,6 +10,13 @@ on:
       - smart_holder
       - v*
 
+concurrency:
+  group: test-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PIP_ONLY_BINARY: numpy
+
 jobs:
   # This is the "main" test suite, which tests a large number of different
   # versions of default compilers and Python versions in GitHub Actions.
@@ -19,13 +26,14 @@ jobs:
       matrix:
         runs-on: [ubuntu-latest, windows-latest, macos-latest]
         python:
-        - 2.7
+        - '2.7'
-        - 3.5
+        - '3.5'
-        - 3.6
+        - '3.6'
-        - 3.9
+        - '3.9'
-        - 3.10-dev
+        - '3.10'
-        - pypy2
+        # - '3.11-dev'
-        - pypy3
+        - 'pypy-3.7-v7.3.5'
+        # - 'pypy-3.8'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the
@@ -43,18 +51,8 @@ jobs:
             python: 3.6
             args: >
               -DPYBIND11_FINDPYTHON=ON
-
+          - runs-on: macos-latest
-        # These items will be removed from the build matrix, keys must match.
+            python: pypy-2.7
-        exclude:
-            # Currently 32bit only, and we build 64bit
-          - runs-on: windows-latest
-            python: pypy2
-          - runs-on: windows-latest
-            python: pypy3
-
-          # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine.
-          - runs-on: ubuntu-latest
-            python: pypy2
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}"
     runs-on: ${{ matrix.runs-on }}
@@ -90,7 +88,8 @@ jobs:
         key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }}
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Setup annotations on Linux
       if: runner.os == 'Linux'
@@ -114,7 +113,7 @@ jobs:
 
     - name: C++11 tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
       run: cmake --build .  --target cpptest -j 2
 
     - name: Interface test C++11
@@ -142,7 +141,7 @@ jobs:
 
     - name: C++ tests
       # TODO: Figure out how to load the DLL on Python 3.8+
-      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))"
+      if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))"
       run: cmake --build build2 --target cpptest
 
     # Third build - C++17 mode with unstable ABI
@@ -193,11 +192,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.9
+        # TODO: Fails on 3.10, investigate
+        - python-version: "3.9"
           python-debug: true
           valgrind: true
-        - python-version: 3.10-dev
+      # - python-version: "3.11-dev"
-          python-debug: false
+      #   python-debug: false
 
     name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64"
     runs-on: ubuntu-latest
@@ -241,7 +241,8 @@ jobs:
         sudo apt-get install libc6-dbg  # Needed by Valgrind
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Configure
       run: >
@@ -518,7 +519,7 @@ jobs:
     - name: Install dependencies
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
-        python3 -m pip install -r tests/requirements.txt --prefer-binary
+        python3 -m pip install -r tests/requirements.txt
 
     - name: Configure C++11
       run: |
@@ -609,7 +610,8 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Install dependencies
-      run: python3 -m pip install cmake -r tests/requirements.txt --prefer-binary
+      run: |
+        python3 -m pip install cmake -r tests/requirements.txt
 
     - name: VAR_BUILD_TYPE 7
       if: matrix.centos == 7
@@ -736,8 +738,7 @@ jobs:
         - 3.7
         - 3.8
         - 3.9
-        - pypy3
+        - pypy-3.6
-        # TODO: fix hang on pypy2
 
         include:
           - python: 3.9
@@ -766,7 +767,8 @@ jobs:
         arch: x86
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure ${{ matrix.args }}
@@ -812,7 +814,8 @@ jobs:
         toolset: 14.0
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure
@@ -848,6 +851,10 @@ jobs:
             std: 17
             args: >
               -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR"
+          - python: 3.7
+            std: 17
+            args: >
+              -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR"
 
     steps:
     - uses: actions/checkout@v2
@@ -861,7 +868,8 @@ jobs:
       uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     # First build - C++11 mode and inplace
     - name: Configure
@@ -893,7 +901,8 @@ jobs:
           - { sys: mingw64, env: x86_64 }
           - { sys: mingw32, env: i686 }
     steps:
-    - uses: msys2/setup-msys2@v2
+    # Force version because of https://github.com/msys2/setup-msys2/issues/167
+    - uses: msys2/setup-msys2@v2.4.2
       with:
         msystem: ${{matrix.sys}}
         install: >-

.github/workflows/pip.yml

@@ -13,6 +13,9 @@ on:
     types:
     - published
 
+env:
+  PIP_ONLY_BINARY: numpy
+
 jobs:
   # This builds the sdists and wheels and makes sure the files are exactly as
   # expected. Using Windows and Python 2.7, since that is often the most
@@ -30,7 +33,8 @@ jobs:
         python-version: 2.7
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt
 
     - name: Python Packaging tests
       run: pytest tests/extra_python_package/
@@ -51,7 +55,8 @@ jobs:
         python-version: 3.8
 
     - name: Prepare env
-      run: python -m pip install -r tests/requirements.txt build twine --prefer-binary
+      run: |
+        python -m pip install -r tests/requirements.txt build twine
 
     - name: Python Packaging tests
       run: pytest tests/extra_python_package/

.pre-commit-config.yaml

@@ -63,12 +63,6 @@ repos:
   - id: remove-tabs
     exclude: (^docs/.*|\.patch)?$
 
-# Autoremoves unused imports
-- repo: https://github.com/hadialqattan/pycln
-  rev: v1.0.3
-  hooks:
-  - id: pycln
-
 - repo: https://github.com/pre-commit/pygrep-hooks
   rev: v1.9.0
   hooks:
@@ -90,7 +84,7 @@ repos:
     exclude: ^(docs/.*|tools/.*)$
 
 - repo: https://github.com/asottile/yesqa
-  rev: v1.2.3
+  rev: v1.3.0
   hooks:
   - id: yesqa
     additional_dependencies: *flake8_dependencies

CMakeLists.txt

@@ -7,13 +7,13 @@
 
 cmake_minimum_required(VERSION 3.4)
 
-# The `cmake_minimum_required(VERSION 3.4...3.21)` syntax does not work with
+# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with
 # some versions of VS that have a patched CMake 3.11. This forces us to emulate
 # the behavior using the following workaround:
-if(${CMAKE_VERSION} VERSION_LESS 3.21)
+if(${CMAKE_VERSION} VERSION_LESS 3.22)
   cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
 else()
-  cmake_policy(VERSION 3.21)
+  cmake_policy(VERSION 3.22)
 endif()
 
 # Extract project version from source

docs/advanced/exceptions.rst

@@ -56,7 +56,9 @@ at its exception handler.
 +--------------------------------------+--------------------------------------+
 | :class:`pybind11::buffer_error`      | ``BufferError``                      |
 +--------------------------------------+--------------------------------------+
-| :class:`pybind11::import_error`      | ``import_error``                     |
+| :class:`pybind11::import_error`      | ``ImportError``                      |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::attribute_error`   | ``AttributeError``                   |
 +--------------------------------------+--------------------------------------+
 | Any other exception                  | ``RuntimeError``                     |
 +--------------------------------------+--------------------------------------+

docs/advanced/pycpp/object.rst

@@ -41,24 +41,17 @@ A tuple of python objects can be instantiated using :func:`py::make_tuple`:
 Each element is converted to a supported Python type.
 
 A `simple namespace`_ can be instantiated using
-:func:`py::make_simple_namespace`:
 
 .. code-block:: cpp
 
     using namespace pybind11::literals;  // to bring in the `_a` literal
-    py::object ns = py::make_simple_namespace("spam"_a=py::none(), "eggs"_a=42);
+    py::object SimpleNamespace = py::module_::import("types").attr("SimpleNamespace");
+    py::object ns = SimpleNamespace("spam"_a=py::none(), "eggs"_a=42);
 
 Attributes on a namespace can be modified with the :func:`py::delattr`,
 :func:`py::getattr`, and :func:`py::setattr` functions. Simple namespaces can
 be useful as lightweight stand-ins for class instances.
 
-.. note::
-
-    ``make_simple_namespace`` is not available in Python 2.
-
-.. versionchanged:: 2.8
-    ``make_simple_namespace`` added.
-
 .. _simple namespace: https://docs.python.org/3/library/types.html#types.SimpleNamespace
 
 .. _casting_back_and_forth:

docs/changelog.rst

@@ -10,6 +10,10 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning
 IN DEVELOPMENT
 --------------
 
+* The simple namespace creation shortcut added in 2.8.0 was deprecated due to
+  usage of CPython internal API, and will be removed soon. Use
+  ``py::module_::import("types").attr("SimpleNamespace")``.
+  `#3374 <https://github.com/pybinyyd/pybind11/pull/3374>`_
 
 
 v2.8.0 (Oct 4, 2021)
@@ -25,10 +29,10 @@ New features:
   ``register_local_exception_translator(ExceptionTranslator&& translator)``
   instead of  ``register_exception_translator(ExceptionTranslator&&
   translator)`` to keep your exception remapping code local to the module.
-  `#2650 <https://github.com/pybind/pybind11/pull/2650>`_
+  `#2650 <https://github.com/pybinyyd/pybind11/pull/2650>`_
 
 * Add ``make_simple_namespace`` function for instantiating Python
-  ``SimpleNamespace`` objects.
+  ``SimpleNamespace`` objects. **Deprecated in 2.8.1.**
   `#2840 <https://github.com/pybind/pybind11/pull/2840>`_
 
 * ``pybind11::scoped_interpreter`` and ``initialize_interpreter`` have new

docs/upgrade.rst

@@ -8,7 +8,17 @@ to a new version. But it goes into more detail. This includes things like
 deprecated APIs and their replacements, build system changes, general code
 modernization and other useful information.
 
-.. _upgrade-guide-2.6:
+.. _upgrade-guide-2.9:
+
+v2.9
+====
+
+* Any usage of the recently added ``py::make_simple_namespace`` should be
+  converted to using ``py::module_::import("types").attr("SimpleNamespace")``
+  instead.
+
+
+.. _upgrade-guide-2.7:
 
 v2.7
 ====
@@ -34,6 +44,7 @@ to be common:
   careful review and custom fixes.
 
 
+.. _upgrade-guide-2.6:
 
 v2.6
 ====

include/pybind11/cast.h

@@ -1063,16 +1063,6 @@ template <return_value_policy policy = return_value_policy::automatic_reference,
     return result;
 }
 
-#if PY_VERSION_HEX >= 0x03030000
-template <typename... Args,
-          typename = detail::enable_if_t<args_are_all_keyword_or_ds<Args...>()>>
-object make_simple_namespace(Args&&... args_) {
-    PyObject *ns = _PyNamespace_New(dict(std::forward<Args>(args_)...).ptr());
-    if (!ns) throw error_already_set();
-    return reinterpret_steal<object>(ns);
-}
-#endif
-
 /// \ingroup annotations
 /// Annotation for arguments
 struct arg {

include/pybind11/detail/common.h

@@ -831,6 +831,7 @@ PYBIND11_RUNTIME_EXCEPTION(value_error, PyExc_ValueError)
 PYBIND11_RUNTIME_EXCEPTION(type_error, PyExc_TypeError)
 PYBIND11_RUNTIME_EXCEPTION(buffer_error, PyExc_BufferError)
 PYBIND11_RUNTIME_EXCEPTION(import_error, PyExc_ImportError)
+PYBIND11_RUNTIME_EXCEPTION(attribute_error, PyExc_AttributeError)
 PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybind11::cast or handle::call fail due to a type casting error
 PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally
 

include/pybind11/detail/type_caster_base.h

@@ -468,12 +468,19 @@ PYBIND11_NOINLINE std::string error_string() {
         PyFrameObject *frame = trace->tb_frame;
         errorString += "\n\nAt:\n";
         while (frame) {
+#if PY_VERSION_HEX >= 0x03090000
+            PyCodeObject *f_code = PyFrame_GetCode(frame);
+#else
+            PyCodeObject *f_code = frame->f_code;
+            Py_INCREF(f_code);
+#endif
             int lineno = PyFrame_GetLineNumber(frame);
             errorString +=
-                "  " + handle(frame->f_code->co_filename).cast<std::string>() +
+                "  " + handle(f_code->co_filename).cast<std::string>() +
                 "(" + std::to_string(lineno) + "): " +
-                handle(frame->f_code->co_name).cast<std::string>() + "\n";
+                handle(f_code->co_name).cast<std::string>() + "\n";
             frame = frame->f_back;
+            Py_DECREF(f_code);
         }
     }
 #endif

include/pybind11/functional.h

@@ -69,7 +69,7 @@ public:
         // ensure GIL is held during functor destruction
         struct func_handle {
             function f;
-#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17) && PY_MAJOR_VERSION < 3)
+#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17))
             // This triggers a syntax error under very special conditions (very weird indeed).
             explicit
 #endif

include/pybind11/numpy.h

@@ -518,7 +518,7 @@ public:
     }
 
     /// Single-character for dtype's type.
-    /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'd'.
+    /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'l'.
     char char_() const {
         // Note: The signature, `dtype::char_` follows the naming of NumPy's
         // public Python API (i.e., ``dtype.char``), rather than its internal

include/pybind11/pybind11.h

@@ -1126,6 +1126,15 @@ inline dict globals() {
     return reinterpret_borrow<dict>(p ? p : module_::import("__main__").attr("__dict__").ptr());
 }
 
+#if PY_VERSION_HEX >= 0x03030000
+template <typename... Args,
+          typename = detail::enable_if_t<args_are_all_keyword_or_ds<Args...>()>>
+PYBIND11_DEPRECATED("make_simple_namespace should be replaced with py::module_::import(\"types\").attr(\"SimpleNamespace\") ")
+object make_simple_namespace(Args&&... args_) {
+    return module_::import("types").attr("SimpleNamespace")(std::forward<Args>(args_)...);
+}
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(detail)
 /// Generic support for creating new Python heap types
 class generic_type : public object {
@@ -2440,6 +2449,29 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
 #if !defined(PYPY_VERSION)
+
+#if PY_VERSION_HEX >= 0x03090000
+    PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
+    if (frame != nullptr) {
+        PyCodeObject *f_code = PyFrame_GetCode(frame);
+        // f_code is guaranteed to not be NULL
+        if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
+            PyObject* locals = PyEval_GetLocals();
+            if (locals != nullptr) {
+                PyObject *self_caller = dict_getitem(
+                    locals, PyTuple_GET_ITEM(f_code->co_varnames, 0)
+                );
+                if (self_caller == self.ptr()) {
+                    Py_DECREF(f_code);
+                    Py_DECREF(frame);
+                    return function();
+                }
+            }
+        }
+        Py_DECREF(f_code);
+        Py_DECREF(frame);
+    }
+#else
     PyFrameObject *frame = PyThreadState_Get()->frame;
     if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name
         && frame->f_code->co_argcount > 0) {
@@ -2449,6 +2481,8 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
         if (self_caller == self.ptr())
             return function();
     }
+#endif
+
 #else
     /* PyPy currently doesn't provide a detailed cpyext emulation of
        frame objects, so we have to emulate this using Python. This

include/pybind11/stl.h

@@ -245,17 +245,17 @@ template <typename Key, typename Value, typename Hash, typename Equal, typename
   : map_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>, Key, Value> { };
 
 // This type caster is intended to be used for std::optional and std::experimental::optional
-template<typename T> struct optional_caster {
+template<typename Type, typename Value = typename Type::value_type> struct optional_caster {
-    using value_conv = make_caster<typename T::value_type>;
+    using value_conv = make_caster<Value>;
 
-    template <typename T_>
+    template <typename T>
-    static handle cast(T_ &&src, return_value_policy policy, handle parent) {
+    static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!src)
             return none().inc_ref();
         if (!std::is_lvalue_reference<T>::value) {
-            policy = return_value_policy_override<T>::policy(policy);
+            policy = return_value_policy_override<Value>::policy(policy);
         }
-        return value_conv::cast(*std::forward<T_>(src), policy, parent);
+        return value_conv::cast(*std::forward<T>(src), policy, parent);
     }
 
     bool load(handle src, bool convert) {
@@ -269,11 +269,11 @@ template<typename T> struct optional_caster {
         if (!inner_caster.load(src, convert))
             return false;
 
-        value.emplace(cast_op<typename T::value_type &&>(std::move(inner_caster)));
+        value.emplace(cast_op<Value &&>(std::move(inner_caster)));
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, _("Optional[") + value_conv::name + _("]"));
 };
 
 #if defined(PYBIND11_HAS_OPTIONAL)

noxfile.py

@@ -2,7 +2,7 @@ import nox
 
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
+PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
 @nox.session(reuse_venv=True)
@@ -20,7 +20,8 @@ def tests(session: nox.Session) -> None:
     Run the tests (requires a compiler).
     """
     tmpdir = session.create_tmp()
-    session.install("pytest", "cmake")
+    session.install("cmake")
+    session.install("-r", "tests/requirements.txt")
     session.run(
         "cmake",
         "-S",

tests/CMakeLists.txt

@@ -192,10 +192,14 @@ set(PYBIND11_CLASS_SH_MODULE_LOCAL_TESTS test_class_sh_module_local.py)
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"
     CACHE STRING "Eigen repository to use for tests")
-# This hash is for 3.4.0, using a hash for security reasons
+# Always use a hash for reconfigure speed and security reasons
-set(PYBIND11_EIGEN_VERSION
+# Include the version number for pretty printing (keep in sync)
-    "929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec"
+set(PYBIND11_EIGEN_VERSION_AND_HASH
-    CACHE STRING "Eigen version to use for tests")
+    "3.4.0;929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec"
+    CACHE STRING "Eigen version to use for tests, format: VERSION;HASH")
+
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 0 PYBIND11_EIGEN_VERSION_STRING)
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH)
 
 # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but
 # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed"
@@ -214,16 +218,22 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
     FetchContent_Declare(
       eigen
       GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}"
-      GIT_TAG "${PYBIND11_EIGEN_VERSION}")
+      GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}")
 
     FetchContent_GetProperties(eigen)
     if(NOT eigen_POPULATED)
-      message(STATUS "Downloading Eigen")
+      message(
+        STATUS
+          "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}"
+      )
       FetchContent_Populate(eigen)
     endif()
 
     set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR})
     set(EIGEN3_FOUND TRUE)
+    # When getting locally, the version is not visible from a superprojet,
+    # so just force it.
+    set(EIGEN3_VERSION "${PYBIND11_EIGEN_VERSION_STRING}")
 
   else()
     find_package(Eigen3 3.2.7 QUIET CONFIG)
@@ -274,7 +284,9 @@ if(Boost_FOUND)
 endif()
 
 # Check if we need to add -lstdc++fs or -lc++fs or nothing
-if(MSVC)
+if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD LESS 17)
+  set(STD_FS_NO_LIB_NEEDED TRUE)
+elseif(MSVC)
   set(STD_FS_NO_LIB_NEEDED TRUE)
 else()
   file(
@@ -304,7 +316,7 @@ elseif(${STD_FS_NEEDS_CXXFS})
 elseif(${STD_FS_NO_LIB_NEEDED})
   set(STD_FS_LIB "")
 else()
-  message(WARNING "Unknown compiler - not passing -lstdc++fs")
+  message(WARNING "Unknown C++17 compiler - not passing -lstdc++fs")
   set(STD_FS_LIB "")
 endif()
 

tests/requirements.txt

@@ -1,12 +1,12 @@
---extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/
+numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" and platform_python_implementation!="PyPy"
-numpy==1.16.6; python_version<"3.6" and sys_platform!="win32"
+numpy==1.19.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.6"
-numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6"
+numpy==1.20.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7"
-numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6"
+numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
-numpy==1.21.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10"
+numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7"
-numpy==1.21.2; platform_python_implementation!="PyPy" and sys_platform=="linux" and python_version=="3.10"
+py @ git+https://github.com/pytest-dev/py; python_version>="3.11"
 pytest==4.6.9; python_version<"3.5"
 pytest==6.1.2; python_version=="3.5"
 pytest==6.2.4; python_version>="3.6"
 pytest-timeout
-scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6"
+scipy==1.2.3; platform_python_implementation!="PyPy" and python_version<"3.6"
-scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10"
+scipy==1.5.4; platform_python_implementation!="PyPy" and python_version>="3.6" and python_version<"3.10"

tests/test_pytypes.cpp

@@ -84,7 +84,7 @@ TEST_SUBMODULE(pytypes, m) {
 #if PY_VERSION_HEX >= 0x03030000
     // test_simple_namespace
     m.def("get_simple_namespace", []() {
-        auto ns = py::make_simple_namespace("attr"_a=42, "x"_a="foo", "wrong"_a=1);
+        auto ns = py::module_::import("types").attr("SimpleNamespace")("attr"_a=42, "x"_a="foo", "wrong"_a=1);
         py::delattr(ns, "wrong");
         py::setattr(ns, "right", py::int_(2));
         return ns;

tests/test_stl.cpp

@@ -19,6 +19,18 @@
 #include <vector>
 #include <string>
 
+#if defined(PYBIND11_TEST_BOOST)
+#include <boost/optional.hpp>
+
+namespace pybind11 { namespace detail {
+template <typename T>
+struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
+
+template <>
+struct type_caster<boost::none_t> : void_caster<boost::none_t> {};
+}} // namespace pybind11::detail
+#endif
+
 // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14
 #if defined(PYBIND11_HAS_VARIANT)
 using std::variant;
@@ -59,7 +71,8 @@ namespace std {
 template <template <typename> class OptionalImpl, typename T>
 struct OptionalHolder
 {
-    OptionalHolder() = default;
+    // NOLINTNEXTLINE(modernize-use-equals-default): breaks GCC 4.8
+    OptionalHolder() {};
     bool member_initialized() const {
         return member && member->initialized;
     }
@@ -67,6 +80,95 @@ struct OptionalHolder
 };
 
 
+enum class EnumType {
+  kSet = 42,
+  kUnset = 85,
+};
+
+// This is used to test that return-by-ref and return-by-copy policies are
+// handled properly for optional types. This is a regression test for a dangling
+// reference issue. The issue seemed to require the enum value type to
+// reproduce - it didn't seem to happen if the value type is just an integer.
+template <template <typename> class OptionalImpl>
+class OptionalProperties {
+public:
+    using OptionalEnumValue = OptionalImpl<EnumType>;
+
+    OptionalProperties() : value(EnumType::kSet) {}
+    ~OptionalProperties() {
+        // Reset value to detect use-after-destruction.
+        // This is set to a specific value rather than nullopt to ensure that
+        // the memory that contains the value gets re-written.
+        value = EnumType::kUnset;
+    }
+
+    OptionalEnumValue& access_by_ref() { return value; }
+    OptionalEnumValue access_by_copy() { return value; }
+
+private:
+    OptionalEnumValue value;
+};
+
+// This type mimics aspects of boost::optional from old versions of Boost,
+// which exposed a dangling reference bug in Pybind11. Recent versions of
+// boost::optional, as well as libstdc++'s std::optional, don't seem to be
+// affected by the same issue. This is meant to be a minimal implementation
+// required to reproduce the issue, not fully standard-compliant.
+// See issue #3330 for more details.
+template <typename T>
+class ReferenceSensitiveOptional {
+public:
+    using value_type = T;
+
+    ReferenceSensitiveOptional() = default;
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    ReferenceSensitiveOptional(const T& value) : storage{value} {}
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    ReferenceSensitiveOptional(T&& value) : storage{std::move(value)} {}
+    ReferenceSensitiveOptional& operator=(const T& value) {
+        storage = {value};
+        return *this;
+    }
+    ReferenceSensitiveOptional& operator=(T&& value) {
+        storage = {std::move(value)};
+        return *this;
+    }
+
+    template <typename... Args>
+    T& emplace(Args&&... args) {
+        storage.clear();
+        storage.emplace_back(std::forward<Args>(args)...);
+        return storage.back();
+    }
+
+    const T& value() const noexcept {
+        assert(!storage.empty());
+        return storage[0];
+    }
+
+    const T& operator*() const noexcept {
+        return value();
+    }
+
+    const T* operator->() const noexcept {
+        return &value();
+    }
+
+    explicit operator bool() const noexcept {
+        return !storage.empty();
+    }
+
+private:
+    std::vector<T> storage;
+};
+
+namespace pybind11 { namespace detail {
+template <typename T>
+struct type_caster<ReferenceSensitiveOptional<T>> : optional_caster<ReferenceSensitiveOptional<T>> {};
+} // namespace detail
+} // namespace pybind11
+
+
 TEST_SUBMODULE(stl, m) {
     // test_vector
     m.def("cast_vector", []() { return std::vector<int>{1}; });
@@ -145,6 +247,10 @@ TEST_SUBMODULE(stl, m) {
         return v;
     });
 
+    pybind11::enum_<EnumType>(m, "EnumType")
+        .value("kSet", EnumType::kSet)
+        .value("kUnset", EnumType::kUnset);
+
     // test_move_out_container
     struct MoveOutContainer {
         struct Value { int value; };
@@ -213,6 +319,12 @@ TEST_SUBMODULE(stl, m) {
         .def(py::init<>())
         .def_readonly("member", &opt_holder::member)
         .def("member_initialized", &opt_holder::member_initialized);
+
+    using opt_props = OptionalProperties<std::optional>;
+    pybind11::class_<opt_props>(m, "OptionalProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_props::access_by_copy);
 #endif
 
 #ifdef PYBIND11_HAS_EXP_OPTIONAL
@@ -239,8 +351,75 @@ TEST_SUBMODULE(stl, m) {
         .def(py::init<>())
         .def_readonly("member", &opt_exp_holder::member)
         .def("member_initialized", &opt_exp_holder::member_initialized);
+
+    using opt_exp_props = OptionalProperties<std::experimental::optional>;
+    pybind11::class_<opt_exp_props>(m, "OptionalExpProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_exp_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_exp_props::access_by_copy);
 #endif
 
+#if defined(PYBIND11_TEST_BOOST)
+    // test_boost_optional
+    m.attr("has_boost_optional") = true;
+
+    using boost_opt_int = boost::optional<int>;
+    using boost_opt_no_assign = boost::optional<NoAssign>;
+    m.def("double_or_zero_boost", [](const boost_opt_int& x) -> int {
+        return x.value_or(0) * 2;
+    });
+    m.def("half_or_none_boost", [](int x) -> boost_opt_int {
+        return x != 0 ? boost_opt_int(x / 2) : boost_opt_int();
+    });
+    m.def("test_nullopt_boost", [](boost_opt_int x) {
+        return x.value_or(42);
+    }, py::arg_v("x", boost::none, "None"));
+    m.def("test_no_assign_boost", [](const boost_opt_no_assign &x) {
+        return x ? x->value : 42;
+    }, py::arg_v("x", boost::none, "None"));
+
+    using opt_boost_holder = OptionalHolder<boost::optional, MoveOutDetector>;
+    py::class_<opt_boost_holder>(m, "OptionalBoostHolder", "Class with optional member")
+        .def(py::init<>())
+        .def_readonly("member", &opt_boost_holder::member)
+        .def("member_initialized", &opt_boost_holder::member_initialized);
+
+    using opt_boost_props = OptionalProperties<boost::optional>;
+    pybind11::class_<opt_boost_props>(m, "OptionalBoostProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_boost_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_boost_props::access_by_copy);
+#endif
+
+    // test_refsensitive_optional
+    using refsensitive_opt_int = ReferenceSensitiveOptional<int>;
+    using refsensitive_opt_no_assign = ReferenceSensitiveOptional<NoAssign>;
+    m.def("double_or_zero_refsensitive", [](const refsensitive_opt_int& x) -> int {
+        return (x ? x.value() : 0) * 2;
+    });
+    m.def("half_or_none_refsensitive", [](int x) -> refsensitive_opt_int {
+        return x != 0 ? refsensitive_opt_int(x / 2) : refsensitive_opt_int();
+    });
+    // NOLINTNEXTLINE(performance-unnecessary-value-param)
+    m.def("test_nullopt_refsensitive", [](refsensitive_opt_int x) {
+        return x ? x.value() : 42;
+    }, py::arg_v("x", refsensitive_opt_int(), "None"));
+    m.def("test_no_assign_refsensitive", [](const refsensitive_opt_no_assign &x) {
+        return x ? x->value : 42;
+    }, py::arg_v("x", refsensitive_opt_no_assign(), "None"));
+
+    using opt_refsensitive_holder = OptionalHolder<ReferenceSensitiveOptional, MoveOutDetector>;
+    py::class_<opt_refsensitive_holder>(m, "OptionalRefSensitiveHolder", "Class with optional member")
+        .def(py::init<>())
+        .def_readonly("member", &opt_refsensitive_holder::member)
+        .def("member_initialized", &opt_refsensitive_holder::member_initialized);
+
+    using opt_refsensitive_props = OptionalProperties<ReferenceSensitiveOptional>;
+    pybind11::class_<opt_refsensitive_props>(m, "OptionalRefSensitiveProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_refsensitive_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_refsensitive_props::access_by_copy);
+
 #ifdef PYBIND11_HAS_FILESYSTEM
     // test_fs_path
     m.attr("has_filesystem") = true;
@@ -280,8 +459,12 @@ TEST_SUBMODULE(stl, m) {
     m.def("tpl_ctor_set", [](std::unordered_set<TplCtorClass> &) {});
 #if defined(PYBIND11_HAS_OPTIONAL)
     m.def("tpl_constr_optional", [](std::optional<TplCtorClass> &) {});
-#elif defined(PYBIND11_HAS_EXP_OPTIONAL)
+#endif
-    m.def("tpl_constr_optional", [](std::experimental::optional<TplCtorClass> &) {});
+#if defined(PYBIND11_HAS_EXP_OPTIONAL)
+    m.def("tpl_constr_optional_exp", [](std::experimental::optional<TplCtorClass> &) {});
+#endif
+#if defined(PYBIND11_TEST_BOOST)
+    m.def("tpl_constr_optional_boost", [](boost::optional<TplCtorClass> &) {});
 #endif
 
     // test_vec_of_reference_wrapper

tests/test_stl.py

@@ -132,6 +132,10 @@ def test_optional():
     assert mvalue.initialized
     assert holder.member_initialized()
 
+    props = m.OptionalProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
 
 @pytest.mark.skipif(
     not hasattr(m, "has_exp_optional"), reason="no <experimental/optional>"
@@ -160,6 +164,69 @@ def test_exp_optional():
     assert mvalue.initialized
     assert holder.member_initialized()
 
+    props = m.OptionalExpProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
+
+@pytest.mark.skipif(not hasattr(m, "has_boost_optional"), reason="no <boost/optional>")
+def test_boost_optional():
+    assert m.double_or_zero_boost(None) == 0
+    assert m.double_or_zero_boost(42) == 84
+    pytest.raises(TypeError, m.double_or_zero_boost, "foo")
+
+    assert m.half_or_none_boost(0) is None
+    assert m.half_or_none_boost(42) == 21
+    pytest.raises(TypeError, m.half_or_none_boost, "foo")
+
+    assert m.test_nullopt_boost() == 42
+    assert m.test_nullopt_boost(None) == 42
+    assert m.test_nullopt_boost(42) == 42
+    assert m.test_nullopt_boost(43) == 43
+
+    assert m.test_no_assign_boost() == 42
+    assert m.test_no_assign_boost(None) == 42
+    assert m.test_no_assign_boost(m.NoAssign(43)) == 43
+    pytest.raises(TypeError, m.test_no_assign_boost, 43)
+
+    holder = m.OptionalBoostHolder()
+    mvalue = holder.member
+    assert mvalue.initialized
+    assert holder.member_initialized()
+
+    props = m.OptionalBoostProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
+
+def test_reference_sensitive_optional():
+    assert m.double_or_zero_refsensitive(None) == 0
+    assert m.double_or_zero_refsensitive(42) == 84
+    pytest.raises(TypeError, m.double_or_zero_refsensitive, "foo")
+
+    assert m.half_or_none_refsensitive(0) is None
+    assert m.half_or_none_refsensitive(42) == 21
+    pytest.raises(TypeError, m.half_or_none_refsensitive, "foo")
+
+    assert m.test_nullopt_refsensitive() == 42
+    assert m.test_nullopt_refsensitive(None) == 42
+    assert m.test_nullopt_refsensitive(42) == 42
+    assert m.test_nullopt_refsensitive(43) == 43
+
+    assert m.test_no_assign_refsensitive() == 42
+    assert m.test_no_assign_refsensitive(None) == 42
+    assert m.test_no_assign_refsensitive(m.NoAssign(43)) == 43
+    pytest.raises(TypeError, m.test_no_assign_refsensitive, 43)
+
+    holder = m.OptionalRefSensitiveHolder()
+    mvalue = holder.member
+    assert mvalue.initialized
+    assert holder.member_initialized()
+
+    props = m.OptionalRefSensitiveProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
 
 @pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no <filesystem>")
 def test_fs_path():

tests/test_virtual_functions.cpp

@@ -174,6 +174,25 @@ struct DispatchIssue : Base {
     }
 };
 
+// An abstract adder class that uses visitor pattern to add two data
+// objects and send the result to the visitor functor
+struct AdderBase {
+    struct Data {};
+    using DataVisitor = std::function<void (const Data&)>;
+
+    virtual void operator()(const Data& first, const Data& second, const DataVisitor& visitor) const = 0;
+    virtual ~AdderBase() = default;
+    AdderBase() = default;
+    AdderBase(const AdderBase&) = delete;
+};
+
+struct Adder : AdderBase {
+    void operator()(const Data& first, const Data& second, const DataVisitor& visitor) const override {
+        PYBIND11_OVERRIDE_PURE_NAME(void, AdderBase, "__call__", operator(), first, second, visitor);
+    }
+};
+
+
 static void test_gil() {
     {
         py::gil_scoped_acquire lock;
@@ -295,6 +314,27 @@ TEST_SUBMODULE(virtual_functions, m) {
 
     m.def("dispatch_issue_go", [](const Base * b) { return b->dispatch(); });
 
+    // test_recursive_dispatch_issue
+    // #3357: Recursive dispatch fails to find python function override
+    pybind11::class_<AdderBase, Adder>(m, "Adder")
+        .def(pybind11::init<>())
+        .def("__call__", &AdderBase::operator());
+
+    pybind11::class_<AdderBase::Data>(m, "Data")
+        .def(pybind11::init<>());
+
+    m.def("add2", [](const AdderBase::Data& first, const AdderBase::Data& second,
+                     const AdderBase& adder, const AdderBase::DataVisitor& visitor) {
+        adder(first, second, visitor);
+    });
+
+    m.def("add3", [](const AdderBase::Data& first, const AdderBase::Data& second, const AdderBase::Data& third,
+                     const AdderBase& adder, const AdderBase::DataVisitor& visitor) {
+        adder(first, second, [&] (const AdderBase::Data& first_plus_second) {
+            adder(first_plus_second, third, visitor);
+        });
+    });
+
     // test_override_ref
     // #392/397: overriding reference-returning functions
     class OverrideTest {

tests/test_virtual_functions.py

@@ -257,6 +257,39 @@ def test_dispatch_issue(msg):
     assert m.dispatch_issue_go(b) == "Yay.."
 
 
+def test_recursive_dispatch_issue(msg):
+    """#3357: Recursive dispatch fails to find python function override"""
+
+    class Data(m.Data):
+        def __init__(self, value):
+            super(Data, self).__init__()
+            self.value = value
+
+    class Adder(m.Adder):
+        def __call__(self, first, second, visitor):
+            # lambda is a workaround, which adds extra frame to the
+            # current CPython thread. Removing lambda reveals the bug
+            # [https://github.com/pybind/pybind11/issues/3357]
+            (lambda: visitor(Data(first.value + second.value)))()
+
+    class StoreResultVisitor:
+        def __init__(self):
+            self.result = None
+
+        def __call__(self, data):
+            self.result = data.value
+
+    store = StoreResultVisitor()
+
+    m.add2(Data(1), Data(2), Adder(), store)
+    assert store.result == 3
+
+    # without lambda in Adder class, this function fails with
+    # RuntimeError: Tried to call pure virtual function "AdderBase::__call__"
+    m.add3(Data(1), Data(2), Data(3), Adder(), store)
+    assert store.result == 6
+
+
 def test_override_ref():
     """#392/397: overriding reference-returning functions"""
     o = m.OverrideTest("asdf")

tools/pybind11Tools.cmake

@@ -38,7 +38,7 @@ endif()
 
 # A user can set versions manually too
 set(Python_ADDITIONAL_VERSIONS
-    "3.10;3.9;3.8;3.7;3.6;3.5;3.4"
+    "3.11;3.10;3.9;3.8;3.7;3.6;3.5;3.4"
     CACHE INTERNAL "")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")