Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -705,14 +705,11 @@ jobs:
     - name: Install Doxygen
       run: sudo apt-get install -y doxygen librsvg2-bin # Changed to rsvg-convert in 20.04
 
-    - name: Install docs & setup requirements
-      run: python3 -m pip install -r docs/requirements.txt
-
     - name: Build docs
-      run: python3 -m sphinx -W -b html docs docs/.build
+      run: pipx run nox -s docs
 
     - name: Make SDist
-      run: python3 setup.py sdist
+      run: pipx run nox -s build -- --sdist
 
     - run: git status --ignored
 
@@ -724,7 +721,7 @@ jobs:
     - name: Compare Dists (headers only)
       working-directory: include
       run: |
-        python3 -m pip install --user -U ../dist/*
+        python3 -m pip install --user -U ../dist/*.tar.gz
         installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')")
         diff -rq $installed ./pybind11
 

.pre-commit-config.yaml

@@ -39,7 +39,7 @@ repos:
   - id: pyupgrade
 
 - repo: https://github.com/PyCQA/isort
-  rev: 5.9.3
+  rev: 5.10.0
   hooks:
   - id: isort
 
@@ -63,6 +63,12 @@ repos:
   - id: remove-tabs
     exclude: (^docs/.*|\.patch)?$
 
+# Autoremoves unused imports
+- repo: https://github.com/hadialqattan/pycln
+  rev: v1.1.0
+  hooks:
+  - id: pycln
+
 - repo: https://github.com/pre-commit/pygrep-hooks
   rev: v1.9.0
   hooks:

docs/Doxyfile

@@ -18,6 +18,5 @@ ALIASES               += "endrst=\endverbatim"
 QUIET                  = YES
 WARNINGS               = YES
 WARN_IF_UNDOCUMENTED   = NO
-PREDEFINED             = DOXYGEN_SHOULD_SKIP_THIS \
+PREDEFINED             = PY_MAJOR_VERSION=3 \
-                         PY_MAJOR_VERSION=3 \
                          PYBIND11_NOINLINE

docs/advanced/cast/eigen.rst

@@ -52,7 +52,7 @@ can be mapped *and* if the numpy array is writeable (that is
 the passed variable will be transparently carried out directly on the
 ``numpy.ndarray``.
 
-This means you can can write code such as the following and have it work as
+This means you can write code such as the following and have it work as
 expected:
 
 .. code-block:: cpp

docs/requirements.txt

@@ -1,8 +1,5 @@
-breathe==4.26.1
+breathe==4.31.0
-# docutils 0.17 breaks HTML tags & RTD theme
+sphinx==3.5.4
-# https://github.com/sphinx-doc/sphinx/issues/9001
+sphinx_rtd_theme==1.0.0
-docutils==0.16
+sphinxcontrib-moderncmakedomain==3.19
-sphinx==3.3.1
+sphinxcontrib-svg2pdfconverter==1.1.1
-sphinx_rtd_theme==0.5.0
-sphinxcontrib-moderncmakedomain==3.17
-sphinxcontrib-svg2pdfconverter==1.1.0

include/pybind11/pybind11.h

@@ -2093,6 +2093,16 @@ inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_t
         // gets destroyed:
         weakref((PyObject *) type, cpp_function([type](handle wr) {
             get_internals().registered_types_py.erase(type);
+
+            // TODO consolidate the erasure code in pybind11_meta_dealloc() in class.h
+            auto &cache = get_internals().inactive_override_cache;
+            for (auto it = cache.begin(), last = cache.end(); it != last; ) {
+                if (it->first == reinterpret_cast<PyObject *>(type))
+                    it = cache.erase(it);
+                else
+                    ++it;
+            }
+
             wr.dec_ref();
         })).release();
     }

include/pybind11/pytypes.h

@@ -287,10 +287,10 @@ protected:
     struct borrowed_t { };
     struct stolen_t { };
 
-#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
+    /// @cond BROKEN
     template <typename T> friend T reinterpret_borrow(handle);
     template <typename T> friend T reinterpret_steal(handle);
-#endif
+    /// @endcond
 
 public:
     // Only accessible from derived classes and the reinterpret_* functions
@@ -1717,7 +1717,7 @@ public:
 #endif
 };
 
-#ifndef DOXYGEN_SHOULD_SKIP_THIS
+/// @cond DUPLICATE
 inline memoryview memoryview::from_buffer(
     void *ptr, ssize_t itemsize, const char* format,
     detail::any_container<ssize_t> shape,
@@ -1745,7 +1745,7 @@ inline memoryview memoryview::from_buffer(
         throw error_already_set();
     return memoryview(object(obj, stolen_t{}));
 }
-#endif  // DOXYGEN_SHOULD_SKIP_THIS
+/// @endcond
 /// @} pytypes
 
 /// \addtogroup python_builtins

noxfile.py

@@ -57,10 +57,10 @@ def docs(session: nox.Session) -> None:
     session.chdir("docs")
 
     if "pdf" in session.posargs:
-        session.run("sphinx-build", "-M", "latexpdf", ".", "_build")
+        session.run("sphinx-build", "-b", "latexpdf", ".", "_build")
         return
 
-    session.run("sphinx-build", "-M", "html", ".", "_build")
+    session.run("sphinx-build", "-b", "html", ".", "_build")
 
     if "serve" in session.posargs:
         session.log("Launching docs at http://localhost:8000/ - use Ctrl-C to quit")
@@ -86,6 +86,8 @@ def build(session: nox.Session) -> None:
 
     session.install("build")
     session.log("Building normal files")
-    session.run("python", "-m", "build")
+    session.run("python", "-m", "build", *session.posargs)
     session.log("Building pybind11-global files (PYBIND11_GLOBAL_SDIST=1)")
-    session.run("python", "-m", "build", env={"PYBIND11_GLOBAL_SDIST": "1"})
+    session.run(
+        "python", "-m", "build", *session.posargs, env={"PYBIND11_GLOBAL_SDIST": "1"}
+    )

pybind11/setup_helpers.py

@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import contextlib
 import os
 import platform
+import shlex
 import shutil
 import sys
 import sysconfig
@@ -143,7 +144,12 @@ class Pybind11Extension(_Extension):
         if WIN:
             cflags += ["/EHsc", "/bigobj"]
         else:
-            cflags += ["-fvisibility=hidden", "-g0"]
+            cflags += ["-fvisibility=hidden"]
+            env_cflags = os.environ.get("CFLAGS", "")
+            env_cppflags = os.environ.get("CPPFLAGS", "")
+            c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags)
+            if not any(opt.startswith("-g") for opt in c_cpp_flags):
+                cflags += ["-g0"]
             if MACOS:
                 cflags += ["-stdlib=libc++"]
                 ldflags += ["-stdlib=libc++"]

tests/extra_setuptools/test_setuphelper.py

@@ -8,6 +8,7 @@ import pytest
 
 DIR = os.path.abspath(os.path.dirname(__file__))
 MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
+WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
 
 
 @pytest.mark.parametrize("parallel", [False, True])
@@ -71,13 +72,20 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std):
         encoding="ascii",
     )
 
-    subprocess.check_call(
+    out = subprocess.check_output(
         [sys.executable, "setup.py", "build_ext", "--inplace"],
-        stdout=sys.stdout,
-        stderr=sys.stderr,
     )
+    if not WIN:
+        assert b"-g0" in out
+    out = subprocess.check_output(
+        [sys.executable, "setup.py", "build_ext", "--inplace", "--force"],
+        env=dict(os.environ, CFLAGS="-g"),
+    )
+    if not WIN:
+        assert b"-g0" not in out
 
     # Debug helper printout, normally hidden
+    print(out)
     for item in tmpdir.listdir():
         print(item.basename)
 

tests/test_embed/CMakeLists.txt

@@ -25,7 +25,7 @@ pybind11_enable_warnings(test_embed)
 target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads)
 
 if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
-  file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+  file(COPY test_interpreter.py test_trampoline.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 endif()
 
 add_custom_target(

tests/test_embed/test_interpreter.cpp

@@ -37,6 +37,22 @@ class PyWidget final : public Widget {
     std::string argv0() const override { PYBIND11_OVERRIDE_PURE(std::string, Widget, argv0); }
 };
 
+class test_override_cache_helper {
+
+public:
+    virtual int func() { return 0; }
+
+    test_override_cache_helper() = default;
+    virtual ~test_override_cache_helper() = default;
+    // Non-copyable
+    test_override_cache_helper &operator=(test_override_cache_helper const &Right) = delete;
+    test_override_cache_helper(test_override_cache_helper const &Copy) = delete;
+};
+
+class test_override_cache_helper_trampoline : public test_override_cache_helper {
+    int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
+};
+
 PYBIND11_EMBEDDED_MODULE(widget_module, m) {
     py::class_<Widget, PyWidget>(m, "Widget")
         .def(py::init<std::string>())
@@ -45,6 +61,12 @@ PYBIND11_EMBEDDED_MODULE(widget_module, m) {
     m.def("add", [](int i, int j) { return i + j; });
 }
 
+PYBIND11_EMBEDDED_MODULE(trampoline_module, m) {
+    py::class_<test_override_cache_helper, test_override_cache_helper_trampoline, std::shared_ptr<test_override_cache_helper>>(m, "test_override_cache_helper")
+        .def(py::init_alias<>())
+        .def("func", &test_override_cache_helper::func);
+}
+
 PYBIND11_EMBEDDED_MODULE(throw_exception, ) {
     throw std::runtime_error("C++ Error");
 }
@@ -73,6 +95,33 @@ TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
     REQUIRE(cpp_widget.the_answer() == 42);
 }
 
+TEST_CASE("Override cache") {
+    auto module_ = py::module_::import("test_trampoline");
+    REQUIRE(py::hasattr(module_, "func"));
+    REQUIRE(py::hasattr(module_, "func2"));
+
+    auto locals = py::dict(**module_.attr("__dict__"));
+
+    int i = 0;
+    for (; i < 1500; ++i) {
+        std::shared_ptr<test_override_cache_helper> p_obj;
+        std::shared_ptr<test_override_cache_helper> p_obj2;
+
+        py::object loc_inst = locals["func"]();
+        p_obj = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst);
+
+        int ret = p_obj->func();
+
+        REQUIRE(ret == 42);
+
+        loc_inst = locals["func2"]();
+
+        p_obj2 = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst);
+
+        p_obj2->func();
+    }
+}
+
 TEST_CASE("Import error handling") {
     REQUIRE_NOTHROW(py::module_::import("widget_module"));
     REQUIRE_THROWS_WITH(py::module_::import("throw_exception"),

tests/test_embed/test_trampoline.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+import trampoline_module
+
+
+def func():
+    class Test(trampoline_module.test_override_cache_helper):
+        def func(self):
+            return 42
+
+    return Test()
+
+
+def func2():
+    class Test(trampoline_module.test_override_cache_helper):
+        pass
+
+    return Test()

tests/test_virtual_functions.cpp

@@ -214,6 +214,25 @@ static void test_gil_from_thread() {
     t.join();
 }
 
+class test_override_cache_helper {
+
+public:
+    virtual int func() { return 0; }
+
+    test_override_cache_helper() = default;
+    virtual ~test_override_cache_helper() = default;
+    // Non-copyable
+    test_override_cache_helper &operator=(test_override_cache_helper const &Right) = delete;
+    test_override_cache_helper(test_override_cache_helper const &Copy) = delete;
+};
+
+class test_override_cache_helper_trampoline : public test_override_cache_helper {
+    int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
+};
+
+inline int test_override_cache(std::shared_ptr<test_override_cache_helper> const &instance) { return instance->func(); }
+
+
 
 // Forward declaration (so that we can put the main tests here; the inherited virtual approaches are
 // rather long).
@@ -378,6 +397,12 @@ TEST_SUBMODULE(virtual_functions, m) {
 //      .def("str_ref", &OverrideTest::str_ref)
         .def("A_value", &OverrideTest::A_value)
         .def("A_ref", &OverrideTest::A_ref);
+
+    py::class_<test_override_cache_helper, test_override_cache_helper_trampoline, std::shared_ptr<test_override_cache_helper>>(m, "test_override_cache_helper")
+        .def(py::init_alias<>())
+        .def("func", &test_override_cache_helper::func);
+
+    m.def("test_override_cache", test_override_cache);
 }
 
 

tests/test_virtual_functions.py

@@ -439,3 +439,22 @@ def test_issue_1454():
     # Fix issue #1454 (crash when acquiring/releasing GIL on another thread in Python 2.7)
     m.test_gil()
     m.test_gil_from_thread()
+
+
+def test_python_override():
+    def func():
+        class Test(m.test_override_cache_helper):
+            def func(self):
+                return 42
+
+        return Test()
+
+    def func2():
+        class Test(m.test_override_cache_helper):
+            pass
+
+        return Test()
+
+    for _ in range(1500):
+        assert m.test_override_cache(func()) == 42
+        assert m.test_override_cache(func2()) == 0