diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 0a8ef8318..0b0533dab 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -58,7 +58,7 @@ PYBIND11_NOINLINE inline internals &get_internals() {
     handle builtins(PyEval_GetBuiltins());
     const char *id = PYBIND11_INTERNALS_ID;
     if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
-        internals_ptr = capsule(builtins[id]);
+        internals_ptr = *static_cast<internals **>(capsule(builtins[id]));
     } else {
         internals_ptr = new internals();
         #if defined(WITH_THREAD)
@@ -68,7 +68,7 @@ PYBIND11_NOINLINE inline internals &get_internals() {
             PyThread_set_key_value(internals_ptr->tstate, tstate);
             internals_ptr->istate = tstate->interp;
         #endif
-        builtins[id] = capsule(internals_ptr);
+        builtins[id] = capsule(&internals_ptr);
         internals_ptr->registered_exception_translators.push_front(
             [](std::exception_ptr p) -> void {
                 try {
diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h
index a92465209..29d0950b2 100644
--- a/include/pybind11/embed.h
+++ b/include/pybind11/embed.h
@@ -131,13 +131,29 @@ inline void initialize_interpreter(bool init_signal_handlers = true) {
 
     .. warning::
 
-        Python cannot unload binary extension modules. If `initialize_interpreter` is
-        called again to restart the interpreter, the initializers of those modules will
-        be executed for a second time and they will fail. This is a known CPython issue.
-        See the Python documentation for details.
+        The interpreter can be restarted by calling `initialize_interpreter` again.
+        Modules created using pybind11 can be safely re-initialized. However, Python
+        itself cannot completely unload binary extension modules and there are several
+        caveats with regard to interpreter restarting. All the details can be found
+        in the CPython documentation. In short, not all interpreter memory may be
+        freed, either due to reference cycles or user-created global data.
 
  \endrst */
-inline void finalize_interpreter() { Py_Finalize(); }
+inline void finalize_interpreter() {
+    handle builtins(PyEval_GetBuiltins());
+    const char *id = PYBIND11_INTERNALS_ID;
+
+    detail::internals **internals_ptr_ptr = nullptr;
+    if (builtins.contains(id) && isinstance<capsule>(builtins[id]))
+        internals_ptr_ptr = capsule(builtins[id]);
+
+    Py_Finalize();
+
+    if (internals_ptr_ptr) {
+        delete *internals_ptr_ptr;
+        *internals_ptr_ptr = nullptr;
+    }
+}
 
 /** \rst
     Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp
index 1069ccac6..01fa98346 100644
--- a/tests/test_embed/test_interpreter.cpp
+++ b/tests/test_embed/test_interpreter.cpp
@@ -26,6 +26,8 @@ PYBIND11_EMBEDDED_MODULE(widget_module, m) {
     py::class_<Widget, PyWidget>(m, "Widget")
         .def(py::init<std::string>())
         .def_property_readonly("the_message", &Widget::the_message);
+
+    m.def("add", [](int i, int j) { return i + j; });
 }
 
 PYBIND11_EMBEDDED_MODULE(throw_exception, ) {
@@ -81,3 +83,73 @@ TEST_CASE("There can be only one interpreter") {
     }
     py::initialize_interpreter();
 }
+
+bool has_pybind11_internals() {
+    auto builtins = py::handle(PyEval_GetBuiltins());
+    return builtins.contains(PYBIND11_INTERNALS_ID);
+};
+
+TEST_CASE("Restart the interpreter") {
+    // Verify pre-restart state.
+    REQUIRE(py::module::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
+    REQUIRE(has_pybind11_internals());
+
+    // Restart the interpreter.
+    py::finalize_interpreter();
+    REQUIRE(Py_IsInitialized() == 0);
+
+    py::initialize_interpreter();
+    REQUIRE(Py_IsInitialized() == 1);
+
+    // Internals are deleted after a restart.
+    REQUIRE_FALSE(has_pybind11_internals());
+    pybind11::detail::get_internals();
+    REQUIRE(has_pybind11_internals());
+
+    // C++ modules can be reloaded.
+    auto cpp_module = py::module::import("widget_module");
+    REQUIRE(cpp_module.attr("add")(1, 2).cast<int>() == 3);
+
+    // C++ type information is reloaded and can be used in python modules.
+    auto py_module = py::module::import("test_interpreter");
+    auto py_widget = py_module.attr("DerivedWidget")("Hello after restart");
+    REQUIRE(py_widget.attr("the_message").cast<std::string>() == "Hello after restart");
+}
+
+TEST_CASE("Subinterpreter") {
+    // Add tags to the modules in the main interpreter and test the basics.
+    py::module::import("__main__").attr("main_tag") = "main interpreter";
+    {
+        auto m = py::module::import("widget_module");
+        m.attr("extension_module_tag") = "added to module in main interpreter";
+
+        REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
+    }
+    REQUIRE(has_pybind11_internals());
+
+    /// Create and switch to a subinterpreter.
+    auto main_tstate = PyThreadState_Get();
+    auto sub_tstate = Py_NewInterpreter();
+
+    // Subinterpreters get their own copy of builtins. detail::get_internals() still
+    // works by returning from the static variable, i.e. all interpreters share a single
+    // global pybind11::internals;
+    REQUIRE_FALSE(has_pybind11_internals());
+
+    // Modules tags should be gone.
+    REQUIRE_FALSE(py::hasattr(py::module::import("__main__"), "tag"));
+    {
+        auto m = py::module::import("widget_module");
+        REQUIRE_FALSE(py::hasattr(m, "extension_module_tag"));
+
+        // Function bindings should still work.
+        REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
+    }
+
+    // Restore main interpreter.
+    Py_EndInterpreter(sub_tstate);
+    PyThreadState_Swap(main_tstate);
+
+    REQUIRE(py::hasattr(py::module::import("__main__"), "main_tag"));
+    REQUIRE(py::hasattr(py::module::import("widget_module"), "extension_module_tag"));
+}