diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h
index 1044db94d..74dc361e3 100644
--- a/include/pybind11/attr.h
+++ b/include/pybind11/attr.h
@@ -331,6 +331,10 @@ struct type_record {
     /// Is the class inheritable from python classes?
     bool is_final : 1;
 
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+    holder_enum_t holder_enum_v = holder_enum_t::undefined;
+#endif
+
     PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) {
         auto *base_info = detail::get_type_info(base, false);
         if (!base_info) {
diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 7cf2a3117..ba9f81c33 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -835,7 +835,7 @@ protected:
     holder_type holder;
 };
 
-#define PYBIND11_HAVE_HOLDER_CASTER_WITH_SMART_HOLDER_SUPPORT
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
 
 template <typename, typename SFINAE = void>
 struct copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled : std::true_type {};
@@ -862,14 +862,14 @@ public:
     }
 
     explicit operator type *() {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             throw std::runtime_error("BAKEIN_WIP: operator type *() shared_ptr");
         }
         return this->value;
     }
 
     explicit operator type &() {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             throw std::runtime_error("BAKEIN_WIP: operator type &() shared_ptr");
         }
         // static_cast works around compiler error with MSVC 17 and CUDA 10.2
@@ -878,14 +878,14 @@ public:
     }
 
     explicit operator std::shared_ptr<type> *() {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             throw std::runtime_error("BAKEIN_WIP: operator std::shared_ptr<type> *()");
         }
         return std::addressof(shared_ptr_holder);
     }
 
     explicit operator std::shared_ptr<type> &() {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             shared_ptr_holder = sh_load_helper.loaded_as_shared_ptr(value);
         }
         return shared_ptr_holder;
@@ -898,7 +898,7 @@ public:
         if (st.second == nullptr) {
             return handle(); // no type info: error will be set already
         }
-        if (st.second->default_holder) {
+        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return smart_holder_type_caster_support::smart_holder_from_shared_ptr(
                 src, policy, parent, st);
         }
@@ -913,7 +913,7 @@ public:
     static std::shared_ptr<type> shared_ptr_with_responsible_parent(handle responsible_parent) {
         copyable_holder_caster loader;
         loader.load(responsible_parent, /*convert=*/false);
-        assert(loader.typeinfo->default_holder);
+        assert(loader.typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder);
         return loader.sh_load_helper.loaded_as_shared_ptr(loader.value, responsible_parent);
     }
 
@@ -921,7 +921,7 @@ protected:
     friend class type_caster_generic;
 
     void load_value(value_and_holder &&v_h) {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
             value = sh_load_helper.get_void_ptr_or_nullptr();
             return;
@@ -932,13 +932,13 @@ protected:
             return;
         }
         throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) "
-#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
+#    if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
                          "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for "
                          "type information)");
-#else
+#    else
                          "of type '"
                          + type_id<std::shared_ptr<type>>() + "''");
-#endif
+#    endif
     }
 
     template <typename T = std::shared_ptr<type>,
@@ -954,7 +954,7 @@ protected:
             copyable_holder_caster sub_caster(*cast.first);
             if (sub_caster.load(src, convert)) {
                 value = cast.second(sub_caster.value);
-                if (typeinfo->default_holder) {
+                if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
                     // BAKEIN_WIP: Copy pointer only?
                     sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
                 } else {
@@ -973,6 +973,8 @@ protected:
     smart_holder_type_caster_support::load_helper<remove_cv_t<type>> sh_load_helper; // Const2Mutbl
 };
 
+#endif // PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
 /// Specialize for the common std::shared_ptr, so users don't need to
 template <typename T>
 class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> {};
@@ -992,6 +994,8 @@ struct move_only_holder_caster {
     static constexpr auto name = type_caster_base<type>::name;
 };
 
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
 template <typename, typename SFINAE = void>
 struct move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled : std::true_type {};
 
@@ -1018,7 +1022,7 @@ public:
         if (st.second == nullptr) {
             return handle(); // no type info: error will be set already
         }
-        if (st.second->default_holder) {
+        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return smart_holder_type_caster_support::smart_holder_from_unique_ptr(
                 std::move(src), policy, parent, st);
         }
@@ -1051,7 +1055,7 @@ public:
     }
 
     void load_value(value_and_holder &&v_h) {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
             sh_load_helper.loaded_v_h.type = typeinfo;
             value = sh_load_helper.get_void_ptr_or_nullptr();
@@ -1064,7 +1068,7 @@ public:
     using cast_op_type = std::unique_ptr<type, deleter>;
 
     explicit operator std::unique_ptr<type, deleter>() {
-        if (typeinfo->default_holder) {
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return sh_load_helper.template loaded_as_unique_ptr<deleter>(value);
         }
         pybind11_fail("Passing std::unique_ptr from Python to C++ requires smart_holder.");
@@ -1075,7 +1079,7 @@ public:
             move_only_holder_caster sub_caster(*cast.first);
             if (sub_caster.load(src, convert)) {
                 value = cast.second(sub_caster.value);
-                if (typeinfo->default_holder) {
+                if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
                     // BAKEIN_WIP: Copy pointer only?
                     sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
                 } else {
@@ -1093,6 +1097,8 @@ public:
     smart_holder_type_caster_support::load_helper<remove_cv_t<type>> sh_load_helper; // Const2Mutbl
 };
 
+#endif // PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
 template <typename type, typename deleter>
 class type_caster<std::unique_ptr<type, deleter>>
     : public move_only_holder_caster<type, std::unique_ptr<type, deleter>> {};
@@ -1128,8 +1134,11 @@ struct is_holder_type
 // Specializations for always-supported holders:
 template <typename base, typename deleter>
 struct is_holder_type<base, std::unique_ptr<base, deleter>> : std::true_type {};
+
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
 template <typename base>
 struct is_holder_type<base, pybindit::memory::smart_holder> : std::true_type {};
+#endif
 
 #ifdef PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION // See PR #4888
 
diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
index 5202a1163..d46d77894 100644
--- a/include/pybind11/detail/internals.h
+++ b/include/pybind11/detail/internals.h
@@ -39,9 +39,9 @@
 #    if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER)
 // Version bump for Python 3.12+, before first 3.12 beta release.
 // Version bump for MSVC piggy-backed on PR #4779. See comments there.
-#        define PYBIND11_INTERNALS_VERSION 5
+#        define PYBIND11_INTERNALS_VERSION 6 // BAKEIN_WIP: Only do this for pybind11 v3.0.0
 #    else
-#        define PYBIND11_INTERNALS_VERSION 4
+#        define PYBIND11_INTERNALS_VERSION 6 // BAKEIN_WIP: Only do this for pybind11 v3.0.0
 #    endif
 #endif
 
@@ -236,6 +236,20 @@ struct internals {
     }
 };
 
+#if PYBIND11_INTERNALS_VERSION >= 6
+
+#    define PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
+enum class holder_enum_t : uint8_t {
+    undefined,
+    std_unique_ptr, // Default, lacking interop with std::shared_ptr.
+    std_shared_ptr, // Lacking interop with std::unique_ptr.
+    smart_holder,   // Full std::unique_ptr / std::shared_ptr interop.
+    custom_holder,
+};
+
+#endif
+
 /// Additional type information which does not fit into the PyTypeObject.
 /// Changes to this struct also require bumping `PYBIND11_INTERNALS_VERSION`.
 struct type_info {
@@ -262,6 +276,9 @@ struct type_info {
     bool default_holder : 1;
     /* true if this is a type registered with py::module_local */
     bool module_local : 1;
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+    holder_enum_t holder_enum_v = holder_enum_t::undefined;
+#endif
 };
 
 /// On MSVC, debug and release builds are not ABI-compatible!
diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h
index c8b2b4f01..cd5bc52c7 100644
--- a/include/pybind11/detail/type_caster_base.h
+++ b/include/pybind11/detail/type_caster_base.h
@@ -935,7 +935,8 @@ public:
 
     // Base methods for generic caster; there are overridden in copyable_holder_caster
     void load_value(value_and_holder &&v_h) {
-        if (typeinfo->default_holder) {
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             smart_holder_type_caster_support::value_and_holder_helper v_h_helper;
             v_h_helper.loaded_v_h = v_h;
             if (v_h_helper.have_holder()) {
@@ -944,6 +945,7 @@ public:
                 return;
             }
         }
+#endif
         auto *&vptr = v_h.value_ptr();
         // Lazy allocation for unallocated values:
         if (vptr == nullptr) {
diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
index 835708401..cad97eaa2 100644
--- a/include/pybind11/pybind11.h
+++ b/include/pybind11/pybind11.h
@@ -1424,6 +1424,9 @@ protected:
         tinfo->simple_ancestors = true;
         tinfo->default_holder = rec.default_holder;
         tinfo->module_local = rec.module_local;
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+        tinfo->holder_enum_v = rec.holder_enum_v;
+#endif
 
         with_internals([&](internals &internals) {
             auto tindex = std::type_index(*rec.type);
@@ -1643,6 +1646,8 @@ struct both_t_and_d_use_type_caster_base<
 
 PYBIND11_NAMESPACE_END(detail)
 
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
 // BAKEIN_WIP: Rewrite comment.
 // smart_holder specializations for raw pointer members.
 // WARNING: Like the classic implementation, this implementation can lead to dangling pointers.
@@ -1662,7 +1667,7 @@ struct property_cpp_function<
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function readonly(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function(
                 [pm](handle c_hdl) -> std::shared_ptr<drp> {
                     std::shared_ptr<T> c_sp = detail::type_caster<
@@ -1683,7 +1688,7 @@ struct property_cpp_function<
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function write(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function([pm](T &c, D value) { c.*pm = std::forward<D>(value); },
                                 is_method(hdl));
         }
@@ -1709,7 +1714,7 @@ struct property_cpp_function<T,
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function readonly(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function(
                 [pm](handle c_hdl) -> std::shared_ptr<typename std::add_const<D>::type> {
                     std::shared_ptr<T> c_sp = detail::type_caster<
@@ -1725,7 +1730,7 @@ struct property_cpp_function<T,
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function read(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function(
                 [pm](handle c_hdl) -> std::shared_ptr<D> {
                     std::shared_ptr<T> c_sp = detail::type_caster<
@@ -1740,7 +1745,7 @@ struct property_cpp_function<T,
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function write(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl));
         }
         return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl));
@@ -1772,7 +1777,7 @@ struct property_cpp_function<
     template <typename PM, detail::must_be_member_function_pointer<PM> = 0>
     static cpp_function read(PM pm, const handle &hdl) {
         detail::type_info *tinfo = detail::get_type_info(typeid(T), /*throw_if_missing=*/true);
-        if (tinfo->default_holder) {
+        if (tinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             return cpp_function(
                 [pm](handle c_hdl) -> D {
                     std::shared_ptr<T> c_sp = detail::type_caster<
@@ -1790,6 +1795,8 @@ struct property_cpp_function<
     }
 };
 
+#endif // PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+
 template <typename>
 using default_holder_type = pybindit::memory::smart_holder;
 
@@ -1843,6 +1850,17 @@ public:
         record.init_instance = init_instance;
         record.dealloc = dealloc;
         record.default_holder = std::is_same<holder_type, pybindit::memory::smart_holder>::value;
+#ifdef PYBIND11_HAVE_INTERNALS_WITH_SMART_HOLDER_SUPPORT
+        if (detail::is_instantiation<std::unique_ptr, holder_type>::value) {
+            record.holder_enum_v = detail::holder_enum_t::std_unique_ptr;
+        } else if (detail::is_instantiation<std::shared_ptr, holder_type>::value) {
+            record.holder_enum_v = detail::holder_enum_t::std_shared_ptr;
+        } else if (std::is_same<holder_type, pybindit::memory::smart_holder>::value) {
+            record.holder_enum_v = detail::holder_enum_t::smart_holder;
+        } else {
+            record.holder_enum_v = detail::holder_enum_t::custom_holder;
+        }
+#endif
 
         set_operator_new<type>(&record);