diff --git a/include/pybind11/smart_holder_poc.h b/include/pybind11/smart_holder_poc.h
index 1f14fba92..43759ed2d 100644
--- a/include/pybind11/smart_holder_poc.h
+++ b/include/pybind11/smart_holder_poc.h
@@ -9,10 +9,19 @@ struct smart_holder {
   std::shared_ptr<void> vptr;
   const std::type_info* rtti_held;
   const std::type_info* rtti_uqp_del;
-  bool have_external_shp;
+  bool vptr_deleter_flag;
+
+  template <typename T>
+  struct vptr_deleter {
+    bool* flag_ptr;
+    explicit vptr_deleter(bool* flag_ptr_) : flag_ptr{flag_ptr_} {}
+    void operator()(T* raw_ptr) {
+      if (*flag_ptr) delete raw_ptr;
+    }
+  };
 
   smart_holder()
-      : rtti_held{nullptr}, rtti_uqp_del{nullptr}, have_external_shp(false) {}
+      : rtti_held{nullptr}, rtti_uqp_del{nullptr}, vptr_deleter_flag{false} {}
 
   template <typename T>
   void ensure_compatible_rtti(const char* context) {
@@ -30,23 +39,17 @@ struct smart_holder {
     }
   }
 
-  void ensure_unique_ptr_default_deleter(const char* context) {
+  void ensure_vptr_deleter_flag_true(const char* context) {
     if (rtti_uqp_del != nullptr) {
-      throw std::runtime_error(
-          std::string("Cannot disown unique_ptr deleter (") + context + ").");
-    }
-  }
-
-  void ensure_internal_shared_ptr(const char* context) {
-    if (have_external_shp) {
-      throw std::runtime_error(
-          std::string("Cannot disown external shared_ptr (") + context + ").");
+      throw std::runtime_error(std::string("Cannot disown this shared_ptr (") +
+                               context + ").");
     }
   }
 
   template <typename T>
   void from_raw_ptr_owned(T* raw_ptr) {
-    vptr.reset(raw_ptr);
+    vptr_deleter_flag = true;
+    vptr.reset(raw_ptr, vptr_deleter<T>(&vptr_deleter_flag));
     rtti_held = &typeid(T);
   }
 
@@ -55,12 +58,11 @@ struct smart_holder {
     static const char* context = "as_raw_ptr_owned";
     ensure_compatible_rtti<T>(context);
     ensure_use_count_1(context);
-    ensure_unique_ptr_default_deleter(context);
-    ensure_internal_shared_ptr(context);
+    ensure_vptr_deleter_flag_true(context);
     std::shared_ptr<T> tptr = std::static_pointer_cast<T>(vptr);
     vptr.reset();
     T* result = tptr.get();
-    // TODO tptr.release();
+    vptr_deleter_flag = false;
     return result;
   }