diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index b84ba4ec5..819f456e6 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -1220,10 +1220,12 @@ struct smart_holder_type_caster_load {
     }
 
     T *loaded_as_raw_ptr_unowned() const {
+        if (!have_value()) return nullptr;
         return convert_type(holder().template as_raw_ptr_unowned<void>());
     }
 
     T &loaded_as_lvalue_ref() const {
+        if (!have_value()) throw reference_cast_error();
         static const char *context = "loaded_as_lvalue_ref";
         holder().ensure_is_populated(context);
         holder().ensure_has_pointee(context);
@@ -1231,6 +1233,7 @@ struct smart_holder_type_caster_load {
     }
 
     T &&loaded_as_rvalue_ref() const {
+        if (!have_value()) throw reference_cast_error();
         static const char *context = "loaded_as_rvalue_ref";
         holder().ensure_is_populated(context);
         holder().ensure_has_pointee(context);
@@ -1238,12 +1241,14 @@ struct smart_holder_type_caster_load {
     }
 
     std::shared_ptr<T> loaded_as_shared_ptr() {
+        if (!have_value()) return nullptr;
         std::shared_ptr<void> void_ptr = holder().template as_shared_ptr<void>();
         return std::shared_ptr<T>(void_ptr, convert_type(void_ptr.get()));
     }
 
     template <typename D>
     std::unique_ptr<T, D> loaded_as_unique_ptr(const char *context = "loaded_as_unique_ptr") {
+        if (!have_value()) return nullptr;
         holder().template ensure_compatible_rtti_uqp_del<T, D>(context);
         holder().ensure_use_count_1(context);
         auto raw_void_ptr = holder().template as_raw_ptr_unowned<void>();
@@ -1268,6 +1273,8 @@ struct smart_holder_type_caster_load {
 private:
     modified_type_caster_generic_load_impl load_impl;
 
+    bool have_value() const { return load_impl.loaded_v_h.vh != nullptr; }
+
     holder_type &holder() const { return load_impl.loaded_v_h.holder<holder_type>(); }
 
     T *convert_type(void *void_ptr) const {