diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2f295614e..8b537e6e1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -315,8 +315,8 @@ jobs:
   # Testing NVCC; forces sources to behave like .cu files
   cuda:
     runs-on: ubuntu-latest
-    name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04"
-    container: nvidia/cuda:11.0-devel-ubuntu20.04
+    name: "🐍 3.8 • CUDA 11.2 • Ubuntu 20.04"
+    container: nvidia/cuda:11.2.2-devel-ubuntu20.04
 
     steps:
     - uses: actions/checkout@v3
diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 376a67954..9a971704e 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -1009,6 +1009,8 @@ struct return_value_policy_override<
 // Basic python -> C++ casting; throws if casting fails
 template <typename T, typename SFINAE>
 type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &handle) {
+    static_assert(!detail::is_pyobject<T>::value,
+                  "Internal error: type_caster should only be used for C++ types");
     if (!conv.load(handle, true)) {
 #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES)
         throw cast_error("Unable to cast Python instance to C++ type (#define "
@@ -1099,21 +1101,30 @@ detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) {
 // - If both movable and copyable, check ref count: if 1, move; otherwise copy
 // - Otherwise (not movable), copy.
 template <typename T>
-detail::enable_if_t<detail::move_always<T>::value, T> cast(object &&object) {
+detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_always<T>::value, T>
+cast(object &&object) {
     return move<T>(std::move(object));
 }
 template <typename T>
-detail::enable_if_t<detail::move_if_unreferenced<T>::value, T> cast(object &&object) {
+detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_if_unreferenced<T>::value, T>
+cast(object &&object) {
     if (object.ref_count() > 1) {
         return cast<T>(object);
     }
     return move<T>(std::move(object));
 }
 template <typename T>
-detail::enable_if_t<detail::move_never<T>::value, T> cast(object &&object) {
+detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_never<T>::value, T>
+cast(object &&object) {
     return cast<T>(object);
 }
 
+// pytype rvalue -> pytype (calls converting constructor)
+template <typename T>
+detail::enable_if_t<detail::is_pyobject<T>::value, T> cast(object &&object) {
+    return T(std::move(object));
+}
+
 template <typename T>
 T object::cast() const & {
     return pybind11::cast<T>(*this);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index a7a2c91b1..f0c9ef54c 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -352,7 +352,7 @@ endif()
 # Compile with compiler warnings turned on
 function(pybind11_enable_warnings target_name)
   if(MSVC)
-    target_compile_options(${target_name} PRIVATE /W4)
+    target_compile_options(${target_name} PRIVATE /W4 /wd4189)
   elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)" AND NOT PYBIND11_CUDA_TESTS)
     target_compile_options(
       ${target_name}
diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp
index 6e1bdd856..28c244564 100644
--- a/tests/test_copy_move.cpp
+++ b/tests/test_copy_move.cpp
@@ -289,4 +289,7 @@ TEST_SUBMODULE(copy_move_policies, m) {
         py::return_value_policy::move);
     m.def(
         "get_moveissue2", [](int i) { return MoveIssue2(i); }, py::return_value_policy::move);
+
+    // Make sure that cast from pytype rvalue to other pytype works
+    m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast<py::int_>(); });
 }
diff --git a/tests/test_copy_move.py b/tests/test_copy_move.py
index f2c0cdceb..9fef08933 100644
--- a/tests/test_copy_move.py
+++ b/tests/test_copy_move.py
@@ -123,3 +123,10 @@ def test_move_fallback():
     assert m1.value == 1
     m2 = m.get_moveissue2(2)
     assert m2.value == 2
+
+
+def test_pytype_rvalue_cast():
+    """Make sure that cast from pytype rvalue to other pytype works"""
+
+    value = m.get_pytype_rvalue_castissue(1.0)
+    assert value == 1