Add TypeVars / method generics typing (#5167)

* typevar prototype

* style: pre-commit fixes

* change to NameT

* style: pre-commit fixes

* make string const

* add missing closing bracket

* style: pre-commit fixes

* clean up handle_type_name

* style: pre-commit fixes

* add back missing <

* style: pre-commit fixes

* add back NameT

* try fixed_string

* style: pre-commit fixes

* std::basic_fixed_string

* test c++20

* style: pre-commit fixes

* cleanup

* fix object to typevar conversion

* style: pre-commit fixes

* And CPP20 checks

* style: pre-commit fixes

* add missing cpp20++ check

* style: pre-commit fixes

* Add C++20 check to python

* Fix python if {

* style: pre-commit fixes

* update test name

* style: pre-commit fixes

* remove call on cpp_std

* make field const

* test nontype_template

* update feature check

* update name of guard

* fix try except in test

* fix pre commit

* remove extra semi colon

* except AttributeError

* fix try except in test

* remove const

* Clean up tests

* style: pre-commit fixes

* use contextlib.suppres

* request changes

* lint

* Add comments

* style: pre-commit fixes

* Add support for unions and optionals to be compatible with object

* lint

* remove comment

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

include/pybind11/typing.h

@@ -70,14 +70,32 @@ class Type : public type {
 
 template <typename... Types>
 class Union : public object {
+    PYBIND11_OBJECT_DEFAULT(Union, object, PyObject_Type)
     using object::object;
 };
 
 template <typename T>
 class Optional : public object {
+    PYBIND11_OBJECT_DEFAULT(Optional, object, PyObject_Type)
     using object::object;
 };
 
+#if defined(__cpp_nontype_template_parameter_class)
+template <size_t N>
+struct StringLiteral {
+    constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, value); }
+    char value[N];
+};
+
+// Example syntax for creating a TypeVar.
+// typedef typing::TypeVar<"T"> TypeVarT;
+template <StringLiteral>
+class TypeVar : public object {
+    PYBIND11_OBJECT_DEFAULT(TypeVar, object, PyObject_Type)
+    using object::object;
+};
+#endif
+
 PYBIND11_NAMESPACE_END(typing)
 
 PYBIND11_NAMESPACE_BEGIN(detail)
@@ -153,5 +171,12 @@ struct handle_type_name<typing::Optional<T>> {
     static constexpr auto name = const_name("Optional[") + make_caster<T>::name + const_name("]");
 };
 
+#if defined(__cpp_nontype_template_parameter_class)
+template <typing::StringLiteral StrLit>
+struct handle_type_name<typing::TypeVar<StrLit>> {
+    static constexpr auto name = const_name(StrLit.value);
+};
+#endif
+
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/test_pytypes.cpp

@@ -109,6 +109,13 @@ void m_defs(py::module_ &m) {
 
 } // namespace handle_from_move_only_type_with_operator_PyObject
 
+#if defined(__cpp_nontype_template_parameter_class)
+namespace typevar {
+typedef py::typing::TypeVar<"T"> TypeVarT;
+typedef py::typing::TypeVar<"V"> TypeVarV;
+} // namespace typevar
+#endif
+
 TEST_SUBMODULE(pytypes, m) {
     m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); });
 
@@ -844,7 +851,7 @@ TEST_SUBMODULE(pytypes, m) {
     m.def("annotate_iterator_int", [](const py::typing::Iterator<int> &) {});
     m.def("annotate_fn",
           [](const py::typing::Callable<int(py::typing::List<py::str>, py::str)> &) {});
-    m.def("annotate_type", [](const py::typing::Type<int> &) {});
+    m.def("annotate_type", [](const py::typing::Type<int> &t) -> py::type { return t; });
 
     m.def("annotate_union",
           [](py::typing::List<py::typing::Union<py::str, py::int_, py::object>> l,
@@ -861,10 +868,29 @@ TEST_SUBMODULE(pytypes, m) {
           [](py::typing::List<py::typing::Union<py::str>> &l)
               -> py::typing::List<py::typing::Union<py::int_>> { return l; });
 
+    m.def("annotate_union_to_object",
+          [](py::typing::Union<int, py::str> &o) -> py::object { return o; });
+
     m.def("annotate_optional",
           [](py::list &list) -> py::typing::List<py::typing::Optional<py::str>> {
               list.append(py::str("hi"));
               list.append(py::none());
               return list;
           });
+    m.def("annotate_optional_to_object",
+          [](py::typing::Optional<int> &o) -> py::object { return o; });
+
+#if defined(__cpp_nontype_template_parameter_class)
+    m.def("annotate_generic_containers",
+          [](const py::typing::List<typevar::TypeVarT> &l) -> py::typing::List<typevar::TypeVarV> {
+              return l;
+          });
+
+    m.def("annotate_listT_to_T",
+          [](const py::typing::List<typevar::TypeVarT> &l) -> typevar::TypeVarT { return l[0]; });
+    m.def("annotate_object_to_T", [](const py::object &o) -> typevar::TypeVarT { return o; });
+    m.attr("if_defined__cpp_nontype_template_parameter_class") = true;
+#else
+    m.attr("if_defined__cpp_nontype_template_parameter_class") = false;
+#endif
 }

tests/test_pytypes.py

@@ -960,7 +960,7 @@ def test_fn_annotations(doc):
 
 
 def test_type_annotation(doc):
-    assert doc(m.annotate_type) == "annotate_type(arg0: type[int]) -> None"
+    assert doc(m.annotate_type) == "annotate_type(arg0: type[int]) -> type"
 
 
 def test_union_annotations(doc):
@@ -977,8 +977,37 @@ def test_union_typing_only(doc):
     )
 
 
+def test_union_object_annotations(doc):
+    assert (
+        doc(m.annotate_union_to_object)
+        == "annotate_union_to_object(arg0: Union[int, str]) -> object"
+    )
+
+
 def test_optional_annotations(doc):
     assert (
         doc(m.annotate_optional)
         == "annotate_optional(arg0: list) -> list[Optional[str]]"
     )
+
+
+def test_optional_object_annotations(doc):
+    assert (
+        doc(m.annotate_optional_to_object)
+        == "annotate_optional_to_object(arg0: Optional[int]) -> object"
+    )
+
+
+@pytest.mark.skipif(
+    not m.if_defined__cpp_nontype_template_parameter_class,
+    reason="C++20 feature not available.",
+)
+def test_typevar(doc):
+    assert (
+        doc(m.annotate_generic_containers)
+        == "annotate_generic_containers(arg0: list[T]) -> list[V]"
+    )
+
+    assert doc(m.annotate_listT_to_T) == "annotate_listT_to_T(arg0: list[T]) -> T"
+
+    assert doc(m.annotate_object_to_T) == "annotate_object_to_T(arg0: object) -> T"