Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -303,6 +303,9 @@ jobs:
           - clang: 15
             std: 20
             container_suffix: "-bullseye"
+          - clang: 16
+            std: 20
+            container_suffix: "-bullseye"
 
     name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64"
     container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}"
@@ -466,6 +469,7 @@ jobs:
           - { gcc: 10, std: 17 }
           - { gcc: 11, std: 20 }
           - { gcc: 12, std: 20 }
+          - { gcc: 13, std: 20 }
 
     name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64"
     container: "gcc:${{ matrix.gcc }}"

.pre-commit-config.yaml

@@ -32,13 +32,13 @@ repos:
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black
-  rev: "23.3.0" # Keep in sync with blacken-docs
+  rev: "23.7.0" # Keep in sync with blacken-docs
   hooks:
   - id: black
 
 # Ruff, the Python auto-correcting linter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.276
+  rev: v0.0.281
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -85,7 +85,7 @@ repos:
 
 # Also code format the docs
 - repo: https://github.com/asottile/blacken-docs
-  rev: "1.14.0"
+  rev: "1.15.0"
   hooks:
   - id: blacken-docs
     additional_dependencies:

CMakeLists.txt

@@ -148,7 +148,8 @@ set(PYBIND11_HEADERS
     include/pybind11/stl_bind.h
     include/pybind11/stl/filesystem.h
     include/pybind11/trampoline_self_life_support.h
-    include/pybind11/type_caster_pyobject_ptr.h)
+    include/pybind11/type_caster_pyobject_ptr.h
+    include/pybind11/typing.h)
 
 # Compare with grep and warn if mismatched
 if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)

docs/advanced/misc.rst

@@ -398,3 +398,32 @@ before they are used as a parameter or return type of a function:
         pyFoo.def(py::init<const ns::Bar&>());
         pyBar.def(py::init<const ns::Foo&>());
     }
+
+Setting inner type hints in docstrings
+======================================
+
+When you use pybind11 wrappers for ``list``, ``dict``, and other generic python
+types, the docstring will just display the generic type. You can convey the
+inner types in the docstring by using a special 'typed' version of the generic
+type.
+
+.. code-block:: cpp
+
+    PYBIND11_MODULE(example, m) {
+        m.def("pass_list_of_str", [](py::typing::List<py::str> arg) {
+            // arg can be used just like py::list
+        ));
+    }
+
+The resulting docstring will be ``pass_list_of_str(arg0: list[str]) -> None``.
+
+The following special types are available in ``pybind11/typing.h``:
+
+* ``py::Tuple<Args...>``
+* ``py::Dict<K, V>``
+* ``py::List<V>``
+* ``py::Set<V>``
+* ``py::Callable<Signature>``
+
+.. warning:: Just like in python, these are merely hints. They don't actually
+             enforce the types of their contents at runtime or compile time.

docs/compiling.rst

@@ -143,7 +143,7 @@ Your ``pyproject.toml`` file will likely look something like this:
 .. code-block:: toml
 
     [build-system]
-    requires = ["setuptools>=42", "wheel", "pybind11~=2.6.1"]
+    requires = ["setuptools>=42", "pybind11>=2.6.1"]
     build-backend = "setuptools.build_meta"
 
 .. note::

docs/release.rst

@@ -15,8 +15,8 @@ For example:
 
 For beta, ``PYBIND11_VERSION_PATCH`` should be ``Z.b1``. RC's can be ``Z.rc1``.
 Always include the dot (even though PEP 440 allows it to be dropped). For a
-final release, this must be a simple integer. There is also a HEX version of
+final release, this must be a simple integer. There is also
-the version just below.
+``PYBIND11_VERSION_HEX`` just below that needs to be updated.
 
 
 To release a new version of pybind11:
@@ -26,56 +26,93 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use
 ``pipx install nox`` or ``brew install nox`` (Unix).
 
 - Update the version number
-    - Update ``PYBIND11_VERSION_MAJOR`` etc. in
+
-      ``include/pybind11/detail/common.h``. PATCH should be a simple integer.
+  - Update ``PYBIND11_VERSION_MAJOR`` etc. in
-    - Update the version HEX just below, as well.
+    ``include/pybind11/detail/common.h``. PATCH should be a simple integer.
-    - Update ``pybind11/_version.py`` (match above)
+
-    - Run ``nox -s tests_packaging`` to ensure this was done correctly.
+  - Update ``PYBIND11_VERSION_HEX`` just below as well.
-    - Ensure that all the information in ``setup.cfg`` is up-to-date, like
+
-      supported Python versions.
+  - Update ``pybind11/_version.py`` (match above).
-    - Add release date in ``docs/changelog.rst`` and integrate the output of
+
-      ``nox -s make_changelog``.
+  - Run ``nox -s tests_packaging`` to ensure this was done correctly.
-          - Note that the ``make_changelog`` command inspects
+
-            `needs changelog <https://github.com/pybind/pybind11/pulls?q=is%3Apr+is%3Aclosed+label%3A%22needs+changelog%22>`_.
+  - Ensure that all the information in ``setup.cfg`` is up-to-date, like
-          - Manually clear the ``needs changelog`` labels using the GitHub web
+    supported Python versions.
-            interface (very easy: start by clicking the link above).
+
-    - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it
+  - Add release date in ``docs/changelog.rst`` and integrate the output of
-      fails due to a known flake issue, either ignore or restart CI.)
+    ``nox -s make_changelog``.
-- Add a release branch if this is a new MINOR version, or update the existing release branch if it is a patch version
+
-    - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y``
+    - Note that the ``make_changelog`` command inspects
-    - Update branch: ``git checkout vX.Y``, ``git merge <release branch>``, ``git push``
+      `needs changelog <https://github.com/pybind/pybind11/pulls?q=is%3Apr+is%3Aclosed+label%3A%22needs+changelog%22>`_.
+
+    - Manually clear the ``needs changelog`` labels using the GitHub web
+      interface (very easy: start by clicking the link above).
+
+  - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it
+    fails due to a known flake issue, either ignore or restart CI.)
+
+- Add a release branch if this is a new MINOR version, or update the existing
+  release branch if it is a patch version
+
+  - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y``
+
+  - Update branch: ``git checkout vX.Y``, ``git merge <release branch>``, ``git push``
+
 - Update tags (optional; if you skip this, the GitHub release makes a
-    non-annotated tag for you)
+  non-annotated tag for you)
-    - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``.
+
-    - ``git push --tags``.
+  - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``
+
+  - ``grep ^__version__ pybind11/_version.py``
+
+    - Last-minute consistency check: same as tag?
+
+  - ``git push --tags``
+
 - Update stable
-    - ``git checkout stable``
+
-    - ``git merge -X theirs vX.Y.Z``
+  - ``git checkout stable``
-    - ``git diff vX.Y.Z``
+
-    - Carefully review and reconcile any diffs. There should be none.
+  - ``git merge -X theirs vX.Y.Z``
-    - ``git push``
+
+  - ``git diff vX.Y.Z``
+
+  - Carefully review and reconcile any diffs. There should be none.
+
+  - ``git push``
+
 - Make a GitHub release (this shows up in the UI, sends new release
   notifications to users watching releases, and also uploads PyPI packages).
   (Note: if you do not use an existing tag, this creates a new lightweight tag
   for you, so you could skip the above step.)
-    - GUI method: Under `releases <https://github.com/pybind/pybind11/releases>`_
+
-      click "Draft a new release" on the far right, fill in the tag name
+  - GUI method: Under `releases <https://github.com/pybind/pybind11/releases>`_
-      (if you didn't tag above, it will be made here), fill in a release name
+    click "Draft a new release" on the far right, fill in the tag name
-      like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog
+    (if you didn't tag above, it will be made here), fill in a release name
-      into the description (usually ``cat docs/changelog.rst | pandoc -f rst -t gfm``).
+    like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog
-      Check "pre-release" if this is a beta/RC.
+    into the description. You can use ``cat docs/changelog.rst | pandoc -f rst -t gfm``,
-    - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"``
+    then manually remove line breaks and strip links to PRs and issues,
-      If this is a pre-release, add ``-p``.
+    e.g. to a bare ``#1234``, without the surrounding ``<...>_`` hyperlink markup.
+    Check "pre-release" if this is a beta/RC.
+
+  - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"``
+    If this is a pre-release, add ``-p``.
 
 - Get back to work
-    - Make sure you are on master, not somewhere else: ``git checkout master``
+
-    - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to
+  - Make sure you are on master, not somewhere else: ``git checkout master``
-      ``0.dev1`` and increment MINOR).
+
-    - Update ``pybind11/_version.py`` to match
+  - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to
-    - Run ``nox -s tests_packaging`` to ensure this was done correctly.
+    ``0.dev1`` and increment MINOR).
-    - If the release was a new MINOR version, add a new `IN DEVELOPMENT`
+
-      section in ``docs/changelog.rst``.
+  - Update ``pybind11/_version.py`` to match.
-    - ``git add``, ``git commit``, ``git push``
+
+  - Run ``nox -s tests_packaging`` to ensure this was done correctly.
+
+  - If the release was a new MINOR version, add a new ``IN DEVELOPMENT``
+    section in ``docs/changelog.rst``.
+
+  - ``git add``, ``git commit``, ``git push``
 
 If a version branch is updated, remember to set PATCH to ``1.dev1``.
 
@@ -92,7 +129,11 @@ merge it if there are no issues.
 Manual packaging
 ^^^^^^^^^^^^^^^^
 
-If you need to manually upload releases, you can download the releases from the job artifacts and upload them with twine. You can also make the files locally (not recommended in general, as your local directory is more likely to be "dirty" and SDists love picking up random unrelated/hidden files); this is the procedure:
+If you need to manually upload releases, you can download the releases from
+the job artifacts and upload them with twine. You can also make the files
+locally (not recommended in general, as your local directory is more likely
+to be "dirty" and SDists love picking up random unrelated/hidden files);
+this is the procedure:
 
 .. code-block:: bash
 

include/pybind11/cast.h

@@ -1419,7 +1419,15 @@ inline namespace literals {
 /** \rst
     String literal version of `arg`
  \endrst */
-constexpr arg operator"" _a(const char *name, size_t) { return arg(name); }
+constexpr arg
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5
+operator"" _a // gcc 4.8.5 insists on having a space (hard error).
+#else
+operator""_a // clang 17 generates a deprecation warning if there is a space.
+#endif
+    (const char *name, size_t) {
+    return arg(name);
+}
 } // namespace literals
 
 PYBIND11_NAMESPACE_BEGIN(detail)

include/pybind11/pytypes.h

@@ -1612,7 +1612,15 @@ inline namespace literals {
 /** \rst
     String literal version of `str`
  \endrst */
-inline str operator"" _s(const char *s, size_t size) { return {s, size}; }
+inline str
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5
+operator"" _s // gcc 4.8.5 insists on having a space (hard error).
+#else
+operator""_s // clang 17 generates a deprecation warning if there is a space.
+#endif
+    (const char *s, size_t size) {
+    return {s, size};
+}
 } // namespace literals
 
 /// \addtogroup pytypes

include/pybind11/typing.h

@@ -0,0 +1,97 @@
+/*
+    pybind11/typing.h: Convenience wrapper classes for basic Python types
+    with more explicit annotations.
+
+    Copyright (c) 2023 Dustin Spicuzza <dustin@virtualroadside.com>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "detail/common.h"
+#include "cast.h"
+#include "pytypes.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(typing)
+
+/*
+    The following types can be used to direct pybind11-generated docstrings
+    to have have more explicit types (e.g., `list[str]` instead of `list`).
+    Just use these in place of existing types.
+
+    There is no additional enforcement of types at runtime.
+*/
+
+template <typename... Types>
+class Tuple : public tuple {
+    using tuple::tuple;
+};
+
+template <typename K, typename V>
+class Dict : public dict {
+    using dict::dict;
+};
+
+template <typename T>
+class List : public list {
+    using list::list;
+};
+
+template <typename T>
+class Set : public set {
+    using set::set;
+};
+
+template <typename Signature>
+class Callable;
+
+template <typename Return, typename... Args>
+class Callable<Return(Args...)> : public function {
+    using function::function;
+};
+
+PYBIND11_NAMESPACE_END(typing)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename... Types>
+struct handle_type_name<typing::Tuple<Types...>> {
+    static constexpr auto name
+        = const_name("tuple[") + concat(make_caster<Types>::name...) + const_name("]");
+};
+
+template <>
+struct handle_type_name<typing::Tuple<>> {
+    // PEP 484 specifies this syntax for an empty tuple
+    static constexpr auto name = const_name("tuple[()]");
+};
+
+template <typename K, typename V>
+struct handle_type_name<typing::Dict<K, V>> {
+    static constexpr auto name = const_name("dict[") + make_caster<K>::name + const_name(", ")
+                                 + make_caster<V>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::List<T>> {
+    static constexpr auto name = const_name("list[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename T>
+struct handle_type_name<typing::Set<T>> {
+    static constexpr auto name = const_name("set[") + make_caster<T>::name + const_name("]");
+};
+
+template <typename Return, typename... Args>
+struct handle_type_name<typing::Callable<Return(Args...)>> {
+    using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>;
+    static constexpr auto name = const_name("Callable[[") + concat(make_caster<Args>::name...)
+                                 + const_name("], ") + make_caster<retval_type>::name
+                                 + const_name("]");
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/extra_python_package/test_files.py

@@ -46,6 +46,7 @@ main_headers = {
     "include/pybind11/stl_bind.h",
     "include/pybind11/trampoline_self_life_support.h",
     "include/pybind11/type_caster_pyobject_ptr.h",
+    "include/pybind11/typing.h",
 }
 
 detail_headers = {

tests/test_pytypes.cpp

@@ -7,6 +7,8 @@
     BSD-style license that can be found in the LICENSE file.
 */
 
+#include <pybind11/typing.h>
+
 #include "pybind11_tests.h"
 
 #include <utility>
@@ -820,4 +822,12 @@ TEST_SUBMODULE(pytypes, m) {
         a >>= b;
         return a;
     });
+
+    m.def("annotate_tuple_float_str", [](const py::typing::Tuple<py::float_, py::str> &) {});
+    m.def("annotate_tuple_empty", [](const py::typing::Tuple<> &) {});
+    m.def("annotate_dict_str_int", [](const py::typing::Dict<py::str, int> &) {});
+    m.def("annotate_list_int", [](const py::typing::List<int> &) {});
+    m.def("annotate_set_str", [](const py::typing::Set<std::string> &) {});
+    m.def("annotate_fn",
+          [](const py::typing::Callable<int(py::typing::List<py::str>, py::str)> &) {});
 }

tests/test_pytypes.py

@@ -896,3 +896,38 @@ def test_inplace_lshift(a, b):
 def test_inplace_rshift(a, b):
     expected = a >> b
     assert m.inplace_rshift(a, b) == expected
+
+
+def test_tuple_nonempty_annotations(doc):
+    assert (
+        doc(m.annotate_tuple_float_str)
+        == "annotate_tuple_float_str(arg0: tuple[float, str]) -> None"
+    )
+
+
+def test_tuple_empty_annotations(doc):
+    assert (
+        doc(m.annotate_tuple_empty) == "annotate_tuple_empty(arg0: tuple[()]) -> None"
+    )
+
+
+def test_dict_annotations(doc):
+    assert (
+        doc(m.annotate_dict_str_int)
+        == "annotate_dict_str_int(arg0: dict[str, int]) -> None"
+    )
+
+
+def test_list_annotations(doc):
+    assert doc(m.annotate_list_int) == "annotate_list_int(arg0: list[int]) -> None"
+
+
+def test_set_annotations(doc):
+    assert doc(m.annotate_set_str) == "annotate_set_str(arg0: set[str]) -> None"
+
+
+def test_fn_annotations(doc):
+    assert (
+        doc(m.annotate_fn)
+        == "annotate_fn(arg0: Callable[[list[str], str], int]) -> None"
+    )