Merge branch 'master' into sh_merge_master

.github/workflows/ci.yml

@@ -76,7 +76,7 @@ jobs:
       run: brew install boost
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Cache wheels
       if: runner.os == 'macOS'
@@ -212,7 +212,7 @@ jobs:
         debug: ${{ matrix.python-debug }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Valgrind cache
       if: matrix.valgrind
@@ -464,7 +464,7 @@ jobs:
       run: python3 -m pip install --upgrade pip
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Configure
       shell: bash
@@ -757,7 +757,7 @@ jobs:
         architecture: x86
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Prepare MSVC
       uses: ilammy/msvc-dev-cmd@v1.9.0
@@ -803,7 +803,7 @@ jobs:
         python-version: ${{ matrix.python }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Prepare MSVC
       uses: ilammy/msvc-dev-cmd@v1.9.0
@@ -857,7 +857,7 @@ jobs:
         python-version: ${{ matrix.python }}
 
     - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
 
     - name: Prepare env
       run: python -m pip install -r tests/requirements.txt --prefer-binary

.github/workflows/configure.yml

@@ -56,7 +56,7 @@ jobs:
     # An action for adding a specific version of CMake:
     #   https://github.com/jwlawson/actions-setup-cmake
     - name: Setup CMake ${{ matrix.cmake }}
-      uses: jwlawson/actions-setup-cmake@v1.10
+      uses: jwlawson/actions-setup-cmake@v1.11
       with:
         cmake-version: ${{ matrix.cmake }}
 

.pre-commit-config.yaml

@@ -19,8 +19,10 @@ repos:
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
+  - id: check-docstring-first
   - id: check-merge-conflict
   - id: check-symlinks
+  - id: check-toml
   - id: check-yaml
   - id: debug-statements
   - id: end-of-file-fixer
@@ -43,12 +45,16 @@ repos:
 
 # Black, the code formatter, natively supports pre-commit
 - repo: https://github.com/psf/black
-  rev: 21.9b0
+  rev: 21.9b0 # Keep in sync with blacken-docs
   hooks:
   - id: black
-    # By default, this ignores pyi files, though black supports them
-    types: [text]
-    files: \.pyi?$
+
+- repo: https://github.com/asottile/blacken-docs
+  rev: v1.11.0
+  hooks:
+  - id: blacken-docs
+    additional_dependencies:
+    - black==21.9b0 # keep in sync with black hook
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
@@ -57,6 +63,12 @@ repos:
   - id: remove-tabs
     exclude: (^docs/.*|\.patch)?$
 
+# Autoremoves unused imports
+- repo: https://github.com/hadialqattan/pycln
+  rev: v1.0.3
+  hooks:
+  - id: pycln
+
 # Flake8 also supports pre-commit natively (same author)
 - repo: https://github.com/PyCQA/flake8
   rev: 3.9.2
@@ -88,7 +100,7 @@ repos:
 
 # Checks the manifest for missing files (native support)
 - repo: https://github.com/mgedmin/check-manifest
-  rev: "0.46"
+  rev: "0.47"
   hooks:
   - id: check-manifest
     # This is a slow hook, so only run this if --hook-stage manual is passed
@@ -102,10 +114,10 @@ repos:
     exclude: ".supp$"
     args: ["-L", "nd,ot,thist", "--exclude-file", ".codespell-ignorelines"]
 
--   repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.7.2.1
-    hooks:
-    -   id: shellcheck
+- repo: https://github.com/shellcheck-py/shellcheck-py
+  rev: v0.7.2.1
+  hooks:
+  - id: shellcheck
 
 # The original pybind11 checks for a few C++ style items
 - repo: local

docs/advanced/cast/custom.rst

@@ -26,7 +26,9 @@ The following Python snippet demonstrates the intended usage from the Python sid
         def __int__(self):
             return 123
 
+
     from example import print
+
     print(A())
 
 To register the necessary conversion routines, it is necessary to add an

docs/advanced/cast/eigen.rst

@@ -112,7 +112,7 @@ example:
 .. code-block:: python
 
     a = MyClass()
-    m = a.get_matrix()   # flags.writeable = True,  flags.owndata = False
+    m = a.get_matrix()  # flags.writeable = True,  flags.owndata = False
     v = a.view_matrix()  # flags.writeable = False, flags.owndata = False
     c = a.copy_matrix()  # flags.writeable = True,  flags.owndata = True
     # m[5,6] and v[5,6] refer to the same element, c[5,6] does not.
@@ -203,7 +203,7 @@ adding the ``order='F'`` option when creating an array:
 
 .. code-block:: python
 
-    myarray = np.array(source, order='F')
+    myarray = np.array(source, order="F")
 
 Such an object will be passable to a bound function accepting an
 ``Eigen::Ref<MatrixXd>`` (or similar column-major Eigen type).

docs/advanced/cast/strings.rst

@@ -36,13 +36,13 @@ everywhere <http://utf8everywhere.org/>`_.
         }
     );
 
-.. code-block:: python
+.. code-block:: pycon
 
-    >>> utf8_test('🎂')
+    >>> utf8_test("🎂")
     utf-8 is icing on the cake.
     🎂
 
-    >>> utf8_charptr('🍕')
+    >>> utf8_charptr("🍕")
     My favorite food is
     🍕
 
@@ -80,7 +80,7 @@ raise a ``UnicodeDecodeError``.
         }
     );
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> isinstance(example.std_string_return(), str)
     True
@@ -114,7 +114,7 @@ conversion has the same overhead as implicit conversion.
         }
     );
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> str_output()
     'Send your résumé to Alice in HR'
@@ -143,7 +143,7 @@ returned to Python as ``bytes``, then one can return the data as a
         }
     );
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> example.return_bytes()
     b'\xba\xd0\xba\xd0'
@@ -160,7 +160,7 @@ encoding, but cannot convert ``std::string`` back to ``bytes`` implicitly.
         }
     );
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> isinstance(example.asymmetry(b"have some bytes"), str)
     True
@@ -229,16 +229,16 @@ character.
     m.def("pass_char", [](char c) { return c; });
     m.def("pass_wchar", [](wchar_t w) { return w; });
 
-.. code-block:: python
+.. code-block:: pycon
 
-    >>> example.pass_char('A')
+    >>> example.pass_char("A")
     'A'
 
 While C++ will cast integers to character types (``char c = 0x65;``), pybind11
 does not convert Python integers to characters implicitly. The Python function
 ``chr()`` can be used to convert integers to characters.
 
-.. code-block:: python
+.. code-block:: pycon
 
     >>> example.pass_char(0x65)
     TypeError
@@ -259,17 +259,17 @@ a combining acute accent). The combining character will be lost if the
 two-character sequence is passed as an argument, even though it renders as a
 single grapheme.
 
-.. code-block:: python
+.. code-block:: pycon
 
-    >>> example.pass_wchar('é')
+    >>> example.pass_wchar("é")
     'é'
 
-    >>> combining_e_acute = 'e' + '\u0301'
+    >>> combining_e_acute = "e" + "\u0301"
 
     >>> combining_e_acute
     'é'
 
-    >>> combining_e_acute == 'é'
+    >>> combining_e_acute == "é"
     False
 
     >>> example.pass_wchar(combining_e_acute)
@@ -278,9 +278,9 @@ single grapheme.
 Normalizing combining characters before passing the character literal to C++
 may resolve *some* of these issues:
 
-.. code-block:: python
+.. code-block:: pycon
 
-    >>> example.pass_wchar(unicodedata.normalize('NFC', combining_e_acute))
+    >>> example.pass_wchar(unicodedata.normalize("NFC", combining_e_acute))
     'é'
 
 In some languages (Thai for example), there are `graphemes that cannot be

docs/advanced/classes.rst

@@ -136,7 +136,7 @@ a virtual method call.
     u'woof! woof! woof! '
     >>> class Cat(Animal):
     ...     def go(self, n_times):
-    ...             return "meow! " * n_times
+    ...         return "meow! " * n_times
     ...
     >>> c = Cat()
     >>> call_go(c)
@@ -159,8 +159,9 @@ Here is an example:
 
     class Dachshund(Dog):
         def __init__(self, name):
-            Dog.__init__(self) # Without this, a TypeError is raised.
+            Dog.__init__(self)  # Without this, a TypeError is raised.
             self.name = name
+
         def bark(self):
             return "yap!"
 
@@ -1153,6 +1154,7 @@ error:
 
     >>> class PyFinalChild(IsFinal):
     ...     pass
+    ...
     TypeError: type 'IsFinal' is not an acceptable base type
 
 .. note:: This attribute is currently ignored on PyPy
@@ -1247,7 +1249,7 @@ Accessing the type object
 
 You can get the type object from a C++ class that has already been registered using:
 
-.. code-block:: python
+.. code-block:: cpp
 
     py::type T_py = py::type::of<T>();
 

docs/advanced/embedding.rst

@@ -122,6 +122,7 @@ embedding the interpreter. This makes it easy to import local Python files:
 
     """calc.py located in the working directory"""
 
+
     def add(i, j):
         return i + j
 

docs/advanced/functions.rst

@@ -272,7 +272,7 @@ And used in Python as usual:
 
 .. code-block:: pycon
 
-    >>> print_dict({'foo': 123, 'bar': 'hello'})
+    >>> print_dict({"foo": 123, "bar": "hello"})
     key=foo, value=123
     key=bar, value=hello
 
@@ -377,10 +377,11 @@ argument in a function definition:
     def f(a, *, b):  # a can be positional or via keyword; b must be via keyword
         pass
 
+
     f(a=1, b=2)  # good
     f(b=2, a=1)  # good
-    f(1, b=2)    # good
-    f(1, 2)      # TypeError: f() takes 1 positional argument but 2 were given
+    f(1, b=2)  # good
+    f(1, 2)  # TypeError: f() takes 1 positional argument but 2 were given
 
 Pybind11 provides a ``py::kw_only`` object that allows you to implement
 the same behaviour by specifying the object between positional and keyword-only

docs/advanced/pycpp/numpy.rst

@@ -258,8 +258,8 @@ by the compiler. The result is returned as a NumPy array of type
 
 .. code-block:: pycon
 
-    >>> x = np.array([[1, 3],[5, 7]])
-    >>> y = np.array([[2, 4],[6, 8]])
+    >>> x = np.array([[1, 3], [5, 7]])
+    >>> y = np.array([[2, 4], [6, 8]])
     >>> z = 3
     >>> result = vectorized_func(x, y, z)
 
@@ -403,7 +403,7 @@ In Python 2, the syntactic sugar ``...`` is not available, but the singleton
 
 .. code-block:: python
 
-   a = # a NumPy array
+   a = ...  # a NumPy array
    b = a[0, ..., 0]
 
 The function ``py::ellipsis()`` function can be used to perform the same

docs/advanced/pycpp/object.rst

@@ -173,6 +173,7 @@ Keyword arguments are also supported. In Python, there is the usual call syntax:
     def f(number, say, to):
         ...  # function code
 
+
     f(1234, say="hello", to=some_instance)  # keyword call in Python
 
 In C++, the same call can be made using:

docs/advanced/pycpp/utilities.rst

@@ -66,7 +66,7 @@ extra type, `py::scoped_estream_redirect <scoped_estream_redirect>`, is identica
 except for defaulting to ``std::cerr`` and ``sys.stderr``; this can be useful with
 `py::call_guard`, which allows multiple items, but uses the default constructor:
 
-.. code-block:: py
+.. code-block:: cpp
 
     // Alternative: Call single function using call guard
     m.def("noisy_func", &call_noisy_function,

docs/advanced/smart_ptrs.rst

@@ -84,6 +84,7 @@ segmentation fault).
 .. code-block:: python
 
    from example import Parent
+
    print(Parent().get_child())
 
 The problem is that ``Parent::get_child()`` returns a pointer to an instance of

docs/benchmark.py

@@ -2,7 +2,6 @@
 import datetime as dt
 import os
 import random
-import time
 
 nfns = 4  # Functions per class
 nargs = 4  # Arguments per function

docs/changelog.rst

@@ -1142,6 +1142,7 @@ v2.2.0 (August 31, 2017)
 
       from cpp_module import CppBase1, CppBase2
 
+
       class PyDerived(CppBase1, CppBase2):
           def __init__(self):
               CppBase1.__init__(self)  # C++ bases must be initialized explicitly

docs/classes.rst

@@ -44,12 +44,12 @@ interactive Python session demonstrating this example is shown below:
 
     % python
     >>> import example
-    >>> p = example.Pet('Molly')
+    >>> p = example.Pet("Molly")
     >>> print(p)
     <example.Pet object at 0x10cd98060>
     >>> p.getName()
     u'Molly'
-    >>> p.setName('Charly')
+    >>> p.setName("Charly")
     >>> p.getName()
     u'Charly'
 
@@ -122,10 +122,10 @@ This makes it possible to write
 
 .. code-block:: pycon
 
-    >>> p = example.Pet('Molly')
+    >>> p = example.Pet("Molly")
     >>> p.name
     u'Molly'
-    >>> p.name = 'Charly'
+    >>> p.name = "Charly"
     >>> p.name
     u'Charly'
 
@@ -174,10 +174,10 @@ Native Python classes can pick up new attributes dynamically:
 .. code-block:: pycon
 
     >>> class Pet:
-    ...     name = 'Molly'
+    ...     name = "Molly"
     ...
     >>> p = Pet()
-    >>> p.name = 'Charly'  # overwrite existing
+    >>> p.name = "Charly"  # overwrite existing
     >>> p.age = 2  # dynamically add a new attribute
 
 By default, classes exported from C++ do not support this and the only writable
@@ -195,7 +195,7 @@ Trying to set any other attribute results in an error:
 .. code-block:: pycon
 
     >>> p = example.Pet()
-    >>> p.name = 'Charly'  # OK, attribute defined in C++
+    >>> p.name = "Charly"  # OK, attribute defined in C++
     >>> p.age = 2  # fail
     AttributeError: 'Pet' object has no attribute 'age'
 
@@ -213,7 +213,7 @@ Now everything works as expected:
 .. code-block:: pycon
 
     >>> p = example.Pet()
-    >>> p.name = 'Charly'  # OK, overwrite value in C++
+    >>> p.name = "Charly"  # OK, overwrite value in C++
     >>> p.age = 2  # OK, dynamically add a new attribute
     >>> p.__dict__  # just like a native Python class
     {'age': 2}
@@ -280,7 +280,7 @@ expose fields and methods of both types:
 
 .. code-block:: pycon
 
-    >>> p = example.Dog('Molly')
+    >>> p = example.Dog("Molly")
     >>> p.name
     u'Molly'
     >>> p.bark()
@@ -486,7 +486,7 @@ typed enums.
 
 .. code-block:: pycon
 
-    >>> p = Pet('Lucy', Pet.Cat)
+    >>> p = Pet("Lucy", Pet.Cat)
     >>> p.type
     Kind.Cat
     >>> int(p.type)
@@ -508,7 +508,7 @@ The ``name`` property returns the name of the enum value as a unicode string.
 
     .. code-block:: pycon
 
-        >>> p = Pet( "Lucy", Pet.Cat )
+        >>> p = Pet("Lucy", Pet.Cat)
         >>> pet_type = p.type
         >>> pet_type
         Pet.Cat

docs/compiling.rst

@@ -42,10 +42,7 @@ An example of a ``setup.py`` using pybind11's helpers:
         ),
     ]
 
-    setup(
-        ...,
-        ext_modules=ext_modules
-    )
+    setup(..., ext_modules=ext_modules)
 
 If you want to do an automatic search for the highest supported C++ standard,
 that is supported via a ``build_ext`` command override; it will only affect
@@ -64,11 +61,7 @@ that is supported via a ``build_ext`` command override; it will only affect
         ),
     ]
 
-    setup(
-        ...,
-        cmdclass={"build_ext": build_ext},
-        ext_modules=ext_modules
-    )
+    setup(..., cmdclass={"build_ext": build_ext}, ext_modules=ext_modules)
 
 If you have single-file extension modules that are directly stored in the
 Python source tree (``foo.cpp`` in the same directory as where a ``foo.py``

docs/conf.py

@@ -15,7 +15,6 @@
 
 import os
 import re
-import shlex
 import subprocess
 import sys
 from pathlib import Path

docs/faq.rst

@@ -54,7 +54,7 @@ provided by the caller -- in fact, it does nothing at all.
 .. code-block:: python
 
     def increment(i):
-        i += 1 # nope..
+        i += 1  # nope..
 
 pybind11 is also affected by such language-level conventions, which means that
 binding ``increment`` or ``increment_ptr`` will also create Python functions

docs/reference.rst

@@ -63,9 +63,6 @@ Convenience functions converting to Python types
 .. doxygenfunction:: make_key_iterator(Iterator, Sentinel, Extra &&...)
 .. doxygenfunction:: make_key_iterator(Type &, Extra&&...)
 
-.. doxygenfunction:: make_value_iterator(Iterator, Sentinel, Extra &&...)
-.. doxygenfunction:: make_value_iterator(Type &, Extra&&...)
-
 .. _extras:
 
 Passing extra arguments to ``def`` or ``class_``

include/pybind11/detail/common.h

@@ -161,7 +161,26 @@
 
 // https://en.cppreference.com/w/c/chrono/localtime
 #if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__)
-#    define __STDC_WANT_LIB_EXT1__
+#  define __STDC_WANT_LIB_EXT1__
+#endif
+
+#ifdef __has_include
+// std::optional (but including it in c++14 mode isn't allowed)
+#  if defined(PYBIND11_CPP17) && __has_include(<optional>)
+#    define PYBIND11_HAS_OPTIONAL 1
+#  endif
+// std::experimental::optional (but not allowed in c++11 mode)
+#  if defined(PYBIND11_CPP14) && (__has_include(<experimental/optional>) && \
+                                 !__has_include(<optional>))
+#    define PYBIND11_HAS_EXP_OPTIONAL 1
+#  endif
+// std::variant
+#  if defined(PYBIND11_CPP17) && __has_include(<variant>)
+#    define PYBIND11_HAS_VARIANT 1
+#  endif
+#elif defined(_MSC_VER) && defined(PYBIND11_CPP17)
+#  define PYBIND11_HAS_OPTIONAL 1
+#  define PYBIND11_HAS_VARIANT 1
 #endif
 
 #include <Python.h>

include/pybind11/pybind11.h

@@ -2069,52 +2069,25 @@ inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_t
     return res;
 }
 
-/* There are a large number of apparently unused template arguments because
- * each combination requires a separate py::class_ registration.
- */
-template <typename Access, return_value_policy Policy, typename Iterator, typename Sentinel, typename ValueType, typename... Extra>
+template <typename Iterator, typename Sentinel, bool KeyIterator, return_value_policy Policy>
 struct iterator_state {
     Iterator it;
     Sentinel end;
     bool first_or_done;
 };
 
-// Note: these helpers take the iterator by non-const reference because some
-// iterators in the wild can't be dereferenced when const.
-template <typename Iterator>
-struct iterator_access {
-    using result_type = decltype((*std::declval<Iterator>()));
-    // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
-    result_type operator()(Iterator &it) const {
-        return *it;
-    }
-};
+PYBIND11_NAMESPACE_END(detail)
 
-template <typename Iterator>
-struct iterator_key_access {
-    using result_type = decltype(((*std::declval<Iterator>()).first));
-    result_type operator()(Iterator &it) const {
-        return (*it).first;
-    }
-};
-
-template <typename Iterator>
-struct iterator_value_access {
-    using result_type = decltype(((*std::declval<Iterator>()).second));
-    result_type operator()(Iterator &it) const {
-        return (*it).second;
-    }
-};
-
-template <typename Access,
-          return_value_policy Policy,
+/// Makes a python iterator from a first and past-the-end C++ InputIterator.
+template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Iterator,
           typename Sentinel,
-          typename ValueType,
+#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
+          typename ValueType = decltype(*std::declval<Iterator>()),
+#endif
           typename... Extra>
-iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&... extra) {
-    using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>;
-    // TODO: state captures only the types of Extra, not the values
+iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
+    using state = detail::iterator_state<Iterator, Sentinel, false, Policy>;
 
     if (!detail::get_type_info(typeid(state), false)) {
         class_<state>(handle(), "iterator", pybind11::module_local())
@@ -2128,7 +2101,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&... extra) {
                     s.first_or_done = true;
                     throw stop_iteration();
                 }
-                return Access()(s.it);
+                return *s.it;
             // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
             }, std::forward<Extra>(extra)..., Policy);
     }
@@ -2136,55 +2109,35 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&... extra) {
     return cast(state{first, last, true});
 }
 
-PYBIND11_NAMESPACE_END(detail)
-
-/// Makes a python iterator from a first and past-the-end C++ InputIterator.
-template <return_value_policy Policy = return_value_policy::reference_internal,
-          typename Iterator,
-          typename Sentinel,
-          typename ValueType = typename detail::iterator_access<Iterator>::result_type,
-          typename... Extra>
-iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
-    return detail::make_iterator_impl<
-        detail::iterator_access<Iterator>,
-        Policy,
-        Iterator,
-        Sentinel,
-        ValueType,
-        Extra...>(first, last, std::forward<Extra>(extra)...);
-}
-
-/// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a
+/// Makes an python iterator over the keys (`.first`) of a iterator over pairs from a
 /// first and past-the-end InputIterator.
 template <return_value_policy Policy = return_value_policy::reference_internal,
           typename Iterator,
           typename Sentinel,
-          typename KeyType = typename detail::iterator_key_access<Iterator>::result_type,
+#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
+          typename KeyType = decltype((*std::declval<Iterator>()).first),
+#endif
           typename... Extra>
 iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) {
-    return detail::make_iterator_impl<
-        detail::iterator_key_access<Iterator>,
-        Policy,
-        Iterator,
-        Sentinel,
-        KeyType,
-        Extra...>(first, last, std::forward<Extra>(extra)...);
-}
+    using state = detail::iterator_state<Iterator, Sentinel, true, Policy>;
 
-/// Makes a python iterator over the values (`.second`) of a iterator over pairs from a
-/// first and past-the-end InputIterator.
-template <return_value_policy Policy = return_value_policy::reference_internal,
-          typename Iterator,
-          typename Sentinel,
-          typename ValueType = typename detail::iterator_value_access<Iterator>::result_type,
-          typename... Extra>
-iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) {
-    return detail::make_iterator_impl<
-        detail::iterator_value_access<Iterator>,
-        Policy, Iterator,
-        Sentinel,
-        ValueType,
-        Extra...>(first, last, std::forward<Extra>(extra)...);
+    if (!detail::get_type_info(typeid(state), false)) {
+        class_<state>(handle(), "iterator", pybind11::module_local())
+            .def("__iter__", [](state &s) -> state& { return s; })
+            .def("__next__", [](state &s) -> detail::remove_cv_t<KeyType> {
+                if (!s.first_or_done)
+                    ++s.it;
+                else
+                    s.first_or_done = false;
+                if (s.it == s.end) {
+                    s.first_or_done = true;
+                    throw stop_iteration();
+                }
+                return (*s.it).first;
+            }, std::forward<Extra>(extra)..., Policy);
+    }
+
+    return cast(state{first, last, true});
 }
 
 /// Makes an iterator over values of an stl container or other container supporting
@@ -2201,13 +2154,6 @@ template <return_value_policy Policy = return_value_policy::reference_internal,
     return make_key_iterator<Policy>(std::begin(value), std::end(value), extra...);
 }
 
-/// Makes an iterator over the values (`.second`) of a stl map-like container supporting
-/// `std::begin()`/`std::end()`
-template <return_value_policy Policy = return_value_policy::reference_internal,
-          typename Type, typename... Extra> iterator make_value_iterator(Type &value, Extra&&... extra) {
-    return make_value_iterator<Policy>(std::begin(value), std::end(value), extra...);
-}
-
 template <typename InputType, typename OutputType> void implicitly_convertible() {
     struct set_flag {
         bool &flag;

include/pybind11/pytypes.h

@@ -14,6 +14,10 @@
 #include <utility>
 #include <type_traits>
 
+#if defined(PYBIND11_HAS_OPTIONAL)
+#  include <optional>
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
 /* A few forward declarations */
@@ -1345,11 +1349,20 @@ private:
 class slice : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(slice, object, PySlice_Check)
-    slice(ssize_t start_, ssize_t stop_, ssize_t step_) {
-        int_ start(start_), stop(stop_), step(step_);
+    slice(handle start, handle stop, handle step) {
         m_ptr = PySlice_New(start.ptr(), stop.ptr(), step.ptr());
-        if (!m_ptr) pybind11_fail("Could not allocate slice object!");
+        if (!m_ptr)
+            pybind11_fail("Could not allocate slice object!");
     }
+
+#ifdef PYBIND11_HAS_OPTIONAL
+    slice(std::optional<ssize_t> start, std::optional<ssize_t> stop, std::optional<ssize_t> step)
+        : slice(index_to_object(start), index_to_object(stop), index_to_object(step)) {}
+#else
+    slice(ssize_t start_, ssize_t stop_, ssize_t step_)
+        : slice(int_(start_), int_(stop_), int_(step_)) {}
+#endif
+
     bool compute(size_t length, size_t *start, size_t *stop, size_t *step,
                  size_t *slicelength) const {
         return PySlice_GetIndicesEx((PYBIND11_SLICE_OBJECT *) m_ptr,
@@ -1364,6 +1377,12 @@ public:
           stop, step,
           slicelength) == 0;
     }
+
+private:
+    template <typename T>
+    static object index_to_object(T index) {
+        return index ? object(int_(*index)) : object(none());
+    }
 };
 
 class capsule : public object {

include/pybind11/stl.h

@@ -9,6 +9,7 @@
 
 #pragma once
 
+#include "detail/common.h"
 #include "pybind11.h"
 #include <set>
 #include <unordered_set>
@@ -19,28 +20,15 @@
 #include <deque>
 #include <valarray>
 
-#ifdef __has_include
-// std::optional (but including it in c++14 mode isn't allowed)
-#  if defined(PYBIND11_CPP17) && __has_include(<optional>)
-#    include <optional>
-#    define PYBIND11_HAS_OPTIONAL 1
-#  endif
-// std::experimental::optional (but not allowed in c++11 mode)
-#  if defined(PYBIND11_CPP14) && (__has_include(<experimental/optional>) && \
-                                 !__has_include(<optional>))
-#    include <experimental/optional>
-#    define PYBIND11_HAS_EXP_OPTIONAL 1
-#  endif
-// std::variant
-#  if defined(PYBIND11_CPP17) && __has_include(<variant>)
-#    include <variant>
-#    define PYBIND11_HAS_VARIANT 1
-#  endif
-#elif defined(_MSC_VER) && defined(PYBIND11_CPP17)
+// See `detail/common.h` for implementation of these guards.
+#if defined(PYBIND11_HAS_OPTIONAL)
 #  include <optional>
+#elif defined(PYBIND11_HAS_EXP_OPTIONAL)
+#  include <experimental/optional>
+#endif
+
+#if defined(PYBIND11_HAS_VARIANT)
 #  include <variant>
-#  define PYBIND11_HAS_OPTIONAL 1
-#  define PYBIND11_HAS_VARIANT 1
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)

tests/test_sequences_and_iterators.cpp

@@ -15,7 +15,11 @@
 
 #include <algorithm>
 #include <utility>
-#include <vector>
+
+#ifdef PYBIND11_HAS_OPTIONAL
+#include <optional>
+#endif  // PYBIND11_HAS_OPTIONAL
+
 
 template<typename T>
 class NonZeroIterator {
@@ -33,29 +37,6 @@ bool operator==(const NonZeroIterator<std::pair<A, B>>& it, const NonZeroSentine
     return !(*it).first || !(*it).second;
 }
 
-class NonCopyableInt {
-public:
-    explicit NonCopyableInt(int value) : value_(value) {}
-    NonCopyableInt(const NonCopyableInt &) = delete;
-    NonCopyableInt(NonCopyableInt &&other) noexcept : value_(other.value_) {
-        other.value_ = -1;  // detect when an unwanted move occurs
-    }
-    NonCopyableInt &operator=(const NonCopyableInt &) = delete;
-    NonCopyableInt &operator=(NonCopyableInt &&other) noexcept {
-        value_ = other.value_;
-        other.value_ = -1;  // detect when an unwanted move occurs
-        return *this;
-    }
-    int get() const { return value_; }
-    void set(int value) { value_ = value; }
-    ~NonCopyableInt() = default;
-private:
-    int value_;
-};
-using NonCopyableIntPair = std::pair<NonCopyableInt, NonCopyableInt>;
-PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableInt>);
-PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableIntPair>);
-
 template <typename PythonType>
 py::list test_random_access_iterator(PythonType x) {
     if (x.size() < 5)
@@ -117,6 +98,18 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
             return std::make_tuple(istart, istop, istep);
         });
 
+    m.def("make_forward_slice_size_t", []() { return py::slice(0, -1, 1); });
+    m.def("make_reversed_slice_object", []() { return py::slice(py::none(), py::none(), py::int_(-1)); });
+#ifdef PYBIND11_HAS_OPTIONAL
+    m.attr("has_optional") = true;
+    m.def("make_reversed_slice_size_t_optional_verbose", []() { return py::slice(std::nullopt, std::nullopt, -1); });
+    // Warning: The following spelling may still compile if optional<> is not present and give wrong answers.
+    // Please use with caution.
+    m.def("make_reversed_slice_size_t_optional", []() { return py::slice({}, {}, -1); });
+#else
+    m.attr("has_optional") = false;
+#endif
+
     // test_sequence
     class Sequence {
     public:
@@ -295,10 +288,6 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
         .def(
             "items",
             [](const StringMap &map) { return py::make_iterator(map.begin(), map.end()); },
-            py::keep_alive<0, 1>())
-        .def(
-            "values",
-            [](const StringMap &map) { return py::make_value_iterator(map.begin(), map.end()); },
             py::keep_alive<0, 1>());
 
     // test_generalized_iterators
@@ -306,6 +295,8 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
     public:
         explicit IntPairs(std::vector<std::pair<int, int>> data) : data_(std::move(data)) {}
         const std::pair<int, int>* begin() const { return data_.data(); }
+        // .end() only required for py::make_iterator(self) overload
+        const std::pair<int, int>* end() const { return data_.data() + data_.size(); }
     private:
         std::vector<std::pair<int, int>> data_;
     };
@@ -317,38 +308,19 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
         .def("nonzero_keys", [](const IntPairs& s) {
             return py::make_key_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
         }, py::keep_alive<0, 1>())
-        .def("nonzero_values", [](const IntPairs& s) {
-            return py::make_value_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
+        .def("simple_iterator", [](IntPairs& self) {
+            return py::make_iterator(self);
+        }, py::keep_alive<0, 1>())
+        .def("simple_keys", [](IntPairs& self) {
+            return py::make_key_iterator(self);
+        }, py::keep_alive<0, 1>())
+
+        // test iterator with keep_alive (doesn't work so not used at runtime, but tests compile)
+        .def("make_iterator_keep_alive", [](IntPairs& self) {
+            return py::make_iterator(self, py::keep_alive<0, 1>());
         }, py::keep_alive<0, 1>())
         ;
 
-    // test_iterater_referencing
-    py::class_<NonCopyableInt>(m, "NonCopyableInt")
-        .def(py::init<int>())
-        .def("set", &NonCopyableInt::set)
-        .def("__int__", &NonCopyableInt::get)
-        ;
-    py::class_<std::vector<NonCopyableInt>>(m, "VectorNonCopyableInt")
-        .def(py::init<>())
-        .def("append", [](std::vector<NonCopyableInt> &vec, int value) {
-            vec.emplace_back(value);
-        })
-        .def("__iter__", [](std::vector<NonCopyableInt> &vec) {
-            return py::make_iterator(vec.begin(), vec.end());
-        })
-        ;
-    py::class_<std::vector<NonCopyableIntPair>>(m, "VectorNonCopyableIntPair")
-        .def(py::init<>())
-        .def("append", [](std::vector<NonCopyableIntPair> &vec, const std::pair<int, int> &value) {
-            vec.emplace_back(NonCopyableInt(value.first), NonCopyableInt(value.second));
-        })
-        .def("keys", [](std::vector<NonCopyableIntPair> &vec) {
-            return py::make_key_iterator(vec.begin(), vec.end());
-        })
-        .def("values", [](std::vector<NonCopyableIntPair> &vec) {
-            return py::make_value_iterator(vec.begin(), vec.end());
-        })
-        ;
 
 #if 0
     // Obsolete: special data structure for exposing custom iterator types to python

tests/test_sequences_and_iterators.py

@@ -16,6 +16,17 @@ def allclose(a_list, b_list, rel_tol=1e-05, abs_tol=0.0):
     )
 
 
+def test_slice_constructors():
+    assert m.make_forward_slice_size_t() == slice(0, -1, 1)
+    assert m.make_reversed_slice_object() == slice(None, None, -1)
+
+
+@pytest.mark.skipif(not m.has_optional, reason="no <optional>")
+def test_slice_constructors_explicit_optional():
+    assert m.make_reversed_slice_size_t_optional() == slice(None, None, -1)
+    assert m.make_reversed_slice_size_t_optional_verbose() == slice(None, None, -1)
+
+
 def test_generalized_iterators():
     assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).nonzero()) == [(1, 2), (3, 4)]
     assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero()) == [(1, 2)]
@@ -25,10 +36,6 @@ def test_generalized_iterators():
     assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_keys()) == [1]
     assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_keys()) == []
 
-    assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).nonzero_values()) == [2, 4]
-    assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_values()) == [2]
-    assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_values()) == []
-
     # __next__ must continue to raise StopIteration
     it = m.IntPairs([(0, 0)]).nonzero()
     for _ in range(3):
@@ -41,28 +48,13 @@ def test_generalized_iterators():
             next(it)
 
 
-def test_iterator_referencing():
-    """Test that iterators reference rather than copy their referents."""
-    vec = m.VectorNonCopyableInt()
-    vec.append(3)
-    vec.append(5)
-    assert [int(x) for x in vec] == [3, 5]
-    # Increment everything to make sure the referents can be mutated
-    for x in vec:
-        x.set(int(x) + 1)
-    assert [int(x) for x in vec] == [4, 6]
-
-    vec = m.VectorNonCopyableIntPair()
-    vec.append([3, 4])
-    vec.append([5, 7])
-    assert [int(x) for x in vec.keys()] == [3, 5]
-    assert [int(x) for x in vec.values()] == [4, 7]
-    for x in vec.keys():
-        x.set(int(x) + 1)
-    for x in vec.values():
-        x.set(int(x) + 10)
-    assert [int(x) for x in vec.keys()] == [4, 6]
-    assert [int(x) for x in vec.values()] == [14, 17]
+def test_generalized_iterators_simple():
+    assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_iterator()) == [
+        (1, 2),
+        (3, 4),
+        (0, 5),
+    ]
+    assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_keys()) == [1, 3, 0]
 
 
 def test_sliceable():
@@ -168,7 +160,6 @@ def test_map_iterator():
         assert sm[k] == expected[k]
     for k, v in sm.items():
         assert v == expected[k]
-    assert list(sm.values()) == [expected[k] for k in sm]
 
     it = iter(m.StringMap({}))
     for _ in range(3):  # __next__ must continue to raise StopIteration