From d06d53694ab30b48865c8f502374243517a768f3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 14 Sep 2023 09:47:34 -0700 Subject: [PATCH 01/18] Fix small bug introduced with PR #4735 (#4845) * Bug fix: `result[0]` called if `result.empty()` * Add unit test that fails without the fix. --- include/pybind11/pybind11.h | 12 +++++++----- tests/test_kwargs_and_defaults.cpp | 2 ++ tests/test_kwargs_and_defaults.py | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 5e3c5c657..25ca508ab 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -57,11 +57,13 @@ inline std::string replace_newlines_and_squash(const char *text) { std::string result(text); bool previous_is_whitespace = false; - // Do not modify string representations - char first_char = result[0]; - char last_char = result[result.size() - 1]; - if (first_char == last_char && first_char == '\'') { - return result; + if (result.size() >= 2) { + // Do not modify string representations + char first_char = result[0]; + char last_char = result[result.size() - 1]; + if (first_char == last_char && first_char == '\'') { + return result; + } } result.clear(); diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 614c826da..9a12c42af 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -85,6 +85,8 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { "kw_lb_func7", [](const std::string &) {}, py::arg("str_arg") = "First line.\n Second line."); + m.def( + "kw_lb_func8", [](const CustomRepr &) {}, py::arg("custom") = CustomRepr("")); // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 9d9738de7..0e3bbdb76 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -55,6 +55,10 @@ def test_function_signatures(doc): doc(m.kw_lb_func7) == "kw_lb_func7(str_arg: str = 'First line.\\n Second line.') -> None" ) + assert ( + doc(m.kw_lb_func8) + == "kw_lb_func8(custom: m.kwargs_and_defaults.CustomRepr = ) -> None" + ) def test_named_arguments(): From 4fb111bd78abe5f94a7462f1f565a4dc41b225a9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Sep 2023 17:59:30 -0400 Subject: [PATCH 02/18] fix(cmake): correctly detect FindPython policy and better warning (#4806) --- CMakeLists.txt | 10 ++++++++++ tools/FindPythonLibsNew.cmake | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15fa79908..65ad0c22f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. +# Propagate this policy (FindPythonInterp removal) so it can be detected later +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) +endif() + cmake_minimum_required(VERSION 3.5) # The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with @@ -16,6 +21,11 @@ else() cmake_policy(VERSION 3.26) endif() +if(_pybind11_cmp0148) + cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) + unset(_pybind11_cmp0148) +endif() + # Avoid infinite recursion if tests include this as a subdirectory if(DEFINED PYBIND11_MASTER_PROJECT) return() diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index ce558d4ec..992f9f52a 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -95,6 +95,22 @@ if(NOT PythonLibsNew_FIND_VERSION) set(PythonLibsNew_FIND_VERSION "3.6") endif() +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) + if(NOT _pybind11_cmp0148) + message( + AUTHOR_WARNING + "Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs " + "modules are removed. Run \"cmake --help-policy CMP0148\" for policy " + "details. Use the cmake_policy command to set the policy and suppress " + "this warning, or preferably upgrade to using FindPython, either by " + "calling it explicitly before pybind11, or by setting " + "PYBIND11_FINDPYTHON ON before pybind11.") + endif() + cmake_policy(SET CMP0148 OLD) + unset(_pybind11_cmp0148) +endif() + find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required} ${_pythonlibs_quiet}) From 5891867ee4ad1d671e0f54c5fb30f62d1322d8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Papp?= <108480845+lpapp-foundry@users.noreply.github.com> Date: Fri, 15 Sep 2023 23:05:43 +0100 Subject: [PATCH 03/18] fix(cmake): support DEBUG_POSTFIX correctly (#4761) * cmake: split extension Into suffix and debug postfix. Pybind11 is currently treating both as suffix, which is problematic when something else defines the DEBUG_POSTFIX because they will be concatenated. pybind11_extension sets SUFFIX to _d.something and if DEBUG_POSTFIX is set to _d. _d + _d.something = _d_d.something The issue has been reported at: https://github.com/pybind/pybind11/issues/4699 * style: pre-commit fixes * fix(cmake): support postfix for old FindPythonInterp mode too Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 9 ++++++++- tools/pybind11NewTools.cmake | 34 ++++++++++++++++++++++++---------- tools/pybind11Tools.cmake | 8 ++++++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index 992f9f52a..8275b9d5a 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -188,13 +188,20 @@ _pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST) _pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX) _pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) _pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) _pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG) _pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) _pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) _pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR) _pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH) +list(GET _PYTHON_VALUES 4 _PYTHON_MODULE_EXT_SUFFIX) +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) +endif() +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) +endif() + # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 3a35a7386..8cee2d460 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -95,25 +95,36 @@ endif() # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is # required for PyPy3 (as of 7.3.1) -if(NOT DEFINED PYTHON_MODULE_EXTENSION) +if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) execute_process( COMMAND "${${_Python}_EXECUTABLE}" "-c" "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" - OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION - ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR + OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX + ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR OUTPUT_STRIP_TRAILING_WHITESPACE) - if(_PYTHON_MODULE_EXTENSION STREQUAL "") + if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") message( FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'" - "package is not installed. Full error message:\n${_PYTHON_MODULE_EXTENSION_ERR}") + "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}" + ) endif() # This needs to be available for the pybind11_extension function - set(PYTHON_MODULE_EXTENSION - "${_PYTHON_MODULE_EXTENSION}" - CACHE INTERNAL "") + if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + endif() + + if(NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + endif() endif() # Python debug libraries expose slightly different objects before 3.8 @@ -253,6 +264,9 @@ endfunction() function(pybind11_extension name) # The extension is precomputed - set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${PYTHON_MODULE_EXTENSION}") - + set_target_properties( + ${name} + PROPERTIES PREFIX "" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 67e710bda..1b1774d09 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -65,6 +65,7 @@ _pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS) _pybind11_promote_to_cache(PYTHON_LIBRARIES) _pybind11_promote_to_cache(PYTHON_MODULE_PREFIX) _pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION) +_pybind11_promote_to_cache(PYTHON_MODULE_DEBUG_POSTFIX) _pybind11_promote_to_cache(PYTHON_VERSION_MAJOR) _pybind11_promote_to_cache(PYTHON_VERSION_MINOR) _pybind11_promote_to_cache(PYTHON_VERSION) @@ -148,8 +149,11 @@ endif() function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake - set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + set_target_properties( + ${name} + PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() # Build a Python extension module: From 7e5edbc947168fa73e25c5b6e58d656f5e2e9863 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 25 Sep 2023 10:38:21 -0500 Subject: [PATCH 04/18] Avoid copy in iteration by using const auto & (#4861) This change is fixing a Coverity AUTO_CAUSES_COPY issues. --- include/pybind11/pybind11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 25ca508ab..4c0069bfb 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1142,7 +1142,7 @@ protected: } msg += "kwargs: "; bool first = true; - for (auto kwarg : kwargs) { + for (const auto &kwarg : kwargs) { if (first) { first = false; } else { From f468b0707ea8eb1b0f91d7a77bdaec6d1c385586 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 27 Sep 2023 09:55:49 -0700 Subject: [PATCH 05/18] Add 2 missing `throw error_already_set();` (#4863) Fixes oversights in PR #4570. --- include/pybind11/detail/internals.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e2c94ea96..ab399016e 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -451,6 +451,7 @@ inline object get_python_state_dict() { #endif if (!state_dict) { raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + throw error_already_set(); } return state_dict; } @@ -463,6 +464,7 @@ inline internals **get_internals_pp_from_capsule(handle obj) { void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); if (raw_ptr == nullptr) { raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + throw error_already_set(); } return static_cast(raw_ptr); } From 0a756c0bb2f7e7ac6cf4e2c4de26211409fadbe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= <8431159+mtsokol@users.noreply.github.com> Date: Wed, 27 Sep 2023 19:22:04 +0200 Subject: [PATCH 06/18] MAINT: Include `numpy._core` imports (#4857) * MAINT: Include numpy._core imports * style: pre-commit fixes * Apply review comments * style: pre-commit fixes * Add no-inline attribute * Select submodule name based on numpy version * style: pre-commit fixes * Update pre-commit check * Add error_already_set and simplify if statement * Update .pre-commit-config.yaml Co-authored-by: Ralf W. Grosse-Kunstleve --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 2 +- include/pybind11/numpy.h | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1fb4ae11..c511ab916 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -142,7 +142,7 @@ repos: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Numpy|Cmake|CCache|PyTest + entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ # PyLint has native support - not always usable, but works for us diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 02f65d740..23c38660e 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,6 +120,20 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; + return module_::import((numpy_core_path + "." + submodule_name).c_str()); +} + template struct same_size { template @@ -263,9 +277,13 @@ private: }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); + if (api_ptr == nullptr) { + raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); @@ -626,11 +644,8 @@ public: private: static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); + module_ m = detail::import_numpy_core_submodule("_internal"); + static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); return reinterpret_borrow(obj); } From dd64df73c31fd333018fec34fa7e35e9df365bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= <8431159+mtsokol@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:12:58 +0200 Subject: [PATCH 07/18] MAINT: Remove np.int_ (#4867) --- tests/test_numpy_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 12e7d17d1..0697daf3e 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -298,7 +298,7 @@ def test_constructors(): results = m.converting_constructors([1, 2, 3]) for a in results.values(): np.testing.assert_array_equal(a, [1, 2, 3]) - assert results["array"].dtype == np.int_ + assert results["array"].dtype == np.dtype(int) assert results["array_t"].dtype == np.int32 assert results["array_t"].dtype == np.float64 From 2b2e4ca4a3fe162bf9c4c95d1621dadab071df86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:13:44 -0400 Subject: [PATCH 08/18] chore(deps): update pre-commit hooks (#4868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/psf/black-pre-commit-mirror: 23.7.0 → 23.9.1](https://github.com/psf/black-pre-commit-mirror/compare/23.7.0...23.9.1) - [github.com/astral-sh/ruff-pre-commit: v0.0.287 → v0.0.292](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.287...v0.0.292) - [github.com/codespell-project/codespell: v2.2.5 → v2.2.6](https://github.com/codespell-project/codespell/compare/v2.2.5...v2.2.6) - [github.com/shellcheck-py/shellcheck-py: v0.9.0.5 → v0.9.0.6](https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.5...v0.9.0.6) - [github.com/PyCQA/pylint: v3.0.0a7 → v3.0.0](https://github.com/PyCQA/pylint/compare/v3.0.0a7...v3.0.0) * Update .pre-commit-config.yaml * style: pre-commit fixes * Update .pre-commit-config.yaml --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/CONTRIBUTING.md | 2 +- .pre-commit-config.yaml | 10 +++++----- include/pybind11/detail/init.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ad7974395..f5a08e2d7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -135,7 +135,7 @@ The valid options are: * Use `-G` and the name of a generator to use something different. `cmake --help` lists the generators available. - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give - you automatic mulithreading on all your CMake projects! + you automatic multithreading on all your CMake projects! * Open the `CMakeLists.txt` with QtCreator to generate for that IDE. * You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file that some tools expect. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c511ab916..e15902045 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,13 +32,13 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black-pre-commit-mirror - rev: "23.7.0" # Keep in sync with blacken-docs + rev: "23.9.1" # Keep in sync with blacken-docs hooks: - id: black # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.287 + rev: v0.0.292 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -124,7 +124,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.5" + rev: "v2.2.6" hooks: - id: codespell exclude: ".supp$" @@ -132,7 +132,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.5" + rev: "v0.9.0.6" hooks: - id: shellcheck @@ -147,7 +147,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a7" + rev: "v3.0.0" hooks: - id: pylint files: ^pybind11 diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index e21171688..4509bd131 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -65,7 +65,7 @@ constexpr bool is_alias(void *) { } // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall -// back to brace aggregate initiailization so that for aggregate initialization can be used with +// back to brace aggregate initialization so that for aggregate initialization can be used with // py::init, e.g. `py::init` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list` constructor). From 6c77208561f23c255bf73e3d76674a6a9496179f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 21:21:22 -0700 Subject: [PATCH 09/18] chore(deps): bump seanmiddleditch/gha-setup-ninja from 3 to 4 (#4875) Bumps [seanmiddleditch/gha-setup-ninja](https://github.com/seanmiddleditch/gha-setup-ninja) from 3 to 4. - [Release notes](https://github.com/seanmiddleditch/gha-setup-ninja/releases) - [Commits](https://github.com/seanmiddleditch/gha-setup-ninja/compare/v3...v4) --- updated-dependencies: - dependency-name: seanmiddleditch/gha-setup-ninja dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fcf0ca52..9d0312b1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1052,7 +1052,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.14 - name: Install ninja-build tool - uses: seanmiddleditch/gha-setup-ninja@v3 + uses: seanmiddleditch/gha-setup-ninja@v4 - name: Run pip installs run: | From 0e2c3e5db41b6b2af4038734c84ab855ccaaa5f0 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 11 Oct 2023 21:05:31 -0700 Subject: [PATCH 10/18] Add pybind11/gil_safe_call_once.h (to fix deadlocks in pybind11/numpy.h) (#4877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * LazyInitializeAtLeastOnceDestroyNever v1 * Go back to using `union` as originally suggested by jbms@. The trick (also suggested by jbms@) is to add empty ctor + dtor. * Revert "Go back to using `union` as originally suggested by jbms@. The trick (also suggested by jbms@) is to add empty ctor + dtor." This reverts commit e7b8c4f0fcd72191e88d1c17abf5da08fe3a9c6f. * Remove `#include ` * `include\pybind11/numpy.h(24,10): fatal error C1083: Cannot open include file: 'stdalign.h': No such file or directory` * @tkoeppe wrote: this is a C interop header (and we're not writing C) * Suppress gcc 4.8.5 (CentOS 7) warning. ``` include/pybind11/eigen/../numpy.h:63:53: error: dereferencing type-punned pointer will break strict-aliasing rules [-Werror=strict-aliasing] return *reinterpret_cast(value_storage_); ^ ``` * Replace comments: Document PRECONDITION. Adopt comment suggested by @tkoeppe: https://github.com/pybind/pybind11/pull/4877#discussion_r1350356093 * Adopt suggestion by @tkoeppe: * https://github.com/pybind/pybind11/pull/4877#issuecomment-1752969127 * https://godbolt.org/z/Wa79nKz6e * Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases: ``` g++ -o pybind11/tests/test_numpy_array.os -c -std=c++20 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.11 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp ``` ``` In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp:10: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h: In static member function ‘static pybind11::detail::npy_api& pybind11::detail::npy_api::get()’: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h:258:82: error: ‘constinit’ variable ‘api_init’ does not have a constant initializer 258 | PYBIND11_CONSTINIT static LazyInitializeAtLeastOnceDestroyNever api_init; | ^~~~~~~~ ``` ``` In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp:10: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h: In static member function ‘static pybind11::object& pybind11::dtype::_dtype_from_pep3118()’: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h:697:13: error: ‘constinit’ variable ‘imported_obj’ does not have a constant initializer 697 | imported_obj; | ^~~~~~~~~~~~ ``` * Revert "Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases:" This reverts commit f07b28bda9f91fb723aa898a21c81b6dd6857072. * Reapply "Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases:" This reverts commit 36be645758aa82b576d24003808386bec6e55bf9. * Add Default Member Initializer on `value_storage_` as suggested by @tkoeppe: https://github.com/pybind/pybind11/pull/4877#issuecomment-1753201342 This fixes the errors reported under commit f07b28bda9f91fb723aa898a21c81b6dd6857072. * Fix copy-paste-missed-a-change mishap in commit 88cec1152ab5576db19bab95c484672f06f5989a. * Semi-paranoid placement new (based on https://github.com/pybind/pybind11/pull/4877#discussion_r1350573114). * Move PYBIND11_CONSTINIT to detail/common.h * Move code to the right places, rename new class and some variables. * Fix oversight: update tests/extra_python_package/test_files.py * Get the name right first. * Use `std::call_once`, `std::atomic`, following a pattern developed by @tkoeppe * Make the API more self-documenting (and possibly more easily reusable). * google-clang-tidy IWYU fixes * Rewrite comment as suggested by @tkoeppe * Update test_exceptions.cpp and exceptions.rst * Fix oversight in previous commit: add `PYBIND11_CONSTINIT` * Make `get_stored()` non-const for simplicity. As suggested by @tkoeppe: not seeing any reasonable use in which `get_stored` has to be const. * Add comment regarding `KeyboardInterrupt` behavior, based heavily on information provided by @jbms. * Add `assert(PyGILState_Check())` in `gil_scoped_release` ctor (simple & non-simple implementation) as suggested by @EthanSteinberg. * Fix oversight in previous commit (missing include cassert). * Remove use of std::atomic, leaving comments with rationale, why it is not needed. * Rewrite comment re `std:optional` based on deeper reflection (aka 2nd thoughts). * Additional comment with the conclusion of a discussion under PR #4877. * https://github.com/pybind/pybind11/pull/4877#issuecomment-1757363179 * Small comment changes suggested by @tkoeppe. --- CMakeLists.txt | 1 + docs/advanced/exceptions.rst | 9 ++- include/pybind11/detail/common.h | 8 +++ include/pybind11/gil.h | 10 ++- include/pybind11/gil_safe_call_once.h | 91 ++++++++++++++++++++++++ include/pybind11/numpy.h | 19 +++-- tests/extra_python_package/test_files.py | 1 + tests/test_exceptions.cpp | 8 ++- 8 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 include/pybind11/gil_safe_call_once.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 65ad0c22f..d2c44dc40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ set(PYBIND11_HEADERS include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h + include/pybind11/gil_safe_call_once.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 616e2d772..e20f42b5f 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -141,15 +141,14 @@ standard python RuntimeError: .. code-block:: cpp - // This is a static object, so we must leak the Python reference: - // It is undefined when the destructor will run, possibly only after the - // Python interpreter is finalized already. - static py::handle exc = py::exception(m, "MyCustomError").release(); + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store exc_storage; + exc_storage.call_once_and_store_result( + [&]() { return py::exception(m, "MyCustomError"); }); py::register_exception_translator([](std::exception_ptr p) { try { if (p) std::rethrow_exception(p); } catch (const MyCustomException &e) { - py::set_error(exc, e.what()); + py::set_error(exc_storage.get_stored(), e.what()); } catch (const OtherException &e) { py::set_error(PyExc_RuntimeError, e.what()); } diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index e79f7693d..83800e960 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -118,6 +118,14 @@ # endif #endif +#if defined(PYBIND11_CPP20) +# define PYBIND11_CONSTINIT constinit +# define PYBIND11_DTOR_CONSTEXPR constexpr +#else +# define PYBIND11_CONSTINIT +# define PYBIND11_DTOR_CONSTEXPR +#endif + // Compiler version assertions #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 570a5581d..da22f48d7 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -11,6 +11,8 @@ #include "detail/common.h" +#include + #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # include "detail/internals.h" #endif @@ -137,7 +139,9 @@ private: class gil_scoped_release { public: + // PRECONDITION: The GIL must be held when this constructor is called. explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + assert(PyGILState_Check()); // `get_internals()` must be called here unconditionally in order to initialize // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an // initialization race could occur as multiple threads try `gil_scoped_acquire`. @@ -201,7 +205,11 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() : state{PyEval_SaveThread()} {} + // PRECONDITION: The GIL must be held when this constructor is called. + gil_scoped_release() { + assert(PyGILState_Check()); + state = PyEval_SaveThread(); + } gil_scoped_release(const gil_scoped_release &) = delete; gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h new file mode 100644 index 000000000..eaf84d16e --- /dev/null +++ b/include/pybind11/gil_safe_call_once.h @@ -0,0 +1,91 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "gil.h" + +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +// Use the `gil_safe_call_once_and_store` class below instead of the naive +// +// static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! +// +// which has two serious issues: +// +// 1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and +// 2. deadlocks in multi-threaded processes (because of missing lock ordering). +// +// The following alternative avoids both problems: +// +// PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; +// auto &imported_obj = storage // Do NOT make this `static`! +// .call_once_and_store_result([]() { +// return py::module_::import("module_name"); +// }) +// .get_stored(); +// +// The parameter of `call_once_and_store_result()` must be callable. It can make +// CPython API calls, and in particular, it can temporarily release the GIL. +// +// `T` can be any C++ type, it does not have to involve CPython API types. +// +// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`), +// is not ideal. If the main thread is the one to actually run the `Callable`, +// then a `KeyboardInterrupt` will interrupt it if it is running normal Python +// code. The situation is different if a non-main thread runs the +// `Callable`, and then the main thread starts waiting for it to complete: +// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will +// get processed only when it is the main thread's turn again and it is running +// normal Python code. However, this will be unnoticeable for quick call-once +// functions, which is usually the case. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) { + if (!is_initialized_) { // This read is guarded by the GIL. + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + std::call_once(once_flag_, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL. + is_initialized_ = true; // This write is guarded by the GIL. + }); + // All threads will observe `is_initialized_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + assert(is_initialized_); + PYBIND11_WARNING_PUSH +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 + // Needed for gcc 4.8.5 + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") +#endif + return *reinterpret_cast(storage_); + PYBIND11_WARNING_POP + } + + constexpr gil_safe_call_once_and_store() = default; + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + +private: + alignas(T) char storage_[sizeof(T)] = {}; + std::once_flag once_flag_ = {}; + bool is_initialized_ = false; + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, + // but the latter does not have the triviality properties of former, + // therefore `std::optional` is not a viable alternative here. +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 23c38660e..8551aa264 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,7 +10,10 @@ #pragma once #include "pybind11.h" +#include "detail/common.h" #include "complex.h" +#include "gil_safe_call_once.h" +#include "pytypes.h" #include #include @@ -206,8 +209,8 @@ struct npy_api { }; static npy_api &get() { - static npy_api api = lookup(); - return api; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result(lookup).get_stored(); } bool PyArray_Check_(PyObject *obj) const { @@ -643,10 +646,14 @@ public: char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } private: - static object _dtype_from_pep3118() { - module_ m = detail::import_numpy_core_submodule("_internal"); - static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); - return reinterpret_borrow(obj); + static object &_dtype_from_pep3118() { + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() { + return detail::import_numpy_core_submodule("_internal") + .attr("_dtype_from_pep3118"); + }) + .get_stored(); } dtype strip_padding(ssize_t itemsize) { diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e3d881c0b..344e70d5d 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -35,6 +35,7 @@ main_headers = { "include/pybind11/eval.h", "include/pybind11/functional.h", "include/pybind11/gil.h", + "include/pybind11/gil_safe_call_once.h", "include/pybind11/iostream.h", "include/pybind11/numpy.h", "include/pybind11/operators.h", diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index f6d9c215c..d3af7ab72 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -6,6 +6,8 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#include + #include "test_exceptions.h" #include "local_bindings.h" @@ -114,7 +116,9 @@ TEST_SUBMODULE(exceptions, m) { []() { throw std::runtime_error("This exception was intentionally thrown."); }); // PLEASE KEEP IN SYNC with docs/advanced/exceptions.rst - static py::handle ex = py::exception(m, "MyException").release(); + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store ex_storage; + ex_storage.call_once_and_store_result( + [&]() { return py::exception(m, "MyException"); }); py::register_exception_translator([](std::exception_ptr p) { try { if (p) { @@ -122,7 +126,7 @@ TEST_SUBMODULE(exceptions, m) { } } catch (const MyException &e) { // Set MyException as the active python error - py::set_error(ex, e.what()); + py::set_error(ex_storage.get_stored(), e.what()); } }); From 0cbd92bab8c940f8dbb9327aa6a8bdfa8215b589 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Mon, 16 Oct 2023 16:42:30 +0200 Subject: [PATCH 11/18] Update pytest to version 7.2.0 (which removes their dependency on py) (#4880) * Update pytest (which removes their dependency on py) The py library through 1.11.0 for Python allows remote attackers to conduct a ReDoS (Regular expression Denial of Service) attack via a Subversion repository with crafted info data, because the InfoSvnCommand argument is mishandled. The particular codepath in question is the regular expression at py._path.svnurl.InfoSvnCommand.lspattern and is only relevant when dealing with subversion (svn) projects. Notably the codepath is not used in the popular pytest project. The developers of the pytest package have released version 7.2.0 which removes their dependency on py. Users of pytest seeing alerts relating to this advisory may update to version 7.2.0 of pytest to resolve this issue. See https://github.com/pytest-dev/py/issues/287#issuecomment-1290407715 for additional context. * Added conditions so that we keep using 7.0.0 on python 3.6 --- tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 4ba101119..09063e768 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,8 @@ numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" and python_version<"3.11" -pytest==7.0.0 +pytest==7.0.0; platform_python_implementation!="PyPy" and python_version=="3.6" +pytest==7.2.0; platform_python_implementation!="PyPy" and python_version>="3.7" pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" From 74439a64a22850f16323f1e1e09ace2a909b0b61 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Wed, 18 Oct 2023 04:04:46 +0900 Subject: [PATCH 12/18] feature: Use typed iterators in `make_*iterator` (#4876) --- include/pybind11/pybind11.h | 19 +++++++++++++------ tests/test_sequences_and_iterators.py | 9 +++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4c0069bfb..a94f6f0f1 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -15,6 +15,7 @@ #include "attr.h" #include "gil.h" #include "options.h" +#include "typing.h" #include #include @@ -2447,7 +2448,7 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2465,7 +2466,7 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2483,7 +2484,7 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2498,8 +2499,10 @@ iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_iterator(Type &value, Extra &&...extra) { return make_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2508,8 +2511,10 @@ iterator make_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_key_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_key_iterator(Type &value, Extra &&...extra) { return make_key_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2518,8 +2523,10 @@ iterator make_key_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_value_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_value_iterator(Type &value, Extra &&...extra) { return make_value_iterator( std::begin(value), std::end(value), std::forward(extra)...); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index acbe9d898..c13f03dd8 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -58,6 +58,15 @@ def test_generalized_iterators_simple(): assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_values()) == [2, 4, 5] +def test_iterator_doc_annotations(): + assert m.IntPairs.nonref.__doc__.endswith("-> Iterator[tuple[int, int]]\n") + assert m.IntPairs.nonref_keys.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.nonref_values.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.simple_iterator.__doc__.endswith("-> Iterator[tuple[int, int]]\n") + assert m.IntPairs.simple_keys.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.simple_values.__doc__.endswith("-> Iterator[int]\n") + + def test_iterator_referencing(): """Test that iterators reference rather than copy their referents.""" vec = m.VectorNonCopyableInt() From 7969049de4c11f1d4955ababd97f340d82d2bf5a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 19 Oct 2023 23:12:37 -0700 Subject: [PATCH 13/18] Comment out failing job, with link to #4889 (#4890) --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d0312b1b..e1946da99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,9 +195,10 @@ jobs: matrix: include: # TODO: Fails on 3.10, investigate - - python-version: "3.9" - python-debug: true - valgrind: true + # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4889 + # - python-version: "3.9" + # python-debug: true + # valgrind: true - python-version: "3.11" python-debug: false From 3414c56b6c7c521d868c9a137ca2ace2e26b5b2e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 21 Oct 2023 19:50:14 +0200 Subject: [PATCH 14/18] Workaround NVCC parse failure in `cast_op` (#4893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Workaround NVCC parse failure in `cast_op` There is a bug in some CUDA versions (observed in CUDA 12.1 and 11.7 w/ GCC 12.2), that makes `cast_op` fail to compile: `cast.h:45:120: error: expected template-name before ‘<’ token` Defining the nested type as an alias and using it allows this to work without any change in semantics. Fixes #4606 * style: pre-commit fixes * Add comments to result_t referencing PR --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/cast.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3c9b7c927..8b5beb0ef 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -42,13 +42,15 @@ using make_caster = type_caster>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T template typename make_caster::template cast_op_type cast_op(make_caster &caster) { - return caster.operator typename make_caster::template cast_op_type(); + using result_t = typename make_caster::template cast_op_type; // See PR #4893 + return caster.operator result_t(); } template typename make_caster::template cast_op_type::type> cast_op(make_caster &&caster) { - return std::move(caster).operator typename make_caster:: - template cast_op_type::type>(); + using result_t = typename make_caster::template cast_op_type< + typename std::add_rvalue_reference::type>; // See PR #4893 + return std::move(caster).operator result_t(); } template From bf88e29c95d20a94528637c1827df757821b5a88 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 23 Oct 2023 12:51:04 -0700 Subject: [PATCH 15/18] Bug fix: Replace bare `static exception` with `gil_safe_call_once_and_store`. (#4897) This is to ensure that `Py_DECREF()` is not called after the Python interpreter was finalized already: https://github.com/pybind/pybind11/blob/3414c56b6c7c521d868c9a137ca2ace2e26b5b2e/include/pybind11/gil_safe_call_once.h#L19 --- include/pybind11/pybind11.h | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index a94f6f0f1..95f7fa2eb 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -14,6 +14,7 @@ #include "detail/init.h" #include "attr.h" #include "gil.h" +#include "gil_safe_call_once.h" #include "options.h" #include "typing.h" @@ -2609,23 +2610,14 @@ public: }; PYBIND11_NAMESPACE_BEGIN(detail) -// Returns a reference to a function-local static exception object used in the simple -// register_exception approach below. (It would be simpler to have the static local variable -// directly in register_exception, but that makes clang <3.5 segfault - issue #1349). -template -exception &get_exception_object() { - static exception ex; - return ex; -} // Helper function for register_exception and register_local_exception template exception & register_exception_impl(handle scope, const char *name, handle base, bool isLocal) { - auto &ex = detail::get_exception_object(); - if (!ex) { - ex = exception(scope, name, base); - } + PYBIND11_CONSTINIT static gil_safe_call_once_and_store> exc_storage; + exc_storage.call_once_and_store_result( + [&]() { return exception(scope, name, base); }); auto register_func = isLocal ? ®ister_local_exception_translator : ®ister_exception_translator; @@ -2637,10 +2629,10 @@ register_exception_impl(handle scope, const char *name, handle base, bool isLoca try { std::rethrow_exception(p); } catch (const CppException &e) { - set_error(detail::get_exception_object(), e.what()); + set_error(exc_storage.get_stored(), e.what()); } }); - return ex; + return exc_storage.get_stored(); } PYBIND11_NAMESPACE_END(detail) From fa27d2fd439cd84a749e2759c0dd21529fac50a4 Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Tue, 24 Oct 2023 18:46:26 +0200 Subject: [PATCH 16/18] Adapt to changed function name in Python 3.13 (#4902) According to https://docs.python.org/3.13/whatsnew/3.13.html: Add PyThreadState_GetUnchecked() function: similar to PyThreadState_Get(), but don't kill the process with a fatal error if it is NULL. The caller is responsible to check if the result is NULL. Previously, the function was private and known as _PyThreadState_UncheckedGet(). --- include/pybind11/detail/type_caster_base.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index fc5f1c88b..68512b5bd 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -486,8 +486,10 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) return PyThreadState_GET(); -#else +#elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); +#else + return PyThreadState_GetUnchecked(); #endif } From 1e28599e41ef5bcf436eb0ce61f7c1e8ebade054 Mon Sep 17 00:00:00 2001 From: Chun Yang Date: Thu, 26 Oct 2023 20:40:19 -0700 Subject: [PATCH 17/18] fix: Add missing spaces to error string (#4906) * [minor] Add a missing space Add a missing space to error message * Add space after period, always add newline. --- include/pybind11/pytypes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index fa2d24984..e7c7c8124 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -305,19 +305,19 @@ private: "https://pybind11.readthedocs.io/en/stable/advanced/" "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" "If you are convinced there is no bug in your code, you can #define " - "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF" + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF " "to disable this check. In that case you have to ensure this #define is consistently " "used for all translation units linked into a given pybind11 extension, otherwise " "there will be ODR violations.", function_name.c_str()); - fflush(stderr); if (Py_TYPE(m_ptr)->tp_name != nullptr) { fprintf(stderr, - "The failing %s call was triggered on a %s object.\n", + " The failing %s call was triggered on a %s object.", function_name.c_str(), Py_TYPE(m_ptr)->tp_name); - fflush(stderr); } + fprintf(stderr, "\n"); + fflush(stderr); throw std::runtime_error(function_name + " PyGILState_Check() failure."); } #endif From 3aece819fd0d0a9f40eed659a088eab60a555202 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 27 Oct 2023 01:26:28 -0400 Subject: [PATCH 18/18] chore: update hooks and Ruff config (#4904) Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 14 +++++++------- pyproject.toml | 16 ++++++++-------- tests/test_numpy_dtypes.cpp | 2 +- tests/test_pytypes.cpp | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e15902045..e40e4fe1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,27 +25,27 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.6" + rev: "v17.0.3" hooks: - id: clang-format types_or: [c++, c, cuda] # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black-pre-commit-mirror - rev: "23.9.1" # Keep in sync with blacken-docs + rev: "23.10.1" # Keep in sync with blacken-docs hooks: - id: black # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.2 hooks: - id: ruff args: ["--fix", "--show-fixes"] # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.5.1" + rev: "v1.6.1" hooks: - id: mypy args: [] @@ -67,7 +67,7 @@ repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.4.0" + rev: "v4.5.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -98,7 +98,7 @@ repos: # Avoid directional quotes - repo: https://github.com/sirosen/texthooks - rev: "0.5.0" + rev: "0.6.2" hooks: - id: fix-ligatures - id: fix-smartquotes @@ -147,7 +147,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.0.0" + rev: "v3.0.1" hooks: - id: pylint files: ^pybind11 diff --git a/pyproject.toml b/pyproject.toml index ef2a8736b..7769860c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ ignore = [ [tool.mypy] files = ["pybind11"] -python_version = "3.6" +python_version = "3.7" strict = true show_error_codes = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] @@ -57,10 +57,13 @@ messages_control.disable = [ "unused-argument", # covered by Ruff ARG ] - [tool.ruff] -select = [ - "E", "F", "W", # flake8 +target-version = "py37" +src = ["src"] +line-length = 120 + +[tool.ruff.lint] +extend-select = [ "B", # flake8-bugbear "I", # isort "N", # pep8-naming @@ -86,13 +89,10 @@ ignore = [ "PT004", # Fixture that doesn't return needs underscore (no, it is fine) "SIM118", # iter(x) is not always the same as iter(x.keys()) ] -target-version = "py37" -src = ["src"] unfixable = ["T20"] exclude = [] -line-length = 120 isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/**" = ["EM", "N", "E721"] "tests/test_call_policies.py" = ["PLC1901"] diff --git a/tests/test_numpy_dtypes.cpp b/tests/test_numpy_dtypes.cpp index 6654f9ed8..053004a22 100644 --- a/tests/test_numpy_dtypes.cpp +++ b/tests/test_numpy_dtypes.cpp @@ -157,7 +157,7 @@ py::array mkarray_via_buffer(size_t n) { do { \ (s).bool_ = (i) % 2 != 0; \ (s).uint_ = (uint32_t) (i); \ - (s).float_ = (float) (i) *1.5f; \ + (s).float_ = (float) (i) * 1.5f; \ (s).ldbl_ = (long double) (i) * -2.5L; \ } while (0) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 0a587680f..0cd480ebb 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -662,8 +662,8 @@ TEST_SUBMODULE(pytypes, m) { // This is "most correct" and enforced on these platforms. # define PYBIND11_AUTO_IT auto it #else -// This works on many platforms and is (unfortunately) reflective of existing user code. -// NOLINTNEXTLINE(bugprone-macro-parentheses) + // This works on many platforms and is (unfortunately) reflective of existing user code. + // NOLINTNEXTLINE(bugprone-macro-parentheses) # define PYBIND11_AUTO_IT auto &it #endif