From 9c18a74e377dece2f2acd22c2c6e63ecb2a59c77 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 22 Nov 2022 17:17:02 -0800 Subject: [PATCH] Use `multiprocessing` `start_method` `"forkserver"` (#4306) * Use `multiprocessing` `start_method` `"forkserver"` Alternative to PR #4305 * Add link to comment under PR #4105 * Unconditionally `pytest.skip("DEADLOCK")` for PyPy Windows * Remove `SKIP_IF_DEADLOCK` entirely, for simplicity. Hopefully this PR will resolve the deadlocks for good. * Add "In a nutshell" comment, in response to request by @EricCousineau-TRI --- tests/conftest.py | 12 ++++++++++++ tests/test_gil_scoped.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f5ddb9f12..96dffc81c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ Adds docstring and exceptions message sanitizers. import contextlib import difflib import gc +import multiprocessing +import os import re import textwrap @@ -15,6 +17,16 @@ import pytest # Early diagnostic for failed imports import pybind11_tests +if os.name != "nt": + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 + # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. + # It is actually a well-known pitfall, unfortunately without guard rails. + # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, + # visit the issuecomment link above for details). + # Windows does not have fork() and the associated pitfall, therefore it is best left + # running with defaults. + multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index e890a7b0c..6af6a472d 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -5,6 +5,7 @@ import time import pytest +import env from pybind11_tests import gil_scoped as m @@ -144,7 +145,6 @@ def _intentional_deadlock(): ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) -SKIP_IF_DEADLOCK = True # See PR #4216 def _run_in_process(target, *args, **kwargs): @@ -181,7 +181,7 @@ def _run_in_process(target, *args, **kwargs): elif process.exitcode is None: assert t_delta > 0.9 * timeout msg = "DEADLOCK, most likely, exactly what this test is meant to detect." - if SKIP_IF_DEADLOCK: + if env.PYPY and env.WIN: pytest.skip(msg) raise RuntimeError(msg) return process.exitcode