From 0f8d5f2eb6b5756920a6927417b040188adca3c7 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 15 Jan 2021 21:07:31 +0100 Subject: [PATCH] Add a Valgrind build on debug Python 3.9 (#2746) * Adding a valgrind build on debug Python 3.9 Co-authored-by: Boris Staletic * Add Valgrind suppression files - Introduce suppression file, populate it with a first suppression taken from CPython, and fix one leak in the tests - Suppress leak in NumPy - More clean tests! - Tests with names a-e passing (except for test_buffer) - Suppress multiprocessing errors - Merge multiprocessing suppressions into other suppression files - Numpy seems to be spelled with a big P - Append single entry from valgrind-misc.supp to valgrind-python.supp, and make clear valgrind-python.supp is only CPython Co-authored-by: Boris Staletic * Enable test_virtual_functions with a workaround * Add a memcheck cmake target - Add a memcheck cmake target - Reformat cmake - Appease the formatting overlords - they are angry - Format CMake valgrind target decently * Update CI config to new action versions * fix: separate memcheck from pytest * ci: cleanup * Merge Valgrind and other deadsnakes builds Co-authored-by: Boris Staletic Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 53 ++++++++++--- tests/CMakeLists.txt | 30 ++++++- tests/requirements.txt | 2 +- tests/test_virtual_functions.py | 3 +- tests/valgrind-numpy-scipy.supp | 118 ++++++++++++++++++++++++++++ tests/valgrind-python.supp | 135 ++++++++++++++++++++++++++++++++ 6 files changed, 327 insertions(+), 14 deletions(-) create mode 100644 tests/valgrind-numpy-scipy.supp create mode 100644 tests/valgrind-python.supp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fa9c4eca..23dfbb0c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,23 +169,53 @@ jobs: strategy: fail-fast: false matrix: - python: - - version: 3.9 - debug: true - - version: 3.10-dev - debug: false + include: + - python-version: 3.9 + python-debug: true + valgrind: true + - python-version: 3.10-dev + python-debug: false - name: "🐍 ${{ matrix.python.version }}${{ matrix.python.debug && ' (debug)' || '' }} • deadsnakes • x64" + name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Setup Python ${{ matrix.python.version }} (deadsnakes) + - name: Setup Python ${{ matrix.python-version }} (deadsnakes) uses: deadsnakes/action@v2.1.1 with: - python-version: ${{ matrix.python.version }} - debug: ${{ matrix.python.debug }} + python-version: ${{ matrix.python-version }} + debug: ${{ matrix.python-debug }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.6 + + - name: Valgrind cache + if: matrix.valgrind + uses: actions/cache@v2 + id: cache-valgrind + with: + path: valgrind + key: 3.16.1 # Valgrind version + + - name: Compile Valgrind + if: matrix.valgrind && steps.cache-valgrind.outputs.cache-hit != 'true' + run: | + VALGRIND_VERSION=3.16.1 + curl https://sourceware.org/pub/valgrind/valgrind-$VALGRIND_VERSION.tar.bz2 -o - | tar xj + mv valgrind-$VALGRIND_VERSION valgrind + cd valgrind + ./configure + make -j 2 > /dev/null + + - name: Install Valgrind + if: matrix.valgrind + working-directory: valgrind + run: | + sudo make install + sudo apt-get update + sudo apt-get install libc6-dbg # Needed by Valgrind - name: Prepare env run: python -m pip install -r tests/requirements.txt --prefer-binary @@ -193,6 +223,7 @@ jobs: - name: Configure run: > cmake -S . -B build + -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -207,6 +238,10 @@ jobs: - name: C++ tests run: cmake --build build --target cpptest + - name: Run Valgrind on Python tests + if: matrix.valgrind + run: cmake --build build --target memcheck + # Testing on clang using the excellent silkeh clang docker images clang: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dae8b5ad4..cf1a984f0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -370,12 +370,17 @@ endif() string(REPLACE "test_" "${CMAKE_CURRENT_SOURCE_DIR}/test_" PYBIND11_ABS_PYTEST_FILES "${PYBIND11_PYTEST_FILES}") +set(PYBIND11_TEST_PREFIX_COMMAND + "" + CACHE STRING "Put this before pytest, use for checkers and such") + # A single command to compile and run the tests add_custom_target( pytest - COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_ABS_PYTEST_FILES} + COMMAND ${PYBIND11_TEST_PREFIX_COMMAND} ${PYTHON_EXECUTABLE} -m pytest + ${PYBIND11_ABS_PYTEST_FILES} DEPENDS ${test_targets} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" USES_TERMINAL) if(PYBIND11_TEST_OVERRIDE) @@ -386,6 +391,27 @@ if(PYBIND11_TEST_OVERRIDE) "Note: not all tests run: -DPYBIND11_TEST_OVERRIDE is in effect") endif() +# cmake-format: off +add_custom_target( + memcheck + COMMAND + PYTHONMALLOC=malloc + valgrind + --leak-check=full + --show-leak-kinds=definite,indirect + --errors-for-leak-kinds=definite,indirect + --error-exitcode=1 + --read-var-info=yes + --track-origins=yes + --suppressions="${CMAKE_CURRENT_SOURCE_DIR}/valgrind-python.supp" + --suppressions="${CMAKE_CURRENT_SOURCE_DIR}/valgrind-numpy-scipy.supp" + --gen-suppressions=all + ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_ABS_PYTEST_FILES} + DEPENDS ${test_targets} + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + USES_TERMINAL) +# cmake-format: on + # Add a check target to run all the tests, starting with pytest (we add dependencies to this below) add_custom_target(check DEPENDS pytest) diff --git a/tests/requirements.txt b/tests/requirements.txt index a6253e004..0b383bf5c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,4 +6,4 @@ pytest==4.6.9; python_version<"3.5" pytest==6.1.2; python_version=="3.5" pytest==6.2.1; python_version>="3.6" scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6" -scipy==1.5.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.9" +scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index ae1993016..f7d3bd1e4 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -251,8 +251,7 @@ def test_dispatch_issue(msg): == 'Tried to call pure virtual function "Base::dispatch"' ) - p = PyClass1() - return m.dispatch_issue_go(p) + return m.dispatch_issue_go(PyClass1()) b = PyClass2() assert m.dispatch_issue_go(b) == "Yay.." diff --git a/tests/valgrind-numpy-scipy.supp b/tests/valgrind-numpy-scipy.supp new file mode 100644 index 000000000..60e9f479e --- /dev/null +++ b/tests/valgrind-numpy-scipy.supp @@ -0,0 +1,118 @@ +# Valgrind suppression file for NumPy & SciPy errors and leaks in pybind11 tests + +{ + Leaks when importing NumPy + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_GC_Alloc + fun:_PyObject_GC_Malloc + fun:_PyObject_GC_NewVar + fun:tuple_alloc + fun:PyTuple_Pack + ... + fun:__pyx_pymod_exec_* +} + +{ + Leaks when importing NumPy (bis) + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_New + fun:PyCode_NewWithPosOnlyArgs + fun:PyCode_New + ... + fun:__pyx_pymod_exec_* +} + +{ + Leaks when importing NumPy (tris) + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_GC_Alloc + fun:_PyObject_GC_Malloc + fun:_PyObject_GC_NewVar + fun:tuple_alloc + fun:_PyTuple_FromArray + fun:_PyObject_MakeTpCall + fun:_PyObject_VectorcallTstate + fun:PyObject_Vectorcall + fun:call_function + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrame + fun:function_code_fastcall + fun:_PyFunction_Vectorcall +} + +{ + Leaks when importing NumPy (quater) + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_GC_Alloc + fun:_PyObject_GC_Malloc + fun:_PyObject_GC_NewVar + fun:tuple_alloc + fun:PyTuple_New + fun:r_object + fun:r_object + fun:r_object + fun:r_object +} + +{ + Leaks when importing NumPy (quinquies) + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_GC_Alloc + fun:_PyObject_GC_Malloc + fun:_PyObject_GC_NewVar + fun:tuple_alloc + fun:PyTuple_New + fun:dictiter_iternextitem + fun:list_extend + fun:_PyList_Extend + fun:PySequence_List +} + +{ + Leak when importing scipy.fft + Memcheck:Leak + fun:_Znwm + fun:PyInit_pypocketfft + fun:_PyImport_LoadDynamicModuleWithSpec + fun:_imp_create_dynamic_impl.constprop.3 + fun:_imp_create_dynamic + fun:cfunction_vectorcall_FASTCALL + fun:PyVectorcall_Call + fun:_PyObject_Call + fun:PyObject_Call + fun:do_call_core + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrame + fun:_PyEval_EvalCode +} + +{ + NumPy leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + ... + fun:_buffer_get_info + fun:array_getbuffer + fun:PyObject_GetBuffer + fun:__Pyx__GetBufferAndValidate* + fun:__pyx_f_5numpy_6random_13bit_generator_12SeedSequence_mix_entropy + fun:__pyx_pw_5numpy_6random_13bit_generator_12SeedSequence_1__init__ + fun:type_call + fun:__Pyx__PyObject_CallOneArg + fun:__pyx_pw_5numpy_6random_13bit_generator_12BitGenerator_1__init__ +} diff --git a/tests/valgrind-python.supp b/tests/valgrind-python.supp new file mode 100644 index 000000000..1dd04fa7b --- /dev/null +++ b/tests/valgrind-python.supp @@ -0,0 +1,135 @@ +# Valgrind suppression file for CPython errors and leaks in pybind11 tests + +# Taken verbatim from https://github.com/python/cpython/blob/3.9/Misc/valgrind-python.supp#L266-L272 +{ + Uninitialised byte(s) false alarm, see bpo-35561 + Memcheck:Param + epoll_ctl(event) + fun:epoll_ctl + fun:pyepoll_internal_ctl +} + +{ + Python leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyMem_RawMalloc + fun:PyThread_allocate_lock + fun:_PyEval_InitState + fun:PyInterpreterState_New + ... + fun:pyinit_core* + fun:Py_InitializeFromConfig + fun:pymain_init + fun:pymain_main +} + +{ + Python leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:_PyMem_DebugRawAlloc + fun:_PyMem_DebugRawMalloc + fun:PyMem_RawMalloc + fun:PyThread_allocate_lock + fun:_PyRuntimeState_Init_impl + fun:_PyRuntimeState_Init + fun:_PyRuntime_Initialize + fun:pymain_init + fun:pymain_main + fun:Py_BytesMain +} + +{ + Python leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyMem_RawMalloc + fun:PyThread_allocate_lock + fun:_PyImport_AcquireLock + fun:_imp_acquire_lock_impl* + fun:_imp_acquire_lock + fun:cfunction_vectorcall_NOARGS + fun:_PyObject_VectorcallTstate + fun:PyObject_Vectorcall + fun:call_function + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrame + fun:function_code_fastcall +} + +{ + Python leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyMem_RawMalloc + fun:PyThread_allocate_lock + fun:newlockobject + ... + fun:cfunction_vectorcall_NOARGS + fun:_PyObject_VectorcallTstate + fun:PyObject_Vectorcall + fun:call_function + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrame + fun:function_code_fastcall + fun:_PyFunction_Vectorcall +} + +{ + Python leaks when spawning a subprocess + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyMem_RawMalloc + fun:PyThread_allocate_lock + fun:rlock_new + fun:type_call + fun:_PyObject_Call + fun:PyObject_Call + fun:do_call_core + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalFrame + fun:_PyEval_EvalCode + fun:_PyFunction_Vectorcall +} + +# Not really CPython-specific, see link +{ + dlopen leak (https://stackoverflow.com/questions/1542457/memory-leak-reported-by-valgrind-in-dlopen) + Memcheck:Leak + fun:malloc + ... + fun:dl_open_worker + fun:_dl_catch_exception + fun:_dl_open + fun:dlopen_doit + fun:_dl_catch_exception + fun:_dl_catch_error + fun:_dlerror_run + fun:dlopen@@GLIBC_2.2.5 + fun:_PyImport_FindSharedFuncptr + fun:_PyImport_LoadDynamicModuleWithSpec +} + +# Not really CPython-specific, see link +{ + dlopen leak (https://stackoverflow.com/questions/1542457/memory-leak-reported-by-valgrind-in-dlopen) + Memcheck:Leak + fun:malloc + ... + fun:dl_open_worker + fun:_dl_catch_exception + fun:_dl_open + fun:dlopen_doit + fun:_dl_catch_exception + fun:_dl_catch_error + fun:_dlerror_run + fun:dlopen@@GLIBC_2.2.5 + fun:_PyImport_FindSharedFuncptr + fun:_PyImport_LoadDynamicModuleWithSpec +}