From 38370a87f417001f3984cfa929860dabce6b69e7 Mon Sep 17 00:00:00 2001 From: andriish Date: Sat, 12 Sep 2020 04:06:52 +0200 Subject: [PATCH] fix: support NVIDIA-PGI HPC SDK (#2475) * Added guards to the includes Added new CI config Added new trigger Changed CI workflow name Debug CI Debug CI Debug CI Debug CI Added flags fro PGI Disable Eigen Removed tests that fail Uncomment lines * fix: missing include fix: minor style cleanup tests: support skipping ci: remove and tighten a bit fix: try msvc workaround for pgic * tests: split up prealoc tests * fix: PGI compiler fix * fix: PGI void_t only * fix: try to appease nvcc * ci: better ordering for slow tests * ci: minor improvements to testing * ci: Add NumPy to testing * ci: Eigen generates CUDA warnings / PGI errors * Added CentOS7 back for a moment * Fix YAML * ci: runs-on missing * centos7 is missing pytest * ci: use C++11 on CentOS 7 * ci: test something else * Try just adding flags on CentOS 7 * fix: CentOS 7 * refactor: move include to shared location * Added verbose flag * Try to use system cmake3 on CI * Try to use system cmake3 on CI, attempt2 * Try to use system cmake3 on CI, attempt3 * tests: not finding pytest should be a warning, not a fatal error * tests: cleanup * Weird issue? * fix: final polish Co-authored-by: Andrii Verbytskyi Co-authored-by: Henry Schreiner Co-authored-by: Andrii Verbytskyi --- .github/CONTRIBUTING.md | 23 +++++ .github/workflows/ci.yml | 131 +++++++++++++++++++++++----- README.md | 1 + docs/changelog.rst | 10 +++ include/pybind11/detail/common.h | 9 ++ tests/CMakeLists.txt | 65 ++++++++++---- tests/test_factory_constructors.cpp | 1 + tests/test_factory_constructors.py | 17 +++- tests/test_smart_ptr.py | 5 +- tests/test_virtual_functions.cpp | 4 +- tests/test_virtual_functions.py | 4 +- 11 files changed, 222 insertions(+), 48 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f61011d54..e83c31565 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -164,6 +164,29 @@ name, pre-commit): pre-commit install ``` +### Build recipes + +This builds with the Intel compiler (assuming it is in your path, along with a +recent CMake and Python 3): + +```bash +python3 -m venv venv +. venv/bin/activate +pip install pytest +cmake -S . -B build-intel -DCMAKE_CXX_COMPILER=$(which icpc) -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DPYBIND11_WERROR=ON +``` + +This will test the PGI compilers: + +```bash +docker run --rm -it -v $PWD:/pybind11 nvcr.io/hpc/pgi-compilers:ce +apt-get update && apt-get install -y python3-dev python3-pip python3-pytest +wget -qO- "https://cmake.org/files/v3.18/cmake-3.18.2-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local +cmake -S pybind11/ -B build +cmake --build build +``` + + [pre-commit]: https://pre-commit.com [pybind11.readthedocs.org]: http://pybind11.readthedocs.org/en/latest [issue tracker]: https://github.com/pybind/pybind11/issues diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3104f49e5..530d8caf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,6 @@ jobs: python: 3.9-dev arch: x64 - name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • ${{ matrix.arch }} ${{ matrix.args }}" runs-on: ${{ matrix.runs-on }} continue-on-error: ${{ endsWith(matrix.python, 'dev') }} @@ -196,6 +195,114 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + + cuda: + runs-on: ubuntu-latest + name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" + container: nvidia/cuda:11.0-devel-ubuntu20.04 + + steps: + - uses: actions/checkout@v2 + + # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND + - name: Install 🐍 3 + run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy + + - name: Configure + run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON + + - name: Build + run: cmake --build build -j2 --verbose + + - name: Python tests + run: cmake --build build --target pytest + + + centos-nvhpc8: + runs-on: ubuntu-latest + name: "🐍 3 • CentOS8 / PGI 20.7 • x64" + container: centos:8 + + steps: + - uses: actions/checkout@v2 + + - name: Add Python 3 and a few requirements + run: yum update -y && yum install -y git python3-devel python3-numpy python3-pytest make environment-modules + + - name: Install CMake with pip + run: | + python3 -m pip install --upgrade pip + python3 -m pip install cmake --prefer-binary + + - name: Install NVidia HPC SDK + run: yum -y install https://developer.download.nvidia.com/hpc-sdk/nvhpc-20-7-20.7-1.x86_64.rpm https://developer.download.nvidia.com/hpc-sdk/nvhpc-2020-20.7-1.x86_64.rpm + + - name: Configure + shell: bash + run: | + source /etc/profile.d/modules.sh + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.7 + cmake -S . -B build -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=14 -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build build -j 2 --verbose + + - name: Python tests + run: cmake --build build --target pytest + + - name: C++ tests + run: cmake --build build --target cpptest + + - name: Interface test + run: cmake --build build --target test_cmake_build + + centos-nvhpc7: + runs-on: ubuntu-latest + name: "🐍 3 • CentOS7 / PGI 20.7 • x64" + container: centos:7 + + steps: + - uses: actions/checkout@v2 + + - name: Add Python 3 and a few requirements + run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 + + - name: Install NVidia HPC SDK + run: yum -y install https://developer.download.nvidia.com/hpc-sdk/nvhpc-20-7-20.7-1.x86_64.rpm https://developer.download.nvidia.com/hpc-sdk/nvhpc-2020-20.7-1.x86_64.rpm + + # On CentOS 7, we have to filter a few tests (compiler internal error) + # and allow deeper templete recursion (not needed on CentOS 8 with a newer + # standard library). On some systems, you many need further workarounds: + # https://github.com/pybind/pybind11/pull/2475 + - name: Configure + shell: bash + run: | + source /etc/profile.d/modules.sh + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.7 + cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ + -DCMAKE_CXX_STANDARD=11 \ + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ + -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + + # Building before installing Pip should produce a warning but not an error + - name: Build + run: cmake3 --build build -j 2 --verbose + + - name: Install CMake with pip + run: | + python3 -m pip install --upgrade pip + python3 -m pip install pytest + + - name: Python tests + run: cmake3 --build build --target pytest + + - name: C++ tests + run: cmake3 --build build --target cpptest + + - name: Interface test + run: cmake3 --build build --target test_cmake_build + gcc: runs-on: ubuntu-latest strategy: @@ -243,6 +350,7 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + centos: runs-on: ubuntu-latest strategy: @@ -289,27 +397,6 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build - cuda: - runs-on: ubuntu-latest - name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" - container: nvidia/cuda:11.0-devel-ubuntu20.04 - - steps: - - uses: actions/checkout@v2 - - # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND - - name: Install 🐍 3 - run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake python3-dev python3-pytest - - - name: Configure - run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON - - - name: Build - run: cmake --build build -j2 -v - - - name: Python tests - run: cmake --build build --target pytest - install-classic: name: "🐍 3.5 • Debian • x86 • Install" diff --git a/README.md b/README.md index 633231f74..69a0fc90b 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ In addition to the core functionality, pybind11 provides some extra goodies: v2.0 and a [workaround][intel-15-workaround]) 5. Cygwin/GCC (tested on 2.5.1) 6. NVCC (CUDA 11 tested) +7. NVIDIA PGI (20.7 tested) ## About diff --git a/docs/changelog.rst b/docs/changelog.rst index ca025f9d8..77fd44173 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -80,6 +80,9 @@ Smaller or developer focused features: * Bugfixes related to more extensive testing `#2321 `_ +* Bug in timezone issue in Eastern hemisphere midnight fixed. + `#2438 `_ + * Pointer to ``std::tuple`` & ``std::pair`` supported in cast. `#2334 `_ @@ -96,6 +99,13 @@ Smaller or developer focused features: * Debug Python interpreter support. `#2025 `_ +* NVCC (CUDA 11) now supported and tested in CI. + `#2461 `_ + +* NVIDIA PGI compilers now supported and tested in CI. + `#2475 `_ + + v2.5.0 (Mar 31, 2020) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 8923faef7..7d6530cc8 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -154,6 +154,7 @@ #include #include #include +#include #include #include #include @@ -501,8 +502,16 @@ template using select_indices = typename select_indices_impl using bool_constant = std::integral_constant; template struct negation : bool_constant { }; +// PGI cannot detect operator delete with the "compatible" void_t impl, so +// using the new one (C++14 defect, so generally works on newer compilers, even +// if not in C++17 mode) +#if defined(__PGIC__) +template using void_t = void; +#else template struct void_t_impl { using type = void; }; template using void_t = typename void_t_impl::type; +#endif + /// Compile-time all/any/none of that check the boolean value of all template types #if defined(__cpp_fold_expressions) && !(defined(_MSC_VER) && (_MSC_VER < 1916)) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e59c75ce8..45e094b08 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,29 @@ else() cmake_policy(VERSION 3.18) endif() +# Only needed for CMake < 3.5 support +include(CMakeParseArguments) + +# Filter out items; print an optional message if any items filtered +# +# Usage: +# pybind11_filter_tests(LISTNAME file1.cpp file2.cpp ... MESSAGE "") +# +macro(PYBIND11_FILTER_TESTS LISTNAME) + cmake_parse_arguments(ARG "" "MESSAGE" "" ${ARGN}) + set(PYBIND11_FILTER_TESTS_FOUND OFF) + foreach(filename IN LISTS ARG_UNPARSED_ARGUMENTS) + list(FIND ${LISTNAME} ${filename} _FILE_FOUND) + if(_FILE_FOUND GREATER -1) + list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND}) + set(PYBIND11_FILTER_TESTS_FOUND ON) + endif() + endforeach() + if(PYBIND11_FILTER_TESTS_FOUND AND ARG_MESSAGE) + message(STATUS "${ARG_MESSAGE}") + endif() +endmacro() + # New Python support if(DEFINED Python_EXECUTABLE) set(PYTHON_EXECUTABLE "${Python_EXECUTABLE}") @@ -34,6 +57,9 @@ option(PYBIND11_CUDA_TESTS "Enable building CUDA tests (requires CMake 3.12+)" O set(PYBIND11_TEST_OVERRIDE "" CACHE STRING "Tests from ;-separated list of *.cpp files will be built instead of all tests") +set(PYBIND11_TEST_FILTER + "" + CACHE STRING "Tests from ;-separated list of *.cpp files will be removed from all tests") if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) # We're being loaded directly, i.e. not via add_subdirectory, so make this @@ -106,21 +132,23 @@ if(PYBIND11_TEST_OVERRIDE) set(PYBIND11_TEST_FILES ${PYBIND11_TEST_OVERRIDE}) endif() -# Skip test_async for Python < 3.5 -list(FIND PYBIND11_TEST_FILES test_async.cpp PYBIND11_TEST_FILES_ASYNC_I) -if((PYBIND11_TEST_FILES_ASYNC_I GREATER -1) AND (PYTHON_VERSION VERSION_LESS 3.5)) - message(STATUS "Skipping test_async because Python version ${PYTHON_VERSION} < 3.5") - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_ASYNC_I}) +# You can also filter tests: +if(PYBIND11_TEST_FILTER) + pybind11_filter_tests(PYBIND11_TEST_FILES ${PYBIND11_TEST_FILTER}) +endif() + +if(PYTHON_VERSION VERSION_LESS 3.5) + pybind11_filter_tests(PYBIND11_TEST_FILES test_async.cpp MESSAGE + "Skipping test_async on Python 2") endif() # Skip tests for CUDA check: # /pybind11/tests/test_constants_and_functions.cpp(125): # error: incompatible exception specifications -list(FIND PYBIND11_TEST_FILES test_constants_and_functions.cpp PYBIND11_TEST_FILES_CAF_I) -if((PYBIND11_TEST_FILES_CAF_I GREATER -1) AND PYBIND11_CUDA_TESTS) - message( - STATUS "Skipping test_constants_and_functions due to incompatible exception specifications") - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_CAF_I}) +if(PYBIND11_CUDA_TESTS) + pybind11_filter_tests( + PYBIND11_TEST_FILES test_constants_and_functions.cpp MESSAGE + "Skipping test_constants_and_functions due to incompatible exception specifications") endif() string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}") @@ -318,7 +346,7 @@ foreach(target ${test_targets}) endif() endforeach() -# Make sure pytest is found or produce a fatal error +# Make sure pytest is found or produce a warning if(NOT PYBIND11_PYTEST_FOUND) execute_process( COMMAND ${PYTHON_EXECUTABLE} -c "import pytest; print(pytest.__version__)" @@ -326,15 +354,16 @@ if(NOT PYBIND11_PYTEST_FOUND) OUTPUT_VARIABLE pytest_version ERROR_QUIET) if(pytest_not_found) - message(FATAL_ERROR "Running the tests requires pytest. Please install it manually" - " (try: ${PYTHON_EXECUTABLE} -m pip install pytest)") + message(WARNING "Running the tests requires pytest. Please install it manually" + " (try: ${PYTHON_EXECUTABLE} -m pip install pytest)") elseif(pytest_version VERSION_LESS 3.1) - message(FATAL_ERROR "Running the tests requires pytest >= 3.1. Found: ${pytest_version}" - "Please update it (try: ${PYTHON_EXECUTABLE} -m pip install -U pytest)") + message(WARNING "Running the tests requires pytest >= 3.1. Found: ${pytest_version}" + "Please update it (try: ${PYTHON_EXECUTABLE} -m pip install -U pytest)") + else() + set(PYBIND11_PYTEST_FOUND + TRUE + CACHE INTERNAL "") endif() - set(PYBIND11_PYTEST_FOUND - TRUE - CACHE INTERNAL "") endif() if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) diff --git a/tests/test_factory_constructors.cpp b/tests/test_factory_constructors.cpp index 61cf33d16..f61474346 100644 --- a/tests/test_factory_constructors.cpp +++ b/tests/test_factory_constructors.cpp @@ -11,6 +11,7 @@ #include "pybind11_tests.h" #include "constructor_stats.h" #include +#include // Classes for testing python construction via C++ factory function: // Not publicly constructible, copyable, or movable: diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 6c4bed165..b141c13de 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -336,10 +336,10 @@ def strip_comments(s): return re.sub(r'\s+#.*', '', s) -def test_reallocations(capture, msg): +def test_reallocation_a(capture, msg): """When the constructor is overloaded, previous overloads can require a preallocated value. This test makes sure that such preallocated values only happen when they might be necessary, - and that they are deallocated properly""" + and that they are deallocated properly.""" pytest.gc_collect() @@ -353,6 +353,9 @@ def test_reallocations(capture, msg): ~NoisyAlloc() noisy delete """ + + +def test_reallocation_b(capture, msg): with capture: create_and_destroy(1.5) assert msg(capture) == strip_comments(""" @@ -365,6 +368,8 @@ def test_reallocations(capture, msg): noisy delete # operator delete """) + +def test_reallocation_c(capture, msg): with capture: create_and_destroy(2, 3) assert msg(capture) == strip_comments(""" @@ -375,6 +380,8 @@ def test_reallocations(capture, msg): noisy delete # operator delete """) + +def test_reallocation_d(capture, msg): with capture: create_and_destroy(2.5, 3) assert msg(capture) == strip_comments(""" @@ -386,6 +393,8 @@ def test_reallocations(capture, msg): noisy delete # operator delete """) + +def test_reallocation_e(capture, msg): with capture: create_and_destroy(3.5, 4.5) assert msg(capture) == strip_comments(""" @@ -397,6 +406,8 @@ def test_reallocations(capture, msg): noisy delete # operator delete """) + +def test_reallocation_f(capture, msg): with capture: create_and_destroy(4, 0.5) assert msg(capture) == strip_comments(""" @@ -409,6 +420,8 @@ def test_reallocations(capture, msg): noisy delete # operator delete """) + +def test_reallocation_g(capture, msg): with capture: create_and_destroy(5, "hi") assert msg(capture) == strip_comments(""" diff --git a/tests/test_smart_ptr.py b/tests/test_smart_ptr.py index c9267f687..0b1ca45b5 100644 --- a/tests/test_smart_ptr.py +++ b/tests/test_smart_ptr.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import pytest -from pybind11_tests import smart_ptr as m -from pybind11_tests import ConstructorStats + +m = pytest.importorskip("pybind11_tests.smart_ptr") +from pybind11_tests import ConstructorStats # noqa: E402 def test_smart_ptr(capture): diff --git a/tests/test_virtual_functions.cpp b/tests/test_virtual_functions.cpp index 6dcf29415..c5ee6365f 100644 --- a/tests/test_virtual_functions.cpp +++ b/tests/test_virtual_functions.cpp @@ -139,7 +139,7 @@ public: std::string print_movable(int a, int b) { return get_movable(a, b).get_value(); } }; class NCVirtTrampoline : public NCVirt { -#if !defined(__INTEL_COMPILER) && !defined(__CUDACC__) +#if !defined(__INTEL_COMPILER) && !defined(__CUDACC__) && !defined(__PGIC__) NonCopyable get_noncopyable(int a, int b) override { PYBIND11_OVERLOAD(NonCopyable, NCVirt, get_noncopyable, a, b); } @@ -205,7 +205,7 @@ TEST_SUBMODULE(virtual_functions, m) { .def(py::init()); // test_move_support -#if !defined(__INTEL_COMPILER) && !defined(__CUDACC__) +#if !defined(__INTEL_COMPILER) && !defined(__CUDACC__) && !defined(__PGIC__) py::class_(m, "NCVirt") .def(py::init<>()) .def("get_noncopyable", &NCVirt::get_noncopyable) diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index b7bd5badf..66a353ae7 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -3,8 +3,8 @@ import pytest import env # noqa: F401 -from pybind11_tests import virtual_functions as m -from pybind11_tests import ConstructorStats +m = pytest.importorskip("pybind11_tests.virtual_functions") +from pybind11_tests import ConstructorStats # noqa: E402 def test_override(capture, msg):