Independent tests (#665)

* Make tests buildable independently

This makes "tests" buildable as a separate project that uses
find_package(pybind11 CONFIG) when invoked independently.

This also moves the WERROR option into tests/CMakeLists.txt, as that's
the only place it is used.

* Use Eigen 3.3.1's cmake target, if available

This changes the eigen finding code to attempt to use Eigen's
system-installed Eigen3Config first.  In Eigen 3.3.1, it exports a cmake
Eigen3::Eigen target to get dependencies from (rather than setting the
include path directly).

If it fails, we fall back to the trying to load allowing modules (i.e.
allowing our tools/FindEigen3.cmake).  If we either fallback, or the
eigen version is older than 3.3.1 (or , we still set the include
directory manually; otherwise, for CONFIG + new Eigen, we get it via
the target.

This is also needed to allow 'tests' to be built independently, when
the find_package(Eigen3) is going to find via the system-installed
Eigen3Config.cmake.

* Add a install-then-build test, using clang on linux

This tests that `make install` to the actual system, followed by a build
of the tests (without the main pybind11 repository available) works as
expected.

To also expand the testing variety a bit, it also builds using
clang-3.9 instead of gcc.

* Don't try loading Eigen3Config in cmake < 3.0

It could FATAL_ERROR as the newer cmake includes a cmake 3.0 required
line.

If doing an independent, out-of-tree "tests" build, the regular
find_package(Eigen3) is likely to fail with the same error, but I think
we can just let that be: if you want a recent Eigen with proper cmake
loading support *and* want to do an independent tests build, you'll
need at least cmake 3.0.

.travis.yml

@@ -39,6 +39,19 @@ matrix:
   - sudo: true
     services: docker
     env: ARCH=i386 PYTHON=3.5 CPP=14 GCC=6
+  # This next one does a make install *before* testing, then builds the tests against the installed version:
+  - sudo: true
+    services: docker
+    env: PYTHON=3.5 CPP=14 CLANG=3.9 INSTALL=1
+    script:
+      - |
+        $SCRIPT_RUN_PREFIX sh -c "set -e
+        cmake ${CMAKE_EXTRA_ARGS} -DPYBIND11_INSTALL=1 -DPYBIND11_TEST=0
+        make install
+        cp -a tests /pybind11-tests
+        mkdir /build-tests && cd /build-tests
+        cmake ../pybind11-tests ${CMAKE_EXTRA_ARGS} -DPYBIND11_WERROR=ON
+        make pytest -j 2"
   # A barebones build makes sure everything still works without optional deps (numpy/scipy/eigen)
   # and also tests the automatic discovery functions in CMake (Python version, C++ standard).
   - os: linux
@@ -72,9 +85,15 @@ before_install:
 - |
   # Configure build variables
   if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-    if [ -z "$GCC" ]; then export GCC=4.8; fi
+    if [ -n "$CLANG" ]; then
-    export CXX=g++-$GCC CC=gcc-$GCC;
+      export CXX=clang++-$CLANG CC=clang-$CLANG COMPILER_PACKAGES="clang-$CLANG llvm-$CLANG-dev"
-    if [ "$GCC" = "6" ]; then export DOCKER=${ARCH:+$ARCH/}debian:testing
+    else
+      if [ -z "$GCC" ]; then export GCC=4.8
+      else export COMPILER_PACKAGES=g++-$GCC
+      fi
+      export CXX=g++-$GCC CC=gcc-$GCC
+    fi
+    if [ "$GCC" = "6" ] || [ -n "$CLANG" ]; then export DOCKER=${ARCH:+$ARCH/}debian:testing
     elif [ "$GCC" = "7" ]; then export DOCKER=debian:experimental APT_GET_EXTRA="-t experimental"
     fi
   elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
@@ -131,7 +150,7 @@ install:
     docker exec --tty "$containerid" sh -c "for s in 0 15; do sleep \$s; \
       apt-get -qy --no-install-recommends $APT_GET_EXTRA install \
         python$PY-dev python$PY-pytest python$PY-scipy \
-        libeigen3-dev cmake make g++-$GCC && break; done"
+        libeigen3-dev cmake make ${COMPILER_PACKAGES} && break; done"
   else
     pip install numpy scipy pytest
 

CMakeLists.txt

@@ -22,7 +22,6 @@ endif()
 
 option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT})
 option(PYBIND11_TEST    "Build pybind11 test suite?"     ${PYBIND11_MASTER_PROJECT})
-option(PYBIND11_WERROR  "Report all warnings as errors"  OFF)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/tools")
 
@@ -35,23 +34,6 @@ set(PYTHON_LIBRARIES ${PYTHON_LIBRARIES} CACHE INTERNAL "")
 set(PYTHON_MODULE_PREFIX ${PYTHON_MODULE_PREFIX} CACHE INTERNAL "")
 set(PYTHON_MODULE_EXTENSION ${PYTHON_MODULE_EXTENSION} CACHE INTERNAL "")
 
-# Compile with compiler warnings turned on
-function(pybind11_enable_warnings target_name)
-  if(MSVC)
-    target_compile_options(${target_name} PRIVATE /W4)
-  else()
-    target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual)
-  endif()
-
-  if(PYBIND11_WERROR)
-    if(MSVC)
-      target_compile_options(${target_name} PRIVATE /WX)
-    else()
-      target_compile_options(${target_name} PRIVATE -Werror)
-    endif()
-  endif()
-endfunction()
-
 set(PYBIND11_HEADERS
   include/pybind11/attr.h
   include/pybind11/cast.h

tests/CMakeLists.txt

@@ -1,3 +1,22 @@
+# CMakeLists.txt -- Build system for the pybind11 test suite
+#
+# Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch>
+#
+# All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+cmake_minimum_required(VERSION 2.8.12)
+
+option(PYBIND11_WERROR  "Report all warnings as errors"  OFF)
+
+if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+    # We're being loaded directly, i.e. not via add_subdirectory, so make this
+    # work as its own project and load the pybind11Config to get the tools we need
+    project(pybind11_tests)
+
+    find_package(pybind11 REQUIRED CONFIG)
+endif()
+
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   message(STATUS "Setting tests build type to MinSizeRel as none was specified")
   set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING "Choose the type of build." FORCE)
@@ -54,9 +73,30 @@ string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}")
 # skip message).
 list(FIND PYBIND11_TEST_FILES test_eigen.cpp PYBIND11_TEST_FILES_EIGEN_I)
 if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
+  # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake).
+  # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also
+  # produces a fatal error if loaded from a pre-3.0 cmake.
+  if (NOT CMAKE_VERSION VERSION_LESS 3.0)
+    find_package(Eigen3 QUIET CONFIG)
+    if (EIGEN3_FOUND)
+      if (EIGEN3_VERSION_STRING AND NOT EIGEN3_VERSION_STRING VERSION_LESS 3.3.1)
+        set(PYBIND11_EIGEN_VIA_TARGET 1)
+      endif()
+    endif()
+  endif()
+  if (NOT EIGEN3_FOUND)
+    # Couldn't load via target, so fall back to allowing module mode finding, which will pick up
+    # tools/FindEigen3.cmake
     find_package(Eigen3 QUIET)
+  endif()
 
   if(EIGEN3_FOUND)
+    # Eigen 3.3.1+ cmake sets EIGEN3_VERSION_STRING (and hard codes the version when installed
+    # rather than looking it up in the cmake script); older versions, and the
+    # tools/FindEigen3.cmake, set EIGEN3_VERSION instead.
+    if(NOT EIGEN3_VERSION AND EIGEN3_VERSION_STRING)
+      set(EIGEN3_VERSION ${EIGEN3_VERSION_STRING})
+    endif()
     message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}")
   else()
     list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I})
@@ -64,6 +104,24 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
   endif()
 endif()
 
+# Compile with compiler warnings turned on
+function(pybind11_enable_warnings target_name)
+  if(MSVC)
+    target_compile_options(${target_name} PRIVATE /W4)
+  else()
+      target_compile_options(${target_name} PRIVATE -Wall -Wextra -Wconversion -Wcast-qual)
+  endif()
+
+  if(PYBIND11_WERROR)
+    if(MSVC)
+      target_compile_options(${target_name} PRIVATE /WX)
+    else()
+      target_compile_options(${target_name} PRIVATE -Werror)
+    endif()
+  endif()
+endfunction()
+
+
 # Create the binding library
 pybind11_add_module(pybind11_tests THIN_LTO pybind11_tests.cpp
   ${PYBIND11_TEST_FILES} ${PYBIND11_HEADERS})
@@ -71,11 +129,15 @@ pybind11_add_module(pybind11_tests THIN_LTO pybind11_tests.cpp
 pybind11_enable_warnings(pybind11_tests)
 
 if(EIGEN3_FOUND)
+  if (PYBIND11_EIGEN_VIA_TARGET)
+    target_link_libraries(pybind11_tests PRIVATE Eigen3::Eigen)
+  else()
     target_include_directories(pybind11_tests PRIVATE ${EIGEN3_INCLUDE_DIR})
+  endif()
   target_compile_definitions(pybind11_tests PRIVATE -DPYBIND11_TEST_EIGEN)
 endif()
 
-set(testdir ${PROJECT_SOURCE_DIR}/tests)
+set(testdir ${CMAKE_CURRENT_SOURCE_DIR})
 
 # Always write the output file directly into the 'tests' directory (even on MSVC)
 if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
@@ -105,7 +167,16 @@ if(PYBIND11_TEST_OVERRIDE)
     COMMAND ${CMAKE_COMMAND} -E echo "Note: not all tests run: -DPYBIND11_TEST_OVERRIDE is in effect")
 endif()
 
-# And another to show the .so size and, if a previous size, compare it:
+# Add a check target to run all the tests, starting with pytest (we add dependencies to this below)
+add_custom_target(check DEPENDS pytest)
+
+# The remaining tests only apply when being built as part of the pybind11 project, but not if the
+# tests are being built independently.
+if (NOT PROJECT_NAME STREQUAL "pybind11")
+  return()
+endif()
+
+# Add a post-build comment to show the .so size and, if a previous size, compare it:
 add_custom_command(TARGET pybind11_tests POST_BUILD
   COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/libsize.py
   $<TARGET_FILE:pybind11_tests> ${CMAKE_CURRENT_BINARY_DIR}/sosize-$<TARGET_FILE_NAME:pybind11_tests>.txt)
@@ -160,5 +231,4 @@ if(NOT CMAKE_VERSION VERSION_LESS 3.1)
   endif()
 endif()
 
-# Run all the tests
+add_dependencies(check test_cmake_build)
-add_custom_target(check DEPENDS pytest test_cmake_build)