diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4f9ebac9f..0f01b1d26 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,6 @@ jobs:
         runs-on: [ubuntu-latest, windows-latest, macos-latest]
         arch: [x64]
         max-cxx-std: [17]
-        dev: [false]
         python:
         - 2.7
         - 3.5
@@ -30,41 +29,34 @@ jobs:
             python: 3.6
             arch: x64
             max-cxx-std: 17
-            dev: false
             args: "-DPYBIND11_FINDPYTHON=ON"
           - runs-on: macos-latest
             python: 3.7
             arch: x64
             max-cxx-std: 17
-            dev: false
             args: "-DPYBIND11_FINDPYTHON=ON"
           - runs-on: windows-2016
             python: 3.7
             arch: x86
             max-cxx-std: 14
-            dev: false
           - runs-on: windows-latest
             python: 3.6
             arch: x64
             max-cxx-std: 17
-            dev: false
             args: "-DPYBIND11_FINDPYTHON=ON"
           - runs-on: windows-latest
             python: 3.7
             arch: x64
             max-cxx-std: 17
-            dev: false
 
           - runs-on: ubuntu-latest
             python: 3.9-dev
             arch: x64
             max-cxx-std: 17
-            dev: true
           - runs-on: macos-latest
             python: 3.9-dev
             arch: x64
             max-cxx-std: 17
-            dev: true
 
         exclude:
             # Currently 32bit only, and we build 64bit
@@ -72,29 +64,24 @@ jobs:
             python: pypy2
             arch: x64
             max-cxx-std: 17
-            dev: false
           - runs-on: windows-latest
             python: pypy3
             arch: x64
             max-cxx-std: 17
-            dev: false
 
             # Currently broken on embed_test
           - runs-on: windows-latest
             python: 3.8
             arch: x64
             max-cxx-std: 17
-            dev: false
           - runs-on: windows-latest
             python: 3.9-dev
             arch: x64
             max-cxx-std: 17
-            dev: false
 
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • ${{ matrix.arch }} ${{ matrix.args }}"
     runs-on: ${{ matrix.runs-on }}
-    continue-on-error: ${{ matrix.dev }}
 
     steps:
     - uses: actions/checkout@v2
@@ -129,7 +116,7 @@ jobs:
     - name: Configure C++11 ${{ matrix.args }}
       shell: bash
       run: >
-        cmake -S . -B build
+        cmake -S . -B .
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
@@ -137,16 +124,19 @@ jobs:
         ${{ matrix.args }}
 
     - name: Build C++11
-      run: cmake --build build -j 2
+      run: cmake --build . -j 2
 
     - name: Python tests C++11
-      run: cmake --build build --target pytest -j 2
+      run: cmake --build . --target pytest -j 2
 
     - name: C++11 tests
-      run: cmake --build build --target cpptest -j 2
+      run: cmake --build .  --target cpptest -j 2
 
     - name: Interface test C++11
-      run: cmake --build build --target test_cmake_build -v
+      run: cmake --build . --target test_cmake_build
+
+    - name: Clean directory
+      run: git clean -fdx
 
     - name: Configure C++${{ matrix.max-cxx-std }} ${{ matrix.args }}
       shell: bash
diff --git a/.gitignore b/.gitignore
index 6d65838b5..47e010ce2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,7 +32,7 @@ MANIFEST
 .*.swp
 .DS_Store
 /dist
-/build*
+/*build*
 .cache/
 sosize-*.txt
 pybind11Config*.cmake
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 427975258..00e39bc0e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -46,7 +46,17 @@ if(NOT pybind11_FIND_QUIETLY)
 endif()
 
 # Check if pybind11 is being used directly or via add_subdirectory
-if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+if(CMAKE_CURRENT_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
+  ### Warn if not an out-of-source builds
+  if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+    set(lines
+        "You are building in-place. If that is not what you intended to "
+        "do, you can clean the source directory with:\n"
+        "rm -r CMakeCache.txt CMakeFiles/ cmake_uninstall.cmake pybind11Config.cmake "
+        "pybind11ConfigVersion.cmake tests/CMakeFiles/\n")
+    message(AUTHOR_WARNING ${lines})
+  endif()
+
   set(PYBIND11_MASTER_PROJECT ON)
 
   if(OSX AND CMAKE_VERSION VERSION_LESS 3.7)
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 895cfe75e..72de21018 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -239,7 +239,6 @@ foreach(t ${PYBIND11_CROSS_MODULE_GIL_TESTS})
   endif()
 endforeach()
 
-set(testdir ${CMAKE_CURRENT_SOURCE_DIR})
 foreach(target ${test_targets})
   set(test_files ${PYBIND11_TEST_FILES})
   if(NOT "${target}" STREQUAL "pybind11_tests")
@@ -250,6 +249,18 @@ foreach(target ${test_targets})
   pybind11_add_module(${target} THIN_LTO ${target}.cpp ${test_files} ${PYBIND11_HEADERS})
   pybind11_enable_warnings(${target})
 
+  if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+    get_property(
+      suffix
+      TARGET ${target}
+      PROPERTY SUFFIX)
+    set(source_output "${CMAKE_CURRENT_SOURCE_DIR}/${target}${suffix}")
+    if(suffix AND EXISTS "${source_output}")
+      message(WARNING "Output file also in source directory; "
+                      "please remove to avoid confusion: ${source_output}")
+    endif()
+  endif()
+
   if(MSVC)
     target_compile_options(${target} PRIVATE /utf-8)
   endif()
@@ -266,10 +277,12 @@ foreach(target ${test_targets})
 
   # Always write the output file directly into the 'tests' directory (even on MSVC)
   if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
-    set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${testdir}")
+    set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
+                                               "${CMAKE_CURRENT_BINARY_DIR}")
     foreach(config ${CMAKE_CONFIGURATION_TYPES})
       string(TOUPPER ${config} config)
-      set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} "${testdir}")
+      set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config}
+                                                 "${CMAKE_CURRENT_BINARY_DIR}")
     endforeach()
   endif()
 endforeach()
@@ -293,12 +306,26 @@ if(NOT PYBIND11_PYTEST_FOUND)
       CACHE INTERNAL "")
 endif()
 
+if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+  # This is not used later in the build, so it's okay to regenerate each time.
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/pytest.ini" "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini"
+                 COPYONLY)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini"
+       "\ntestpaths = \"${CMAKE_CURRENT_SOURCE_DIR}\"")
+
+endif()
+
+# cmake 3.12 added list(transform <list> prepend
+# but we can't use it yet
+string(REPLACE "test_" "${CMAKE_CURRENT_BINARY_DIR}/test_" PYBIND11_BINARY_TEST_FILES
+               "${PYBIND11_PYTEST_FILES}")
+
 # A single command to compile and run the tests
 add_custom_target(
   pytest
-  COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_PYTEST_FILES}
+  COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_BINARY_PYTEST_FILES}
   DEPENDS ${test_targets}
-  WORKING_DIRECTORY ${testdir}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   USES_TERMINAL)
 
 if(PYBIND11_TEST_OVERRIDE)
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b6e47dc2..a2350d041 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,6 +13,8 @@ import textwrap
 
 import pytest
 
+import env
+
 # Early diagnostic for failed imports
 import pybind11_tests  # noqa: F401
 
@@ -20,6 +22,11 @@ _unicode_marker = re.compile(r'u(\'[^\']*\')')
 _long_marker = re.compile(r'([0-9])L')
 _hexadecimal = re.compile(r'0x[0-9a-fA-F]+')
 
+# Avoid collecting Python3 only files
+collect_ignore = []
+if env.PY2:
+    collect_ignore.append("test_async.py")
+
 
 def _strip_and_dedent(s):
     """For triple-quote strings"""
diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt
index 1495c7775..2e298fa7e 100644
--- a/tests/test_embed/CMakeLists.txt
+++ b/tests/test_embed/CMakeLists.txt
@@ -21,18 +21,22 @@ pybind11_enable_warnings(test_embed)
 
 target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads)
 
+if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+  file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+endif()
+
 add_custom_target(
   cpptest
   COMMAND "$<TARGET_FILE:test_embed>"
-  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
 
 pybind11_add_module(external_module THIN_LTO external_module.cpp)
 set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY
-                                                 "${CMAKE_CURRENT_SOURCE_DIR}")
+                                                 "${CMAKE_CURRENT_BINARY_DIR}")
 foreach(config ${CMAKE_CONFIGURATION_TYPES})
   string(TOUPPER ${config} config)
   set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config}
-                                                   "${CMAKE_CURRENT_SOURCE_DIR}")
+                                                   "${CMAKE_CURRENT_BINARY_DIR}")
 endforeach()
 add_dependencies(cpptest external_module)