diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fc4717c16..9673485dd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -87,9 +87,7 @@ jobs:
         # for ways to do this more generally
         path: ~/Library/Caches/pip
         # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        key: ${{ runner.os }}-pip-${{ matrix.python }}-${{ matrix.arch }}-${{ hashFiles('tests/requirements.txt') }}
 
     - name: Prepare env
       run: python -m pip install -r tests/requirements.txt
diff --git a/tests/requirements.txt b/tests/requirements.txt
index b0248f010..c9750e473 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,5 +1,8 @@
 --extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/
-numpy; platform_python_implementation!="PyPy" or sys_platform!="darwin" or python_version<"3.0"
-numpy<1.19; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.0"
-pytest
-scipy; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version<"3.9"
+numpy==1.16.6; python_version<"3.6"
+numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6"
+numpy==1.19.1; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version>="3.6"
+pytest==4.6.9; python_version<"3.5"
+pytest==5.4.3; python_version>="3.5"
+scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version<"3.6"
+scipy==1.5.2; (platform_python_implementation!="PyPy" or sys_platform!="darwin") and python_version>="3.6" and python_version<"3.9"