diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d1acf5e..6ae2e84 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,7 +21,11 @@ jobs:
           python-version: "3.8"
           cache: "pip"
       - name: Install dependencies and package
-        run: pip install -e . mypy
+        run: |
+          # Get the CPU only torch and torchaudio packages first
+          pip install $(grep torch requirements.txt | sed 's/$/+cpu/') --find-links https://download.pytorch.org/whl/torch_stable.html
+          # Install everything else
+          pip install -e . mypy
       - name: Minimal test, --help should work
         run: ctc-segmenter --help
       - name: Code quality test, mypy should pass