diff --git a/.github/workflows/_build_plugin.yml b/.github/workflows/_build_plugin.yml
index e30b88aed1e..bd5dd39936e 100644
--- a/.github/workflows/_build_plugin.yml
+++ b/.github/workflows/_build_plugin.yml
@@ -25,7 +25,7 @@ jobs:
       GCLOUD_SERVICE_KEY: ${{ secrets.gcloud-service-key }}
       GOOGLE_APPLICATION_CREDENTIALS: /tmp/default_credentials.json
       BAZEL_JOBS: 16
-      BAZEL_REMOTE_CACHE: 1
+      BAZEL_REMOTE_CACHE: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
       - name: Setup gcloud
         shell: bash
diff --git a/.github/workflows/_build_torch_xla.yml b/.github/workflows/_build_torch_xla.yml
index 58a783216e4..0de6a8de39b 100644
--- a/.github/workflows/_build_torch_xla.yml
+++ b/.github/workflows/_build_torch_xla.yml
@@ -25,7 +25,7 @@ jobs:
       GCLOUD_SERVICE_KEY: ${{ secrets.gcloud-service-key }}
       GOOGLE_APPLICATION_CREDENTIALS: /tmp/default_credentials.json
       BAZEL_JOBS: 16
-      BAZEL_REMOTE_CACHE: 1
+      BAZEL_REMOTE_CACHE: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
       BUILD_CPP_TESTS: 1
     steps:
       - name: Setup gcloud