diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml
index 01e662f2dd6..e7f40400ba2 100644
--- a/.azure/gpu-unittests.yml
+++ b/.azure/gpu-unittests.yml
@@ -130,10 +130,14 @@ jobs:
           #py-tree $(PYTEST_REFERENCE_CACHE) --show_hidden
           # make sure the cache exists even it is empty
           mkdir -p /var/tmp/cached-references
-          # copy the cache to the tests folder to be used in the next steps
+        displayName: "Show caches"
+
+      - bash: |
           cp -r /var/tmp/cached-references tests/_cache-references
           du -h --max-depth=1 tests/
-        displayName: "Show caches"
+        condition: eq(variables['Build.Reason'], 'PullRequest')
+        # if pull request, copy the cache to the tests folder to be used in the next steps
+        displayName: "Copy cached refs"
 
       - bash: |
           python -m pytest torchmetrics --cov=torchmetrics \
diff --git a/.github/actions/pull-caches/action.yml b/.github/actions/pull-caches/action.yml
index fec93a58bde..2f7f29ea0cc 100644
--- a/.github/actions/pull-caches/action.yml
+++ b/.github/actions/pull-caches/action.yml
@@ -82,6 +82,8 @@ runs:
       shell: bash
 
     - name: Cache References
+      # do not use this cache for dispatch and crone, to enable rebuild caches if needed
+      if: github.event_name != 'workflow_dispatch' && github.event_name != 'schedule'
       continue-on-error: true
       uses: actions/cache/restore@v3
       with:
@@ -90,5 +92,8 @@ runs:
 
     - name: Restored References
       continue-on-error: true
-      run: py-tree tests/_cache-references/ --show_hidden
+      working-directory: tests/
+      run: |
+        mkdir -p _cache-references
+        ls -lh _cache-references/
       shell: bash