[MNT] CI add pip dependency caching

.github/actions/numba_cache/action.yml

@@ -44,12 +44,6 @@ runs:
       run: echo "CURRENT_DATE=$(date +'%d/%m/%Y')" >> $GITHUB_ENV
       shell: bash
 
-      # GNU tar on windows runs much faster than the default BSD tar
-    - name: Use GNU tar instead BSD tar if Windows
-      if: ${{ inputs.runner_os == 'Windows' }}
-      shell: cmd
-      run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
-
     # Restore cache if restore_cache is true
     - name: Restore cache
       uses: actions/cache/restore@v3

.github/actions/pip_cache/action.yml

@@ -0,0 +1,58 @@
+name: Sets up optimal pip environment and restores cache
+description: "Sets up optimal pip environment and restores cache"
+
+inputs:
+  runner_os:
+    description: "The runner os"
+    required: true
+  python_version:
+    description: "The python version"
+    required: true
+  restore_cache:
+    description: "Whether to restore the cache"
+    required: false
+
+
+runs:
+  using: "composite"
+  steps:
+      # Get current date for cache restore
+    - name: Get Current Date
+      run: echo "CURRENT_DATE=$(date +'%d/%m/%Y')" >> $GITHUB_ENV
+      shell: bash
+
+      # For some reason on windows I/O on the C drive is really slow. To get around this
+      # we set the python, and the cache directories to the D drive which doesn't
+      # have slow I/O.
+      # See: https://github.com/actions/runner-images/issues/8755 for more details
+    - name: Set Python environment for Windows in powershell
+      if: ${{ inputs.runner_os == 'Windows' }}
+      shell: pwsh
+      # See https://docs.python.org/3/using/cmdline.html for env variable details
+      run: |
+        $Env:PYTHONUSERBASE = "D:\python\userbase"
+        $Env:PIP_CACHE_DIR = "D:\pip\cache"
+        $Env:PYTHONPYCACHEPREFIX = "D:\pycache"
+
+    - if: ${{ inputs.runner_os == 'Windows' }}
+      name: Set Python environment for windows
+      shell: bash
+      run: |
+        echo "PYTHONUSERBASE=D:\python\userbase" >> $GITHUB_ENV
+        echo "PIP_CACHE_DIR=D:\pip\cache" >> $GITHUB_ENV
+        echo "PYTHONPYCACHEPREFIX=D:\pycache" >> $GITHUB_ENV
+    - if: ${{ inputs.runner_os == 'Linux' }}
+      name: Set Python environment for linux
+      run: echo "PIP_CACHE_DIR=~/.cache/pip" >> $GITHUB_ENV
+      shell: bash
+    - if: ${{ inputs.runner_os == 'macOS' }}
+      name: Set Python environment for macos
+      run: echo "PIP_CACHE_DIR=~/Library/Caches/pip" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Restore cache
+      uses: actions/cache/restore@v3
+      if: ${{ inputs.restore_cache == 'true' }}
+      with:
+        path: ${{ env.PIP_CACHE_DIR }}
+        key: ${{ inputs.runner_os }}-${{ inputs.python_version }}-pip-${{ env.CURRENT_DATE }}

.github/workflows/periodic_pip_cache.yml

@@ -0,0 +1,73 @@
+name: Update pip cache every night
+
+on:
+  schedule:
+    # every day at 1:30 AM UTC
+    - cron:  "30 1 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  create-dependency-cache:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
+        python-version: [ "3.8", "3.10", "3.12" ]
+
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+
+      - name: Set optimal environment variables for pip cache
+        uses: ./.github/actions/pip_cache
+        with:
+          runner_os: ${{ runner.os }}
+          python_version: ${{ matrix.python-version }}
+          restore_cache: "false"
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install CPU version of pytorch on linux
+        uses: nick-fields/retry@v3
+        if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'Linux'
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+
+      # Install extras and dev on everything except Linux 3.10
+      - name: Install aeon and dependencies
+        uses: nick-fields/retry@v3
+        if: runner.os != 'Linux' || matrix.python-version != '3.10'
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: python -m pip install .[all_extras,dev]
+
+      # Install extras, dev and binder on Linux 3.10 (Examples run on this)
+      - name: Install notebook if on linux 3.10
+        uses: nick-fields/retry@v3
+        if: runner.os == 'Linux' && matrix.python-version == '3.10'
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: python -m pip install .[all_extras,dev,binder]
+
+      - name: Show dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: python -m pip list
+
+      - name: Save new cache
+        uses: actions/cache/save@v4
+        if: steps.cache.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.PIP_CACHE_DIR }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ env.CURRENT_DATE }}

.github/workflows/pr_examples.yml

@@ -30,7 +30,14 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Use numba cache to set env variables but not restore cache
+      - name: Set optimal environment variables for pip cache and restore
+        uses: ./.github/actions/pip_cache
+        with:
+          runner_os: ${{ runner.os }}
+          python_version: "3.10"
+          restore_cache: "true"
+
+      - name: Use numba cache to set env variables and restore cache
         uses: ./.github/actions/numba_cache
         with:
           cache_name: "run-notebook-examples"

.github/workflows/pr_pytest.yml

@@ -104,6 +104,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set optimal environment variables for pip cache and restore
+        uses: ./.github/actions/pip_cache
+        with:
+          runner_os: ${{ runner.os }}
+          python_version: ${{ matrix.python-version }}
+          restore_cache: "true"
+
       - name: Use numba cache to set env variables and restore cache
         uses: ./.github/actions/numba_cache
         with: