Merge branch 'EnricoMi-master'

im-open · Sep 20, 2024 · 8bca85a · 8bca85a
2 parents caf82b6 + 0fbfebc
commit 8bca85a
Show file tree

Hide file tree

Showing 120 changed files with 8,791 additions and 1,308 deletions.
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
@@ -22,7 +22,7 @@ runs:
 
     - name: Setup Python
       if: inputs.python-version != 'installed'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
 
@@ -49,8 +49,9 @@ runs:
       shell: bash
 
     - name: Cache PIP Packages
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
+      if: github.event_name != 'schedule'
       with:
         path: ${{ steps.os.outputs.pip-cache }}
         key: ${{ inputs.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles('**/requirements.txt', '**/constraints.txt') }}-${{ steps.os.outputs.date }}
@@ -61,16 +62,37 @@ runs:
   
     - name: Install Python dependencies
       run: |
-        python3 -V
-        python3 -m pip freeze | sort
-        python3 -m pip cache info || true
-        python3 -m pip cache list || true
-        python3 -m pip install --upgrade --force pip wheel
-        python3 -m pip install --force -r python/requirements.txt
-        python3 -m pip install --force -r python/test/requirements.txt -c python/test/constraints.txt
-        python3 -m pip freeze | sort
-        python3 -m pip cache info || true
-        python3 -m pip cache list || true
+        python -m venv "$RUNNER_TEMP/venv"
+        echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
+        echo "$RUNNER_TEMP/venv/Scripts" >> "$GITHUB_PATH"
+        export PATH="$RUNNER_TEMP/venv/bin:$RUNNER_TEMP/venv/Scripts:$PATH"
+        which python
+
+        # inspect pip cache
+        python -m pip freeze | sort
+        python -m pip cache info || true
+        python -m pip cache list || true
+
+        # remove cached built whl files
+        rm -rf "$(python -m pip cache info | grep ".*[Ww]heels location: " | cut -d ":" -f 2- | cut -d " " -f 2-)"
+        python -m pip cache list || true
+
+        # install dependencies
+        python_minor_version="$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+        python -m pip install --force -r python/requirements-$python_minor_version.txt
+        python -m pip install --force -r python/test/requirements.txt -c python/test/constraints.txt
+
+        # inspect pip cache
+        python -m pip freeze | sort
+        python -m pip cache info || true
+        python -m pip cache list || true
+
+        # assert no whl files have been built
+        if python -m pip cache info && [[ "$(python -m pip cache info | grep "Number of .*wheels:")" != "Number of "*"wheels: 0" ]]
+        then
+          echo "Dependency whl files have been built"
+          exit 1
+        fi
       shell: bash
 
     - name: Update expectation files
@@ -85,16 +107,16 @@ runs:
           # we only upload the changed files if we can find zip
           if which zip
           then
-            (git diff --name-only && git ls-files -o  --exclude-standard) | xargs -d "\n" zip changed-expectations.zip
+            (git diff --name-only && git ls-files -o  --exclude-standard) | xargs zip changed-expectations.zip
             exit 1
           fi
         fi
       shell: bash
     - name: Upload changed expectation files
       if: steps.changes.outcome == 'failure'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: Changed expectations
+        name: Changed expectations (python-${{ inputs.python-version }}, ${{ inputs.os }})
         path: changed-expectations.zip
         if-no-files-found: error
 
@@ -103,7 +125,7 @@ runs:
         PYTHONPATH: ..
       run: |
         cd python/test
-        python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest.xml
+        python -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest.xml
       shell: bash
 
     - name: PyTest (EST)
@@ -113,7 +135,7 @@ runs:
         PYTHONPATH: ..
       run: |
         cd python/test
-        python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-est.xml
+        python -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-est.xml
       shell: bash
 
     - name: PyTest (CET)
@@ -123,12 +145,12 @@ runs:
         PYTHONPATH: ..
       run: |
         cd python/test
-        python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-cet.xml
+        python -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-cet.xml
       shell: bash
 
     - name: Upload Test Results
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Test Results (python-${{ inputs.python-version }}, ${{ inputs.os }})
         path: |

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,7 +4,3 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
diff --git a/.github/upgrade-pip-packages.sh b/.github/upgrade-pip-packages.sh
@@ -2,12 +2,13 @@
 set -euo pipefail
 
 base="$(dirname "$0")"
+python_minor_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
 
-pip install --upgrade --force pip==22.0.0
-pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements-direct.txt"
+pip install --upgrade --force pip==24.0.0
+pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements.txt"
 
 pip install pipdeptree
-pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements-direct.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements.txt"
+pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements-$python_minor_version.txt"
 
-git diff "$base/../python/requirements.txt"
+git diff "$base/../python/requirements-$python_minor_version.txt"
 
diff --git a/.github/workflows/badges.yml b/.github/workflows/badges.yml
@@ -28,13 +28,13 @@ jobs:
         uses: emibcn/badge-action@4209421db54c8764d8932070ffd0f81715a629bf
         with:
           label: Docker pulls
-          status: ${{ steps.downloads.outputs.total_downloads }} (${{ steps.downloads.outputs.recent_downloads_per_day }}/day)
+          status: ${{ steps.downloads.outputs.total_downloads }} (${{ steps.downloads.outputs.recent_downloads_per_day }}/d)
           color: blue
           path: downloads.svg
 
       - name: Create JSON
         run: |
-          echo '{"subject": "Docker pulls", "status": "${{ steps.downloads.outputs.total_downloads }} (${{ steps.downloads.outputs.recent_downloads_per_day }}/day)", "color": "blue"}' > downloads.json
+          echo '{"subject": "Docker pulls", "status": "${{ steps.downloads.outputs.total_downloads }} (${{ steps.downloads.outputs.recent_downloads_per_day }}/d)", "color": "blue"}' > downloads.json
 
       - name: Upload badge to Gist
         uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
@@ -65,7 +65,7 @@ jobs:
         uses: ./misc/action/fetch-workflows
 
       - name: Create badge
-        uses: emibcn/badge-action@4209421db54c8764d8932070ffd0f81715a629bf
+        uses: emibcn/badge-action@808173dd03e2f30c980d03ee49e181626088eee8
         with:
           label: GitHub Workflows
           status: ${{ steps.workflows.outputs.total_workflows }}
@@ -77,14 +77,14 @@ jobs:
           echo '{"subject": "GitHub Workflows", "status": "${{ steps.workflows.outputs.total_workflows }}", "color": "blue"}' > workflows.json
 
       - name: Upload badge to Gist
-        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        uses: andymckay/append-gist-action@6e8d64427fe47cbacf4ab6b890411f1d67c07f3e
         with:
           token: ${{ secrets.GIST_TOKEN }}
           gistURL: https://gist.githubusercontent.com/im-open/612cb538c14731f1a8fefe504f519395
           file: workflows.svg
 
       - name: Upload JSON to Gist
-        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        uses: andymckay/append-gist-action@6e8d64427fe47cbacf4ab6b890411f1d67c07f3e
         with:
           token: ${{ secrets.GIST_TOKEN }}
           gistURL: https://gist.githubusercontent.com/im-open/612cb538c14731f1a8fefe504f519395

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -9,7 +9,7 @@ on:
       - '*'
   pull_request:
   schedule:
-    - cron: '0 16 * * *'
+    - cron: '10 9 * * *'
   workflow_dispatch:
 permissions: {}
 
@@ -20,33 +20,53 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Check requirements.txt against requirements-direct.txt
+      - name: Check requirements.txt
         run: |
-          (diff -w python/requirements-direct.txt python/requirements.txt || true) | (! grep -e "^<")
+          (diff -w <(grep -v "python_version >  '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-3.7.txt || true) | (! grep -e "^<")
+          (diff -w <(grep -v "python_version <= '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-post-3.7.txt || true) | (! grep -e "^<")
         shell: bash
       - name: Check for dependency updates
         continue-on-error: true
-        run:
-          .github/upgrade-pip-packages.sh
+        run: |
+          pip install tox
+          tox
+          git diff --exit-code
         shell: bash
 
   test-mac:
     name: "Test macOS"
     uses: "./.github/workflows/test-os.yml"
     with:
-      os: '["macos-11", "macos-12", "macos-13"]'
+      os: '["macos-12", "macos-13", "macos-14"]'
+      python-version: '["3.10", "3.11", "3.12", "installed"]'
+      include: >
+        [
+          {"os": "macos-12", "python-version": "3.8"},
+          {"os": "macos-12", "python-version": "3.9"},
+          {"os": "macos-13", "python-version": "3.8"},
+          {"os": "macos-13", "python-version": "3.9"},
+        ]
 
   test-lnx:
     name: "Test Ubuntu"
     uses: "./.github/workflows/test-os.yml"
     with:
-      os: '["ubuntu-20.04", "ubuntu-22.04"]'
+      os: '["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04"]'
+      python-version: '["3.9", "3.10", "3.11", "3.12", "installed"]'
+      include: >
+        [
+          {"os": "ubuntu-20.04", "python-version": "3.7"},
+          {"os": "ubuntu-20.04", "python-version": "3.8"},
+          {"os": "ubuntu-22.04", "python-version": "3.8"},
+        ]
 
   test-win:
     name: "Test Windows"
     uses: "./.github/workflows/test-os.yml"
     with:
       os: '["windows-2019", "windows-2022"]'
+      python-version: '["3.8", "3.9", "3.10", "3.11", "3.12", "installed"]'
+      include: '[{"os": "windows-2019", "python-version": "3.7"}]'
 
   publish:
     name: "Publish"
@@ -136,7 +156,7 @@ jobs:
           password: ${{ secrets.CR_PAT }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
@@ -151,13 +171,15 @@ jobs:
                 needs.config-deploy.outputs.image == 'Dockerfile' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '.')
               )
             }}
+        env:
+          DOCKER_BUILD_RECORD_UPLOAD: false
 
   event_file:
     name: "Event File"
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Event File
           path: ${{ github.event_path }}