ci(framework): Use uv to run e2e tests with Python 3.10

.github/workflows/framework-e2e.yml

@@ -143,45 +143,55 @@ jobs:
         include:
           - directory: e2e-bare
             e2e: e2e_bare
+            pytorch_cpu: false
 
           - directory: e2e-bare-https
             e2e: e2e_bare_https
+            pytorch_cpu: false
 
           - directory: e2e-bare-auth
             e2e: e2e_bare_auth
+            pytorch_cpu: false
 
           - directory: e2e-jax
             e2e: e2e_jax
+            pytorch_cpu: false
 
           - directory: e2e-pytorch
             e2e: e2e_pytorch
             dataset: |
               from torchvision.datasets import CIFAR10
               CIFAR10('./data', download=True)
+            pytorch_cpu: true
 
           - directory: e2e-tensorflow
             e2e: e2e_tensorflow
             dataset: |
               import tensorflow_datasets as tfds
               tfds.load('cifar10', split=['train', 'test'], as_supervised=True)
+            pytorch_cpu: false
 
           - directory: e2e-opacus
             e2e: e2e_opacus
             dataset: |
               from torchvision.datasets import CIFAR10
               CIFAR10('./data', download=True)
+            pytorch_cpu: true
 
           - directory: e2e-pytorch-lightning
             e2e: e2e_pytorch_lightning
+            pytorch_cpu: true
 
           - directory: e2e-scikit-learn
             e2e: e2e_scikit_learn
+            pytorch_cpu: false
 
           - directory: e2e-fastai
             e2e: e2e_fastai
             dataset: |
               from fastai.vision.all import untar_data, URLs
               untar_data(URLs.MNIST)
+            pytorch_cpu: true
 
           - directory: e2e-pandas
             e2e: e2e_pandas
@@ -190,6 +200,7 @@ jobs:
               from sklearn.datasets import load_iris
               Path('data').mkdir(exist_ok=True)
               load_iris(as_frame=True)['data'].to_csv('./data/client.csv')
+            pytorch_cpu: false
 
     name: Framework / ${{ matrix.directory }}
 
@@ -199,45 +210,42 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        if: ${{ needs.changes.outputs.framework == 'true' }}
-        uses: actions/setup-python@v5
+      - name: Install uv and set the Python version
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ (matrix.directory == 'e2e-fastai') && '3.10' || '3.9' }}
-      - name: Install build tools
-        if: ${{ needs.changes.outputs.framework == 'true' }}
-        run: |
-            python -m pip install -U pip==23.3.1
-        shell: bash
+          python-version: "3.10"
       # Using approach described here for Python location caching:
       # https://blog.allenai.org/python-caching-in-github-actions-e9452698e98d
-      - name: Cache Python location
-        if: ${{ needs.changes.outputs.framework == 'true' }}
-        id: cache-restore-python
-        uses: actions/cache/restore@v4
-        with:
-          path: ${{ env.pythonLocation }}
-          key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./framework/e2e/{0}/pyproject.toml', matrix.directory)) }}
-      - name: Install dependencies
+      # - name: Cache Python location
+      #   if: ${{ needs.changes.outputs.framework == 'true' }}
+      #   id: cache-restore-python
+      #   uses: actions/cache/restore@v4
+      #   with:
+      #     path: ${{ env.pythonLocation }}
+      #     key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./framework/e2e/{0}/pyproject.toml', matrix.directory)) }}
+      - name: Install the project
         if: ${{ needs.changes.outputs.framework == 'true' }}
-        run: python -m pip install --upgrade .
+        run: uv sync --all-extras --dev
+      - name: Install CPU PyTorch
+        if: ${{ needs.changes.outputs.framework == 'true' && matrix.pytorch_cpu }}
+        run: uv pip install --upgrade --reinstall torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          uv pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
       - name: Download dataset
         if: ${{ needs.changes.outputs.framework == 'true' && matrix.dataset }}
-        run: python -c "${{ matrix.dataset }}"
+        run: uv run --no-sync python -c "${{ matrix.dataset }}"
       - name: Run edge client test
         if: ${{ needs.changes.outputs.framework == 'true' && matrix.directory != 'e2e-bare-auth' }}
         working-directory: framework/e2e/${{ matrix.directory }}/${{ matrix.e2e }}
         run: ./../../test_legacy.sh "${{ matrix.directory }}"
       - name: Run virtual client test
         if: ${{ needs.changes.outputs.framework == 'true' && matrix.directory != 'e2e-bare-auth' }}
-        run: python simulation.py
+        run: uv run --no-sync simulation.py
       - name: Run simulation engine test
         if: ${{ needs.changes.outputs.framework == 'true' && (matrix.directory == 'e2e-pytorch' || matrix.directory == 'e2e-tensorflow') }}
-        run: python simulation_next.py
+        run: uv run --no-sync simulation_next.py
       - name: Run driver test
         if: ${{ needs.changes.outputs.framework == 'true' && matrix.directory != 'e2e-bare-auth' }}
         run: ./../test_superlink.sh "${{ matrix.directory }}"
@@ -253,13 +261,13 @@ jobs:
       # - name: Run reconnection test with SQLite database (Temporarily disabled due to in-memory ObjectStore)
       #   if: ${{ needs.changes.outputs.framework == 'true' && matrix.directory == 'e2e-bare' }}
       #   run: ./../test_reconnection.sh sqlite
-      - name: Cache save Python location
-        id: cache-save-python
-        uses: actions/cache/save@v4
-        if: ${{ needs.changes.outputs.framework == 'true' && github.ref_name == 'main' &&  !steps.cache-restore-python.outputs.cache-hit }}
-        with:
-          path: ${{ env.pythonLocation }}
-          key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./framework/e2e/{0}/pyproject.toml', matrix.directory)) }}
+      # - name: Cache save Python location
+      #   id: cache-save-python
+      #   uses: actions/cache/save@v4
+      #   if: ${{ needs.changes.outputs.framework == 'true' && github.ref_name == 'main' &&  !steps.cache-restore-python.outputs.cache-hit }}
+      #   with:
+      #     path: ${{ env.pythonLocation }}
+      #     key: pythonloc-${{ runner.os }}-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./framework/e2e/{0}/pyproject.toml', matrix.directory)) }}
 
   strategies:
     runs-on: ubuntu-22.04

framework/e2e/e2e-bare-auth/pyproject.toml

@@ -8,6 +8,7 @@ version = "1.0.0"
 description = "Auth-enabled bare Federated Learning test with Flower"
 license = "Apache-2.0"
 dependencies = ["flwr @ {root:parent:parent:uri}"]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-bare-https/pyproject.toml

@@ -8,6 +8,7 @@ version = "1.0.0"
 description = "HTTPS-enabled bare Federated Learning test with Flower"
 license = "Apache-2.0"
 dependencies = ["flwr @ {root:parent:parent:uri}"]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-bare/pyproject.toml

@@ -8,6 +8,7 @@ version = "1.0.0"
 description = "Bare Federated Learning test with Flower"
 license = "Apache-2.0"
 dependencies = ["flwr[simulation,rest] @ {root:parent:parent:uri}"]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-fastai/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "torch>=2.0.0,!=2.0.1,<2.1.0",
     "spacy==3.7.6",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-jax/pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "scikit-learn>=1.1.1,<2.0.0",
     "numpy>=1.21.4,<2.0.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-opacus/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "torch>=1.13.1,<3.0.0",
     "torchvision>=0.14.0,<2.0.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-pandas/pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pandas>=2.0.0,<3.0.0",
     "scikit-learn>=1.1.1,<2.0.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-pytorch-lightning/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "pytorch-lightning==2.4.0",
     "torchvision>=0.20.1,<0.21.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-pytorch/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "torchvision>=0.20.1,<0.21.0",
     "tqdm>=4.63.0,<5.0.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-scikit-learn/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "flwr-datasets[vision]>=0.5.0,<1.0.0",
     "scikit-learn>=1.1.1,<2.0.0",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-serverapp-heartbeat/pyproject.toml

@@ -8,6 +8,7 @@ version = "1.0.0"
 description = "Test heartbeat for runs."
 license = "Apache-2.0"
 dependencies = ["flwr @ {root:parent:parent:uri}"]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/e2e-tensorflow/pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "tensorflow-io-gcs-filesystem<0.35.0",
     "Pillow>=11.2.1",
 ]
+requires-python = ">=3.9.2"
 
 [tool.hatch.build.targets.wheel]
 packages = ["."]

framework/e2e/test_legacy.sh

@@ -6,14 +6,14 @@ if [ "$1" = "e2e-bare-https" ]; then
 fi
 
 # run the first command in background and save output to a temporary file:
-timeout 3m python server_app.py &
+timeout 3m uv run --no-sync server_app.py &
 pid=$!
 sleep 3
 
-python client_app.py &
+uv run --no-sync client_app.py &
 sleep 3
 
-python client_app.py &
+uv run --no-sync client_app.py &
 sleep 3
 
 wait $pid

framework/e2e/test_superlink.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+source .venv/bin/activate
 
 case "$1" in
   e2e-bare-https | e2e-bare-auth)
@@ -59,11 +60,14 @@ case "$2" in
 esac
 
 # Install Flower app
-pip install -e . --no-deps
+uv pip install -e . --no-deps
 
 # Remove any duplicates
 sed -i '/^\[tool\.flwr\.federations\.e2e\]/,/^$/d' pyproject.toml
 
+# Create .gitignore to ignore .venv folder
+echo ".venv" > .gitignore
+
 # Check if the first argument is 'insecure'
 if [ "$server_arg" = "--insecure" ]; then
   # If $server_arg is '--insecure', append the first line