chore(ci): optimize caching to reduce cache count

.github/workflows/build.yml

@@ -8,8 +8,12 @@ on:
   merge_group:
 
 env:
-  SCCACHE_GHA_ENABLED: "true"
-  RUSTC_WRAPPER: "sccache"
+  DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs "
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  RUSTFLAGS: -C debuginfo=line-tables-only
+  # Disable incremental builds by cargo for CI which should save disk space
+  # and hopefully avoid final link "No space left on device"
+  CARGO_INCREMENTAL: 0
 
 jobs:
   check:
@@ -20,20 +24,20 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: Setup sccache
-        uses: mozilla-actions/sccache-action@v0.0.9
-
-      - name: Setup Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           components: rustfmt,clippy
           cache: true
           # if we ever get rid of all the clippy warnings, we can remove this line
           rustflags: ""
 
-      - name: Rustfmt Check
-        uses: actions-rust-lang/rustfmt@v1
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: "rust-cache-${{ matrix.os }}"
+          cache-all-crates: true
+          cache-targets: false
 
       - name: build and lint with clippy
         run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} --tests
@@ -64,23 +68,153 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.test-matrix.outputs.targets) }}
     runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.86"
+          override: true
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: "rust-cache-${{ matrix.os }}"
+          cache-all-crates: true
+          cache-targets: false
+
+      - name: Run tests
+        run: |
+          make setup-dat
+          cargo test --features ${{ env.DEFAULT_FEATURES }}
+
+  integration_test:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    env:
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      AWS_DEFAULT_REGION: "us-east-1"
+      AWS_ACCESS_KEY_ID: deltalake
+      AWS_SECRET_ACCESS_KEY: weloverust
+      AWS_ENDPOINT_URL: http://localhost:4566
+      AWS_ALLOW_HTTP: "1"
+      AZURE_USE_EMULATOR: "1"
+      AZURE_STORAGE_ALLOW_HTTP: "1"
+      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
+      AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.86"
+          override: true
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "rust-cache-integration-tests"
+          cache-all-crates: true
+          cache-targets: false
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - name: Start emulated services
+        run: docker compose up -d
+
+      - name: Run tests with rustls (default)
+        run: |
+          gmake setup-dat
+          cargo llvm-cov \
+            --features integration_test,${{ env.DEFAULT_FEATURES }} \
+            --workspace \
+            --exclude delta-inspect \
+            --exclude deltalake-hdfs \
+            --exclude deltalake-lakefs \
+            --codecov \
+            --output-path codecov.json
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: codecov.json
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  integration_test_native_tls:
+    name: Integration Tests (Native TLS)
+    runs-on: ubuntu-latest
+    env:
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      AWS_DEFAULT_REGION: "us-east-1"
+      AWS_ACCESS_KEY_ID: deltalake
+      AWS_SECRET_ACCESS_KEY: weloverust
+      AWS_ENDPOINT_URL: http://localhost:4566
+      AWS_ALLOW_HTTP: "1"
+      AZURE_USE_EMULATOR: "1"
+      AZURE_STORAGE_ALLOW_HTTP: "1"
+      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
+      AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.86"
+          override: true
+
+      # Install Java and Hadoop for HDFS integration tests
+      - uses: actions/setup-java@v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Download Hadoop
+        run: |
+          wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz
+          tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE
+          echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH
+
+      - name: Start emulated services
+        run: docker compose up -d
+
+      - name: Run tests with native-tls
+        run: |
+          gmake setup-dat
+          cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
+
+  integration_test_hdfs:
+    name: Integration Tests (HDFS)
+    runs-on: ubuntu-latest
     env:
       DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs"
 
     steps:
       - uses: actions/checkout@v5
 
-      - name: Setup sccache
-        uses: mozilla-actions/sccache-action@v0.0.9
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.86"
+          override: true
 
-      - name: Setup Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: Swatinem/rust-cache@v2
         with:
-          toolchain: stable
-          components: clippy
-          cache: true
-          # if we ever get rid of all the clippy warnings, we can remove this line
-          rustflags: ""
+          shared-key: "rust-cache-integration-tests-hdfs"
+          cache-all-crates: true
+          cache-targets: false
 
       - name: Default build
         run: cargo build --package deltalake
@@ -108,16 +242,18 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: Setup sccache
-        uses: mozilla-actions/sccache-action@v0.0.9
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.86"
+          override: true
 
-      - name: Setup Rust toolchain
-        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses: Swatinem/rust-cache@v2
         with:
-          toolchain: stable
-          cache: true
-          # if we ever get rid of all the clippy warnings, we can remove this line
-          rustflags: ""
+          shared-key: "rust-cache-integration-tests-lakefs"
+          cache-all-crates: true
+          cache-targets: false
 
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov

.github/workflows/python_build.yml

@@ -19,7 +19,13 @@ jobs:
   test-minimal:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "rust-cache-python-builds"
+          cache-all-crates: true
+          cache-targets: false
 
       - name: Setup Environment
         uses: ./.github/actions/setup-env
@@ -43,8 +49,15 @@ jobs:
   test:
     name: Python Build (Python 3.10 PyArrow latest)
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "rust-cache-python-builds"
+          cache-all-crates: true
+          cache-targets: false
 
       - name: Setup Environment
         uses: ./.github/actions/setup-env