fix: Sccache dist tests broken after bump to tokio 1.21 and later

.github/actions/artifact_failure/action.yml

@@ -0,0 +1,25 @@
+name: "Upload failure Artifacts"
+description: "Upload failure Artifacts"
+inputs:
+  name:
+    description: ""
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: pack failure artifacts
+      shell: bash
+      run: |
+        killall sccache || true
+        killall sccache-dist || true
+
+        tar --exclude='target' \
+            --exclude='docs' \
+            --exclude='bins' \
+            --exclude='.git' \
+            --exclude='failure-*' \
+            -zcf failure-${{ inputs.name }}.tar.gz .
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.name }}
+        path: failure-${{ inputs.name }}.tar.gz

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
         include:
           - os: ubuntu-20.04
             rustc: 1.60.0 # Oldest supported version, keep in sync with README.md
-          - os: ubuntu-18.04
+          - os: ubuntu-22.04
             rustc: 1.60.0
             extra_desc: dist-server
             extra_args: --no-default-features --features=dist-tests test_dist_ -- --test-threads 1
@@ -87,6 +87,12 @@ jobs:
       - name: Run tests
         run: cargo test --locked --all-targets --verbose ${{ matrix.extra_args }}
 
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-${{ matrix.os }}-${{ matrix.rustc || 'stable' }}-${{ matrix.extra_desc }}
+
   build:
     name: build ${{ matrix.binary || 'sccache' }} ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
@@ -182,10 +188,10 @@ jobs:
             extra_args: --features=unstable
           - os: macOS-11
             rustc: nightly
-# Disable on Windows for now as it fails with:
-# found invalid metadata files for crate `vte_generate_state_changes`
-#          - os: windows-2019
-#            rustc: nightly
+    # Disable on Windows for now as it fails with:
+    # found invalid metadata files for crate `vte_generate_state_changes`
+    #          - os: windows-2019
+    #            rustc: nightly
     env:
       RUST_BACKTRACE: 1
     steps:
@@ -207,9 +213,9 @@ jobs:
       - name: Execute tests
         run: cargo test --no-fail-fast --locked --all-targets --verbose ${{ matrix.extra_args }}
         env:
-          CARGO_INCREMENTAL: '0'
-          RUSTC_WRAPPER: ''
-          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off'
+          CARGO_INCREMENTAL: "0"
+          RUSTC_WRAPPER: ""
+          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off"
 
       - name: Generate coverage data (via `grcov`)
         id: coverage
@@ -245,53 +251,58 @@ jobs:
       matrix:
         job:
           - { os: macos-12 }
-        release: [ "13.1" ]
+        release: ["13.1"]
     steps:
-    - uses: actions/checkout@v3
-    - name: Prepare, build and test
-      uses: vmactions/freebsd-vm@v0
-      with:
-        mem: 8192
-        usesh: true
-        copyback: false
-        prepare: pkg install -y ca_root_nss curl gmake gtar pot sudo
-        run: |
-          #####################################################################################
-          ###  Prepare, build, and test
-          #####################################################################################
-          ###  based on ref: <https://github.com/rust-lang/rustup/pull/2783>
-          ###  and on ref: <https://github.com/uutils/coreutils/commit/86c610a84b8b6c>
-          ###  * NOTE: All steps need to be run in this block, otherwise, we are operating back
-          ###    on the mac host.
-          set -exo pipefail
-          #
-          ### Basic user setup ################################################################
-          TEST_USER=tester
-          TEST_USER_HOME="/opt/$TEST_USER"
-          REPO_NAME=${GITHUB_WORKSPACE##*/}
-          WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}"
-          WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}"
-          export WORKSPACE
-          #
-          mkdir -p "$TEST_USER_HOME"
-          pw adduser -n "$TEST_USER" -d "$TEST_USER_HOME" -c "Tester" -h -
-          chown -R "$TEST_USER":"$TEST_USER" "$TEST_USER_HOME"
-          chown -R "$TEST_USER":"$TEST_USER" "/$WORKSPACE_PARENT"/
-          cat > /usr/local/etc/sudoers.d/wheel<<EOF
-          $TEST_USER ALL=(ALL) NOPASSWD: ALL
-          EOF
-          #
-          ### Install rust stable from rustup  ################################################
-          su "$TEST_USER" -c "/bin/sh -exo pipefail" <<"EOH"
-          whoami
-          echo "$HOME"
-          fetch -o /tmp/rustup.sh https://sh.rustup.rs
-          sh /tmp/rustup.sh -y --profile=minimal
-          ### Run tests #######################################################################
-          . "$HOME/.cargo/env"
-          "$WORKSPACE/scripts/freebsd-ci-test.sh"
-          EOH
-          # end
+      - uses: actions/checkout@v3
+      - name: Prepare, build and test
+        uses: vmactions/freebsd-vm@v0
+        with:
+          mem: 8192
+          usesh: true
+          copyback: false
+          prepare: pkg install -y ca_root_nss curl gmake gtar pot sudo
+          run: |
+            #####################################################################################
+            ###  Prepare, build, and test
+            #####################################################################################
+            ###  based on ref: <https://github.com/rust-lang/rustup/pull/2783>
+            ###  and on ref: <https://github.com/uutils/coreutils/commit/86c610a84b8b6c>
+            ###  * NOTE: All steps need to be run in this block, otherwise, we are operating back
+            ###    on the mac host.
+            set -exo pipefail
+            #
+            ### Basic user setup ################################################################
+            TEST_USER=tester
+            TEST_USER_HOME="/opt/$TEST_USER"
+            REPO_NAME=${GITHUB_WORKSPACE##*/}
+            WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}"
+            WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}"
+            export WORKSPACE
+            #
+            mkdir -p "$TEST_USER_HOME"
+            pw adduser -n "$TEST_USER" -d "$TEST_USER_HOME" -c "Tester" -h -
+            chown -R "$TEST_USER":"$TEST_USER" "$TEST_USER_HOME"
+            chown -R "$TEST_USER":"$TEST_USER" "/$WORKSPACE_PARENT"/
+            cat > /usr/local/etc/sudoers.d/wheel<<EOF
+            $TEST_USER ALL=(ALL) NOPASSWD: ALL
+            EOF
+            #
+            ### Install rust stable from rustup  ################################################
+            su "$TEST_USER" -c "/bin/sh -exo pipefail" <<"EOH"
+            whoami
+            echo "$HOME"
+            fetch -o /tmp/rustup.sh https://sh.rustup.rs
+            sh /tmp/rustup.sh -y --profile=minimal
+            ### Run tests #######################################################################
+            . "$HOME/.cargo/env"
+            "$WORKSPACE/scripts/freebsd-ci-test.sh"
+            EOH
+            # end
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-freebsd-13.1-stable
 
   release:
     name: release

Cargo.toml

@@ -41,7 +41,6 @@ env_logger = "0.10"
 filetime = "0.2"
 flate2 = { version = "1.0", optional = true, default-features = false, features = ["rust_backend"] }
 futures = "0.3"
-futures-locks = "0.7"
 gzp = { version = "0.11.1", default-features = false, features = ["deflate_rust"]  }
 http = "0.2"
 hyper = { version = "0.14.10", optional = true, features = ["server"] }

scripts/freebsd-ci-test.sh

@@ -247,6 +247,7 @@ test_sccache_dist_01()
 	# one failed remote build.
 	if [ "$FAILED_DIST" -gt 1 ]; then
 		2>&1 echo "More than one distributed compilations failed"
+		cat "$TEST_TMPDIR"/sccache_log.txt
 		return 1
 	fi
 	if [ "$SUCCEEDED_DIST" -eq 0 ]; then

src/dist/http.rs

@@ -556,7 +556,10 @@ mod server {
 
         rouille::Response {
             status_code: 200,
-            headers: vec![("Content-Type".into(), "application/octet-stream".into())],
+            headers: vec![
+                ("Content-Type".into(), "application/octet-stream".into()),
+                ("Content-Length".into(), data.len().to_string().into()),
+            ],
             data: rouille::ResponseBody::from_data(data),
             upgrade: None,
         }
@@ -744,6 +747,9 @@ mod server {
                 }
                 // Finish the client
                 let new_client = client_builder
+                    // Disable connection pool to avoid broken connection
+                    // between runtime
+                    .pool_max_idle_per_host(0)
                     .build()
                     .context("failed to create a HTTP client")?;
                 // Use the updated certificates
@@ -783,12 +789,16 @@ mod server {
                         prepare_response(request, &res)
                     },
                     (GET) (/api/v1/scheduler/server_certificate/{server_id: ServerId}) => {
-                        let certs = server_certificates.lock().unwrap();
-                        let (cert_digest, cert_pem) = try_or_500_log!(req_id, certs.get(&server_id)
+                        let certs = {
+                            let guard = server_certificates.lock().unwrap();
+                            guard.get(&server_id).map(|v|v.to_owned())
+                        };
+
+                        let (cert_digest, cert_pem) = try_or_500_log!(req_id, certs
                             .context("server cert not available"));
                         let res = ServerCertificateHttpResponse {
-                            cert_digest: cert_digest.clone(),
-                            cert_pem: cert_pem.clone(),
+                            cert_digest,
+                            cert_pem,
                         };
                         prepare_response(request, &res)
                     },
@@ -1100,6 +1110,9 @@ mod client {
             let client = reqwest::ClientBuilder::new()
                 .timeout(timeout)
                 .connect_timeout(connect_timeout)
+                // Disable connection pool to avoid broken connection
+                // between runtime
+                .pool_max_idle_per_host(0)
                 .build()
                 .context("failed to create an async HTTP client")?;
             let client_toolchains =