test: fix invalid deserialization (#85)

* test: fix invalid deserialization

* feat: install node or python only when needed

* feat: when action is rerun, run testbed with logs in debug

* feat(ci): update actions to node 20

* feat: log stdout & stderr from setup & build cmd as debug

* fix: bump helix revision

* fix: use helix fork for the time being

* fix(ci): install missing setuptools

* fix(ci): revert back to python 3.10

* fix(ci): disable lance

.github/workflows/test.yml

@@ -14,19 +14,19 @@ jobs:
     strategy:
       matrix:
         repo:
-          - { name: simple, key: simple, parallel: 8 }
-          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8 }
-          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8 }
-          - { name: tiangolo/fastapi, key: fastapi, parallel: 8 }
-          - { name: encode/starlette, key: starlette, parallel: 8 }
-          - { name: lancedb/lancedb, key: lancedb, parallel: 2 }
-          - { name: lancedb/lance, key: lance, parallel: 2 }
-          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8 }
-          - { name: jaemk/cached, key: cached, parallel: 4 }
-          - { name: smol-rs/async-executor, key: async-executor, parallel: 4 }
-          - { name: gcanti/io-ts, key: io-ts, parallel: 8 }
-          - { name: colinhacks/zod, key: zod, parallel: 8 }
-          - { name: helix-editor/helix, key: helix, parallel: 2 }
+          - { name: simple, key: simple, parallel: 8, node: false, python: false }
+          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8, node: false, python: true }
+          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8, node: false, python: true }
+          - { name: tiangolo/fastapi, key: fastapi, parallel: 8, node: false, python: true }
+          - { name: encode/starlette, key: starlette, parallel: 8, node: false, python: true }
+          - { name: lancedb/lancedb, key: lancedb, parallel: 2, node: false, python: false }
+          # - { name: lancedb/lance, key: lance, parallel: 2, node: false, python: false }
+          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8, node: false, python: false }
+          - { name: jaemk/cached, key: cached, parallel: 4, node: false, python: false }
+          - { name: smol-rs/async-executor, key: async-executor, parallel: 4, node: false, python: false }
+          - { name: gcanti/io-ts, key: io-ts, parallel: 8, node: true, python: false }
+          - { name: colinhacks/zod, key: zod, parallel: 8, node: true, python: false }
+          - { name: helix-editor/helix, key: helix, parallel: 2, node: false, python: false }
     runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
     container:
       image: ubuntu:22.04
@@ -46,21 +46,24 @@ jobs:
           toolchain: nightly
 
       - name: Install Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
+        if: matrix.repo.python
         with:
           python-version: '3.10'
 
       - name: Install node 18
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
+        if: matrix.repo.node
         with:
           node-version: 18
 
       - name: Install yarn
+        if: matrix.repo.node
         run: |
           npm i -g yarn
 
       - name: Set up cargo cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         continue-on-error: false
         with:
           path: |
@@ -80,15 +83,17 @@ jobs:
         if: github.event_name == 'push' || github.event_name == 'pull_request'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
+          LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}
 
       - name: Run testbed
         run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
         if: github.event_name == 'workflow_dispatch'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
+          LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }}
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: results-${{ matrix.repo.key }}
           path: ./results.json
@@ -104,67 +109,67 @@ jobs:
           apt update
           apt install -y jq
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-simple
           path: results-simple
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-picklescan
           path: results-picklescan
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-huggingface_hub
           path: results-huggingface_hub
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-fastapi
           path: results-fastapi
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-starlette
           path: results-starlette
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-lancedb
           path: results-lancedb
 
-      - uses: actions/download-artifact@v1
-        with:
-          name: results-lance
-          path: results-lance
+      # - uses: actions/download-artifact@v4
+      #   with:
+      #     name: results-lance
+      #     path: results-lance
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-constrandom
           path: results-constrandom
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-cached
           path: results-cached
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-async-executor
           path: results-async-executor
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-io-ts
           path: results-io-ts
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-zod
           path: results-zod
 
-      - uses: actions/download-artifact@v1
+      - uses: actions/download-artifact@v4
         with:
           name: results-helix
           path: results-helix

crates/lsp-client/src/msg.rs

@@ -1,7 +1,6 @@
 use std::{
     fmt::{self, Display},
     io,
-    marker::Unpin,
 };
 
 use serde::{de::DeserializeOwned, Deserialize, Serialize};

crates/testbed/repositories-ci.yaml

@@ -30,7 +30,7 @@ repositories:
       type: github
       owner: mmaitre314
       name: picklescan
-      revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52
+      revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
       src_path: src/picklescan
     build_command: picklescan-venv/bin/python3
     build_args: ["-m", "compileall", "-q", "."]
@@ -201,9 +201,9 @@ repositories:
     holes_file: zod-smol.json
   - source:
       type: github
-      owner: helix-editor
+      owner: McPatate
       name: helix
-      revision: a1272bdb17a63361342a318982e46129d558743c
+      revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
       exclude_paths:
         - .cargo
         - .github

crates/testbed/repositories.yaml

@@ -30,7 +30,7 @@ repositories:
       type: github
       owner: mmaitre314
       name: picklescan
-      revision: 40001cd1caa9e041b1bce1b80f3707056cd8be52
+      revision: 108b06040de644c7aeb5f22d8b10dd9c5d282386
       src_path: src/picklescan
     build_command: picklescan-venv/bin/python3
     build_args: ["-m", "compileall", "-q", "."]
@@ -201,9 +201,9 @@ repositories:
     holes_file: zod.json
   - source:
       type: github
-      owner: helix-editor
+      owner: McPatate
       name: helix
-      revision: a1272bdb17a63361342a318982e46129d558743c
+      revision: 01c1ebebd813c9034059a966fae2bbc487b1a69f
       exclude_paths:
         - .cargo
         - .github

crates/testbed/src/main.rs

@@ -28,7 +28,11 @@ use serde_json::{Map, Value};
 use tempfile::TempDir;
 use tokio::{
     fs::{self, read_to_string, File, OpenOptions},
-    io::{self, AsyncReadExt, AsyncWriteExt},
+    io::{
+        self, AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt,
+        BufReader as TokioBufReader,
+    },
+    join,
     process::Command,
     sync::{OnceCell, RwLock, Semaphore},
 };
@@ -198,6 +202,7 @@ struct RepositoriesConfig {
     context_window: usize,
     fim: FimParams,
     model: String,
+    #[serde(flatten)]
     backend: Backend,
     repositories: Vec<Repository>,
     tls_skip_verify_insecure: bool,
@@ -400,15 +405,20 @@ async fn run_setup(
             command.0,
             command.1.join(" ")
         );
-        let status = status_cmd
+        let mut child = status_cmd
             .args(&command.1)
             .current_dir(&repo_path)
-            .stdout(Stdio::null())
-            .stderr(Stdio::null())
-            .spawn()?
-            .wait()
-            .await?;
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;
+
+        if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
+            let stdout = TokioBufReader::new(stdout);
+            let stderr = TokioBufReader::new(stderr);
+            join!(log_lines(stdout), log_lines(stderr));
+        }
 
+        let status = child.wait().await?;
         if !status.success() {
             return Err(anyhow!(
                 "error running: \"{} {}\"",
@@ -432,17 +442,31 @@ async fn build(
         status_cmd.env(name, value);
     }
     debug!("building repo: {command} {args:?}");
-    let status = status_cmd
+
+    let mut child = status_cmd
         .args(args)
         .current_dir(repo_path)
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .spawn()?
-        .wait()
-        .await?;
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+
+    if let (Some(stdout), Some(stderr)) = (child.stdout.take(), child.stderr.take()) {
+        let stdout = TokioBufReader::new(stdout);
+        let stderr = TokioBufReader::new(stderr);
+        join!(log_lines(stdout), log_lines(stderr));
+    }
+
+    let status = child.wait().await?;
     Ok(status.success())
 }
 
+async fn log_lines<R: AsyncReadExt + AsyncBufRead + Unpin>(stdio: R) {
+    let mut lines = stdio.lines();
+    while let Ok(Some(log)) = lines.next_line().await {
+        debug!("{log}");
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn complete_holes(
     hole: Hole,