LLM Benchmarking

.cargo/config.toml

@@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"]
 
 [alias]
 bump-versions = "run -p upgrade-version --"
+llm = "run --package xtask-llm-benchmark --bin llm_benchmark --"
 ci = "run -p ci --"
 
 [target.x86_64-pc-windows-msvc]

.github/workflows/ci.yml

@@ -7,9 +7,9 @@
   workflow_dispatch:
     inputs:
       pr_number:
-        description: 'Pull Request Number'
+        description: "Pull Request Number"
         required: false
-        default: ''
+        default: ""
 
 name: CI
 
@@ -19,164 +19,166 @@
 
 jobs:
   docker_smoketests:
+    needs: [lints, llm_ci_check]
     name: Smoketests
     strategy:
       matrix:
         runner: [spacetimedb-new-runner, windows-latest]
         include:
           - runner: spacetimedb-new-runner
             smoketest_args: --docker
             container:
               image: localhost:5000/spacetimedb-ci:latest
               options: --privileged
           - runner: windows-latest
             smoketest_args: --no-build-cli
             container: null
     runs-on: ${{ matrix.runner }}
     container: ${{ matrix.container }}
     timeout-minutes: 120
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
         run: |
           PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
           if test -n "${PR_NUMBER}"; then
             GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
           else
             GIT_REF="${{ github.ref }}"
           fi
           echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
       - name: Checkout sources
         uses: actions/checkout@v4
         with:
           ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
           workspaces: ${{ github.workspace }}
           shared-key: spacetimedb
           cache-on-failure: false
           cache-all-crates: true
           cache-workspace-crates: true
           prefix-key: v1
 
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
 
       # nodejs and pnpm are required for the typescript quickstart smoketest
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - uses: pnpm/action-setup@v4
         with:
           run_install: true
 
       - name: Install psql (Windows)
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
       - name: Build crates
         run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
       - name: Start Docker daemon
         if: runner.os == 'Linux'
         run: /usr/local/bin/start-docker.sh
 
       - name: Build and start database (Linux)
         if: runner.os == 'Linux'
         run: |
           # Our .dockerignore omits `target`, which our CI Dockerfile needs.
           rm .dockerignore
           docker compose -f .github/docker-compose.yml up -d
       - name: Build and start database (Windows)
         if: runner.os == 'Windows'
         run: |
           # Fail properly if any individual command fails
           $ErrorActionPreference = 'Stop'
           $PSNativeCommandUseErrorActionPreference = $true
 
           Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432'
           cd modules
           # the sdk-manifests on windows-latest are messed up, so we need to update them
           dotnet workload config --update-mode manifests
           dotnet workload update
       - uses: actions/setup-python@v5
-        with: { python-version: '3.12' }
+        with: { python-version: "3.12" }
         if: runner.os == 'Windows'
       - name: Install python deps
         run: python -m pip install -r smoketests/requirements.txt
       - name: Run smoketests
         # Note: clear_database and replication only work in private
         run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
         run: docker compose -f .github/docker-compose.yml down
 
   test:
+    needs: [lints, llm_ci_check]
     name: Test Suite
     runs-on: spacetimedb-new-runner
     container:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Find Git ref
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
           if test -n "${PR_NUMBER}"; then
             GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
           else
             GIT_REF="${{ github.ref }}"
           fi
           echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
 
       - name: Checkout sources
         uses: actions/checkout@v4
         with:
           ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
           workspaces: ${{ github.workspace }}
           shared-key: spacetimedb
           # Let the smoketests job save the cache since it builds the most things
           save-if: false
           prefix-key: v1
 
       - uses: actions/setup-dotnet@v3
         with:
           global-json-file: global.json
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - uses: pnpm/action-setup@v4
         with:
           run_install: true
 
       - name: Build typescript module sdk
         working-directory: crates/bindings-typescript
         run: pnpm build
 
       - name: Run tests
         run: cargo ci test
 
   lints:
     name: Lints
     runs-on: spacetimedb-new-runner
     container:
@@ -500,7 +502,22 @@
         run: |
           cargo ci cli-docs
 
+  llm_ci_check:
+    name: Verify LLM benchmark is up to date
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run hash check (both langs)
+        run: cargo llm ci-check
+
   unity-testsuite:
+    needs: [lints, llm_ci_check]
     # Skip if this is an external contribution.
     # The license secrets will be empty, so the step would fail anyway.
     if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }}
@@ -585,7 +602,7 @@
           enable_pr_comment: ${{ github.event_name == 'pull_request' }}
           target_path: sdks/csharp
         env:
-          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
       - name: Start SpacetimeDB
         run: |
@@ -624,116 +641,243 @@
           githubToken: ${{ secrets.GITHUB_TOKEN }}
           testMode: playmode
           useHostNetwork: true
-          artifactsPath: ''
+          artifactsPath: ""
         env:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
           UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
 
   csharp-testsuite:
+    needs: [lints, llm_ci_check]
     runs-on: spacetimedb-new-runner
     container:
       image: localhost:5000/spacetimedb-ci:latest
       options: >-
         --privileged
         --cgroupns=host
     timeout-minutes: 30
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout repository
         id: checkout-stdb
         uses: actions/checkout@v4
 
       # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests.
 
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
           global-json-file: global.json
 
       - name: Override NuGet packages
         run: |
           dotnet pack crates/bindings-csharp/BSATN.Runtime
           dotnet pack crates/bindings-csharp/Runtime
 
           # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository
           # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if
           # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages.
           # This means that (if version numbers match) we will test the local versions of the C# packages, even
           # if they're not pushed to NuGet.
           # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file.
           cd sdks/csharp
           ./tools~/write-nuget-config.sh ../..
 
       - name: Run .NET tests
         working-directory: sdks/csharp
         run: dotnet test -warnaserror
 
       - name: Verify C# formatting
         working-directory: sdks/csharp
         run: dotnet format --no-restore --verify-no-changes SpacetimeDB.ClientSDK.sln
 
       - name: Install Rust toolchain
         uses: dsherret/rust-toolchain-file@v1
 
       - name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
           workspaces: ${{ github.workspace }}
           shared-key: spacetimedb
           # Let the main CI job save the cache since it builds the most things
           save-if: false
           prefix-key: v1
 
       - name: Install SpacetimeDB CLI from the local checkout
         run: |
           cargo install --force --path crates/cli --locked --message-format=short
           cargo install --force --path crates/standalone --locked --message-format=short
           # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules).
           ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime
 
       # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
       # ChatGPT suspects that this could be due to different build invocations using the same target dir,
       # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
       # `cargo build --manifest-path` (which apparently build different dependency trees).
       # However, we've been unable to fix it so... /shrug
       - name: Check v8 outputs
         run: |
           find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
           if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
             echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
             cargo clean -p v8 || true
             cargo build -p v8
           fi
 
       - name: Check quickstart-chat bindings are up to date
         working-directory: sdks/csharp
         run: |
           bash tools~/gen-quickstart.sh
           "${GITHUB_WORKSPACE}"/tools/check-diff.sh examples~/quickstart-chat || {
               echo 'Error: quickstart-chat bindings have changed. Please run `sdks/csharp/tools~/gen-quickstart.sh`.'
               exit 1
           }
 
       - name: Check client-api bindings are up to date
         working-directory: sdks/csharp
         run: |
           bash tools~/gen-client-api.sh
           "${GITHUB_WORKSPACE}"/tools/check-diff.sh src/SpacetimeDB/ClientApi || {
               echo 'Error: Client API bindings are dirty. Please run `sdks/csharp/tools~/gen-client-api.sh`.'
               exit 1
           }
 
       - name: Start SpacetimeDB
         run: |
           spacetime start &
           disown
 
       - name: Run regression tests
         run: |
           bash sdks/csharp/tools~/run-regression-tests.sh
           tools/check-diff.sh sdks/csharp/examples~/regression-tests || {
               echo 'Error: Bindings are dirty. Please run `sdks/csharp/tools~/gen-regression-tests.sh`.'
               exit 1
           }
+
+  internal-tests:
+    name: Internal Tests
+    needs: [lints, llm_ci_check]
+    # Skip if not a PR or a push to master
+    # Skip if this is an external contribution. GitHub secrets will be empty, so the step would fail anyway.
+    if: ${{ (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master'))
+      && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    env:
+      TARGET_OWNER: clockworklabs
+      TARGET_REPO: SpacetimeDBPrivate
+    steps:
+      - id: dispatch
+        name: Trigger tests
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const workflowId = 'ci.yml';
+            const targetRef = 'master';
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            // Use the ref for pull requests because the head sha is brittle (github does some extra dance where it merges in master).
+            const publicRef = (context.eventName === 'pull_request') ? context.payload.pull_request.head.ref : context.sha;
+            const preDispatch = new Date().toISOString();
+
+            // Dispatch the workflow in the target repository
+            await github.rest.actions.createWorkflowDispatch({
+              owner: targetOwner,
+              repo: targetRepo,
+              workflow_id: workflowId,
+              ref: targetRef,
+              inputs: { public_ref: publicRef }
+            });
+
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+            // Find the dispatched run by name
+            let runId = null;
+            for (let attempt = 0; attempt < 20 && !runId; attempt++) { // up to ~10 minutes to locate the run
+              await sleep(5000);
+              const runsResp = await github.rest.actions.listWorkflowRuns({
+                owner: targetOwner,
+                repo: targetRepo,
+                workflow_id: workflowId,
+                event: 'workflow_dispatch',
+                branch: targetRef,
+                per_page: 50,
+              });
+
+              const expectedName = `CI [public_ref=${publicRef}]`;
+              const candidates = runsResp.data.workflow_runs
+                .filter(r => r.name === expectedName && new Date(r.created_at) >= new Date(preDispatch))
+                .sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+              if (candidates.length > 0) {
+                runId = candidates[0].id;
+                break;
+              }
+            }
+
+            if (!runId) {
+              core.setFailed('Failed to locate dispatched run in the private repository.');
+              return;
+            }
+
+            const runUrl = `https://github.com/${targetOwner}/${targetRepo}/actions/runs/${runId}`;
+            core.info(`View run: ${runUrl}`);
+            core.setOutput('run_id', String(runId));
+            core.setOutput('run_url', runUrl);
+
+      - name: Wait for Internal Tests to complete
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            const runUrl = `${{ steps.dispatch.outputs.run_url }}`;
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+            core.info(`Waiting for workflow result... ${runUrl}`);
+
+            let conclusion = null;
+            for (let attempt = 0; attempt < 240; attempt++) { // up to ~2 hours
+              const runResp = await github.rest.actions.getWorkflowRun({
+                owner: targetOwner,
+                repo: targetRepo,
+                run_id: runId
+              });
+              const { status, conclusion: c } = runResp.data;
+              if (status === 'completed') {
+                conclusion = c || 'success';
+                break;
+              }
+              await sleep(30000);
+            }
+
+            if (!conclusion) {
+              core.setFailed('Timed out waiting for private workflow to complete.');
+              return;
+            }
+
+            if (conclusion !== 'success') {
+              core.setFailed(`Private workflow failed with conclusion: ${conclusion}`);
+            }
+
+      - name: Cancel invoked run if workflow cancelled
+        if: ${{ cancelled() }}
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.SPACETIMEDB_PRIVATE_TOKEN }}
+          script: |
+            const targetOwner = process.env.TARGET_OWNER;
+            const targetRepo = process.env.TARGET_REPO;
+            const runId = Number(`${{ steps.dispatch.outputs.run_id }}`);
+            if (!runId) return;
+            await github.rest.actions.cancelWorkflowRun({
+              owner: targetOwner,
+              repo: targetRepo,
+              run_id: runId,
+            });