Trigger buffbench remotely

Author: jahooma

.github/workflows/buffbench.yml

@@ -0,0 +1,50 @@
+name: BuffBench
+
+on:
+  workflow_dispatch: # Manual triggering only
+
+jobs:
+  run-buffbench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: '1.3.5'
+
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            node_modules
+            */node_modules
+            packages/*/node_modules
+          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lock*') }}
+          restore-keys: |
+            ${{ runner.os }}-deps-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Set environment variables
+        env:
+          SECRETS_CONTEXT: ${{ toJSON(secrets) }}
+        run: |
+          VAR_NAMES=$(bun scripts/generate-ci-env.ts)
+          echo "$SECRETS_CONTEXT" | jq -r --argjson vars "$VAR_NAMES" '
+            to_entries | .[] | select(.key as $k | $vars | index($k)) | .key + "=" + .value
+          ' >> $GITHUB_ENV
+          echo "CODEBUFF_GITHUB_ACTIONS=true" >> $GITHUB_ENV
+          echo "NEXT_PUBLIC_CB_ENVIRONMENT=test" >> $GITHUB_ENV
+          echo "NEXT_PUBLIC_INFISICAL_UP=true" >> $GITHUB_ENV
+          echo "CODEBUFF_GITHUB_TOKEN=${{ secrets.CODEBUFF_GITHUB_TOKEN }}" >> $GITHUB_ENV
+
+      - name: Run buffbench
+        run: cd evals && bun run-buffbench
+
+      - name: Workflow completed
+        run: echo "BuffBench workflow completed successfully"

evals/buffbench/main.ts

@@ -8,8 +8,8 @@ async function main() {
   // Use 'external:codex' for OpenAI Codex CLI
   await runBuffBench({
     evalDataPaths: [path.join(__dirname, 'eval-codebuff.json')],
-    agents: ['base2', 'external:claude', 'external:codex'],
-    taskConcurrency: 1,
+    agents: ['base2'],
+    taskConcurrency: 5,
   })
 
   process.exit(0)

evals/package.json

@@ -23,6 +23,7 @@
     "run-eval-set": "bun run git-evals/run-eval-set.ts",
     "run-buffbench": "bun run buffbench/main.ts",
     "run-buffbench-nightly": "bun run buffbench/main-nightly.ts",
+    "trigger-buffbench": "bun run scripts/trigger-buffbench.ts",
     "setup-codebuff-repo": "bun run setup-codebuff-repo.ts"
   },
   "sideEffects": false,

evals/scripts/trigger-buffbench.ts

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+
+const { execSync } = require('child_process')
+
+function log(message: string) {
+  console.log(`${message}`)
+}
+
+function error(message: string) {
+  console.error(`❌ ${message}`)
+  process.exit(1)
+}
+
+function checkGitHubToken() {
+  const token = process.env.CODEBUFF_GITHUB_TOKEN
+  if (!token) {
+    error(
+      'CODEBUFF_GITHUB_TOKEN environment variable is required but not set.\n' +
+      'Please set it with your GitHub personal access token or use the infisical setup.'
+    )
+  }
+  return token
+}
+
+function getCurrentBranch(): string {
+  try {
+    return execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8' }).trim()
+  } catch {
+    return 'main'
+  }
+}
+
+async function triggerWorkflow(token: string, branch: string) {
+  try {
+    const triggerCmd = `curl -s -w "HTTP Status: %{http_code}" -X POST \
+      -H "Accept: application/vnd.github.v3+json" \
+      -H "Authorization: token ${token}" \
+      -H "Content-Type: application/json" \
+      https://api.github.com/repos/CodebuffAI/codebuff/actions/workflows/buffbench.yml/dispatches \
+      -d '{"ref":"${branch}"}'`
+
+    const response = execSync(triggerCmd, { encoding: 'utf8' })
+
+    if (response.includes('workflow_dispatch')) {
+      log(`⚠️  Workflow dispatch failed: ${response}`)
+      log(
+        'Please manually trigger the workflow at: https://github.com/CodebuffAI/codebuff/actions/workflows/buffbench.yml',
+      )
+    } else {
+      log('🎉 BuffBench workflow triggered!')
+    }
+  } catch (err: any) {
+    log(`⚠️  Failed to trigger workflow automatically: ${err.message}`)
+    log(
+      'You may need to trigger it manually at: https://github.com/CodebuffAI/codebuff/actions/workflows/buffbench.yml',
+    )
+  }
+}
+
+async function main() {
+  const branch = process.argv[2] || getCurrentBranch()
+
+  log('🧪 Triggering BuffBench workflow...')
+  log(`Branch: ${branch}`)
+
+  const token = checkGitHubToken()
+  if (!token) return
+  log('✅ Using CODEBUFF_GITHUB_TOKEN')
+
+  await triggerWorkflow(token, branch)
+
+  log('')
+  log('Monitor progress at: https://github.com/CodebuffAI/codebuff/actions/workflows/buffbench.yml')
+}
+
+main().catch((err) => {
+  error(`Failed to trigger BuffBench: ${err.message}`)
+})