feat: add GitHub Actions CI and reusable benchmark workflow

.github/actions/benchmark/action.yml

@@ -0,0 +1,103 @@
+name: "SDK Benchmark"
+description: "Run Sentry SDK benchmarks comparing baseline vs instrumented overhead"
+
+inputs:
+  app:
+    description: "App to benchmark (e.g. python/django, go/net-http)"
+    required: true
+  sdk-version:
+    description: "SDK version or install specifier"
+    required: true
+  iterations:
+    description: "Number of benchmark iterations"
+    required: false
+    default: "5"
+  post-comment:
+    description: "Post results as a PR comment (true/false)"
+    required: false
+    default: "true"
+
+outputs:
+  overhead-p50:
+    description: "P50 latency overhead percentage"
+    value: ${{ steps.bench.outputs.overhead-p50 }}
+  overhead-p99:
+    description: "P99 latency overhead percentage"
+    value: ${{ steps.bench.outputs.overhead-p99 }}
+  regression:
+    description: "Whether a regression was detected (true/false)"
+    value: ${{ steps.bench.outputs.regression }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install bench CLI
+      shell: bash
+      run: pip install -e ${{ github.action_path }}/../../..
+
+    - name: Run benchmark
+      id: bench
+      shell: bash
+      env:
+        BENCH_APP: ${{ inputs.app }}
+        BENCH_SDK_VERSION: ${{ inputs.sdk-version }}
+        BENCH_ITERATIONS: ${{ inputs.iterations }}
+      run: |
+        bench run "$BENCH_APP" \
+          --sdk-version="$BENCH_SDK_VERSION" \
+          --iterations="$BENCH_ITERATIONS" \
+          --output-dir=results/
+
+        # Runner writes to results/{lang}-{framework}/results.json
+        RESULTS_FILE=$(find results/ -name 'results.json' -type f | head -1)
+        # Sanitize app name for artifact upload (slashes not allowed)
+        echo "artifact-name=benchmark-results-$(echo "$BENCH_APP" | tr '/' '-')" >> "$GITHUB_OUTPUT"
+        if [ -n "$RESULTS_FILE" ]; then
+          echo "results-file=${RESULTS_FILE}" >> "$GITHUB_OUTPUT"
+          echo "overhead-p50=$(python -c "
+        import json
+        with open('${RESULTS_FILE}') as f:
+            data = json.load(f)
+        overhead = data.get('summary', {}).get('overhead', {})
+        print(overhead.get('p50', 'N/A'))
+        ")" >> "$GITHUB_OUTPUT"
+          echo "overhead-p99=$(python -c "
+        import json
+        with open('${RESULTS_FILE}') as f:
+            data = json.load(f)
+        overhead = data.get('summary', {}).get('overhead', {})
+        print(overhead.get('p99', 'N/A'))
+        ")" >> "$GITHUB_OUTPUT"
+          echo "regression=$(python -c "
+        import json
+        with open('${RESULTS_FILE}') as f:
+            data = json.load(f)
+        print(str(data.get('summary', {}).get('regression', False)).lower())
+        ")" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Upload results
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: ${{ steps.bench.outputs.artifact-name }}
+        path: results/
+
+    - name: Post PR comment
+      if: inputs.post-comment == 'true' && github.event_name == 'pull_request'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        RESULTS_FILE="${{ steps.bench.outputs.results-file }}"
+        if [ -n "$RESULTS_FILE" ]; then
+          bench post-comment \
+            --repo="${{ github.repository }}" \
+            --pr="${{ github.event.pull_request.number }}" \
+            --results-file="$RESULTS_FILE"
+        fi

.github/workflows/benchmark.yml

@@ -0,0 +1,125 @@
+name: Benchmark
+
+on:
+  workflow_call:
+    inputs:
+      app:
+        description: "App to benchmark (e.g. python/django, go/net-http)"
+        required: true
+        type: string
+      sdk-version:
+        description: "SDK version or install specifier to benchmark"
+        required: true
+        type: string
+      iterations:
+        description: "Number of benchmark iterations"
+        required: false
+        type: number
+        default: 5
+      post-comment:
+        description: "Post results as a PR comment"
+        required: false
+        type: boolean
+        default: true
+    outputs:
+      overhead-p50:
+        description: "P50 latency overhead percentage"
+        value: ${{ jobs.bench.outputs.overhead-p50 }}
+      overhead-p99:
+        description: "P99 latency overhead percentage"
+        value: ${{ jobs.bench.outputs.overhead-p99 }}
+      regression:
+        description: "Whether a regression was detected"
+        value: ${{ jobs.bench.outputs.regression }}
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    outputs:
+      overhead-p50: ${{ steps.results.outputs.overhead-p50 }}
+      overhead-p99: ${{ steps.results.outputs.overhead-p99 }}
+      regression: ${{ steps.results.outputs.regression }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: getsentry/sdk-benchmarks
+
+      - uses: actions/setup-go@v5
+        if: startsWith(inputs.app, 'go/')
+        with:
+          go-version: "1.25"
+          cache: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install bench CLI
+        run: pip install -e .
+
+      - name: Run benchmark
+        env:
+          BENCH_APP: ${{ inputs.app }}
+          BENCH_SDK_VERSION: ${{ inputs.sdk-version }}
+          BENCH_ITERATIONS: ${{ inputs.iterations }}
+        run: |
+          bench run "$BENCH_APP" \
+            --sdk-version="$BENCH_SDK_VERSION" \
+            --iterations="$BENCH_ITERATIONS" \
+            --output-dir=results/
+
+      - name: Find results file
+        id: find-results
+        env:
+          BENCH_APP: ${{ inputs.app }}
+        run: |
+          # Runner writes to results/{lang}-{framework}/results.json (depth 2);
+          # deeper results.json files are raw vegeta NDJSON.
+          RESULTS_FILE=$(find results/ -maxdepth 2 -name 'results.json' -type f | head -1)
+          echo "path=${RESULTS_FILE}" >> "$GITHUB_OUTPUT"
+          # Sanitize app name for artifact upload (slashes not allowed)
+          ARTIFACT_NAME="benchmark-results-$(echo "$BENCH_APP" | tr '/' '-')"
+          echo "artifact-name=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT"
+
+      - name: Parse results
+        id: results
+        if: steps.find-results.outputs.path != ''
+        run: |
+          RESULTS_FILE="${{ steps.find-results.outputs.path }}"
+          echo "overhead-p50=$(python -c "
+          import json
+          with open('$RESULTS_FILE') as f:
+              data = json.load(f)
+          overhead = data.get('summary', {}).get('overhead', {})
+          print(overhead.get('p50', 'N/A'))
+          ")" >> "$GITHUB_OUTPUT"
+          echo "overhead-p99=$(python -c "
+          import json
+          with open('$RESULTS_FILE') as f:
+              data = json.load(f)
+          overhead = data.get('summary', {}).get('overhead', {})
+          print(overhead.get('p99', 'N/A'))
+          ")" >> "$GITHUB_OUTPUT"
+          echo "regression=$(python -c "
+          import json
+          with open('$RESULTS_FILE') as f:
+              data = json.load(f)
+          print(str(data.get('summary', {}).get('regression', False)).lower())
+          ")" >> "$GITHUB_OUTPUT"
+
+      - name: Upload results artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ steps.find-results.outputs.artifact-name }}
+          path: results/
+
+      - name: Post PR comment
+        if: inputs.post-comment && github.event_name == 'pull_request' && steps.find-results.outputs.path != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          bench post-comment \
+            --repo="${{ github.repository }}" \
+            --pr="${{ github.event.pull_request.number }}" \
+            --results-file="${{ steps.find-results.outputs.path }}"

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install ruff
+      - run: ruff check .
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[dev]"
+      - run: pytest tests/
+
+  docker-build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Python apps
+          - name: django-baseline
+            dockerfile: apps/python/django/Dockerfile.baseline
+            context: apps/python/django
+          - name: django-instrumented
+            dockerfile: apps/python/django/Dockerfile.instrumented
+            context: apps/python/django
+          # Tools
+          - name: fakerelay
+            dockerfile: tools/fakerelay/Dockerfile
+            context: tools/fakerelay
+          - name: loadgen
+            dockerfile: tools/loadgen/Dockerfile
+            context: tools/loadgen
+          - name: postgres
+            dockerfile: apps/python/common/postgres/Dockerfile
+            context: apps/python/common/postgres
+          # Go apps
+          - name: go-net-http-baseline
+            dockerfile: apps/go/net-http/Dockerfile.baseline
+            context: apps/go/net-http
+          - name: go-net-http-instrumented
+            dockerfile: apps/go/net-http/Dockerfile.instrumented
+            context: apps/go/net-http
+          - name: go-gin-baseline
+            dockerfile: apps/go/gin/Dockerfile.baseline
+            context: apps/go/gin
+          - name: go-gin-instrumented
+            dockerfile: apps/go/gin/Dockerfile.instrumented
+            context: apps/go/gin
+          - name: go-echo-baseline
+            dockerfile: apps/go/echo/Dockerfile.baseline
+            context: apps/go/echo
+          - name: go-echo-instrumented
+            dockerfile: apps/go/echo/Dockerfile.instrumented
+            context: apps/go/echo
+    steps:
+      - uses: actions/checkout@v4
+      - name: Render SDK requirements templates
+        run: |
+          # Instrumented Dockerfiles need requirements-sentry.txt rendered from .tmpl
+          for tmpl in $(find apps -name 'requirements-sentry.txt.tmpl' 2>/dev/null); do
+            dir=$(dirname "$tmpl")
+            sed 's/=={{ sdk_version }}//' "$tmpl" > "$dir/requirements-sentry.txt"
+          done
+      - run: docker build -f ${{ matrix.dockerfile }} ${{ matrix.context }}
+        name: Build ${{ matrix.name }}

.github/workflows/test-benchmark.yml

@@ -0,0 +1,43 @@
+name: Test Benchmark (smoke test)
+
+on:
+  workflow_dispatch:
+    inputs:
+      app:
+        description: "App to benchmark"
+        default: "python/django"
+      sdk-version:
+        description: "SDK version"
+        default: "2.0.0"
+
+jobs:
+  smoke-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install bench CLI
+        run: pip install -e .
+
+      - name: Verify CLI works
+        run: |
+          bench --version
+          bench --help
+          bench run --help
+          bench post-comment --help
+
+      - name: Verify bench run invokes runner
+        run: |
+          # Run with a non-existent config to verify the runner is wired up
+          # (should fail with a config-not-found error, NOT "Not implemented yet")
+          output=$(bench run python/django --sdk-version=2.0.0 --iterations=1 2>&1 || true)
+          if echo "$output" | grep -q "Not implemented yet"; then
+            echo "FAIL: bench run is still a stub"
+            exit 1
+          fi
+          echo "OK: bench run is wired to the runner"
+          echo "Output: $output"

.gitignore

@@ -16,3 +16,6 @@ results/
 
 # Ruff
 .ruff_cache/
+
+# Jetbrains
+.idea

bench.py

@@ -18,7 +18,13 @@ def cli():
 @click.option("--output-dir", default="results/", help="Directory to write results to.")
 def run(app, sdk_version, iterations, output_dir):
     """Run benchmarks for an APP (e.g. 'python/django')."""
-    click.echo("Not implemented yet")
+    import logging
+
+    from lib.runner import run_benchmark
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+    results = run_benchmark(app, sdk_version, iterations=iterations, output_dir=output_dir)
+    click.echo(f"Benchmark complete. {len(results.get('iterations', []))} iterations recorded.")
 
 
 @cli.command()
@@ -27,3 +33,15 @@ def run(app, sdk_version, iterations, output_dir):
 def compare(baseline, candidate):
     """Compare two result JSON files (BASELINE vs CANDIDATE)."""
     click.echo("Not implemented yet")
+
+
+@cli.command("post-comment")
+@click.option("--repo", required=True, help="GitHub repo (owner/name).")
+@click.option("--pr", required=True, type=int, help="PR number.")
+@click.option("--results-file", required=True, type=click.Path(exists=True), help="Results JSON.")
+def post_comment(repo, pr, results_file):
+    """Post benchmark results as a PR comment."""
+    from lib.github import post_results
+
+    post_results(repo, pr, results_file)
+    click.echo(f"Posted benchmark results to {repo}#{pr}")