feat: generic e2e reusable workflow with auto-detection

.github/workflows/e2e.yml

@@ -0,0 +1,96 @@
+name: "E2E benchmarks"
+
+on:
+  workflow_call:
+    inputs:
+      language:
+        description: "Language to benchmark (auto-detected from repo name if not provided)"
+        required: false
+        type: string
+      sdk-version:
+        description: "SDK version (resolves latest if not provided)"
+        required: false
+        type: string
+      iterations:
+        description: "Number of benchmark iterations"
+        required: false
+        type: number
+        default: 1
+      post-comment:
+        description: "Post results as a PR comment"
+        required: false
+        type: boolean
+        default: true
+  workflow_dispatch:
+    inputs:
+      language:
+        description: "Language to benchmark (e.g. go, python)"
+        required: true
+      sdk-version:
+        description: "SDK version (e.g. 0.42.0 or latest)"
+        default: "latest"
+
+jobs:
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.apps }}
+      sdk-version: ${{ steps.version.outputs.sdk-version }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install bench CLI
+        run: pip install -e .
+
+      - name: Detect language
+        id: lang
+        env:
+          INPUT_LANGUAGE: ${{ inputs.language }}
+          CALLER_REPO: ${{ github.repository }}
+        run: |
+          if [ -n "$INPUT_LANGUAGE" ]; then
+            echo "language=$INPUT_LANGUAGE" >> "$GITHUB_OUTPUT"
+          else
+            # Extract language from repo name: getsentry/sentry-go → go
+            REPO_NAME="${CALLER_REPO##*/}"
+            LANGUAGE="${REPO_NAME#sentry-}"
+            echo "language=$LANGUAGE" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Resolve SDK version
+        id: version
+        env:
+          INPUT_VERSION: ${{ inputs.sdk-version }}
+          LANGUAGE: ${{ steps.lang.outputs.language }}
+        run: |
+          if [ -n "$INPUT_VERSION" ] && [ "$INPUT_VERSION" != "latest" ]; then
+            echo "sdk-version=$INPUT_VERSION" >> "$GITHUB_OUTPUT"
+          else
+            VERSION=$(bench resolve-version "$LANGUAGE")
+            echo "sdk-version=$VERSION" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build matrix
+        id: matrix
+        env:
+          LANGUAGE: ${{ steps.lang.outputs.language }}
+        run: |
+          APPS=$(bench list-apps "$LANGUAGE")
+          echo "apps=$APPS" >> "$GITHUB_OUTPUT"
+
+  bench:
+    needs: discover
+    strategy:
+      fail-fast: false
+      matrix:
+        app: ${{ fromJSON(needs.discover.outputs.matrix) }}
+    uses: ./.github/workflows/benchmark.yml
+    with:
+      app: ${{ matrix.app }}
+      sdk-version: ${{ needs.discover.outputs.sdk-version }}
+      iterations: ${{ inputs.iterations || 1 }}
+      post-comment: ${{ inputs.post-comment == '' && true || inputs.post-comment }}

bench.py

@@ -1,16 +1,72 @@
 """CLI entrypoint for sdk-benchmarks."""
 
+import json
+from pathlib import Path
+from urllib.request import urlopen
+
 import click
+import yaml
 
 __version__ = "0.1.0"
 
+CONFIGS_DIR = Path(__file__).parent / "configs"
+
+VERSION_RESOLVERS = {
+    "go": {
+        "url": "https://proxy.golang.org/github.com/getsentry/sentry-go/@latest",
+        "parse": lambda data: json.loads(data)["Version"].lstrip("v"),
+    },
+    "python": {
+        "url": "https://pypi.org/pypi/sentry-sdk/json",
+        "parse": lambda data: json.loads(data)["info"]["version"],
+    },
+}
+
+
+def _list_apps(language: str) -> list[str]:
+    """Scan configs/*.yaml and return app names matching the given language."""
+    apps = []
+    for config_path in sorted(CONFIGS_DIR.glob("*.yaml")):
+        with open(config_path) as f:
+            config = yaml.safe_load(f)
+        if config.get("language") == language:
+            apps.append(f"{config['language']}/{config['framework']}")
+    return apps
+
+
+def _resolve_version(language: str) -> str:
+    """Resolve the latest Sentry SDK version for a language."""
+    resolver = VERSION_RESOLVERS.get(language)
+    if not resolver:
+        raise click.ClickException(f"No version resolver configured for language: {language}")
+
+    with urlopen(resolver["url"]) as resp:
+        data = resp.read().decode()
+    return resolver["parse"](data)
+
 
 @click.group()
 @click.version_option(version=__version__)
 def cli():
     """Benchmark suite for Sentry SDKs."""
 
 
+@cli.command("list-apps")
+@click.argument("language")
+def list_apps(language):
+    """List benchmark apps for a LANGUAGE (e.g. 'go', 'python')."""
+    apps = _list_apps(language)
+    click.echo(json.dumps(apps))
+
+
+@cli.command("resolve-version")
+@click.argument("language")
+def resolve_version(language):
+    """Resolve the latest Sentry SDK version for a LANGUAGE."""
+    version = _resolve_version(language)
+    click.echo(version)
+
+
 @cli.command()
 @click.argument("app")
 @click.option("--sdk-version", required=True, help="SDK version to benchmark.")

lib/runner.py

@@ -4,6 +4,7 @@
 
 import json
 import logging
+import math
 import os
 import subprocess
 import threading
@@ -222,6 +223,13 @@ def run_benchmark(
             result = run_variant(config, variant, i, results_base)
             all_results["iterations"].append(result)
 
+    # Compute summary with overhead metrics
+    all_results["summary"] = _compute_summary(all_results["iterations"])
+    all_results["load"] = {
+        "rps": config.get("load", {}).get("rps"),
+        "duration": config.get("load", {}).get("duration"),
+    }
+
     # Write aggregated results
     results_path = os.path.join(results_base, "results.json")
     with open(results_path, "w") as f:
@@ -231,9 +239,67 @@ def run_benchmark(
     return all_results
 
 
+def _percentile(sorted_values: list[float], p: float) -> float:
+    """Compute the p-th percentile from a sorted list of values."""
+    if not sorted_values:
+        return 0.0
+    k = (len(sorted_values) - 1) * (p / 100.0)
+    f = math.floor(k)
+    c = math.ceil(k)
+    if f == c:
+        return sorted_values[int(k)]
+    return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
+
+
+def _extract_latencies(iterations: list[dict], variant: str) -> list[float]:
+    """Extract all latency values (in nanoseconds) for a given variant."""
+    latencies = []
+    for it in iterations:
+        if it.get("variant") != variant:
+            continue
+        for req in it.get("vegeta_results") or []:
+            latencies.append(req["latency"])
+    return latencies
+
+
+def _compute_summary(iterations: list[dict]) -> dict:
+    """Compute overhead summary by comparing baseline vs instrumented latencies."""
+    baseline_latencies = sorted(_extract_latencies(iterations, "baseline"))
+    instrumented_latencies = sorted(_extract_latencies(iterations, "instrumented"))
+
+    if not baseline_latencies or not instrumented_latencies:
+        logger.warning("Missing baseline or instrumented data for summary computation")
+        return {"overhead": {}, "regression": False}
+
+    overhead = {}
+    regression = False
+    for name, p in [("p50", 50), ("p90", 90), ("p95", 95), ("p99", 99)]:
+        base_val = _percentile(baseline_latencies, p)
+        inst_val = _percentile(instrumented_latencies, p)
+        if base_val > 0:
+            pct = (inst_val - base_val) / base_val * 100.0
+            overhead[name] = round(pct, 2)
+
+    base_mean = sum(baseline_latencies) / len(baseline_latencies)
+    inst_mean = sum(instrumented_latencies) / len(instrumented_latencies)
+    if base_mean > 0:
+        overhead["mean"] = round((inst_mean - base_mean) / base_mean * 100.0, 2)
+
+    # Flag regression if p99 overhead exceeds 10%
+    if overhead.get("p99", 0) > 10:
+        regression = True
+
+    return {"overhead": overhead, "regression": regression}
+
+
 def _prepare_sdk_version(config: dict, sdk_version: str) -> None:
     """Template the SDK version into requirements files."""
     app_dir = PROJECT_ROOT / config["app_dir"]
+    language = config.get("language", "")
+
+    if language == "go":
+        _prepare_go_sdk_version(app_dir, sdk_version)
+        return
 
     # Python: render requirements-sentry.txt from template
     tmpl_path = app_dir / "requirements-sentry.txt.tmpl"
@@ -245,3 +311,47 @@ def _prepare_sdk_version(config: dict, sdk_version: str) -> None:
         with open(output_path, "w") as f:
             f.write(rendered)
         logger.info("Rendered %s with sdk_version=%s", output_path, sdk_version)
+
+
+def _prepare_go_sdk_version(app_dir: Path, sdk_version: str) -> None:
+    """Update sentry-go module versions in a Go app's go.mod."""
+    go_mod = app_dir / "go.mod"
+    if not go_mod.exists():
+        return
+
+    # Find all sentry-go modules referenced in go.mod
+    with open(go_mod) as f:
+        content = f.read()
+
+    sentry_modules = []
+    for line in content.splitlines():
+        line = line.strip()
+        # Match lines like: github.com/getsentry/sentry-go v0.42.0
+        # or: github.com/getsentry/sentry-go/gin v0.42.0
+        if "github.com/getsentry/sentry-go" in line and not line.startswith("module"):
+            parts = line.split()
+            if parts:
+                mod = parts[0]
+                if mod.startswith("github.com/getsentry/sentry-go"):
+                    sentry_modules.append(mod)
+
+    if not sentry_modules:
+        logger.warning("No sentry-go modules found in %s", go_mod)
+        return
+
+    # Use go get to update each sentry-go module to the target version
+    version_spec = f"@v{sdk_version}" if sdk_version != "latest" and not sdk_version.startswith("v") else f"@{sdk_version}"
+    get_args = [f"{mod}{version_spec}" for mod in sentry_modules]
+
+    logger.info("Updating sentry-go modules in %s: %s", app_dir, get_args)
+    subprocess.run(
+        ["go", "get"] + get_args,
+        cwd=str(app_dir),
+        check=True,
+    )
+    subprocess.run(
+        ["go", "mod", "tidy"],
+        cwd=str(app_dir),
+        check=True,
+    )
+    logger.info("Updated go.mod in %s to sentry-go %s", app_dir, sdk_version)

tests/test_cli.py

@@ -1,8 +1,11 @@
 """Tests for the CLI entrypoint."""
 
+import json
+from unittest.mock import patch
+
 from click.testing import CliRunner
 
-from bench import cli
+from bench import _list_apps, _resolve_version, cli
 
 
 def test_cli_group_exists():
@@ -26,3 +29,73 @@ def test_cli_version():
     result = runner.invoke(cli, ["--version"])
     assert result.exit_code == 0
     assert "0.1.0" in result.output
+
+
+def test_cli_has_list_apps_command():
+    assert "list-apps" in cli.commands
+
+
+def test_cli_has_resolve_version_command():
+    assert "resolve-version" in cli.commands
+
+
+def test_list_apps_go():
+    apps = _list_apps("go")
+    assert apps == ["go/echo", "go/gin", "go/net-http"]
+
+
+def test_list_apps_python():
+    apps = _list_apps("python")
+    assert apps == ["python/django"]
+
+
+def test_list_apps_unknown_language():
+    apps = _list_apps("rust")
+    assert apps == []
+
+
+def test_list_apps_cli():
+    runner = CliRunner()
+    result = runner.invoke(cli, ["list-apps", "go"])
+    assert result.exit_code == 0
+    apps = json.loads(result.output)
+    assert apps == ["go/echo", "go/gin", "go/net-http"]
+
+
+def test_resolve_version_go():
+    mock_response = json.dumps({"Version": "v0.42.0", "Time": "2024-01-01T00:00:00Z"})
+    with patch("bench.urlopen") as mock_urlopen:
+        mock_urlopen.return_value.__enter__ = lambda s: s
+        mock_urlopen.return_value.__exit__ = lambda s, *a: None
+        mock_urlopen.return_value.read.return_value = mock_response.encode()
+        version = _resolve_version("go")
+    assert version == "0.42.0"
+
+
+def test_resolve_version_python():
+    mock_response = json.dumps({"info": {"version": "2.1.0"}})
+    with patch("bench.urlopen") as mock_urlopen:
+        mock_urlopen.return_value.__enter__ = lambda s: s
+        mock_urlopen.return_value.__exit__ = lambda s, *a: None
+        mock_urlopen.return_value.read.return_value = mock_response.encode()
+        version = _resolve_version("python")
+    assert version == "2.1.0"
+
+
+def test_resolve_version_unknown_language():
+    runner = CliRunner()
+    result = runner.invoke(cli, ["resolve-version", "rust"])
+    assert result.exit_code != 0
+    assert "No version resolver configured" in result.output
+
+
+def test_resolve_version_cli():
+    mock_response = json.dumps({"Version": "v0.42.0", "Time": "2024-01-01T00:00:00Z"})
+    with patch("bench.urlopen") as mock_urlopen:
+        mock_urlopen.return_value.__enter__ = lambda s: s
+        mock_urlopen.return_value.__exit__ = lambda s, *a: None
+        mock_urlopen.return_value.read.return_value = mock_response.encode()
+        runner = CliRunner()
+        result = runner.invoke(cli, ["resolve-version", "go"])
+    assert result.exit_code == 0
+    assert "0.42.0" in result.output