feat: add HTTP/2 load testing infrastructure (#819)

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Reflex <reflex@runloop.ai>

Author: 3 people

loadtest/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

loadtest/README.md

@@ -0,0 +1,58 @@
+# Load tests
+
+Manual transport comparison scripts for the Runloop Python SDK. These hit the
+real API and are not part of `pytest`; run on demand.
+
+## End-to-end transport comparison (real API)
+
+All scripts require `RUNLOOP_API_KEY`. Set `RUNLOOP_BASE_URL` to override the
+default endpoint (`https://api.runloop.ai`).
+
+Every script sends `devboxes.create` requests against a **deliberately
+nonexistent blueprint** (`bp_nonexistent_loadtest_00000`), so every request
+fails fast server-side (HTTP `400`) and **no devboxes are created** — isolating
+client + server _request handling_ from provisioning.
+
+| Script | Transport under test |
+| --- | --- |
+| `loadtest.py` | The **SDK** itself (`AsyncRunloop`). `USE_HTTP2=0` → HTTP/1.1; default / `USE_HTTP2=1` → HTTP/2 (shared httpx pool). Always runs against the installed package in this checkout. |
+| `h2_test.py` | Raw `httpx` HTTP/2, bypassing the SDK. Configurable connection count. |
+| `h2_single_conn.py` | Raw `httpx` HTTP/2 on a single warmed connection (50-request burst). |
+| `raw_fetch_test.py` | Raw `httpx` HTTP/1.1 keep-alive baseline. |
+| `alpn_check.py` | Confirms the origin negotiates `h2` via TLS ALPN. |
+
+The raw-transport probes compare httpx HTTP/2 multiplexing against HTTP/1.1
+directly — the same comparison that motivates the SDK's shared HTTP/2 pool.
+They're kept so the comparison stays reproducible.
+
+```sh
+# Install the SDK from this checkout (editable):
+cd /path/to/api-client-python && uv sync
+
+# SDK: HTTP/2 (default) vs HTTP/1.1, 2000-request burst
+source ~/env && REQUEST_COUNT=2000 uv run python loadtest/loadtest.py            # HTTP/2
+source ~/env && REQUEST_COUNT=2000 USE_HTTP2=0 uv run python loadtest/loadtest.py # HTTP/1.1
+
+# Raw httpx HTTP/2 vs HTTP/1.1 comparison
+source ~/env && uv run python loadtest/h2_test.py
+source ~/env && uv run python loadtest/raw_fetch_test.py
+
+# Single-connection burst
+source ~/env && uv run python loadtest/h2_single_conn.py
+
+# ALPN check
+source ~/env && uv run python loadtest/alpn_check.py
+```
+
+HTTP/1.1 opens a socket per in-flight request; for large bursts raise the
+file-descriptor limit (`ulimit -n 65536`) or keep `REQUEST_COUNT` small.
+
+## Environment variables
+
+| Variable | Default | Description |
+| --- | --- | --- |
+| `RUNLOOP_API_KEY` | *(required)* | API key |
+| `RUNLOOP_BASE_URL` | `https://api.runloop.ai` | Override API endpoint |
+| `REQUEST_COUNT` | `100000` (`loadtest.py`) / `10000` (`h2_test.py`) / `500` (`raw_fetch_test.py`) | Total requests |
+| `NUM_CONNECTIONS` | `10` (`h2_test.py`) / `20` (`raw_fetch_test.py`) | Parallel connections |
+| `USE_HTTP2` | `1` | `0` to force HTTP/1.1 in `loadtest.py` |

loadtest/alpn_check.py

@@ -0,0 +1,22 @@
+"""Confirm the origin negotiates h2 via TLS ALPN."""
+
+from __future__ import annotations
+
+import os
+import ssl
+import socket
+
+BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai")
+url_parts = BASE_URL.split("://", 1)
+host = url_parts[1].split("/")[0] if len(url_parts) > 1 else url_parts[0]
+port = 443
+
+print(f"Checking ALPN for {host}:{port}")
+
+ctx = ssl.create_default_context()
+ctx.set_alpn_protocols(["h2", "http/1.1"])
+
+with socket.create_connection((host, port)) as sock:
+    with ctx.wrap_socket(sock, server_hostname=host) as tls:
+        print(f"Negotiated protocol: {tls.selected_alpn_protocol()}")
+        print(f"TLS version:         {tls.version()}")

loadtest/h2_single_conn.py

@@ -0,0 +1,59 @@
+"""Raw httpx HTTP/2 single-connection burst: 50 requests on one warmed connection."""
+
+from __future__ import annotations
+
+import os
+import time
+import asyncio
+from typing import cast
+
+import httpx
+
+BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai")
+API_KEY = os.environ["RUNLOOP_API_KEY"]
+
+BODY = {
+    "blueprint_id": "bp_nonexistent_loadtest_00000",
+    "name": "loadtest-h2s-0",
+    "environment_variables": {"TEST_VAR_1": "value_one"},
+    "launch_parameters": {"resource_size_request": "SMALL"},
+}
+
+
+async def send_request(client: httpx.AsyncClient) -> dict[str, object]:
+    start = time.perf_counter()
+    response = await client.post(
+        "/v1/devboxes",
+        json=BODY,
+        headers={"authorization": f"Bearer {API_KEY}"},
+    )
+    return {"latency_ms": (time.perf_counter() - start) * 1000, "status": response.status_code}
+
+
+async def main() -> None:
+    client = httpx.AsyncClient(
+        base_url=BASE_URL,
+        http2=True,
+        limits=httpx.Limits(max_connections=1, max_keepalive_connections=1),
+        timeout=httpx.Timeout(120.0),
+    )
+
+    # Warmup
+    w = await send_request(client)
+    print(f"Warmup: status={w['status']}, latency={w['latency_ms']:.0f}ms")
+
+    count = 50
+    print(f"\nBursting {count} requests on 1 warmed connection...")
+    wall_start = time.perf_counter()
+    results = await asyncio.gather(*(send_request(client) for _ in range(count)))
+    wall_ms = (time.perf_counter() - wall_start) * 1000
+
+    await client.aclose()
+
+    lats: list[float] = sorted(cast(float, r["latency_ms"]) for r in results)
+    print(f"{count} requests in {wall_ms:.0f}ms ({count / (wall_ms / 1000):.1f} req/s)")
+    print(f"Latency: min={lats[0]:.0f}ms  p50={lats[count // 2]:.0f}ms  max={lats[-1]:.0f}ms")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

loadtest/h2_test.py

@@ -0,0 +1,111 @@
+"""Raw httpx HTTP/2 load test: REQUEST_COUNT requests across NUM_CONNECTIONS connections."""
+
+from __future__ import annotations
+
+import os
+import math
+import time
+import asyncio
+from typing import cast
+
+import httpx
+
+REQUEST_COUNT = int(os.environ.get("REQUEST_COUNT", "10000"))
+NUM_CONNECTIONS = int(os.environ.get("NUM_CONNECTIONS", "10"))
+BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai")
+API_KEY = os.environ["RUNLOOP_API_KEY"]
+
+BODY = {
+    "blueprint_id": "bp_nonexistent_loadtest_00000",
+    "name": "loadtest-h2-0",
+    "environment_variables": {"TEST_VAR_1": "value_one", "TEST_VAR_2": "value_two"},
+    "metadata": {"test_run": "h2", "index": "0"},
+    "launch_parameters": {"resource_size_request": "SMALL", "keep_alive_time_seconds": 300},
+}
+
+
+def percentile(sorted_vals: list[float], p: float) -> float:
+    idx = math.ceil(p / 100 * len(sorted_vals)) - 1
+    return sorted_vals[max(0, idx)]
+
+
+async def send_request(client: httpx.AsyncClient) -> dict[str, object]:
+    start = time.perf_counter()
+    response = await client.post(
+        "/v1/devboxes",
+        json=BODY,
+        headers={"authorization": f"Bearer {API_KEY}"},
+    )
+    return {"latency_ms": (time.perf_counter() - start) * 1000, "status": response.status_code}
+
+
+async def main() -> None:
+    if NUM_CONNECTIONS < 1:
+        print(f'NUM_CONNECTIONS must be a positive integer (got "{os.environ.get("NUM_CONNECTIONS")}")')
+        return
+
+    print(f"HTTP/2 test: {REQUEST_COUNT} requests, {NUM_CONNECTIONS} connections to {BASE_URL}")
+
+    # One client per logical connection — each capped at 1 connection so requests
+    # are distributed across NUM_CONNECTIONS distinct HTTP/2 sessions.
+    clients = [
+        httpx.AsyncClient(
+            base_url=BASE_URL,
+            http2=True,
+            limits=httpx.Limits(max_connections=1, max_keepalive_connections=1),
+            timeout=httpx.Timeout(120.0),
+        )
+        for _ in range(NUM_CONNECTIONS)
+    ]
+
+    print(f"{NUM_CONNECTIONS} connections established\n")
+
+    completed = 0
+
+    async def progress_printer() -> None:
+        while True:
+            await asyncio.sleep(2)
+            pct = completed / REQUEST_COUNT * 100
+            print(f"  progress: {completed}/{REQUEST_COUNT} ({pct:.1f}%)")
+
+    async def wrapped(idx: int) -> dict[str, object]:
+        nonlocal completed
+        r = await send_request(clients[idx % NUM_CONNECTIONS])
+        completed += 1
+        return r
+
+    wall_start = time.perf_counter()
+    progress_task = asyncio.create_task(progress_printer())
+    results = await asyncio.gather(*(wrapped(i) for i in range(REQUEST_COUNT)))
+    progress_task.cancel()
+    wall_ms = (time.perf_counter() - wall_start) * 1000
+
+    for c in clients:
+        await c.aclose()
+
+    latencies: list[float] = sorted(cast(float, r["latency_ms"]) for r in results)
+    status_counts: dict[int, int] = {}
+    for r in results:
+        s = cast(int, r["status"])
+        status_counts[s] = status_counts.get(s, 0) + 1
+
+    print(f"\n=== HTTP/2 Results ===")
+    print(f"Requests:    {REQUEST_COUNT}")
+    print(f"Connections: {NUM_CONNECTIONS}")
+    print(f"Wall clock:  {wall_ms / 1000:.2f}s")
+    print(f"Throughput:  {REQUEST_COUNT / (wall_ms / 1000):.1f} req/s")
+    if latencies:
+        print("\nLatency (ms):")
+        print(f"  min: {latencies[0]:.1f}")
+        print(f"  p50: {percentile(latencies, 50):.1f}")
+        print(f"  p90: {percentile(latencies, 90):.1f}")
+        print(f"  p95: {percentile(latencies, 95):.1f}")
+        print(f"  p99: {percentile(latencies, 99):.1f}")
+        print(f"  max: {latencies[-1]:.1f}")
+    print("\nStatus codes:")
+    for s, c_count in sorted(status_counts.items()):
+        print(f"  {s}: {c_count}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())