feat: gated tracemalloc heap-dump for one-pod prod profiling

chart/templates/s3proxy/configmap.yaml

@@ -22,3 +22,7 @@ data:
   S3PROXY_DASHBOARD_UI: "true"
   S3PROXY_DASHBOARD_PATH: {{ .Values.dashboard.path | quote }}
   {{- end }}
+  {{- /* Arbitrary extra env (e.g. diagnostics like S3PROXY_TRACEMALLOC=1). */ -}}
+  {{- range $k, $v := .Values.extraConfig }}
+  {{ $k }}: {{ $v | quote }}
+  {{- end }}

chart/values.yaml

@@ -255,4 +255,7 @@ topologySpreadConstraints: []
 podDisruptionBudget:
   enabled: true
   minAvailable: 1
-  # maxUnavailable: 1  # Alternative to minAvailable
+  # maxUnavailable: 1  # Alternative to minAvailable
+# Arbitrary extra S3PROXY_* env, injected via the config ConfigMap (envFrom).
+# Used for time-boxed diagnostics, e.g. extraConfig: { S3PROXY_TRACEMALLOC: "1" }.
+extraConfig: {}

s3proxy/app.py

@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+import asyncio
+import contextlib
 import logging
 import os
+import signal
 import sys
 import time
+import tracemalloc
 import uuid
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
@@ -65,6 +69,52 @@ def _silence_health_probe_access_logs() -> None:
         access_logger.addFilter(_health_probe_filter)
 
 
+def _dump_tracemalloc(limit: int = 20) -> None:
+    """Log the top live Python allocations by size, with location.
+
+    Diagnostic only (gated by S3PROXY_TRACEMALLOC). Lets us see, under real
+    backup load, exactly which call sites hold the resident memory that drives
+    the OOM -- instead of guessing.
+    """
+    if not tracemalloc.is_tracing():
+        return
+    snap = tracemalloc.take_snapshot()
+    stats = snap.statistics("lineno")
+    total_mb = sum(s.size for s in stats) / 1024 / 1024
+    logger.warning("TRACEMALLOC_SNAPSHOT", total_tracked_mb=round(total_mb, 1), shown=limit)
+    for i, st in enumerate(stats[:limit], 1):
+        fr = st.traceback[0]
+        logger.warning(
+            "TRACEMALLOC_TOP",
+            rank=i,
+            size_mb=round(st.size / 1024 / 1024, 2),
+            count=st.count,
+            loc=f"{fr.filename}:{fr.lineno}",
+        )
+
+
+async def _periodic_tracemalloc(interval: int) -> None:
+    while True:
+        await asyncio.sleep(interval)
+        _dump_tracemalloc()
+
+
+def _maybe_start_tracemalloc() -> asyncio.Task | None:
+    """Enable tracemalloc + periodic/SIGUSR1 heap dumps when S3PROXY_TRACEMALLOC is set.
+
+    No-op (zero overhead) when unset. Used for one-pod, time-boxed profiling.
+    """
+    if not os.environ.get("S3PROXY_TRACEMALLOC"):
+        return None
+    frames = int(os.environ.get("S3PROXY_TRACEMALLOC_FRAMES", "4"))
+    interval = int(os.environ.get("S3PROXY_TRACEMALLOC_INTERVAL", "15"))
+    tracemalloc.start(frames)
+    logger.warning("TRACEMALLOC_ENABLED", frames=frames, interval_sec=interval)
+    with contextlib.suppress(NotImplementedError, RuntimeError):
+        asyncio.get_running_loop().add_signal_handler(signal.SIGUSR1, _dump_tracemalloc)
+    return asyncio.create_task(_periodic_tracemalloc(interval))
+
+
 def create_lifespan(settings: Settings, credentials_store: dict[str, str]) -> AsyncIterator[None]:
     """Create lifespan context manager for FastAPI app.
 
@@ -114,8 +164,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         app.state.stats_store = stats_store
         app.state.start_time = time.monotonic()
 
+        tracemalloc_task = _maybe_start_tracemalloc()
+
         yield
 
+        if tracemalloc_task is not None:
+            tracemalloc_task.cancel()
         await stats_store.aclose()  # flush buffered samples before Redis closes
         await close_redis()
         await close_http_client()

tests/unit/test_tracemalloc_profiling.py

@@ -0,0 +1,41 @@
+"""Self-check for the gated tracemalloc heap-dump diagnostic.
+
+Off by default (no env) => zero overhead, no tracing started. When enabled it
+must take a snapshot and not raise. Used for one-pod, time-boxed prod profiling
+to identify the live allocations driving the OOM.
+"""
+
+import tracemalloc
+
+from s3proxy import app
+
+
+def test_disabled_by_default(monkeypatch):
+    monkeypatch.delenv("S3PROXY_TRACEMALLOC", raising=False)
+    assert app._maybe_start_tracemalloc() is None
+
+
+def test_dump_is_noop_when_not_tracing():
+    # Should not raise even if tracemalloc isn't running.
+    if tracemalloc.is_tracing():
+        tracemalloc.stop()
+    app._dump_tracemalloc()  # no exception = pass
+
+
+def test_dump_reports_allocations_when_tracing():
+    tracemalloc.start(2)
+    try:
+        blob = bytearray(4 * 1024 * 1024)  # 4MB, should show up
+        # Capture warning logs to confirm it emits a snapshot + top lines.
+        events = []
+        import structlog
+
+        app.logger = structlog.wrap_logger(
+            app.logger, processors=[lambda _l, _m, ev: events.append(ev) or ev]
+        )
+        app._dump_tracemalloc(limit=5)
+        assert blob is not None
+        assert any(e.get("event") == "TRACEMALLOC_SNAPSHOT" for e in events)
+        assert any(e.get("event") == "TRACEMALLOC_TOP" for e in events)
+    finally:
+        tracemalloc.stop()