Add reference-chain repro, sweep, and reporting tooling

Standalone repro/run scripts, JFR metrics extraction and multi-run HTML
reports, budget-sweep drivers, the chaos-harness ReferenceChainLeakAntagonist
and ReferenceChainLeakDemo scenarios, and the build-summarize helper
updates they rely on. Also ignores AI scratch state (.investigations/).

Author: jbachorik

.claude/commands/build-and-summarize

@@ -67,13 +67,164 @@ else
   echo "✖ Gradle failed with status $status. Full log at: $LOG"
 fi
 
-# Hand over to your logs analyst agent — keep the main session output tiny.
+# Generate the summary artifacts deterministically — no LLM, no claude spawn.
+# (The old tail spawned a headless `claude` per build; managed settings pin that
+# to sonnet-5 via the AI gateway, billing real money for every log parse.)
 echo
-echo "Delegating to gradle-logs-analyst agent…"
-# If your CLI supports non-streaming, set it here to avoid verbose output.
-# Example (uncomment if supported): export CLAUDE_NO_STREAM=1
-
-# Sub-agents would inherit the full parent env, including CLAUDECODE, which
-# apparently causes problems with some versions of Claude (e.g. 4.6).
-unset CLAUDECODE
-claude "Act as the gradle-logs-analyst agent to parse the build log at: $LOG. Generate the required Gradle summary artifacts as specified in the gradle-logs-analyst agent definition."
+echo "Generating gradle summary…"
+python3 - "$LOG" <<'PYEOF' || echo "⚠ gradle summary generation failed — full log at: $LOG"
+import json, os, re, sys
+
+log_path = sys.argv[1] if len(sys.argv) > 1 else ""
+OUT_DIR = os.path.join("build", "reports", "claude")
+MD = os.path.join(OUT_DIR, "gradle-summary.md")
+JS = os.path.join(OUT_DIR, "gradle-summary.json")
+
+status, total_time, cur_task = "UNKNOWN", None, None
+failed_tasks, headlines, dep_issues, test_failures = [], [], [], []
+warnings, seen_warnings = [], set()
+tests = {"total": 0, "failed": 0, "skipped": 0, "modules": {}}
+
+RE_STATUS   = re.compile(r"^BUILD (SUCCESSFUL|FAILED)(?: in (.+?))?\s*$")
+RE_TASK     = re.compile(r"^> Task (\S+)")
+RE_GSUM     = re.compile(r"(\d+) tests? (successful|failed|skipped)")
+RE_COMP     = re.compile(r"(\d+) tests? completed(?:, (\d+) failed)?(?:, (\d+) skipped)?")
+RE_GTEST_P  = re.compile(r"\[  PASSED  \] (\d+) tests?")
+RE_GTEST_F  = re.compile(r"\[  FAILED  \] (\d+) tests?")
+RE_DEP      = re.compile(r"Could not (?:resolve|find|get)|timed out|status code: 40[13]|artifact .* not found", re.I)
+RE_WARN     = re.compile(r"\bw: |warning:|deprecat", re.I)
+
+def mod(name):
+    return tests["modules"].setdefault(name, {"total": 0, "failed": 0, "skipped": 0})
+
+def add_counts(name, ok, failed, skipped):
+    m = mod(name)
+    m["total"] += ok + failed + skipped
+    m["failed"] += failed
+    m["skipped"] += skipped
+    tests["total"] += ok + failed + skipped
+    tests["failed"] += failed
+    tests["skipped"] += skipped
+
+if not log_path or not os.path.isfile(log_path) or os.path.getsize(log_path) == 0:
+    why = f"log not found: {log_path!r}" if not log_path or not os.path.isfile(log_path) else "log is empty"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    open(MD, "w").write(f"# Gradle Summary\n\n- Status: UNKNOWN ({why})\n\nFull log unavailable or empty.\n")
+    json.dump({"status": "UNKNOWN", "totalTime": None, "failedTasks": [], "warnings": [],
+               "tests": tests, "slowTasks": [], "depIssues": [why], "actions": []}, open(JS, "w"), indent=1)
+    print(f"Gradle log unusable ({why}); wrote {MD} and {JS}")
+    sys.exit(0)
+
+grab_headline = False
+with open(log_path, encoding="utf-8", errors="replace") as f:
+    for line in f:
+        line = line.rstrip("\n")
+        m = RE_STATUS.match(line)
+        if m:
+            status = m.group(1)
+            total_time = m.group(2)
+            continue
+        m = RE_TASK.match(line)
+        if m:
+            cur_task = m.group(1)
+            if line.endswith("FAILED") and m.group(1) not in failed_tasks:
+                failed_tasks.append(m.group(1))
+            continue
+        if "FAILURE: Build failed with an exception" in line:
+            grab_headline = True
+            continue
+        if grab_headline and line.strip().startswith("* What went wrong"):
+            headlines.append("")
+            continue
+        if grab_headline:
+            s = line.strip()
+            if s.startswith("*") or not s:
+                if headlines and headlines[-1]:
+                    grab_headline = False
+                continue
+            if len(headlines[-1]) < 200:
+                headlines[-1] += (" " if headlines[-1] else "") + s
+            continue
+        m = RE_COMP.search(line)
+        if m:
+            total = int(m.group(1))
+            failed = int(m.group(2) or 0)
+            skipped = int(m.group(3) or 0)
+            add_counts(cur_task or "(unknown)", total - failed - skipped, failed, skipped)
+            continue
+        m = RE_GSUM.search(line)
+        if m:
+            n, verb = int(m.group(1)), m.group(2)
+            mm = mod(cur_task or "(unknown)")
+            mm["total"] += n
+            tests["total"] += n
+            if verb == "failed":
+                mm["failed"] += n
+                tests["failed"] += n
+            elif verb == "skipped":
+                mm["skipped"] += n
+                tests["skipped"] += n
+            continue
+        m = RE_GTEST_P.search(line)
+        if m:
+            add_counts(cur_task or "(gtest)", int(m.group(1)), 0, 0)
+            continue
+        m = RE_GTEST_F.search(line)
+        if m:
+            add_counts(cur_task or "(gtest)", 0, int(m.group(1)), 0)
+            continue
+        if re.search(r" > \S+(?:\.\S+)+.* FAILED$", line) or re.search(r"\[  FAILED  \] (\S+)", line):
+            test_failures.append(line.strip())
+            continue
+        if RE_DEP.search(line) and len(dep_issues) < 20:
+            if line not in dep_issues:
+                dep_issues.append(line.strip())
+            continue
+        wm = RE_WARN.search(line)
+        if wm and len(warnings) < 40:
+            key = line.strip()[:160]
+            if key not in seen_warnings:
+                seen_warnings.add(key)
+                warnings.append(key)
+
+failed = [t + (" — " + h if h else "") for t, h in zip(failed_tasks, headlines + [""] * len(failed_tasks))]
+data = {"status": status, "totalTime": total_time, "failedTasks": failed, "warnings": warnings,
+        "tests": tests, "slowTasks": [], "depIssues": dep_issues, "actions": []}
+
+os.makedirs(OUT_DIR, exist_ok=True)
+with open(JS, "w") as f:
+    json.dump(data, f, indent=1)
+
+L = ["# Gradle Summary\n", f"- Status: {status}", f"- Total time: {total_time or 'UNKNOWN'}",
+     f"- Log: {log_path}\n"]
+if failed:
+    L.append("## Failing Tasks")
+    L += [f"- {t}" for t in failed]
+    L.append("")
+if headlines:
+    L.append("## Primary Failure")
+    L += [f"```text\n{h}\n```" for h in headlines[:3] if h]
+    L.append("")
+if tests["total"]:
+    L.append("## Tests")
+    L.append(f"- Total: {tests['total']}, failed: {tests['failed']}, skipped: {tests['skipped']}")
+    for name, m in tests["modules"].items():
+        L.append(f"  - {name}: {m['total']} total, {m['failed']} failed, {m['skipped']} skipped")
+    if test_failures:
+        L.append("- Top failing tests:")
+        L += [f"  - {t}" for t in test_failures[:10]]
+    L.append("")
+if warnings:
+    L.append("## Warnings")
+    L += [f"- {w}" for w in warnings[:20]]
+    L.append("")
+if dep_issues:
+    L.append("## Dependency / Network Issues")
+    L += [f"- {d}" for d in dep_issues[:10]]
+    L.append("")
+L.append("- Per-task durations are not present in plain `-i` console logs; slowTasks omitted.")
+open(MD, "w").write("\n".join(L) + "\n")
+
+print(f"{status} (time: {total_time or 'n/a'}); {len(failed)} failing task(s), {tests['failed']} test failure(s)")
+print(f"Wrote {MD} and {JS}")
+PYEOF

.gitignore

@@ -44,4 +44,17 @@ doc/temp/
 # CLAUDE.md is auto-generated from AGENTS.md bootstrap instructions
 CLAUDE.md
 
-.sphinx
+# OS/editor cruft
+.DS_Store
+
+# AI review/agent tooling scratch state
+.sphinx/
+.skill-builder-temp/
+.claude/scheduled_tasks.lock
+
+# Python bytecode cache
+__pycache__/
+*.pyc
+
+# AI investigation scratch state (never committed)
+.investigations/

ddprof-stresstest/build.gradle.kts

@@ -101,6 +101,45 @@ tasks.register<Jar>("chaosJar") {
   }
 }
 
+// --- reference-chains repro app ---------------------------------------------
+// Plain, dependency-free demo app for manually reproducing the reference-chains
+// feature: launched directly with `-agentpath:libjavaProfiler.so=start,...`
+// (no dd-trace-java agent, no dynamic attach), so it deliberately has zero
+// compile/runtime dependency on ddprof-lib or dd-trace - the profiler is
+// entirely out-of-process from this app's own point of view.
+
+sourceSets {
+  create("repro")
+}
+
+dependencies {
+  // ddprof-lib's Java API only - not the native library itself (that's the one
+  // already loaded in-process via -agentpath). Used solely to call
+  // JavaProfiler.getInstance().dump(...) periodically: Profiler::dump()
+  // (profiler.cpp) is the only code path that drains ReferenceChainTracker's
+  // pending chain-event queue and actually writes datadog.ReferenceChain into
+  // the JFR file - nothing does this automatically on a timer, and in
+  // production it's dd-trace-java's own recording-chunk rotation that calls
+  // it. Without this dependency+call, chain events get built and enqueued
+  // (visible in TEST_LOG output) but are silently dropped when the process
+  // exits, and the JFR file never shows a single datadog.ReferenceChain event.
+  "reproImplementation"(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug")))
+}
+
+tasks.register<Jar>("reproJar") {
+  group = "build"
+  description = "Demo app that leaks memory in a controlled way, for manually reproducing reference-chains via -agentpath"
+  archiveFileName.set("refchains_repro.jar")
+  from(sourceSets["repro"].output)
+  from({
+    configurations["reproRuntimeClasspath"].map { if (it.isDirectory) it else zipTree(it) }
+  })
+  duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  manifest {
+    attributes("Main-Class" to "com.datadoghq.profiler.repro.ReferenceChainLeakDemo")
+  }
+}
+
 tasks.register<Exec>("runStressTests") {
   dependsOn(tasks.named("jmhJar"))
 

ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java

@@ -92,6 +92,8 @@ private static Antagonist create(String name) {
                 return new WeakRefWaveAntagonist();
             case "dump-storm":
                 return new DumpStormAntagonist();
+            case "reference-chain-leak":
+                return new ReferenceChainLeakAntagonist();
             case "reapply-context-value":
                 return new ReapplyContextValueAntagonist();
             // Deferred: dlopen-churn (needs per-arch dummy .so built in CI prep).

ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReferenceChainLeakAntagonist.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2026, Datadog, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.datadoghq.profiler.chaos;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Grows an ever-referenced, never-evicted cache to give the reference-chains
+ * feature (LivenessTracker/ReferenceChainTracker) a real, steadily growing
+ * population to detect and walk. Growth is capped, not open-ended: once
+ * {@link #CAP_ENTRIES} is reached the cache stops growing and the antagonist
+ * just holds it there for the rest of the run, so a long chaos duration
+ * (hours) still produces the same bounded leak instead of eventually OOMing
+ * the JVM out from under the other antagonists sharing its heap.
+ *
+ * <p>Same object shape as {@code LeakingCacheScenario.CachedPayload}
+ * (ddprof-test's reference-chain integration test) — plain long fields
+ * rather than a nested array, so a single instance clears the allocation
+ * sampler's real size floor without a second, competing heap allocation per
+ * entry.
+ */
+public final class ReferenceChainLeakAntagonist implements Antagonist {
+
+    // ~64MiB of CachedEntry payloads at steady state (entries are a little
+    // over 300 bytes with object header/field overhead) — enough for the
+    // liveness tracker's population-growth trend detection to have a real,
+    // sustained trend to lock onto, without meaningfully competing with the
+    // other antagonists' own memory budget for the run's whole duration.
+    private static final int CAP_ENTRIES = 200_000;
+    private static final int BATCH_SIZE = 200;
+    private static final long BATCH_INTERVAL_MS = 50L;
+
+    private final Map<String, CachedEntry> cache = new ConcurrentHashMap<>();
+    private volatile boolean running;
+    private Thread growthDriver;
+    private long nextKey;
+
+    @Override
+    public String name() {
+        return "reference-chain-leak";
+    }
+
+    @Override
+    public void start() {
+        running = true;
+        growthDriver = new Thread(this::growthLoop, "chaos-refchain-leak");
+        growthDriver.setDaemon(true);
+        growthDriver.start();
+    }
+
+    @Override
+    public void stopGracefully(Duration timeout) {
+        running = false;
+        try {
+            growthDriver.join(timeout.toMillis());
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    private void growthLoop() {
+        while (running && cache.size() < CAP_ENTRIES) {
+            for (int i = 0; i < BATCH_SIZE && cache.size() < CAP_ENTRIES; i++) {
+                String key = "refchain-leak-" + (nextKey++);
+                cache.put(key, new CachedEntry(key));
+                MemoryGovernor.pace();
+            }
+            try {
+                Thread.sleep(BATCH_INTERVAL_MS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                return;
+            }
+        }
+        // Cap reached: keep the reference alive (do nothing) for the rest of
+        // the run instead of exiting the thread, so `cache` stays reachable
+        // via this antagonist's own field for as long as the harness runs.
+        while (running) {
+            try {
+                Thread.sleep(1_000L);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                return;
+            }
+        }
+    }
+
+    static final class CachedEntry {
+        final String key;
+        long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15;
+        long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31;
+
+        CachedEntry(String key) {
+            this.key = key;
+        }
+    }
+}