weight_loop.py

"""walt.weight_loop — measure Walt's HF-paper scoring outcomes per axis.

Two-level autoresearch lifted from arXiv 2605.30003. INNER LOOP is prompt /
weight tuning (deferred to the GEPA-vs-MIPROv2 decision sitting at gary id
2bdfe880). OUTER LOOP, which this module implements, is "do Walt's current
scoring weights actually predict the outcomes Workloft cares about, on a
per-axis basis."

Inputs:
- /home/workloft/walt/data/hf-papers/hf-YYYY-MM-DD.top.json — Walt's daily
  top picks (papers scoring >= 8). Each carries the gary_id Walt filed.

Outcomes pulled from Gary:
- shipped         — there's a downstream ship or commit tied to that todo
- in_progress     — Bob has started it
- still_open      — listed but untouched
- killed/cancelled — Vera or Alfred dropped it

Per axis we compute:
  n_picks           how many papers Walt scored >= 8 in that axis (last N days)
  conversion        share of those picks whose Gary todo reached
                    shipped/in_progress (i.e. actually moved)
  walt_mean_score   Walt's average score across those picks
  axis_health       conversion / walt_mean_score (normalised to 1.0)

The report names the axes where Walt is being too generous (high mean
score, low conversion) and the axes where Walt is being too stingy
(potentially under-filing; harder to measure tonight, flagged for v0.2).

This is the gate-keeping signal for INNER-LOOP weight tuning. We do not
re-prompt Walt tonight — we measure first.

Usage:
    python3 -m walt.weight_loop [--days 30] [--json]
"""
from __future__ import annotations

import argparse
import json
import os
import re
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from urllib import parse as urlparse, request as urlrequest

WALT_DATA = Path("/home/workloft/walt/data/hf-papers")
REPORT_DIR = Path("/home/workloft/walt/reports")
REPORT_DIR.mkdir(parents=True, exist_ok=True)


def _supabase_creds() -> tuple[str, str]:
    env_path = Path("/home/workloft/conexus/.env")
    base = os.environ.get("NEXT_PUBLIC_SUPABASE_URL", "")
    key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY", "")
    if (not base or not key) and env_path.exists():
        text = env_path.read_text()
        for var, val in re.findall(r"^([A-Z_]+)=(.*)$", text, re.M):
            if not base and var == "NEXT_PUBLIC_SUPABASE_URL":
                base = val.strip('"')
            if not key and var == "SUPABASE_SERVICE_ROLE_KEY":
                key = val.strip('"')
    return base, key


def _walt_picks_in_window(days: int) -> list[dict]:
    """Returns the union of Walt's top-pick JSON files within `days`."""
    cutoff = datetime.now(timezone.utc).date() - timedelta(days=days)
    picks: list[dict] = []
    for path in sorted(WALT_DATA.glob("hf-*.top.json")):
        m = re.search(r"hf-(\d{4}-\d{2}-\d{2})\.top\.json$", path.name)
        if not m:
            continue
        pick_date = datetime.strptime(m.group(1), "%Y-%m-%d").date()
        if pick_date < cutoff:
            continue
        try:
            data = json.loads(path.read_text())
        except Exception:
            continue
        for item in data:
            item.setdefault("_pick_date", pick_date.isoformat())
            picks.append(item)
    return picks


def _fetch_gary_todos(ids: list[str]) -> dict[str, dict]:
    """Fetch Gary todos and index by 8-char short_id prefix. We pull recent
    todos and match locally — PostgREST has no efficient prefix-OR clause."""
    if not ids:
        return {}
    base, key = _supabase_creds()
    if not base or not key:
        raise RuntimeError("Supabase creds missing for Gary lookup")
    wanted = set(ids)
    out: dict[str, dict] = {}
    cutoff = (datetime.now(timezone.utc) - timedelta(days=90)).isoformat()
    q = [
        "select=id,status,stage,title,created_at,updated_at",
        f"created_at=gte.{urlparse.quote(cutoff, safe='')}",
        "order=created_at.desc",
        "limit=2000",
    ]
    req = urlrequest.Request(
        f"{base.rstrip('/')}/rest/v1/gary_todos?" + "&".join(q),
        headers={"apikey": key, "Authorization": f"Bearer {key}"},
    )
    with urlrequest.urlopen(req, timeout=20) as r:
        rows = json.loads(r.read())
    for row in rows:
        short = row["id"][:8]
        if short in wanted:
            out[short] = row
    return out


SHIPPED_STATES = {"shipped", "done", "in_progress"}
KILLED_STATES = {"cancelled", "killed"}


def _outcome_for(status: str) -> str:
    if status in SHIPPED_STATES:
        return "moved"
    if status in KILLED_STATES:
        return "killed"
    return "open"


def aggregate(picks: list[dict], gary_rows: dict[str, dict]) -> dict[str, Any]:
    by_axis: dict[str, dict[str, Any]] = defaultdict(
        lambda: {"n": 0, "scores": [], "moved": 0, "killed": 0, "open": 0,
                  "examples_moved": [], "examples_open": []}
    )
    unmatched = 0
    for pick in picks:
        axis = pick.get("axis") or "other"
        score = float(pick.get("score") or 0)
        gid = pick.get("gary_id") or ""
        row = gary_rows.get(gid)
        if not row:
            unmatched += 1
            continue
        outcome = _outcome_for(row.get("status") or "open")
        bucket = by_axis[axis]
        bucket["n"] += 1
        bucket["scores"].append(score)
        bucket[outcome] += 1
        title = pick.get("title", "")[:80]
        if outcome == "moved" and len(bucket["examples_moved"]) < 2:
            bucket["examples_moved"].append(title)
        elif outcome == "open" and len(bucket["examples_open"]) < 2:
            bucket["examples_open"].append(title)
    rows: list[dict[str, Any]] = []
    for axis, b in by_axis.items():
        mean_score = sum(b["scores"]) / b["n"] if b["n"] else 0.0
        conversion = b["moved"] / b["n"] if b["n"] else 0.0
        rows.append({
            "axis": axis,
            "n_picks": b["n"],
            "walt_mean_score": round(mean_score, 2),
            "moved": b["moved"],
            "killed": b["killed"],
            "open": b["open"],
            "conversion": round(conversion, 3),
            "axis_health": round(conversion / (mean_score / 10), 3) if mean_score else 0.0,
            "examples_moved": b["examples_moved"],
            "examples_open": b["examples_open"],
        })
    rows.sort(key=lambda r: -r["n_picks"])
    return {"axis_rows": rows, "unmatched_picks": unmatched,
            "total_picks": len(picks)}


def render_text(report: dict[str, Any]) -> str:
    out: list[str] = []
    out.append("Walt outcome tracker — per-axis health\n")
    out.append(f"Picks analysed: {report['total_picks']}  "
                f"(unmatched in Gary: {report['unmatched_picks']})\n")
    out.append(f"{'axis':50s} {'n':>4s} {'mean':>6s} {'mvd':>4s} "
                f"{'kld':>4s} {'opn':>4s} {'conv':>6s} {'health':>7s}")
    out.append("-" * 95)
    for r in report["axis_rows"]:
        out.append(f"{r['axis'][:50]:50s} {r['n_picks']:>4d} "
                    f"{r['walt_mean_score']:>6.2f} {r['moved']:>4d} "
                    f"{r['killed']:>4d} {r['open']:>4d} "
                    f"{r['conversion']:>6.2%} {r['axis_health']:>7.3f}")
    out.append("\nReading:")
    out.append("  conv  = moved / n_picks (share of Walt's picks that actually advanced)")
    out.append("  health = conv normalised by Walt's mean score for the axis (= 1.0 if perfectly calibrated)")
    out.append("  health << 1: Walt over-scoring (filing things that stall)")
    out.append("  health >> 1: Walt under-scoring (occasional picks routinely moving — could file more aggressively)")
    return "\n".join(out)


def run(*, days: int, emit_json: bool) -> int:
    picks = _walt_picks_in_window(days)
    if not picks:
        print(f"[weight-loop] no Walt picks found in last {days} days under "
                f"{WALT_DATA}", file=sys.stderr)
        return 1
    print(f"[weight-loop] loaded {len(picks)} Walt picks (last {days} days)",
            file=sys.stderr)
    gary_ids = sorted({p.get("gary_id", "")[:8] for p in picks if p.get("gary_id")})
    print(f"[weight-loop] fetching {len(gary_ids)} Gary todos…", file=sys.stderr)
    gary_rows = _fetch_gary_todos(gary_ids)
    print(f"[weight-loop] matched {len(gary_rows)} / {len(gary_ids)}",
            file=sys.stderr)
    report = aggregate(picks, gary_rows)
    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
    if emit_json:
        out_path = REPORT_DIR / f"walt-axis-health-{timestamp}.json"
        out_path.write_text(json.dumps(report, indent=2, default=str))
        print(json.dumps(report, indent=2, default=str))
        print(f"\n[weight-loop] wrote {out_path}", file=sys.stderr)
    else:
        text = render_text(report)
        out_path = REPORT_DIR / f"walt-axis-health-{timestamp}.txt"
        out_path.write_text(text)
        print(text)
        print(f"\n[weight-loop] wrote {out_path}", file=sys.stderr)
    return 0


def main() -> int:
    p = argparse.ArgumentParser(prog="walt.weight_loop")
    p.add_argument("--days", type=int, default=30,
                    help="window in days to analyse (default 30)")
    p.add_argument("--json", action="store_true",
                    help="emit JSON instead of human-readable table")
    a = p.parse_args()
    return run(days=a.days, emit_json=a.json)


if __name__ == "__main__":
    sys.exit(main())