fix(server): keep thread titles tied to user intent

apps/server/scripts/evaluate-thread-titles.ts

@@ -0,0 +1,143 @@
+#!/usr/bin/env node
+// This CLI uses Node argument parsing and random ordering at the application boundary.
+// @effect-diagnostics nodeBuiltinImport:off
+// Run with --model <configured-model> --out /tmp/title-eval.
+// Pass --baseline /tmp/previous-eval/results.json to compare two generation runs.
+// Add --initial to evaluate only the opening request.
+import * as NodeUtil from "node:util";
+import * as NodeCrypto from "node:crypto";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { CodexSettings, ProviderInstanceId } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Duration from "effect/Duration";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+import { makeCodexTextGeneration } from "../src/textGeneration/CodexTextGeneration.ts";
+import { threadTitleEvaluationCases } from "./threadTitleEvaluationCases.ts";
+import {
+  formatThreadTitleContext,
+  type ThreadTitleMessage,
+} from "../src/textGeneration/ThreadTitleContext.ts";
+import { resolveThreadTitleLinks } from "../src/textGeneration/ThreadTitleLinks.ts";
+import * as ProcessRunner from "../src/processRunner.ts";
+import * as ServerConfig from "../src/config.ts";
+
+const { values } = NodeUtil.parseArgs({
+  options: {
+    model: { type: "string" },
+    out: { type: "string" },
+    baseline: { type: "string" },
+    initial: { type: "boolean", default: false },
+  },
+});
+if (!values.model || !values.out)
+  throw new Error("Use --model <configured-model> --out <directory>.");
+const model = values.model;
+const outputDirectory = values.out;
+const Results = Schema.fromJsonString(
+  Schema.Array(
+    Schema.Struct({
+      id: Schema.String,
+      title: Schema.String,
+      latencyMs: Schema.Number,
+      linkedContextDigest: Schema.String,
+    }),
+  ),
+);
+const decodeResults = Schema.decodeUnknownEffect(Results);
+const decodeSettings = Schema.decodeUnknownEffect(CodexSettings);
+const encodeReport = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
+
+await Effect.runPromise(
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-title-evaluation-" });
+    const generation = yield* makeCodexTextGeneration(yield* decodeSettings({}));
+    const baseline = values.baseline
+      ? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
+      : [];
+    const results = [];
+    const review = [];
+    const answerKey = [];
+    for (const fixture of threadTitleEvaluationCases) {
+      const previous = baseline.find((entry) => entry.id === fixture.id);
+      if (values.baseline && !previous) throw new Error(`Baseline is missing ${fixture.id}.`);
+      const firstMessage: ThreadTitleMessage | undefined = fixture.messages.find(
+        (message) => message.role === "user",
+      );
+      if (!firstMessage) throw new Error(`Fixture ${fixture.id} has no user message.`);
+      const context = formatThreadTitleContext(fixture.messages);
+      const message = values.initial ? firstMessage.text : context.message;
+      const attachments = values.initial ? firstMessage.attachments : context.attachments;
+      const [elapsed, { generated, linkedContextDigest }] = yield* Effect.gen(function* () {
+        const linkedContext = yield* resolveThreadTitleLinks({
+          cwd,
+          message,
+        });
+        const linkedContextDigest = NodeCrypto.createHash("sha256")
+          .update(linkedContext ?? "")
+          .digest("hex");
+        if (previous && previous.linkedContextDigest !== linkedContextDigest) {
+          throw new Error(
+            `Linked context changed for ${fixture.id}. Record a new baseline before comparing titles.`,
+          );
+        }
+        const generated = yield* generation.generateThreadTitle({
+          cwd,
+          message,
+          previousTitle: values.initial ? undefined : fixture.previousTitle,
+          attachments,
+          linkedContext,
+          modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
+        });
+        return { generated, linkedContextDigest };
+      }).pipe(Effect.timed);
+      const oldTitle = previous?.title ?? fixture.previousTitle;
+      const newFirst = NodeCrypto.randomInt(2) === 0;
+      results.push({
+        id: fixture.id,
+        title: generated.title,
+        latencyMs: Duration.toMillis(elapsed),
+        needsRefinement: generated.needsRefinement ?? false,
+        linkedContextDigest,
+      });
+      review.push({
+        id: fixture.id,
+        source: fixture.source,
+        request: fixture.request,
+        rubric: fixture.rubric,
+        A: newFirst ? generated.title : oldTitle,
+        B: newFirst ? oldTitle : generated.title,
+        preferred: "",
+        subjectAccuracy: "",
+        recognitionAmongNearbyThreads: "",
+      });
+      answerKey.push({ id: fixture.id, candidate: newFirst ? "A" : "B" });
+    }
+    yield* fs.makeDirectory(outputDirectory, { recursive: true });
+    for (const [name, report] of [
+      ["results", results],
+      ["review", review],
+      ["answer-key", answerKey],
+    ] as const) {
+      yield* fs.writeFileString(
+        path.join(outputDirectory, `${name}.json`),
+        yield* encodeReport(report),
+      );
+    }
+    yield* Effect.log(
+      `Wrote ${results.length} cases to ${outputDirectory}. Score review.json before opening answer-key.json. Latency is in results.json.`,
+    );
+  }).pipe(
+    Effect.provide(
+      Layer.mergeAll(
+        ProcessRunner.layer,
+        ServerConfig.layerTest(process.cwd(), { prefix: "t3-title-evaluation-state-" }),
+      ).pipe(Layer.provideMerge(NodeServices.layer)),
+    ),
+    Effect.scoped,
+  ),
+);

apps/server/scripts/threadTitleEvaluationCases.ts

@@ -0,0 +1,117 @@
+import type { ThreadTitleMessage } from "../src/textGeneration/ThreadTitleContext.ts";
+
+// Public PR subjects and existing title scenarios. Repeated text adds context pressure.
+export const threadTitleEvaluationCases = [
+  {
+    id: "linked-reset-credits",
+    source: "https://github.com/pingdotgg/t3code/pull/10462",
+    request: "Review the reset credit routing change.",
+    previousTitle: "Review PR 10462",
+    messages: [{ role: "user", text: "Review https://github.com/pingdotgg/t3code/pull/10462" }],
+    rubric: "Name reset credit routing. Distinguish it from displaying credit balances.",
+  },
+  {
+    id: "onboarding-merge",
+    source: "https://github.com/pingdotgg/t3code/pull/10465",
+    request: "Make onboarding one shared wizard across computers, then merge when green.",
+    previousTitle: "Finish onboarding PR",
+    messages: [
+      { role: "user", text: "Make onboarding one shared wizard across computers." },
+      {
+        role: "assistant",
+        text: "The wizard now handles pairing, agent selection, and project import.",
+      },
+      { role: "user", text: "File a PR and merge it when green." },
+    ],
+    rubric: "Keep the multi-computer onboarding subject. Do not title it after merging.",
+  },
+  {
+    id: "vague-opening",
+    source: "Existing lazy thread feed title scenario",
+    request: "A failing test is later identified as a lazy thread feed mismatch.",
+    previousTitle: "Fix failing test",
+    messages: [
+      { role: "user", text: "Fix this failing test." },
+      {
+        role: "assistant",
+        text: "The lazy thread feed test expects a full message body before the client requests it.",
+      },
+    ],
+    rubric: "Name the lazy thread feed test. Do not invent a wider mobile regression.",
+  },
+  {
+    id: "scope-change",
+    source: "Title context budget scenario",
+    request: "Change the goal from QR layout to pairing expiry, despite long assistant replies.",
+    previousTitle: "Improve QR layout",
+    messages: [
+      { role: "user", text: "Improve QR sharing layout." },
+      {
+        role: "user",
+        text: "Change of plan. Fix pairing token expiry. Keep remote access working.",
+      },
+      {
+        role: "assistant",
+        text: "The token expires before redemption. " + "Implementation detail. ".repeat(800),
+      },
+      { role: "user", text: "Ship it." },
+    ],
+    rubric: "Name pairing expiry and honor the explicit scope change.",
+  },
+  {
+    id: "review-umbrella",
+    source: "Existing subagent monitoring title scenario",
+    request: "Review subagent monitoring risks. A Codex roster issue is one finding.",
+    previousTitle: "Review subagent monitoring risks",
+    messages: [
+      { role: "user", text: "Review subagent monitoring risks." },
+      {
+        role: "assistant",
+        text: "One finding is a stale Codex roster. " + "Roster detail. ".repeat(800),
+      },
+      { role: "user", text: "Fix the findings and babysit CI." },
+    ],
+    rubric: "Preserve the monitoring review scope. The previous title can stay unchanged.",
+  },
+  {
+    id: "long-opening",
+    source: "Title message truncation scenario",
+    request: "Investigate Android pairing while preserving the iOS flow.",
+    previousTitle: "Inspect logs",
+    messages: [
+      {
+        role: "user",
+        text:
+          "Investigate Android pairing. " +
+          "Connection logs. ".repeat(800) +
+          " Preserve the iOS pairing flow.",
+      },
+    ],
+    rubric: "Name Android pairing. Logs are supporting evidence.",
+  },
+  {
+    id: "research",
+    source: "Maintainer title generation request",
+    request: "How can we improve title generation in T3 Code?",
+    previousTitle: "Research title gen improvements",
+    messages: [
+      { role: "user", text: "How can we improve title gen further in T3 Code?" },
+      {
+        role: "assistant",
+        text: "Prioritize user messages, refine vague titles once, and resolve PR subjects.",
+      },
+      {
+        role: "user",
+        text: "Make these changes and file a PR. Babysit until everything is green.",
+      },
+    ],
+    rubric: "Keep title generation as the subject. Do not focus on filing the PR.",
+  },
+] satisfies ReadonlyArray<{
+  id: string;
+  source: string;
+  request: string;
+  previousTitle: string;
+  messages: ReadonlyArray<ThreadTitleMessage>;
+  rubric: string;
+}>;

apps/server/src/orchestration/Layers/ProjectionPipeline.ts

@@ -810,6 +810,9 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
             ...(event.payload.activeOrderKey !== undefined
               ? { activeOrderKey: event.payload.activeOrderKey }
               : {}),
+            ...(event.payload.titleState !== undefined
+              ? { titleState: event.payload.titleState }
+              : {}),
             ...(event.payload.titleRegeneration !== undefined
               ? {
                   titleRegenerationRequestId: event.payload.titleRegeneration?.requestId ?? null,

apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts

@@ -486,6 +486,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
           pinOrderKey: "gm",
           activeOrderKey: "hq",
           titleRegeneration: null,
+          titleState: null,
           deletedAt: null,
           messages: [
             {
@@ -611,6 +612,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
           pinOrderKey: "gm",
           activeOrderKey: "hq",
           titleRegeneration: null,
+          titleState: null,
           session: {
             threadId: ThreadId.make("thread-1"),
             status: "running",
@@ -741,7 +743,8 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
           id: ThreadId.make("thread-1"),
           projectId: asProjectId("project-1"),
           title: "Thread 1",
-          session: snapshot.threads[0]?.session,
+          titleState: null,
+          session: snapshot.threads[0]?.session ?? null,
         });
       }