feat(preview): show the agent's cursor on server-browser tabs

Server tabs showed a page changing with no sign of where the agent acted,
which made agent clicks, hovers, and drags hard to follow. The server now
sends viewers the agent's pointer before each click, hover, and drag
lands, gliding to the target and pulsing on a click like desktop tabs.
Web draws it with the desktop cursor mark, mobile draws it in the stream
viewer, and recordings now show hovers and drags too. Unwatched tabs skip
the glide delay.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: juliusmarminge

apps/mobile/src/features/browser/preview-stream-document.ts

@@ -48,6 +48,21 @@ export function previewStreamDocument(configuration: string, script: string) {
   canvas, video { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; }
   canvas { touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; }
   video { pointer-events: none; }
+  /* The agent's pointer, positioned over the letterboxed frame by the viewer script. */
+  .agent-cursor {
+    position: absolute; left: 0; top: 0; width: 20px; height: 20px; pointer-events: none;
+    opacity: 0; transition: opacity 150ms ease-out, transform 150ms ease-out;
+  }
+  .agent-cursor svg { position: relative; display: block; filter: drop-shadow(0 1px 1px rgba(0,0,0,.35)); }
+  .agent-cursor .ping {
+    position: absolute; left: 2px; top: 2px; width: 16px; height: 16px; border-radius: 50%;
+    background: rgba(59,130,246,.3); animation: agent-cursor-ping 600ms ease-out forwards;
+  }
+  @keyframes agent-cursor-ping { from { transform: scale(.6); opacity: 1; } to { transform: scale(2.2); opacity: 0; } }
+  @media (prefers-reduced-motion: reduce) {
+    .agent-cursor { transition: none; }
+    .agent-cursor .ping { animation: none; opacity: 0; }
+  }
   /* Pinned so focus never scrolls; 16px keeps iOS from zooming on focus. */
   textarea {
     position: fixed; left: 0; top: 0; width: 1px; height: 1px;

apps/mobile/src/features/browser/preview-stream.browser.ts

@@ -5,6 +5,7 @@ import {
   type PreviewStreamClient,
   type PreviewStreamControl,
   type PreviewStreamInput,
+  type PreviewStreamPointer,
   type PreviewStreamViewport,
 } from "@t3tools/client-runtime/preview/server-browser-stream";
 
@@ -41,6 +42,10 @@ const PROBE_REUSE_MS = 10_000;
 // Kept in the hidden textarea so a soft keyboard's backspace has something to
 // delete and fires `input`; Gboard's keydown carries keyCode 229 and no key.
 const SENTINEL = "\u200b";
+// The agent cursor stays bright while the agent acts, then dims like the web panel's.
+const AGENT_CURSOR_ACTIVE_MS = 700;
+const AGENT_CURSOR_SVG =
+  '<svg width="20" height="20" viewBox="0 0 24 24" fill="#fff" stroke="#3b82f6" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
 
 interface Viewer {
   readonly stop: () => void;
@@ -76,7 +81,11 @@ export function start(configuration: PreviewStreamConfiguration) {
   const canvas = document.createElement("canvas");
   canvas.setAttribute("role", "img");
   canvas.setAttribute("aria-label", "Browser page");
-  container.append(canvas);
+  const agentCursor = document.createElement("div");
+  agentCursor.className = "agent-cursor";
+  agentCursor.setAttribute("aria-hidden", "true");
+  agentCursor.innerHTML = AGENT_CURSOR_SVG;
+  container.append(canvas, agentCursor);
   const input = document.createElement("textarea");
   input.setAttribute("aria-label", "Browser page input");
   input.autocapitalize = "off";
@@ -170,6 +179,7 @@ export function start(configuration: PreviewStreamConfiguration) {
       },
       {
         onFrame: (jpeg) => painter.paint(jpeg),
+        onPointer: showAgentCursor,
         onClipboard: (text) => post({ type: "clipboard", text }),
         onDownload: (download) => post({ type: "download", ...download }),
         onFileChooser: (chooser) => post({ type: "fileChooser", chooser }),
@@ -204,6 +214,8 @@ export function start(configuration: PreviewStreamConfiguration) {
           const previous = control;
           control = nextControl;
           post({ type: "control", ...nextControl });
+          // Taking over hides the agent cursor; the person's own touch is the pointer now.
+          if (nextControl.controller === "you") agentCursor.style.opacity = "0";
           if (nextControl.controller !== "you") clearInput();
           else {
             input.disabled = !interactive;
@@ -263,6 +275,31 @@ export function start(configuration: PreviewStreamConfiguration) {
     visualViewport?.addEventListener("scroll", followVisualViewport);
   }
 
+  let agentCursorTimer: ReturnType<typeof setTimeout> | null = null;
+  /** Mirrors the frame's letterbox so the cursor lands on the element the agent targets. */
+  const showAgentCursor = (pointer: PreviewStreamPointer) => {
+    if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller === "you")
+      return;
+    const box = { width: canvas.clientWidth, height: canvas.clientHeight };
+    const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
+    const width = canvas.width * fit;
+    const height = canvas.height * fit;
+    const left = (box.width - width) / 2 + (pointer.x * width) / viewport.width;
+    const top = (box.height - height) / 2 + (pointer.y * height) / viewport.height;
+    agentCursor.style.transform = `translate3d(${left}px, ${top}px, 0)`;
+    agentCursor.style.opacity = "1";
+    if (pointer.phase === "click") {
+      agentCursor.querySelector(".ping")?.remove();
+      const ping = document.createElement("span");
+      ping.className = "ping";
+      agentCursor.prepend(ping);
+    }
+    if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
+    agentCursorTimer = setTimeout(() => {
+      agentCursor.style.opacity = control?.controller === "agent" ? "0.35" : "0";
+    }, AGENT_CURSOR_ACTIVE_MS);
+  };
+
   const pagePoint = (clientX: number, clientY: number, clamp: boolean) => {
     if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller !== "you")
       return null;
@@ -647,6 +684,7 @@ export function start(configuration: PreviewStreamConfiguration) {
       if (resizeTimer !== null) clearTimeout(resizeTimer);
       if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
       if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
+      if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
       painter.stop();
       client?.stop();
       client = null;

apps/server/src/preview/ServerBrowser.test.ts

@@ -69,6 +69,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
   let contextClosed = false;
   const page = {
     on: (name: string, callback: (...args: unknown[]) => void) => events.on(name, callback),
+    once: (name: string, callback: (...args: unknown[]) => void) => events.once(name, callback),
+    off: (name: string, callback: (...args: unknown[]) => void) => events.off(name, callback),
     emit: (name: string, ...args: unknown[]) => events.emit(name, ...args),
     emitAsync: (name: string, ...args: unknown[]) =>
       Promise.all(events.listeners(name).map((listener) => listener(...args))),
@@ -840,3 +842,28 @@ it.live("an agent answers the page's file picker or sets files on a file input",
     }),
   ).pipe(Effect.provide(layer)),
 );
+
+it.live("viewers see the agent's pointer move to its target and click there", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { browser, broker, tabId } = yield* ready;
+      const page = contexts[0]!.page;
+      const viewer = yield* browser.attachViewer(viewerInput(tabId, false));
+      const click = vi.fn(async () => {});
+      page.locator.mockReturnValue({
+        scrollIntoViewIfNeeded: async () => {},
+        boundingBox: async () => ({ x: 100, y: 40, width: 80, height: 20 }),
+        click,
+      } as never);
+      yield* broker.invoke({ scope, tabId, operation: "click", input: { locator: "#go" } });
+      const pointers = (yield* Queue.clear(viewer.output)).filter(
+        (item) => item._tag === "pointer",
+      );
+      expect(pointers).toEqual([
+        expect.objectContaining({ phase: "move", x: 140, y: 50 }),
+        expect.objectContaining({ phase: "click", x: 140, y: 50 }),
+      ]);
+      expect(click).toHaveBeenCalledOnce();
+    }),
+  ).pipe(Effect.provide(layer)),
+);

apps/server/src/preview/ServerBrowser.ts

@@ -82,6 +82,9 @@ const HOST_RECONNECT_DELAY = "1 second";
 const VIEWER_OUTPUT_LIMIT = 64;
 const RECORDING_SCREENCAST = { format: "jpeg", quality: 90, everyNthFrame: 1 } as const;
 const decodeViewportSetting = Schema.decodeUnknownSync(PreviewViewportSettingSchema);
+/** The agent cursor glides to its target, then pulses just before the press, like desktop tabs. */
+const AGENT_CURSOR_MOVE_MS = 160;
+const AGENT_CURSOR_CLICK_LEAD_MS = 40;
 
 const sleepUntil = (deadline: number) =>
   new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, deadline - Date.now())));
@@ -153,6 +156,13 @@ export type ServerBrowserViewerOutput =
       readonly editable: boolean;
     }
   | { readonly _tag: "clipboard"; readonly text: string }
+  | {
+      readonly _tag: "pointer";
+      readonly phase: "move" | "click";
+      readonly x: number;
+      readonly y: number;
+      readonly sequence: number;
+    }
   | {
       readonly _tag: "fileChooser";
       readonly id: string;
@@ -1187,6 +1197,35 @@ const make = Effect.gen(function* () {
       };
     });
 
+  let pointerSequence = 0;
+  /**
+   * Shows viewers and recordings where the agent is about to act. Nobody is
+   * watching a headless agent tab, so it pays no glide delay.
+   */
+  const pointerFor =
+    (tab: ServerTab): ServerBrowserPage.PointerReporter =>
+    async ({ x, y }, phase) => {
+      const encoder = tab.recording?.encoder;
+      if (tab.viewers.size === 0 && !encoder) return;
+      const show = (next: "move" | "click") => {
+        const sequence = ++pointerSequence;
+        for (const viewer of tab.viewers)
+          viewer.push({ _tag: "pointer", phase: next, x, y, sequence });
+        void encoder
+          ?.evaluate(
+            ([px, py, click]) =>
+              (globalThis as unknown as EncoderWindow).__t3Recorder?.cursor(px, py, click),
+            [x, y, next === "click"] as const,
+          )
+          .catch(constVoid);
+      };
+      show("move");
+      await sleepUntil(Date.now() + AGENT_CURSOR_MOVE_MS);
+      if (phase !== "click") return;
+      show("click");
+      await sleepUntil(Date.now() + AGENT_CURSOR_CLICK_LEAD_MS);
+    };
+
   const recordAction = <A>(tab: ServerTab, action: string, run: () => Promise<A>): Promise<A> => {
     const event: {
       -readonly [K in keyof PreviewAutomationActionEvent]: PreviewAutomationActionEvent[K];
@@ -1454,28 +1493,22 @@ const make = Effect.gen(function* () {
       }
       case "click": {
         const clickInput = input as PreviewAutomationClickInput;
-        const point = await recordAction(tab, "click", () =>
-          ServerBrowserPage.click(tab.page, clickInput),
+        await recordAction(tab, "click", () =>
+          ServerBrowserPage.click(tab.page, clickInput, pointerFor(tab)),
         );
-        void tab.recording?.encoder
-          .evaluate(
-            ([x, y]) => (globalThis as unknown as EncoderWindow).__t3Recorder?.cursor(x, y, true),
-            [point.x, point.y] as const,
-          )
-          .catch(constVoid);
         return undefined;
       }
       case "hover":
         return recordAction(tab, "hover", () =>
-          ServerBrowserPage.hover(tab.page, input as PreviewAutomationHoverInput),
+          ServerBrowserPage.hover(tab.page, input as PreviewAutomationHoverInput, pointerFor(tab)),
         );
       case "select":
         return recordAction(tab, "select", () =>
           ServerBrowserPage.select(tab.page, input as PreviewAutomationSelectInput),
         );
       case "drag":
         return recordAction(tab, "drag", () =>
-          ServerBrowserPage.drag(tab.page, input as PreviewAutomationDragInput),
+          ServerBrowserPage.drag(tab.page, input as PreviewAutomationDragInput, pointerFor(tab)),
         );
       case "upload":
         return recordAction(tab, "upload", () =>

apps/server/src/preview/ServerBrowserPage.test.ts

@@ -214,4 +214,35 @@ describe("server browser element refs", () => {
       ServerBrowserPage.select(page, { locator: "#target", values: ["s"] }),
     ).rejects.toMatchObject({ tag: "PreviewAutomationTargetNotEditableError" });
   });
+
+  it("shows the agent's pointer at each target before the action reaches the page", async () => {
+    await page.setContent(`
+      <button id="go" style="position:absolute;left:100px;top:40px;width:80px;height:20px">go</button>
+      <div id="tip" style="position:absolute;left:300px;top:40px;width:60px;height:20px">tip</div>
+      <div id="card" draggable="true" style="position:absolute;left:20px;top:120px;width:40px;height:40px">card</div>
+      <div id="lane" style="position:absolute;left:220px;top:120px;width:100px;height:40px">lane</div>
+      <script>
+        window.seen = [];
+        go.onclick = () => seen.push("click");
+        tip.onmouseenter = () => seen.push("hover");
+        card.ondragstart = (event) => event.dataTransfer.setData("text/plain", "card");
+        lane.ondragover = (event) => event.preventDefault();
+        lane.ondrop = () => seen.push("drop");
+      </script>`);
+    const shown: Array<string> = [];
+    const pointer: ServerBrowserPage.PointerReporter = async ({ x, y }, phase) => {
+      const seen = await page.evaluate("window.seen.length");
+      shown.push(`${phase}@${Math.round(x)},${Math.round(y)} after ${seen}`);
+    };
+    await ServerBrowserPage.click(page, { locator: "#go" }, pointer);
+    await ServerBrowserPage.hover(page, { locator: "#tip" }, pointer);
+    await ServerBrowserPage.drag(page, { source: "#card", target: "#lane" }, pointer);
+    expect(shown).toEqual([
+      "click@140,50 after 0",
+      "move@330,50 after 1",
+      "move@40,140 after 2",
+      "move@270,140 after 2",
+    ]);
+    expect(await page.evaluate("window.seen")).toEqual(["click", "hover", "drop"]);
+  });
 });

apps/server/src/preview/ServerBrowserPage.ts

@@ -118,6 +118,28 @@ const targetLocator = (
   return page.locator(selector);
 };
 
+/** Shows the agent's pointer to viewers before an action lands at that point. */
+export type PointerReporter = (
+  point: { readonly x: number; readonly y: number },
+  phase: "move" | "click",
+) => Promise<void>;
+const noPointer: PointerReporter = async () => {};
+
+/** The viewport point an action will hit: the target's center after scrolling it into view. */
+const targetPoint = async (
+  page: Page,
+  locator: Locator | null,
+  input: { readonly x?: number | undefined; readonly y?: number | undefined },
+  timeout: number,
+) => {
+  if (locator === null) return { x: input.x ?? 0, y: input.y ?? 0 };
+  await locator.scrollIntoViewIfNeeded({ timeout });
+  const box = await locator.boundingBox({ timeout });
+  if (box) return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+  const viewport = page.viewportSize();
+  return { x: (viewport?.width ?? 0) / 2, y: (viewport?.height ?? 0) / 2 };
+};
+
 const SNAPSHOT_SCRIPT = `(() => {
   return {
     url: location.href,
@@ -211,15 +233,12 @@ export const snapshot = async (input: {
 export const click = async (
   page: Page,
   input: PreviewAutomationClickInput,
+  pointer: PointerReporter = noPointer,
 ): Promise<{ readonly x: number; readonly y: number }> => {
   const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const locator = targetLocator(page, input);
-  let point = { x: input.x ?? 0, y: input.y ?? 0 };
-  if (locator !== null) {
-    await locator.scrollIntoViewIfNeeded({ timeout });
-    const box = await locator.boundingBox({ timeout });
-    if (box) point = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
-  }
+  const point = await targetPoint(page, locator, input, timeout);
+  await pointer(point, "click");
   const options = { button: input.button ?? "left", clickCount: input.clickCount ?? 1 } as const;
   const clicked =
     locator === null
@@ -301,13 +320,19 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => {
 };
 
 /** Hovering by locator moves the real mouse, so CSS :hover and pointer events both apply. */
-export const hover = async (page: Page, input: PreviewAutomationHoverInput) => {
+export const hover = async (
+  page: Page,
+  input: PreviewAutomationHoverInput,
+  pointer: PointerReporter = noPointer,
+) => {
+  const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const locator = targetLocator(page, input);
+  await pointer(await targetPoint(page, locator, input, timeout), "move");
   if (locator === null) {
     await page.mouse.move(input.x ?? 0, input.y ?? 0);
     return;
   }
-  await locator.hover({ timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
+  await locator.hover({ timeout });
 };
 
 export const select = async (
@@ -346,10 +371,20 @@ export const select = async (
   return { selected: await locator.selectOption(values, { timeout }) };
 };
 
-export const drag = async (page: Page, input: PreviewAutomationDragInput) => {
+export const drag = async (
+  page: Page,
+  input: PreviewAutomationDragInput,
+  pointer: PointerReporter = noPointer,
+) => {
+  const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const source = targetLocator(page, { locator: input.source })!;
   const target = targetLocator(page, { locator: input.target })!;
-  await source.dragTo(target, { timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
+  await pointer(await targetPoint(page, source, {}, timeout), "move");
+  // The cursor travels with the drag; the page sees one continuous gesture.
+  const dropped = source.dragTo(target, { timeout });
+  const end = await target.boundingBox({ timeout }).catch(() => null);
+  if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move");
+  await dropped;
 };
 
 /** Sets files on one file input; false when no locator or selector names one. */

apps/server/src/preview/ServerBrowserStream.ts

@@ -135,6 +135,7 @@ const makeHandler = (browser: ServerBrowser.ServerBrowser["Service"]) =>
               case "download":
               case "fileChooser":
               case "fileChooserClosed":
+              case "pointer":
               case "probe": {
                 const { _tag: type, ...data } = output;
                 return write(JSON.stringify({ type, ...data }));