adamd9 · adamd9 · Aug 9, 2025
diff --git a/webapp/components/enhanced-transcript.tsx b/webapp/components/enhanced-transcript.tsx
@@ -74,6 +74,27 @@ export function EnhancedTranscript({
     }
   };
 
+  function renderContentWithLinks(text: string) {
+    const urlRegex = /(https?:\/\/[^\s]+)/g;
+    const parts = text.split(urlRegex);
+    return parts.map((part, index) => {
+      if (/^https?:\/\/[^\s]+$/.test(part)) {
+        return (
+          <a
+            key={index}
+            href={part}
+            target="_blank"
+            rel="noopener noreferrer"
+            className="text-blue-600 underline"
+          >
+            {part}
+          </a>
+        );
+      }
+      return part;
+    });
+  }
+
   return (
     <div className="flex flex-col h-full bg-white rounded-xl border">
       {/* Header */}
@@ -156,12 +177,36 @@ export function EnhancedTranscript({
                         {!isUser && (channelBadge || supervisorBadge)}
                       </div>
                       <div className="whitespace-pre-wrap">
-                        {title}
+                        {renderContentWithLinks(title)}
                       </div>
                     </div>
                   </div>
                 </div>
               );
+            } else if (type === "CANVAS") {
+              const url = typeof data?.url === "string" ? data.url : undefined;
+              return (
+                <div key={itemId} className="flex justify-center">
+                  <div className="bg-blue-50 text-blue-800 border border-blue-200 px-3 py-2 rounded-md text-sm">
+                    {url ? (
+                      <a
+                        href={url}
+                        target="_blank"
+                        rel="noopener noreferrer"
+                        className="underline flex items-center gap-1"
+                      >
+                        <span>🖼️</span>
+                        <span>{title || "Open canvas"}</span>
+                      </a>
+                    ) : (
+                      <div className="flex items-center gap-1">
+                        <span>🖼️</span>
+                        <span>{title || "Canvas"}</span>
+                      </div>
+                    )}
+                  </div>
+                </div>
+              );
             } else if (type === "BREADCRUMB") {
               return (
                 <div

diff --git a/webapp/contexts/TranscriptContext.tsx b/webapp/contexts/TranscriptContext.tsx
@@ -92,6 +92,26 @@ export const TranscriptProvider: FC<PropsWithChildren> = ({ children }) => {
     ]);
   };
 
+  const addTranscriptCanvas: TranscriptContextValue["addTranscriptCanvas"] = (
+    title,
+    url
+  ) => {
+    setTranscriptItems((prev) => [
+      ...prev,
+      {
+        itemId: `canvas-${uuidv4()}`,
+        type: "CANVAS",
+        title,
+        data: { url },
+        expanded: false,
+        timestamp: newTimestampPretty(),
+        createdAtMs: Date.now(),
+        status: "DONE",
+        isHidden: false,
+      },
+    ]);
+  };
+
   const toggleTranscriptItemExpand: TranscriptContextValue["toggleTranscriptItemExpand"] = (itemId) => {
     setTranscriptItems((prev) =>
       prev.map((log) =>
@@ -119,6 +139,7 @@ export const TranscriptProvider: FC<PropsWithChildren> = ({ children }) => {
         addTranscriptMessage,
         updateTranscriptMessage,
         addTranscriptBreadcrumb,
+        addTranscriptCanvas,
         toggleTranscriptItemExpand,
         updateTranscriptItem,
         clearTranscript,

diff --git a/webapp/lib/handle-enhanced-realtime-event.ts b/webapp/lib/handle-enhanced-realtime-event.ts
@@ -27,10 +27,11 @@ export default function handleEnhancedRealtimeEvent(
   event: any,
   transcript: TranscriptContextValue
 ) {
-  const { 
-    addTranscriptMessage, 
-    updateTranscriptMessage, 
-    addTranscriptBreadcrumb 
+  const {
+    addTranscriptMessage,
+    updateTranscriptMessage,
+    addTranscriptBreadcrumb,
+    addTranscriptCanvas
   } = transcript;
 
   console.log("Enhanced event handler:", event.type, event);
@@ -407,16 +408,18 @@ export default function handleEnhancedRealtimeEvent(
       );
       break;
 
-    case "chat.canvas":
-      addTranscriptBreadcrumb(
-        "📝 Canvas response",
-        {
-          content: event.content,
-          timestamp: event.timestamp,
-          supervisor: event.supervisor || false
-        }
-      );
+    case "chat.canvas": {
+      const url =
+        typeof event.content === "string"
+          ? event.content
+          : typeof event.url === "string"
+          ? event.url
+          : typeof event.content?.url === "string"
+          ? event.content.url
+          : "";
+      addTranscriptCanvas(event.title || "Canvas", url);
       break;
+    }
 
     case "chat.error":
       addTranscriptBreadcrumb(

diff --git a/webapp/types/transcript.ts b/webapp/types/transcript.ts
@@ -2,7 +2,7 @@
 
 export interface TranscriptItem {
   itemId: string;
-  type: "MESSAGE" | "BREADCRUMB";
+  type: "MESSAGE" | "BREADCRUMB" | "CANVAS";
   role?: "user" | "assistant";
   title?: string;
   data?: Record<string, any>;
@@ -27,6 +27,7 @@ export interface TranscriptContextValue {
   ) => void;
   updateTranscriptMessage: (itemId: string, text: string, isDelta: boolean) => void;
   addTranscriptBreadcrumb: (title: string, data?: Record<string, any>) => void;
+  addTranscriptCanvas: (title: string, url: string) => void;
   toggleTranscriptItemExpand: (itemId: string) => void;
   updateTranscriptItem: (itemId: string, updatedProperties: Partial<TranscriptItem>) => void;
   clearTranscript: () => void;

diff --git a/websocket-server/src/agentConfigs/baseAgentConfig.ts b/websocket-server/src/agentConfigs/baseAgentConfig.ts
@@ -24,6 +24,8 @@ Keep responses concise—no more than two or three sentences. If that would omit
 
 In particular, if you need to output URLs or other details that are too long for a voice response, use the sendCanvas tool to share the full response.
 
+The client will display any canvas link separately, so do not mention the link in your spoken responses.
+
 Be conversational and natural in speech. When escalating, choose the appropriate reasoning_type and provide good context.
 
 When invoking tools or waiting on longer operations, provide a brief, natural backchannel once at the start (e.g., "One moment…", "Let me check that…"). Keep it short, avoid repetition, and stop as soon as the tool output is ready or the user begins speaking.`,

diff --git a/websocket-server/src/agentConfigs/canvasTool.ts b/websocket-server/src/agentConfigs/canvasTool.ts
@@ -2,7 +2,8 @@ import { FunctionHandler } from './types';
 import { WebSocket } from 'ws';
 import { storeCanvas } from '../canvasStore';
 
-const PUBLIC_URL = process.env.PUBLIC_URL || '';
+const PORT = process.env.PORT || '8081';
+const PUBLIC_URL = process.env.PUBLIC_URL || `http://localhost:${PORT}`;
 
 function jsonSend(ws: WebSocket | undefined, obj: unknown) {
   if (!ws || ws.readyState !== WebSocket.OPEN) return;
@@ -13,7 +14,8 @@ export const sendCanvas: FunctionHandler = {
   schema: {
     name: "send_canvas",
     type: "function",
-    description: "Send detailed content to the canvas UI.",
+    description:
+      "Send detailed content to the canvas UI. The server returns a URL that the client shows separately, so do not mention this URL in your response.",
     parameters: {
       type: "object",
       properties: {
@@ -40,6 +42,6 @@ export const sendCanvas: FunctionHandler = {
       jsonSend(client, message);
     }
 
-    return "canvas_sent";
+    return { status: "sent", url: link };
   }
 };
diff --git a/websocket-server/src/agentConfigs/types.ts b/websocket-server/src/agentConfigs/types.ts
@@ -12,7 +12,7 @@ export interface FunctionHandler {
       additionalProperties?: boolean;
     };
   };
-  handler: (args: any, addBreadcrumb?: (title: string, data?: any) => void) => Promise<string>;
+  handler: (args: any, addBreadcrumb?: (title: string, data?: any) => void) => Promise<any>;
 }
 
 export interface AgentConfig {

diff --git a/websocket-server/src/session/chat.ts b/websocket-server/src/session/chat.ts
@@ -6,6 +6,7 @@ import { getAllFunctions, getDefaultAgent, FunctionHandler } from "../agentConfi
 import { isSmsWindowOpen, getNumbers } from "../smsState";
 import { sendSms } from "../sms";
 import { session, parseMessage, jsonSend, isOpen } from "./state";
+import { sendCanvas } from "../agentConfigs/canvasTool";
 
 export function establishChatSocket(
   ws: WebSocket,
@@ -186,6 +187,10 @@ export async function handleTextChatMessage(
                 arguments: functionCall.arguments,
                 call_id: functionCall.call_id,
                 status: "completed",
+                output:
+                  typeof functionResult === "string"
+                    ? functionResult
+                    : JSON.stringify(functionResult),
               });
           }
           const fnSchemas = allFns.map((f: FunctionHandler) => ({ ...f.schema, strict: false }));
@@ -239,42 +244,21 @@ export async function handleTextChatMessage(
               } catch {}
               const title: string = args?.title || "Canvas";
               const contentStr: string = typeof args?.content === "string" ? args.content : JSON.stringify(args?.content ?? {});
-              // Persist to conversation history as assistant text (until a dedicated canvas channel exists)
+              const result = await (sendCanvas as any).handler({ content: contentStr, title });
+              const link =
+                typeof result === "object" && (result as any).url
+                  ? (result as any).url
+                  : typeof result === "string"
+                  ? result
+                  : "";
               const assistantMessage = {
                 type: "assistant" as const,
-                content: contentStr,
+                content: link,
                 timestamp: Date.now(),
                 channel: "text" as const,
                 supervisor: true,
               };
               session.conversationHistory.push(assistantMessage);
-              // Notify chat clients with the content
-              for (const ws of chatClients) {
-                if (isOpen(ws))
-                  jsonSend(ws, {
-                    type: "chat.response",
-                    content: contentStr,
-                    timestamp: Date.now(),
-                    supervisor: true,
-                    title,
-                  });
-              }
-              // Mirror as a normal assistant message to logs for transcript UI
-              for (const ws of logsClients) {
-                if (isOpen(ws))
-                  jsonSend(ws, {
-                    type: "conversation.item.created",
-                    item: {
-                      id: `msg_${Date.now()}`,
-                      type: "message",
-                      role: "assistant",
-                      content: [{ type: "text", text: `[Canvas] ${title}\n${contentStr}` }],
-                      channel: "text",
-                      supervisor: true,
-                    },
-                  });
-              }
-              // Finish the breadcrumb for completeness
               for (const ws of logsClients) {
                 if (isOpen(ws))
                   jsonSend(ws, {
@@ -283,6 +267,10 @@ export async function handleTextChatMessage(
                     arguments: canvasCall.arguments,
                     call_id: canvasCall.call_id,
                     status: "completed",
+                    output:
+                      typeof result === "string"
+                        ? result
+                        : JSON.stringify(result),
                   });
               }
               // Solution A: confirm tool execution with Responses API to elicit a concise final text
@@ -300,7 +288,7 @@ export async function handleTextChatMessage(
                     {
                       type: "function_call_output",
                       call_id: canvasCall.call_id,
-                      output: JSON.stringify({ status: "sent" }),
+                      output: JSON.stringify({ status: "sent", url: link }),
                     },
                   ],
                   instructions:

diff --git a/websocket-server/src/types.ts b/websocket-server/src/types.ts
@@ -28,7 +28,7 @@ export interface FunctionSchema {
 
 export interface FunctionHandler {
   schema: FunctionSchema;
-  handler: (args: any, addBreadcrumb?: (title: string, data?: any) => void) => Promise<string>;
+  handler: (args: any, addBreadcrumb?: (title: string, data?: any) => void) => Promise<any>;
 }
 
 // New Responses API types