feat: Example shows the hydrated conversation from the client in WebRTC SDK

apps/browser-example/src/components/ConversationView.tsx

@@ -0,0 +1,140 @@
+import { type ReactNode, useRef, useEffect } from "react"
+import { debounce } from "lodash-es"
+import type {
+  RealtimeConversationItem,
+  RealtimeConversationItemContent,
+} from "@tsorta/browser/openai"
+import { DefinedRole, simplifyItem } from "./simpleConversation"
+
+const log = console
+
+export interface ConversationProps {
+  conversation: RealtimeConversationItem[]
+}
+
+export const ConversationView = ({
+  conversation,
+}: ConversationProps): ReactNode => {
+  return (
+    <div className="conversation d-flex flex-column overflow-y-scroll flex-grow-1">
+      {conversation
+        .filter((item) => item.role !== "system")
+        .map((item, index, arr) => (
+          <ConversationItem
+            key={item.id}
+            item={item}
+            doScrollIntoView={index === arr.length - 1}
+          />
+        ))}
+    </div>
+  )
+}
+
+const RoleBgColorMap: Record<DefinedRole, string> = {
+  user: "bg-primary-subtle",
+  assistant: "bg-secondary-subtle",
+  system: "bg-secondary",
+}
+const RoleTextColorMap: Record<DefinedRole, string> = {
+  user: "text-primary",
+  assistant: "text-secondary",
+  system: "text-secondary",
+}
+const RoleLabelMap: Record<DefinedRole, string> = {
+  user: "You",
+  assistant: "Assistant",
+  system: "System",
+}
+
+interface ConversationItemProps {
+  item: RealtimeConversationItem
+  doScrollIntoView: boolean
+}
+
+const ConversationItem = ({
+  item,
+  doScrollIntoView,
+}: ConversationItemProps): ReactNode => {
+  const simpleItem = simplifyItem(item)
+  const { id, role, content } = simpleItem
+  const alignClass = role === "user" ? "align-self-end" : "align-self-start"
+
+  const divRef = useRef<HTMLDivElement>(null)
+
+  useEffect(() => {
+    const executeScroll = () => {
+      // behavior: smooth works fine on FF+macOS, but not on Chrome+macOS, so we detect and force "instant" on chrome
+      const behavior = /Chrome/.test(navigator.userAgent) ? "instant" : "smooth"
+      divRef.current?.scrollIntoView({
+        behavior,
+        block: "end",
+        inline: "nearest",
+      })
+    }
+
+    if (doScrollIntoView) {
+      debounce(executeScroll, 500)()
+    }
+  }, [item, doScrollIntoView, item.content, simpleItem.content.length])
+
+  return (
+    <div
+      className={`conversation-item ${alignClass}`}
+      style={{ maxWidth: "60%" }}
+      ref={divRef}
+    >
+      <div className="attribution mx-8 mb-1 mt-6 text-xs text-gray-500 text-muted">
+        <small>{RoleLabelMap[role]}</small>
+      </div>
+
+      <div
+        key={id}
+        className={`mb-5 ${RoleBgColorMap[role]} ${RoleTextColorMap[role]}`}
+        style={{ padding: "1.5rem 1.5rem", borderRadius: "5rem" }}
+      >
+        <div className="my-content">
+          {content.map((contentItem, index) => (
+            <ConversationItemContent
+              key={index}
+              content={contentItem}
+              doScrollIntoView={doScrollIntoView}
+            />
+          ))}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+interface ConversationItemContentProps {
+  content: RealtimeConversationItemContent
+  doScrollIntoView: boolean
+}
+
+const ConversationItemContent = ({
+  content,
+}: ConversationItemContentProps): ReactNode => {
+  if (!["text", "input_text", "input_audio"].includes(content.type)) {
+    // NOTE: we find content.type="audio" coming in here in logging though it is not in the types!
+    if ((content.type as string) !== "audio") {
+      log.warn(
+        `Unexpected type for RealtimeConversationItemContent '${content.type}'. Will not be rendered: %o`,
+        content
+      )
+    }
+    return null
+  }
+
+  return (
+    <div className={`item-content item-type-${content.type}`}>
+      {(content.type == "text" || content.type == "input_text") && (
+        <span>{content.text}</span>
+      )}
+      {content.type == "input_audio" && (
+        <span className="input_audio transcript">
+          {content.transcript ? content.transcript : "..."}
+        </span>
+      )}
+    </div>
+  )
+}

apps/browser-example/src/components/RealtimeSessionView.tsx

@@ -2,7 +2,11 @@ import { ReactNode, useState } from "react"
 import { BootstrapIcon } from "./BootstrapIcon"
 import { EventList } from "./EventList"
 import { useModal } from "../hooks/useModal"
-import { RealtimeSessionCreateRequest } from "@tsorta/browser/openai"
+import {
+  RealtimeConversationItem,
+  RealtimeSessionCreateRequest,
+} from "@tsorta/browser/openai"
+import { ConversationView } from "./ConversationView"
 
 type PartialSessionRequestWithModel = Partial<RealtimeSessionCreateRequest> &
   Pick<Required<RealtimeSessionCreateRequest>, "model">
@@ -15,13 +19,15 @@ interface RealtimeSessionViewProps {
   stopSession: () => Promise<void>
   sessionStatus: "unavailable" | "stopped" | "recording"
   events: { type: string }[]
+  conversation?: RealtimeConversationItem[]
 }
 
 export function RealtimeSessionView({
   startSession,
   stopSession,
   sessionStatus,
   events,
+  conversation,
 }: RealtimeSessionViewProps): ReactNode {
   // TODO: allow user to select the model
   const model = "gpt-4o-realtime-preview-2024-12-17"
@@ -30,6 +36,10 @@ export function RealtimeSessionView({
     undefined
   )
 
+  const [activeTab, setActiveTab] = useState<"events" | "conversation">(
+    "events"
+  )
+
   const modal = useModal({
     title: "Edit Instructions",
     children: (
@@ -131,8 +141,66 @@ export function RealtimeSessionView({
         </li>
       </ul>
 
-      <h2>Events:</h2>
-      <EventList events={events} />
+      <ul className="nav nav-tabs mt-3" role="tablist">
+        <li className="nav-item" role="presentation">
+          <button
+            className={`nav-link ${activeTab === "events" ? "active" : ""}`}
+            id="events-tab"
+            type="button"
+            role="tab"
+            aria-controls="events"
+            aria-selected={activeTab === "conversation"}
+            onClick={() => setActiveTab("events")}
+          >
+            Events
+          </button>
+        </li>
+        <li className="nav-item" role="presentation">
+          <button
+            className={`nav-link ${
+              activeTab === "conversation" ? "active" : ""
+            }`}
+            id="conversation-tab"
+            type="button"
+            role="tab"
+            aria-controls="conversation"
+            aria-selected={activeTab === "conversation"}
+            onClick={() => setActiveTab("conversation")}
+          >
+            Conversation
+          </button>
+        </li>
+      </ul>
+      <div className="tab-content">
+        <div
+          className={`tab-events tab-pane fade ${
+            activeTab === "events" ? "show active" : ""
+          }`}
+          id="events"
+          role="tabpanel"
+          aria-labelledby="events-tab"
+        >
+          <EventList events={events} />
+        </div>
+        <div
+          className={`tab-events tab-pane fade ${
+            activeTab === "conversation" ? "show active" : ""
+          }`}
+          id="conversation"
+          role="tabpanel"
+          aria-labelledby="conversation-tab"
+        >
+          {conversation && conversation.length > 0 ? (
+            <ConversationView conversation={conversation} />
+          ) : (
+            <div className="alert alert-info m-2" role="alert">
+              {conversation !== undefined
+                ? "Conversation data not yet available. Start a session and talk and they should appear."
+                : "Conversations not available in this SDK"}
+            </div>
+          )}
+        </div>
+      </div>
     </div>
   )
 }

apps/browser-example/src/components/simpleConversation.ts

@@ -0,0 +1,43 @@
+import {
+  RealtimeConversationItem,
+  RealtimeConversationItemContent,
+} from "@tsorta/browser/openai"
+
+/** Removes the possibility of `undefined` in @see RealtimeConversationItem.role. */
+export type DefinedRole = NonNullable<RealtimeConversationItem["role"]>
+
+/** Removes the possibility of `undefined` in @see RealtimeConversationItem.type. */
+type DefinedRealtimeConversationItemType = NonNullable<
+  RealtimeConversationItem["type"]
+>
+
+/**
+ * A simplified form of @see RealtimeConversationItem for rendering.
+ */
+export type RealtimeConversationItemSimple = Pick<
+  RealtimeConversationItem,
+  "id"
+> & {
+  role: DefinedRole
+  type: DefinedRealtimeConversationItemType
+  content: RealtimeConversationItemContent[]
+}
+
+/**
+ * Simplifies the @see RealtimeConversationItem for rendering purposes. Mostly removes the possibility of `undefined` values in places where they are unlikely (impossible) at render time.
+ */
+export function simplifyItem(
+  item: RealtimeConversationItem
+): RealtimeConversationItemSimple {
+  if (!item.role) {
+    throw new Error("Role missing in conversation item")
+  }
+  const role: DefinedRole = item.role
+  const id = item.id || "id-missing"
+  const type = item.type as DefinedRealtimeConversationItemType
+  // NOTE: There may be no contents on the initial creation while the model is still replying.
+  const content: RealtimeConversationItemContent[] = (item.content ||
+    []) as RealtimeConversationItemContent[]
+
+  return { id, type, role, content }
+}

apps/browser-example/src/pages/WebRTCExample.tsx

@@ -5,6 +5,7 @@ import {
 } from "../components/RealtimeSessionView"
 import { RealtimeClient } from "@tsorta/browser/WebRTC"
 import { PageProps } from "./props"
+import { RealtimeConversationItem } from "@tsorta/browser/openai"
 
 export function WebRTCExample({
   apiKey,
@@ -15,6 +16,9 @@ export function WebRTCExample({
 
   const [client, setClient] = useState<RealtimeClient | undefined>(undefined)
   const [events, setEvents] = useState<any[]>([])
+  const [conversation, setConversation] = useState<RealtimeConversationItem[]>(
+    []
+  )
 
   const startSession = useCallback(
     async function startSession({
@@ -48,10 +52,13 @@ export function WebRTCExample({
       setClient(client)
 
       client.addEventListener("serverEvent", (event) => {
-        console.debug("serverEvent event:", event)
         setEvents((events) => [...events, event.event])
       })
 
+      client.addEventListener("conversationChanged", (event) => {
+        setConversation(event.conversation)
+      })
+
       await client.start()
 
       onSessionStatusChanged("recording")
@@ -79,11 +86,13 @@ export function WebRTCExample({
         by Scott Willeke.
       </p>
       <audio ref={audioElementRef}></audio>
+
       <RealtimeSessionView
         startSession={startSession}
         stopSession={stopSession}
         sessionStatus={sessionStatus}
         events={events}
+        conversation={conversation}
       />
     </div>
   )