Update to handle text-only

src/lib/realtime-api-beta/dist/lib/conversation.d.ts

@@ -69,6 +69,12 @@ export class RealtimeConversation {
                 audio: Int16Array;
             };
         };
+        'response.text.delta': (event: any) => {
+            item: any;
+            delta: {
+                text: any;
+            };
+        };
         'response.function_call_arguments.delta': (event: any) => {
             item: any;
             delta: {

src/lib/realtime-api-beta/lib/client.js

@@ -346,6 +346,7 @@ export class RealtimeClient extends RealtimeEventHandler {
       handlerWithDispatch,
     );
     this.realtime.on('server.response.audio.delta', handlerWithDispatch);
+    this.realtime.on('server.response.text.delta', handlerWithDispatch);
     this.realtime.on(
       'server.response.function_call_arguments.delta',
       handlerWithDispatch,

src/lib/realtime-api-beta/lib/conversation.js

@@ -211,6 +211,16 @@ export class RealtimeConversation {
       );
       return { item, delta: { audio: appendValues } };
     },
+    'response.text.delta': (event) => {
+      const { item_id, content_index, delta } = event;
+      const item = this.itemLookup[item_id];
+      if (!item) {
+        throw new Error(`response.text.delta: Item "${item_id}" not found`);
+      }
+      item.content[content_index].text += delta;
+      item.formatted.text += delta;
+      return { item, delta: { text: delta } };
+    },
     'response.function_call_arguments.delta': (event) => {
       const { item_id, delta } = event;
       const item = this.itemLookup[item_id];

src/utils/conversation_config.js

@@ -2,7 +2,8 @@ export const instructions = `System settings:
 Tool use: enabled.
 
 Instructions:
-- You are an artificial intelligence agent responsible for helping test realtime voice capabilities.
+- You are an artificial intelligence agent responsible for helping test realtime voice capabilities
+- Please make sure to respond with a helpful voice via audio
 - Be kind, helpful, and curteous
 - It is okay to ask the user questions
 - Use tools and functions you have available liberally, it is part of the training apparatus