diff --git a/src/lib/realtime-api-beta/dist/lib/client.d.ts.map b/src/lib/realtime-api-beta/dist/lib/client.d.ts.map index 2cd988d..78b6ed3 100644 --- a/src/lib/realtime-api-beta/dist/lib/client.d.ts.map +++ b/src/lib/realtime-api-beta/dist/lib/client.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../lib/client.js"],"names":[],"mappings":"AAKA;;;GAGG;AAEH;;;;GAIG;AAEH;;;;;;GAMG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;;;;;;;GAcG;AAEH;;GAEG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;GAQG;AAEH;;;;;;GAMG;AAEH;;;;;;GAMG;AAEH;;;;;;;;GAQG;AAEH;;;;;;GAMG;AAEH;;GAEG;AAEH;;GAEG;AAEH;;;;GAIG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;;;GAMG;AAEH;;;GAGG;AACH;IACE;;;OAGG;IACH,sEAFW;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,+BAA+B,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAC,EAsCrG;IAlCC;;;;;;;;;;;;MAYC;IACD,kBAAuB;IACvB;;QAIC;IACD;;;;;MAKC;IACD,sBAKE;IACF,mCAA8C;IAKhD;;;;OAIG;IACH,qBAMC;IALC,wBAA2B;IAC3B,UAAe;IAEf,sBAAyC;IAI3C;;;;OAIG;IACH,6BAiHC;IAED;;;OAGG;IACH,eAFa,OAAO,CAInB;IAED;;;OAGG;IACH,SAFa,IAAI,CAShB;IAED;;;;OAIG;IACH,WAFa,OAAO,CAAC,IAAI,CAAC,CASzB;IAED;;;OAGG;IACH,yBAFa,OAAO,CAAC,IAAI,CAAC,CAUzB;IAED;;OAEG;IACH,mBAIC;IAED;;;OAGG;IACH,wBAFa,YAAY,GAAC,IAAI,CAI7B;IAED;;;;;OAKG;IACH,oBAJW,kBAAkB,sBAEhB;QAAC,UAAU,EAAE,kBAAkB,CAAC;QAAC,OAAO,WAAU;KAAC,CAkB/D;IAED;;;;OAIG;IACH,iBAHW,MAAM,GACJ,IAAI,CAQhB;IAED;;;;OAIG;IACH,eAHW,MAAM,GACJ,IAAI,CAKhB;IAED;;;;OAIG;IACH,sMAFW,mBAAmB,WA4D7B;IAED;;;;OAIG;IACH,iCAHW,KAAK,CAAC,oBAAoB,GAAC,qBAAqB,CAAC,GAC/C,IAAI,CAqBhB;IAED;;;;OAIG;IACH,8BAHW,UAAU,GAAC,WAAW,GACpB,IAAI,CAahB;IAED;;;OAGG;IACH,kBAFa,IAAI,CAahB;IAED;;;;;;OAMG;IACH,mBAJW,MAAM,gBACN,MAAM,GACJ;QAAC,IAAI,EAAE,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAA;KAAC,CAgC9C;IAED;;;OAGG;IACH,mBAFa,OAAO,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAC,CAAC,CAMrC;IAED;;;OAGG;IACH,4BAFa,OAAO,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAC,CAAC,CAMrC;CACF;;;;8BA/oBY,OAAO,GAAC,WAAW,GAAC,WAAW;;cAK9B,OAAO;WACP,WAAW;;;UAKX,YAAY;gBACZ,MAAM;wBACN,MAAM;0BACN,MAAM;;;;;;WAMN,UAAU;UACV,MAAM;iBACN,MAAM;gBACN;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC;;;YAKpB,MAAM;iBACN,MAAM,EAAE;mBACR,MAAM;YACN,OAAO,GAAC,SAAS,GAAC,MAAM;yBACxB,eAAe;0BACf,eAAe;gCACf,sBAAsB,GAAC,IAAI;qBAC3B,0BAA0B,GAAC,IAAI;YAC/B,kBAAkB,EAAE;kBACpB,MAAM,GAAC,MAAM,GAAC,UAAU,GAAC;QAAC,IAAI,EAAC,UAAU,CAAC;QAAA,IAAI,EAAC,MAAM,CAAA;KAAC;kBACtD,MAAM;iCACN,MAAM,GAAC,KAAK;;6BAIb,aAAa,GAAC,WAAW,GAAC,YAAY;;UAKrC,YAAY;UACZ,MAAM;;;UAKN,aAAa;;;;YACb,MAAM;iBACN,MAAM,GAAC,IAAI;;;UAKX,MAAM;UACN,MAAM;;;UAKN,OAAO;;;;YACP,MAAM;iBACN,MAAM,GAAC,IAAI;;;uBAKX,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,QAAQ;aACR,KAAK,CAAC,oBAAoB,CAAC;;;uBAK3B,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,QAAQ;aACR,KAAK,CAAC,oBAAoB,GAAC,qBAAqB,CAAC;;;uBAKjD,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,WAAW;aACX,KAAK,CAAC,eAAe,GAAC,gBAAgB,CAAC;;;uBAKvC,MAAM,GAAC,IAAI;UACX,eAAe;YACf,cAAc;aACd,MAAM;UACN,MAAM;eACN,MAAM;;;uBAKN,MAAM,GAAC,IAAI;UACX,sBAAsB;aACtB,MAAM;YACN,MAAM;;;UAKN,UAAU;UACV,MAAM;aACN,MAAM;eACN,MAAM;;;YAKN,UAAU;WACV,MAAM;iBACN,MAAM;WACN,iBAAiB;aACjB,MAAM;WACN,GAAG;;;QAKH,MAAM;YACN,MAAM;WACN,MAAM,GAAC,WAAW,GAAC,QAAQ;eAC3B,qBAAqB;;2BAItB,cAAc,GAAC,YAAY,GAAC,iBAAiB,GAAC,oBAAoB,GAAC,0BAA0B;uBAI7F,iBAAiB,GAAG,YAAY;;UAK/B,YAAY;YACZ,cAAc,GAAC,mBAAmB,GAAC,gBAAgB;;;UAKnD,QAAQ;WACR;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC,GAAC,IAAI;;;kBAKpC,MAAM;kBACN,MAAM;mBACN,MAAM;;;YAKN,aAAa,GAAC,WAAW,GAAC,YAAY,GAAC,WAAW,GAAC,QAAQ;oBAC3D,4BAA4B,GAAC,wBAAwB,GAAC,IAAI;YAC1D,QAAQ,EAAE;WACV,SAAS,GAAC,IAAI;;qCAtLS,oBAAoB;4BAC7B,UAAU;qCACD,mBAAmB"} \ No newline at end of file +{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../lib/client.js"],"names":[],"mappings":"AAKA;;;GAGG;AAEH;;;;GAIG;AAEH;;;;;;GAMG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;;;;;;;GAcG;AAEH;;GAEG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;GAOG;AAEH;;;;;;;;GAQG;AAEH;;;;;;GAMG;AAEH;;;;;;GAMG;AAEH;;;;;;;;GAQG;AAEH;;;;;;GAMG;AAEH;;GAEG;AAEH;;GAEG;AAEH;;;;GAIG;AAEH;;;;GAIG;AAEH;;;;;GAKG;AAEH;;;;;;GAMG;AAEH;;;GAGG;AACH;IACE;;;OAGG;IACH,sEAFW;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,+BAA+B,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAC,EAsCrG;IAlCC;;;;;;;;;;;;MAYC;IACD,kBAAuB;IACvB;;QAIC;IACD;;;;;MAKC;IACD,sBAKE;IACF,mCAA8C;IAKhD;;;;OAIG;IACH,qBAMC;IALC,wBAA2B;IAC3B,UAAe;IAEf,sBAAyC;IAI3C;;;;OAIG;IACH,6BAkHC;IAED;;;OAGG;IACH,eAFa,OAAO,CAInB;IAED;;;OAGG;IACH,SAFa,IAAI,CAShB;IAED;;;;OAIG;IACH,WAFa,OAAO,CAAC,IAAI,CAAC,CASzB;IAED;;;OAGG;IACH,yBAFa,OAAO,CAAC,IAAI,CAAC,CAUzB;IAED;;OAEG;IACH,mBAIC;IAED;;;OAGG;IACH,wBAFa,YAAY,GAAC,IAAI,CAI7B;IAED;;;;;OAKG;IACH,oBAJW,kBAAkB,sBAEhB;QAAC,UAAU,EAAE,kBAAkB,CAAC;QAAC,OAAO,WAAU;KAAC,CAkB/D;IAED;;;;OAIG;IACH,iBAHW,MAAM,GACJ,IAAI,CAQhB;IAED;;;;OAIG;IACH,eAHW,MAAM,GACJ,IAAI,CAKhB;IAED;;;;OAIG;IACH,sMAFW,mBAAmB,WA4D7B;IAED;;;;OAIG;IACH,iCAHW,KAAK,CAAC,oBAAoB,GAAC,qBAAqB,CAAC,GAC/C,IAAI,CAqBhB;IAED;;;;OAIG;IACH,8BAHW,UAAU,GAAC,WAAW,GACpB,IAAI,CAahB;IAED;;;OAGG;IACH,kBAFa,IAAI,CAahB;IAED;;;;;;OAMG;IACH,mBAJW,MAAM,gBACN,MAAM,GACJ;QAAC,IAAI,EAAE,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAA;KAAC,CAgC9C;IAED;;;OAGG;IACH,mBAFa,OAAO,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAC,CAAC,CAMrC;IAED;;;OAGG;IACH,4BAFa,OAAO,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAC,CAAC,CAMrC;CACF;;;;8BAhpBY,OAAO,GAAC,WAAW,GAAC,WAAW;;cAK9B,OAAO;WACP,WAAW;;;UAKX,YAAY;gBACZ,MAAM;wBACN,MAAM;0BACN,MAAM;;;;;;WAMN,UAAU;UACV,MAAM;iBACN,MAAM;gBACN;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC;;;YAKpB,MAAM;iBACN,MAAM,EAAE;mBACR,MAAM;YACN,OAAO,GAAC,SAAS,GAAC,MAAM;yBACxB,eAAe;0BACf,eAAe;gCACf,sBAAsB,GAAC,IAAI;qBAC3B,0BAA0B,GAAC,IAAI;YAC/B,kBAAkB,EAAE;kBACpB,MAAM,GAAC,MAAM,GAAC,UAAU,GAAC;QAAC,IAAI,EAAC,UAAU,CAAC;QAAA,IAAI,EAAC,MAAM,CAAA;KAAC;kBACtD,MAAM;iCACN,MAAM,GAAC,KAAK;;6BAIb,aAAa,GAAC,WAAW,GAAC,YAAY;;UAKrC,YAAY;UACZ,MAAM;;;UAKN,aAAa;;;;YACb,MAAM;iBACN,MAAM,GAAC,IAAI;;;UAKX,MAAM;UACN,MAAM;;;UAKN,OAAO;;;;YACP,MAAM;iBACN,MAAM,GAAC,IAAI;;;uBAKX,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,QAAQ;aACR,KAAK,CAAC,oBAAoB,CAAC;;;uBAK3B,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,QAAQ;aACR,KAAK,CAAC,oBAAoB,GAAC,qBAAqB,CAAC;;;uBAKjD,MAAM,GAAC,IAAI;UACX,SAAS;YACT,cAAc;UACd,WAAW;aACX,KAAK,CAAC,eAAe,GAAC,gBAAgB,CAAC;;;uBAKvC,MAAM,GAAC,IAAI;UACX,eAAe;YACf,cAAc;aACd,MAAM;UACN,MAAM;eACN,MAAM;;;uBAKN,MAAM,GAAC,IAAI;UACX,sBAAsB;aACtB,MAAM;YACN,MAAM;;;UAKN,UAAU;UACV,MAAM;aACN,MAAM;eACN,MAAM;;;YAKN,UAAU;WACV,MAAM;iBACN,MAAM;WACN,iBAAiB;aACjB,MAAM;WACN,GAAG;;;QAKH,MAAM;YACN,MAAM;WACN,MAAM,GAAC,WAAW,GAAC,QAAQ;eAC3B,qBAAqB;;2BAItB,cAAc,GAAC,YAAY,GAAC,iBAAiB,GAAC,oBAAoB,GAAC,0BAA0B;uBAI7F,iBAAiB,GAAG,YAAY;;UAK/B,YAAY;YACZ,cAAc,GAAC,mBAAmB,GAAC,gBAAgB;;;UAKnD,QAAQ;WACR;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC,GAAC,IAAI;;;kBAKpC,MAAM;kBACN,MAAM;mBACN,MAAM;;;YAKN,aAAa,GAAC,WAAW,GAAC,YAAY,GAAC,WAAW,GAAC,QAAQ;oBAC3D,4BAA4B,GAAC,wBAAwB,GAAC,IAAI;YAC1D,QAAQ,EAAE;WACV,SAAS,GAAC,IAAI;;qCAtLS,oBAAoB;4BAC7B,UAAU;qCACD,mBAAmB"} \ No newline at end of file diff --git a/src/lib/realtime-api-beta/dist/lib/conversation.d.ts b/src/lib/realtime-api-beta/dist/lib/conversation.d.ts index 4a85279..ab8b055 100644 --- a/src/lib/realtime-api-beta/dist/lib/conversation.d.ts +++ b/src/lib/realtime-api-beta/dist/lib/conversation.d.ts @@ -69,6 +69,12 @@ export class RealtimeConversation { audio: Int16Array; }; }; + 'response.text.delta': (event: any) => { + item: any; + delta: { + text: any; + }; + }; 'response.function_call_arguments.delta': (event: any) => { item: any; delta: { diff --git a/src/lib/realtime-api-beta/dist/lib/conversation.d.ts.map b/src/lib/realtime-api-beta/dist/lib/conversation.d.ts.map index 9951534..a929c88 100644 --- a/src/lib/realtime-api-beta/dist/lib/conversation.d.ts.map +++ b/src/lib/realtime-api-beta/dist/lib/conversation.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../../lib/conversation.js"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AAEH;;;;GAIG;AACH;IACE,yBAA0B;IAE1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MA6ME;IAxKQ,6BAA4B;IAkLtC;;;OAGG;IACH,SAFa,IAAI,CAWhB;IARC,eAAoB;IACpB,aAAe;IACf,mBAAwB;IACxB,iBAAmB;IACnB,sBAA2B;IAC3B,0BAA+B;IAKjC;;;;OAIG;IACH,4BAHW,UAAU,GACR,UAAU,CAKtB;IAED;;;;;OAKG;IACH,kCAHe,GAAG,EAAA,GACL,IAAI,CAkBhB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,OAAO,aAAa,EAAE,QAAQ,CAI1C;IAED;;;OAGG;IACH,YAFa,OAAO,aAAa,EAAE,QAAQ,EAAE,CAI5C;CACF;;;;;;WAtSa,MAAM;YACN,UAAU;gBACV,MAAM;iBACN,MAAM"} \ No newline at end of file +{"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../../lib/conversation.js"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AAEH;;;;GAIG;AACH;IACE,yBAA0B;IAE1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAuNE;IAlLQ,6BAA4B;IA4LtC;;;OAGG;IACH,SAFa,IAAI,CAWhB;IARC,eAAoB;IACpB,aAAe;IACf,mBAAwB;IACxB,iBAAmB;IACnB,sBAA2B;IAC3B,0BAA+B;IAKjC;;;;OAIG;IACH,4BAHW,UAAU,GACR,UAAU,CAKtB;IAED;;;;;OAKG;IACH,kCAHe,GAAG,EAAA,GACL,IAAI,CAkBhB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,OAAO,aAAa,EAAE,QAAQ,CAI1C;IAED;;;OAGG;IACH,YAFa,OAAO,aAAa,EAAE,QAAQ,EAAE,CAI5C;CACF;;;;;;WAhTa,MAAM;YACN,UAAU;gBACV,MAAM;iBACN,MAAM"} \ No newline at end of file diff --git a/src/lib/realtime-api-beta/lib/client.js b/src/lib/realtime-api-beta/lib/client.js index 1ef0cf2..949a2bc 100644 --- a/src/lib/realtime-api-beta/lib/client.js +++ b/src/lib/realtime-api-beta/lib/client.js @@ -346,6 +346,7 @@ export class RealtimeClient extends RealtimeEventHandler { handlerWithDispatch, ); this.realtime.on('server.response.audio.delta', handlerWithDispatch); + this.realtime.on('server.response.text.delta', handlerWithDispatch); this.realtime.on( 'server.response.function_call_arguments.delta', handlerWithDispatch, diff --git a/src/lib/realtime-api-beta/lib/conversation.js b/src/lib/realtime-api-beta/lib/conversation.js index 0c66daf..ffc5183 100644 --- a/src/lib/realtime-api-beta/lib/conversation.js +++ b/src/lib/realtime-api-beta/lib/conversation.js @@ -211,6 +211,16 @@ export class RealtimeConversation { ); return { item, delta: { audio: appendValues } }; }, + 'response.text.delta': (event) => { + const { item_id, content_index, delta } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error(`response.text.delta: Item "${item_id}" not found`); + } + item.content[content_index].text += delta; + item.formatted.text += delta; + return { item, delta: { text: delta } }; + }, 'response.function_call_arguments.delta': (event) => { const { item_id, delta } = event; const item = this.itemLookup[item_id]; diff --git a/src/utils/conversation_config.js b/src/utils/conversation_config.js index 04996f5..bb9a049 100644 --- a/src/utils/conversation_config.js +++ b/src/utils/conversation_config.js @@ -2,7 +2,8 @@ export const instructions = `System settings: Tool use: enabled. Instructions: -- You are an artificial intelligence agent responsible for helping test realtime voice capabilities. +- You are an artificial intelligence agent responsible for helping test realtime voice capabilities +- Please make sure to respond with a helpful voice via audio - Be kind, helpful, and curteous - It is okay to ask the user questions - Use tools and functions you have available liberally, it is part of the training apparatus