ai tts instructions

Author: tschellenbach

docs/ai/instructions/ai-tts.md

@@ -0,0 +1,40 @@
+## TTS 
+
+Here's a minimal example for building a new TTS plugin
+
+```python
+
+
+class MyTTS(tts.TTS):
+    def __init__(
+        self,
+        voice_id: str = "VR6AewLTigWG4xSOukaG",  # Default ElevenLabs voice
+        model_id: str = "eleven_multilingual_v2",
+        client: Optional[MyClient] = None,
+    ):
+        # it should be possible to pass the client (makes it easier for users to customize things)
+        # settings that are common to change, like voice id or model id should be configurable as well
+        super().__init__()
+        self.voice_id = voice_id
+        self.client = client if client is not None else MyClient(api_key=api_key)
+
+    async def stream_audio(self, text: str, *_, **__) -> AsyncIterator[bytes]:
+
+        audio_stream = self.client.text_to_speech.stream(
+            text=text,
+            voice_id=self.voice_id,
+            output_format=self.output_format,
+            model_id=self.model_id,
+            request_options={"chunk_size": 64000},
+        )
+
+        return audio_stream
+
+```
+
+TODO: the stop part can be generic
+TODO: Track handling can be improved
+
+## Testing your TTS
+
+TOOD: no good test suite yet

plugins/bedrock/vision_agents/plugins/bedrock/bedrock_realtime.py

@@ -52,7 +52,12 @@ class Realtime(realtime.Realtime):
     Input event docs: https://docs.aws.amazon.com/nova/latest/userguide/input-events.html
     Available voices are documented here:
     https://docs.aws.amazon.com/nova/latest/userguide/available-voices.html
-    
+
+    Resumption example:
+    https://github.com/aws-samples/amazon-nova-samples/tree/main/speech-to-speech/repeatable-patterns/resume-conversation
+
+
+
     Examples:
     
         from vision_agents.plugins import bedrock
@@ -160,7 +165,6 @@ async def simple_audio_response(self, pcm: PcmData):
         if not self.connected:
             self.logger.warning("realtime is not active. can't call simple_audio_response")
 
-        logger.info("sar")
         # Resample from 48kHz to 24kHz if needed
         pcm = pcm.resample(24000)
 
@@ -268,6 +272,7 @@ async def start_prompt(self):
             }
           }
         }
+        # TODO: tool support
         await self.send_event(event)
 
 
@@ -351,13 +356,11 @@ async def _handle_events(self):
         """Process incoming responses from Bedrock."""
         try:
             while True:
-                logger.info("iter")
                 try:
                     output = await self.stream.await_output()
                     result = await output[1].receive()
                     if result.value and result.value.bytes_:
                         try:
-                            logger.info("received...")
                             response_data = result.value.bytes_.decode('utf-8')
                             json_data = json.loads(response_data)
 
@@ -385,7 +388,6 @@ async def _handle_events(self):
                                 elif 'completionStart' in json_data['event']:
                                     logger.info("Completion start from Bedrock", json_data['event']['completionStart'])
                                 elif 'audioOutput' in json_data['event']:
-                                    logger.info("Audio output from Bedrock")
                                     audio_content = json_data['event']['audioOutput']['content']
                                     audio_bytes = base64.b64decode(audio_content)
                                     #await self.audio_output_queue.put(audio_bytes)
@@ -422,7 +424,12 @@ async def _handle_events(self):
                                     await self.send_tool_result_event(toolContent, toolResult)
                                     await self.send_tool_content_end_event(toolContent)
                                 elif 'contentEnd' in json_data['event']:
-                                    logger.info(f"Content end from Bedrock: {json_data['event']['contentEnd']}")
+
+
+                                    stopReason = json_data['event']['contentEnd']['stopReason']
+                                    if stopReason == "INTERRUPTED":
+                                        logger.info("TODO: should flush audio buffer")
+                                    logger.info(f"Content end from Bedrock {stopReason}: {json_data['event']['contentEnd']}")
 
                                 elif 'completionEnd' in json_data['event']:
                                     logger.info(f"Completion end from Bedrock: {json_data['event']['completionEnd']}")