wip

Author: tschellenbach

examples/other_examples/bedrock_nova/bedrock_qwen_example.py

@@ -6,7 +6,7 @@
 
 from vision_agents.core import User
 from vision_agents.core.agents import Agent
-from vision_agents.plugins import bedrock, getstream
+from vision_agents.plugins import bedrock, getstream, cartesia, deepgram, smart_turn
 
 load_dotenv()
 
@@ -20,6 +20,10 @@ async def start_agent() -> None:
         agent_user=User(name="Friendly AI"),
         instructions="Be nice to the user",
         llm=bedrock.LLM(model="anthropic.claude-3-5-sonnet-20241022-v2:0"),
+        tts=cartesia.TTS(),
+        stt=deepgram.STT(),
+        turn_detection=smart_turn.TurnDetection(buffer_duration=2.0, confidence_threshold=0.5),
+        # Enable turn detection with FAL/ Smart turn
     )
     await agent.create_user()
 
@@ -29,6 +33,7 @@ async def start_agent() -> None:
     with await agent.join(call):
         await asyncio.sleep(5)
         await agent.llm.simple_response(text="Say hi")
+        await agent.finish()
 
 
 if __name__ == "__main__":

plugins/bedrock/vision_agents/plugins/bedrock/bedrock_realtime.py

@@ -162,6 +162,7 @@ async def simple_audio_response(self, pcm: PcmData):
         if not self.connected:
             self.logger.warning("realtime is not active. can't call simple_audio_response")
 
+        logger.info("sar")
         #self.logger.info("Simple audio response for model %s", self.model)
         content_name = str(uuid.uuid4())
         audio_bytes = pcm
@@ -354,6 +355,7 @@ async def _handle_events(self):
                     result = await output[1].receive()
                     if result.value and result.value.bytes_:
                         try:
+                            logger.info("received...")
                             response_data = result.value.bytes_.decode('utf-8')
                             json_data = json.loads(response_data)
 
@@ -382,18 +384,21 @@ async def _handle_events(self):
                                     logger.info("Completion start from Bedrock", json_data['event']['completionStart'])
                                 elif 'audioOutput' in json_data['event']:
                                     logger.info("Audio output from Bedrock")
-                                    audio_content = json_data['event']['audioOutput']['content']
-                                    audio_bytes = base64.b64decode(audio_content)
-                                    #await self.audio_output_queue.put(audio_bytes)
-
-                                    audio_event = RealtimeAudioOutputEvent(
-                                        plugin_name="gemini",
-                                        audio_data=audio_content,
-                                        sample_rate=24000
-                                    )
-                                    self.events.send(audio_event)
-
-                                    await self.output_track.write(audio_content)
+                                    try:
+                                        audio_content = json_data['event']['audioOutput']['content']
+                                        audio_bytes = base64.b64decode(audio_content)
+                                        #await self.audio_output_queue.put(audio_bytes)
+
+                                        audio_event = RealtimeAudioOutputEvent(
+                                            plugin_name="gemini",
+                                            audio_data=audio_bytes,
+                                            sample_rate=24000
+                                        )
+                                        self.events.send(audio_event)
+
+                                        await self.output_track.write(audio_bytes)
+                                    except Exception as e:
+                                        import pdb; pdb.set_trace()
 
                                 elif 'toolUse' in json_data['event']:
                                     logger.info(f"Tool use from Bedrock: {json_data['event']['toolUse']}")