Merge branch 'dev' into UI-updates

ae/core/autogen_wrapper.py

@@ -132,7 +132,9 @@ def reflection_message(recipient, messages, sender, config): # type: ignore
             last_message=messages[-1]["content"] # type: ignore
             content_json = parse_response(last_message) # type: ignore
             next_step = content_json.get('next_step', None)
-            if next_step is None: 
+
+            if next_step is None:
+                print ("Message to nested chat returned None")
                 return None
             else:
                 next_step = next_step.strip() +" " + get_url() # type: ignore
@@ -337,7 +339,6 @@ async def process_command(self, command: str, current_url: str | None = None) ->
         try:
             if self.agents_map is None:
                 raise ValueError("Agents map is not initialized.")
-            print(self.agents_map["browser_nav_executor"].function_map) # type: ignore
 
             result=await self.agents_map["user"].a_initiate_chat( # type: ignore
                 self.agents_map["planner_agent"], # self.manager # type: ignore

ae/core/post_process_responses.py

@@ -35,9 +35,8 @@ def final_reply_callback_user_proxy(recipient: autogen.ConversableAgent, message
 
     return False, None
 
-
 def final_reply_callback_planner_agent(message:str, message_type:MessageType = MessageType.STEP): # type: ignore 
-            browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
-            loop = asyncio.get_event_loop()
-            loop.run_until_complete(browser_manager.notify_user(message, message_type=message_type))
-            return False, None  # required to ensure the agent communication flow continues
+        browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(browser_manager.notify_user(message, message_type=message_type))
+        return False, None  # required to ensure the agent communication flow continues

ae/core/prompts.py

@@ -20,6 +20,7 @@
 4. Very Important: Helper cannot go back to previous pages. If you need the helper to return to a previous page, you must explicitly add the URL of the previous page in the step (e.g. return to the search result page by navigating to the url https://www.google.com/search?q=Finland")
 
 Guidelines:
+
 1. If you know a URL, you can provide it to the helper to navigate to a new page (e.g. go to www.espn.com).  
 2. Do not assume any capability exists on the webpage. Ask questions to the helper to confirm the presence of features (e.g. is there a sort by price feature available on the page?). This will help you revise the plan as needed and also establish common ground with the helper.
 3. Do not combine multiple steps into one. A step should be strictly as simple as interacting with a single element or navigating to a page. If you need to interact with multiple elements or perform multiple actions, you will break it down into multiple steps.
@@ -84,10 +85,9 @@
    Execute function sequentially to avoid navigation timing issues. Once a task is completed, confirm completion with ##TERMINATE TASK##.
    The given actions are NOT parallelizable. They are intended for sequential execution.
    If you need to call multiple functions in a task step, call one function at a time. Wait for the function's response before invoking the next function. This is important to avoid collision.
-   Prefer to submit a form by clicking the submit button. Use enter key press only strictly when the submit button is not available.
-   Important: Always press Enter key after inputting dates as it is the most common way to submit dates.
-   Important: Unless otherwise specified, the task must be performed strictly  the current page. Do not use openurl unless explicitly instructed to navigate to a page.
-   You will NOT provide any URLs of links on webpage. If user asks for URLs, you will instead provide the text of the hyperlink on the page and offer to click on it. This is very very important. 
+   Strictly for search fields, submit the field by pressing Enter key. For other forms, click on the submit button.
+   Unless otherwise specified, the task must be performed on the current page. Use openurl only when explicitly instructed to navigate to a new page with a url specified. If you do not know the URL ask for it.
+   You will NOT provide any URLs of links on webpage. If user asks for URLs, you will instead provide the text of the hyperlink on the page and offer to click on it. This is very very important.
    When inputing information, remember to follow the format of the input field. For example, if the input field is a date field, you will enter the date in the correct format (e.g. YYYY-MM-DD), you may get clues from the placeholder text in the input field.
    if the task is ambigous or there are multiple options to choose from, you will ask the user for clarification. You will not make any assumptions.
    Individual function will reply with action success and if any changes were observed as a consequence. Adjust your approach based on this feedback.

ae/core/ui_manager.py

@@ -93,6 +93,7 @@ def update_overlay_state(self, is_collapsed: bool):
         self.overlay_is_collapsed = is_collapsed
 
 
+
     async def update_overlay_show_details(self, show_details: bool, page: Page):
         """
         Updates the state of the overlay to either show steps or not.

ae/utils/autogen_sequential_function_call.py

@@ -2,6 +2,7 @@
 import asyncio
 import inspect
 from typing import Any
+
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -12,6 +13,7 @@
 from autogen import UserProxyAgent
 
 
+
 class UserProxyAgent_SequentialFunctionExecution(UserProxyAgent):
     def __init__(self, *args, **kwargs): # type: ignore
         super().__init__(*args, **kwargs) # type: ignore

ae/utils/dom_mutation_observer.py

@@ -1,6 +1,11 @@
 
 import asyncio
 
+import json
+from typing import Callable  # noqa: UP035
+
+from playwright.async_api import Page
+
 # Create an event loop
 loop = asyncio.get_event_loop()
 

ae/utils/response_parser.py

@@ -8,7 +8,6 @@ def parse_response(message: str) -> dict[str, Any]:
     """
     # Parse the response content
     json_response = {}
-
     #if message starts with ``` and ends with ``` then remove them
     if message.startswith("```"):
         message = message[3:]
@@ -23,10 +22,8 @@ def parse_response(message: str) -> dict[str, Any]:
     except Exception as e:
         # If the response is not a valid JSON, try pass it using string matching.
         #This should seldom be triggered
-
         message = message.replace("\\n", "\n")
         message = message.replace("\n", "") # type: ignore
-
         if ("plan" in message and "next_step" in message):
             start = message.index("plan") + len("plan")
             end = message.index("next_step")