fallback to use js click

Skyvern-AI · Nov 27, 2024 · 52f0c71 · 52f0c71
1 parent a2d9b05
commit 52f0c71
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 10 deletions.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
@@ -1262,18 +1262,34 @@ async def fc_func(fc: FileChooser) -> None:
             )
             return action_results
 
-        blocking_element = await skyvern_element.find_blocking_element(
+        blocking_element, blocked = await skyvern_element.find_blocking_element(
             dom=DomUtil(scraped_page=scraped_page, page=page)
         )
         if blocking_element is None:
+            if not blocked:
+                LOG.info(
+                    "Chain click: exit since the element is not blocking by any element",
+                    task_id=task.task_id,
+                    action=action,
+                    element=str(skyvern_element),
+                    locator=locator,
+                )
+                return action_results
+
             LOG.info(
-                "Chain click: exit since the element is not blocking by any skyvern element",
+                "Chain click: element is blocked by an non-interactable element, going to use javascript click instead of playwright click",
                 task_id=task.task_id,
                 action=action,
                 element=str(skyvern_element),
                 locator=locator,
             )
-            return action_results
+            try:
+                await skyvern_element.click_in_javascript()
+                action_results.append(ActionSuccess())
+                return action_results
+            except Exception as e:
+                action_results.append(ActionFailure(FailToClick(action.element_id, anchor="self_js", msg=str(e))))
+                return action_results
 
         try:
             LOG.debug(

diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
@@ -427,10 +427,10 @@ function getBlockElementUniqueID(element) {
   );
 
   if (!hitElement) {
-    return "";
+    return ["", false];
   }
 
-  return hitElement.getAttribute("unique_id") ?? "";
+  return [hitElement.getAttribute("unique_id") ?? "", true];
 }
 
 function isHidden(element) {

diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
@@ -320,12 +320,12 @@ async def get_element_handler(
         assert handler is not None
         return handler
 
-    async def find_blocking_element(self, dom: DomUtil) -> SkyvernElement | None:
+    async def find_blocking_element(self, dom: DomUtil) -> tuple[SkyvernElement | None, bool]:
         skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
-        blocking_element_id = await skyvern_frame.get_blocking_element_id(await self.get_element_handler())
+        blocking_element_id, blocked = await skyvern_frame.get_blocking_element_id(await self.get_element_handler())
         if not blocking_element_id:
-            return None
-        return await dom.get_skyvern_element_by_id(blocking_element_id)
+            return None, blocked
+        return await dom.get_skyvern_element_by_id(blocking_element_id), blocked
 
     async def find_element_in_label_children(
         self, dom: DomUtil, element_type: InteractiveElement
@@ -576,6 +576,10 @@ async def move_mouse_to(
 
         return dest_x, dest_y
 
+    async def click_in_javascript(self) -> None:
+        skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
+        await skyvern_frame.click_element_in_javascript(await self.get_element_handler())
+
     async def coordinate_click(
         self, page: Page, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
     ) -> None:

diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
@@ -182,7 +182,7 @@ async def get_disabled_from_style(self, element: ElementHandle) -> bool:
         js_script = "(element) => checkDisabledFromStyle(element)"
         return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
 
-    async def get_blocking_element_id(self, element: ElementHandle) -> str:
+    async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:
         js_script = "(element) => getBlockElementUniqueID(element)"
         return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
 
@@ -239,3 +239,7 @@ async def is_sibling(self, el1: ElementHandle, el2: ElementHandle) -> bool:
     async def has_ASP_client_control(self) -> bool:
         js_script = "() => hasASPClientControl()"
         return await self.evaluate(frame=self.frame, expression=js_script)
+
+    async def click_element_in_javascript(self, element: ElementHandle) -> None:
+        js_script = "(element) => element.click()"
+        return await self.evaluate(frame=self.frame, expression=js_script, arg=element)