feat: request_login

src/extension/content/index.ts

@@ -44,6 +44,11 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
           sendResponse(result);
           break;
         }
+        case 'request_user_help': {
+          request_user_help(request.task_id, request.failure_type, request.failure_message);
+          sendResponse(true);
+          break;
+        }
         case 'computer:type': {
           sendResponse(type(request));
           break;
@@ -225,7 +230,7 @@ function scroll_to(request: any): boolean {
       return false;
     }
     element.scrollIntoView({
-      behavior: 'smooth'
+      behavior: 'smooth',
     });
   } else if (request.xpath) {
     let xpath = request.xpath as string;
@@ -241,7 +246,7 @@ function scroll_to(request: any): boolean {
       return false;
     }
     element.scrollIntoView({
-      behavior: 'smooth'
+      behavior: 'smooth',
     });
   } else {
     const to_coordinate = request.to_coordinate as [number, number];
@@ -298,7 +303,9 @@ function select_dropdown_option(request: any): any {
   if (!select || select.tagName.toUpperCase() !== 'SELECT') {
     return { success: false, error: 'Select not found or invalid element type' };
   }
-  const option = Array.from(select.options).find((opt: any) => opt.text.trim() === request.text) as any;
+  const option = Array.from(select.options).find(
+    (opt: any) => opt.text.trim() === request.text
+  ) as any;
   if (!option) {
     return {
       success: false,
@@ -314,3 +321,138 @@ function select_dropdown_option(request: any): any {
     selectedText: option.text.trim(),
   };
 }
+
+function request_user_help(task_id: string, failure_type: string, failure_message: string) {
+  const domId = 'eko-request-user-help';
+  if (document.getElementById(domId)) {
+    return;
+  }
+
+  const failureTitleMap: any = {
+    login_required: 'Login Required',
+    captcha: 'Captcha Detected',
+    blocked: 'Blocked',
+    other: 'Error',
+    rate_limited: 'Rate Limited',
+  };
+
+  const notification = document.createElement('div');
+  notification.id = domId;
+  notification.style.cssText = `
+    position: fixed;
+    top: 5px;
+    left: 18px;
+    z-index: 9999;
+    background-color: #FEF0ED;
+    color: white;
+    padding: 16px;
+    border-radius: 12px;
+    border: 1px solid #FBB8A5;
+    font-family: Arial, sans-serif;
+    width: 350px;
+    display: flex;
+    flex-direction: row;
+    gap: 10px;
+    cursor: move;
+    user-select: none;
+  `;
+
+  let isDragging = false;
+  let xOffset = 0;
+  let yOffset = 0;
+  let initialX = 0;
+  let initialY = 0;
+
+  notification.addEventListener('mousedown', (e) => {
+    isDragging = true;
+    initialX = e.clientX - xOffset;
+    initialY = e.clientY - yOffset;
+    e.preventDefault();
+  });
+
+  document.addEventListener('mousemove', (e) => {
+    if (!isDragging) return;
+    const currentX = e.clientX - initialX;
+    const currentY = e.clientY - initialY;
+    xOffset = currentX;
+    yOffset = currentY;
+    notification.style.transform = `translate(${xOffset}px, ${yOffset}px)`;
+  });
+
+  document.addEventListener('mouseup', () => {
+    isDragging = false;
+  });
+
+  const leftContainer = document.createElement('div');
+  leftContainer.style.cssText = `
+    width: 28px;
+    height: 28px;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    border-radius: 99px;
+    background: #FDCCCC;
+    justify-content: center;
+  `;
+  leftContainer.innerHTML = ``;
+
+  const rightContainer = document.createElement('div');
+  rightContainer.style.cssText = `
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+  `;
+
+  const title = document.createElement('div');
+  title.style.cssText = `
+    font-size: 16px;
+    font-weight: 700;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+  title.innerText = failureTitleMap[failure_type] || failure_type;
+
+  const message2 = document.createElement('div');
+  message2.style.cssText = `
+    font-size: 16px;
+    font-weight: 400;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+  message2.innerText = failure_message + '\nWhen you resolve the issue, click the button below.';
+
+  const buttonDiv = document.createElement('div');
+  buttonDiv.style.cssText = `
+    margin-top: 16px;
+    display: flex;
+    flex-direction: row-reverse;
+    justify-content: flex-start;
+    align-items: center;
+  `;
+
+  const resolvedBut = document.createElement('div');
+  resolvedBut.innerText = 'Resolved';
+  resolvedBut.style.cssText = `
+    border-radius: 8px;
+    background: #DD342D;
+    color: white;
+    padding: 10px;
+    border: none;
+    cursor: pointer;
+  `;
+  resolvedBut.onclick = () => {
+    chrome.runtime.sendMessage({ type: 'issue_resolved', task_id, failure_type }, () => {
+      notification.remove();
+    });
+  };
+
+  buttonDiv.appendChild(resolvedBut);
+  rightContainer.appendChild(title);
+  rightContainer.appendChild(message2);
+  rightContainer.appendChild(buttonDiv);
+  notification.appendChild(leftContainer);
+  notification.appendChild(rightContainer);
+  document.body.appendChild(notification);
+}

src/extension/tools/index.ts

@@ -7,6 +7,7 @@ import { OpenUrl } from './open_url';
 import { Screenshot } from './screenshot';
 import { TabManagement } from './tab_management';
 import { WebSearch } from './web_search';
+import { RequestLogin } from './request_login';
 
 export {
   BrowserUse,
@@ -18,4 +19,5 @@ export {
   Screenshot,
   TabManagement,
   WebSearch,
+  RequestLogin,
 };

src/extension/tools/request_login.ts

@@ -0,0 +1,63 @@
+import { Tool, InputSchema, ExecutionContext } from '../../types/action.types';
+import { getTabId, doesTabExists } from '../utils';
+
+export class RequestLogin implements Tool<any, any> {
+  name: string;
+  description: string;
+  input_schema: InputSchema;
+
+  constructor() {
+    this.name = 'request_login';
+    this.description = 'request the user to log in to the current website, assist with identity verification when manual intervention is needed, guide users through the login process, and wait for their confirmation of successful login.';
+    this.input_schema = {
+      type: 'object',
+      properties: {},
+    };
+  }
+
+  async execute(context: ExecutionContext, params: any): Promise<any> {
+    let tabId = await getTabId(context);
+    let task_id = 'login_required_' + tabId;
+    const request_user_help = async () => {
+      await chrome.tabs.sendMessage(tabId, {
+        type: 'request_user_help',
+        task_id,
+        failure_type: 'login_required',
+        failure_message: 'Access page require user authentication.',
+      });
+    };
+    const login_interval = setInterval(async () => {
+      try {
+        request_user_help();
+      } catch (e) {
+        clearInterval(login_interval);
+      }
+    }, 2000);
+    try {
+      let result = await this.awaitLogin(tabId, task_id);
+      return { result };
+    } finally {
+      clearInterval(login_interval);
+    }
+  }
+
+  async awaitLogin(tabId: number, task_id: string): Promise<boolean> {
+    return new Promise((resolve) => {
+      const checkTabClosedInterval = setInterval(async () => {
+        const tabExists = await doesTabExists(tabId);
+        if (!tabExists) {
+          clearInterval(checkTabClosedInterval);
+          resolve(false);
+          chrome.runtime.onMessage.removeListener(listener);
+        }
+      }, 1000);
+      const listener = (message: any) => {
+        if (message.type === 'issue_resolved' && message.task_id === task_id) {
+          resolve(true);
+          clearInterval(checkTabClosedInterval);
+        }
+      };
+      chrome.runtime.onMessage.addListener(listener);
+    });
+  }
+}

src/extension/utils.ts

@@ -143,6 +143,19 @@ export async function waitForTabComplete(
   });
 }
 
+export async function doesTabExists(tabId: number) {
+  const tabExists = await new Promise((resolve) => {
+    chrome.tabs.get(tabId, (tab) => {
+      if (chrome.runtime.lastError) {
+        resolve(false);
+      } else {
+        resolve(true);
+      }
+    });
+  });
+  return tabExists;
+}
+
 export async function getPageSize(tabId?: number): Promise<[number, number]> {
   if (!tabId) {
     tabId = await getCurrentTabId();