Merge branch 'main' into screenshots

Author: lhuanyu

.husky/commit-msg

@@ -1 +1,6 @@
-npx --no -- commitlint --edit "$1" 
+#!/bin/sh
+
+# Ensure node is in PATH (for fnm users)
+export PATH="$HOME/Library/Application Support/fnm/aliases/default/bin:$PATH"
+
+npx --no -- commitlint --edit "$1"

apps/chrome-extension/static/manifest.json

@@ -1,7 +1,7 @@
 {
   "name": "Midscene.js",
   "description": "Open-source SDK for automating web pages using natural language through AI.",
-  "version": "0.130",
+  "version": "0.132",
   "manifest_version": 3,
   "permissions": [
     "activeTab",

apps/playground/src/App.tsx

@@ -127,7 +127,10 @@ export default function App() {
                 <div className="playground-panel-header">
                   <div className="header-row">
                     <Logo />
-                    <NavActions showEnvConfig={false} />
+                    <NavActions
+                      showTooltipWhenEmpty={false}
+                      showModelName={false}
+                    />
                   </div>
                 </div>
 

apps/report/src/components/detail-side/index.tsx

@@ -212,6 +212,16 @@ const DetailSide = (): JSX.Element => {
             },
           ]
         : []),
+      ...(task?.searchAreaUsage
+        ? [
+            {
+              key: 'searchAreaUsage',
+              content: (
+                <pre>{JSON.stringify(task.searchAreaUsage, undefined, 2)}</pre>
+              ),
+            },
+          ]
+        : []),
       ...(task?.usage
         ? [
             {

apps/report/src/components/sidebar/index.less

@@ -73,7 +73,7 @@
 
   .task-meta-tokens {
     display: grid;
-    grid-template-columns: 1fr auto 112px 26px;
+    grid-template-columns: 1fr auto minmax(100px, 1fr) 55px 65px;
     gap: 8px;
     align-items: center;
     font-size: 12px;
@@ -91,11 +91,11 @@
       justify-content: center;
 
       &:nth-child(2) {
-        grid-column: 3;
+        grid-column: 4;
       }
 
       &:nth-child(3) {
-        grid-column: 4;
+        grid-column: 5;
       }
 
       .token-value {
@@ -168,7 +168,7 @@
 
 
     &.pro-mode {
-      grid-template-columns: 1fr auto 100px 55px 65px;
+      grid-template-columns: 1fr auto minmax(100px, 1fr) 55px 65px;
     }
 
     .status-icon {
@@ -230,7 +230,7 @@
     flex-shrink: 0;
 
     &.pro-mode {
-      grid-template-columns: 1fr auto 100px 55px 65px;
+      grid-template-columns: 1fr auto minmax(100px, 1fr) 55px 65px;
     }
 
     .header-name {

apps/report/src/components/sidebar/index.tsx

@@ -1,7 +1,16 @@
 import './index.less';
 import { useAllCurrentTasks, useExecutionDump } from '@/components/store';
-import type { ExecutionTask, ExecutionTaskInsightLocate } from '@midscene/core';
+import type {
+  AIUsageInfo,
+  ExecutionTask,
+  ExecutionTaskInsightLocate,
+} from '@midscene/core';
 import { typeStr } from '@midscene/core/agent';
+
+// Extended task type with searchAreaUsage
+type ExecutionTaskWithSearchAreaUsage = ExecutionTask & {
+  searchAreaUsage?: AIUsageInfo;
+};
 import {
   type AnimationScript,
   iconForStatus,
@@ -16,7 +25,7 @@ import type { PlaywrightTasks } from '../../types';
 import ReportOverview from '../report-overview';
 
 const SideItem = (props: {
-  task: ExecutionTask;
+  task: ExecutionTaskWithSearchAreaUsage;
   selected?: boolean;
   onClick?: () => void;
   onItemHover?: (task: ExecutionTask | null, x?: number, y?: number) => any;
@@ -138,19 +147,31 @@ const SideItem = (props: {
         {/* Display usage info in table style if available and Pro mode is enabled */}
         {props.proModeEnabled && (
           <>
-            <Tooltip title="The name of the model used this time">
+            <Tooltip title={task.usage?.model_name || ''}>
               <div className="usage-column model-name">
                 {task.usage?.model_name || '-'}
               </div>
             </Tooltip>
             <Tooltip title="Input tokens sent to the AI model">
               <div className="usage-column prompt-tokens">
-                {task.usage?.prompt_tokens || '-'}
+                {(() => {
+                  const mainUsage = task.usage?.prompt_tokens || 0;
+                  const searchAreaUsage =
+                    task.searchAreaUsage?.prompt_tokens || 0;
+                  const total = mainUsage + searchAreaUsage;
+                  return total > 0 ? total : '-';
+                })()}
               </div>
             </Tooltip>
             <Tooltip title="Output tokens generated by the AI model">
               <div className="usage-column completion-tokens">
-                {task.usage?.completion_tokens || '-'}
+                {(() => {
+                  const mainUsage = task.usage?.completion_tokens || 0;
+                  const searchAreaUsage =
+                    task.searchAreaUsage?.completion_tokens || 0;
+                  const total = mainUsage + searchAreaUsage;
+                  return total > 0 ? total : '-';
+                })()}
               </div>
             </Tooltip>
           </>
@@ -235,13 +256,24 @@ const Sidebar = (props: SidebarProps = {}): JSX.Element => {
   const totalPromptTokens =
     groupedDump?.executions
       .flatMap((e) => e.tasks)
-      .reduce((acc, task) => acc + (task.usage?.prompt_tokens || 0), 0) || 0;
+      .reduce((acc, task) => {
+        const mainUsage = task.usage?.prompt_tokens || 0;
+        const searchAreaUsage =
+          (task as ExecutionTaskWithSearchAreaUsage).searchAreaUsage
+            ?.prompt_tokens || 0;
+        return acc + mainUsage + searchAreaUsage;
+      }, 0) || 0;
 
   const totalCompletionTokens =
     groupedDump?.executions
       .flatMap((e) => e.tasks)
-      .reduce((acc, task) => acc + (task.usage?.completion_tokens || 0), 0) ||
-    0;
+      .reduce((acc, task) => {
+        const mainUsage = task.usage?.completion_tokens || 0;
+        const searchAreaUsage =
+          (task as ExecutionTaskWithSearchAreaUsage).searchAreaUsage
+            ?.completion_tokens || 0;
+        return acc + mainUsage + searchAreaUsage;
+      }, 0) || 0;
 
   const executionContent = groupedDump ? (
     <div className="execution-info-section">

apps/report/test-data/ai-model.json

@@ -174,11 +174,50 @@ web:
 
   # Whether to ignore HTTPS certificate errors, optional, defaults to false.
   acceptInsecureCerts: <boolean>
+```
+
+### Global agent configuration
+
+If you need to use the `aiActionContext` parameter, you can set it through the global `agent` configuration:
 
+```yaml
+# Global AI agent configuration
+agent:
   # Background knowledge to send to the AI model when calling aiAction, optional.
   aiActionContext: <string>
 ```
 
+:::tip aiActionContext configuration
+
+- **Applicable environments**: Web, iOS, and Android environments can all use the global `agent` configuration to set `aiActionContext`
+- **Purpose**: Provides background knowledge to the AI model, like how to handle popups, business introduction, etc.
+
+:::
+
+#### Usage example
+
+```yaml
+# Global agent configuration, applies to all environments
+agent:
+  aiActionContext: "If any popup appears, click agree. If login page appears, skip it."
+
+# iOS environment configuration
+ios:
+  launch: https://www.bing.com
+  wdaPort: 8100
+
+# Or Android environment configuration
+android:
+  deviceId: s4ey59
+  launch: https://www.bing.com
+
+tasks:
+  - name: Search for weather
+    flow:
+      - ai: Search for "today's weather"
+      - aiAssert: The results show weather information
+```
+
 ### The `android` part
 
 ```yaml

apps/site/docs/en/automate-with-scripts-in-yaml.mdx

@@ -8,7 +8,7 @@ The bridge mode in the Midscene Chrome extension is a tool that allows you to us
 
 Using the desktop version of Chrome allows you to reuse all cookies, plugins, page status, and everything else you want. You can work with automation scripts to complete your tasks. This mode is commonly referred to as 'man-in-the-loop' in the context of automation.
 
-![bridge mode](/midscene-bridge-mode.jpg)
+![bridge mode](/midscene-bridge-mode.png)
 
 :::info Demo Project
 check the demo project of bridge mode: [https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo](https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo)
@@ -57,7 +57,7 @@ Promise.resolve(
 Start the chrome extension and switch to 'Bridge Mode' tab. Click "Allow connection".
 
 <p align="center">
-  <img src="/bridge_in_extension.jpg" alt="bridge in extension" width="400"/>
+  <img src="/bridge_in_extension.png" alt="bridge in extension" width="400"/>
 </p>
 
 ## Step 5. Run the script