Inspector UI and action improvements

Apps/Web/AI_Recorder/background/back.js

@@ -1,6 +1,6 @@
 var metaData = {};
 
-fetch("data.json")
+fetch("./data.json")
     .then(Response => Response.json())
     .then(data => {
         metaData = data;
@@ -9,6 +9,8 @@ fetch("data.json")
 const browserAppData = chrome || browser;
 
 import './back_zeuz.js';
+import './sentiment_analyzer.js';
+import './back_reocrder.js';
 // import '../common_files/poly_fill.js';
 
 /* Zeuz function start */
@@ -19,7 +21,7 @@ var clickEnabled = true;
 
 // import {getWindowSize} from "/back_zeuz.js";
 function getWindowSize(callback) {
-    chrome.storage.local.get('window', function(result) {
+    browserAppData.storage.local.get('window', function(result) {
         var height = 740;
         //var width = 780;
         var width = 1110;
@@ -112,60 +114,25 @@ function open_panel(tab) {
 
 /* Create menu */
 function create_menus() {
-    browserAppData.contextMenus.create({
-        id: "verifyText",
-        title: "verifyText",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "verifyTitle",
-        title: "verifyTitle",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "verifyValue",
-        title: "verifyValue",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "assertText",
-        title: "assertText",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "assertTitle",
-        title: "assertTitle",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "assertValue",
-        title: "assertValue",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "storeText",
-        title: "storeText",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "storeTitle",
-        title: "storeTitle",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
-    browserAppData.contextMenus.create({
-        id: "storeValue",
-        title: "storeValue",
-        documentUrlPatterns: ["<all_urls>"],
-        contexts: ["all"]
-    });
+    let menus = [
+        ["Go_to_link", "Go to link"],
+        ["Save_Text", "Save Text"],
+        ["Validate_Text", "Validate Text"],
+        ["Validate_Text_By_AI", "Validate Text by AI"],
+        ["Wait_For_Element_To_Appear", "Wait for Element to Appear"],
+        ["Wait_For_Element_To_Disappear", "Wait for Element to Disappear"],
+    ]
+    for(let i =0; i< menus.length; i++){
+        browserAppData.contextMenus.create({
+            id: menus[i][0],
+            title: menus[i][1],
+            documentUrlPatterns: [
+                "http://*/*",
+                "https://*/*"
+            ],
+            contexts: ["all"]
+        })
+    }
 }
 
 
@@ -189,47 +156,12 @@ browserAppData.contextMenus.onClicked.addListener(function(info, tab) {
 
 browserAppData.runtime.onConnect.addListener(function(m) {
     port = m;
+    console.log(port);
 });
 
 /* After install open the url */
-chrome.runtime.onInstalled.addListener(function (details) {
+browserAppData.runtime.onInstalled.addListener(function (details) {
     if (details.reason === 'install') {
         console.log("Recorder Installed");
     }
 });
-
-browserAppData.runtime.onMessage.addListener(
-    function(request, sender, sendResponse) {
-      if (request.apiName == 'ai_single_action') {
-        var url = `${metaData.url}/ai_record_single_action/`
-        var url_get = `${url}?${new URLSearchParams(request.dataj)}`;
-        console.log(url)
-        console.log(metaData.apiKey)
-        console.log(request.data)
-        console.log(request.dataj)
-        // fetch(url, {
-        // method: "GET",
-        // headers: {
-        //     "Content-Type": "application/json",
-        //     "X-Api-Key": metaData.apiKey,
-        // },
-        // })
-        // .then(response => response.text())
-        // .then(text => {console.log(text);sendResponse(text);})
-        // .catch(error => console.error(error))
-
-        fetch(url, {
-            method: "POST",
-            headers: {
-                // "Content-Type": "application/json",
-                "X-Api-Key": metaData.apiKey,
-            },
-            body: request.data,
-        })
-        .then(response => response.json())
-        .then(text => {console.log(text);sendResponse(text);})
-
-        return true;  // Will respond asynchronously.
-      }
-    }
-  );

Apps/Web/AI_Recorder/background/back_reocrder.js

@@ -0,0 +1,174 @@
+var metaData = {};
+
+fetch("./data.json")
+    .then(Response => Response.json())
+    .then(data => {
+        metaData = data;
+    });
+
+const browserAppData = chrome || browser;
+
+var idx = 0;
+var recorded_actions = [];
+var action_name_convert = {
+    type: "text",
+    open: "go to link",
+    Go_to_link: "go to link",
+    doubleClick: "double click",
+    Validate_Text: "validate full text",
+    Validate_Text_By_AI: "validate full text by ai",
+    Save_Text: "save attribute",
+    Wait_For_Element_To_Appear: "wait",
+    Wait_For_Element_To_Disappear: "wait disable",
+}
+
+async function start_recording(){
+    idx = 0;
+    recorded_actions = [];    
+}
+async function stop_recording(){
+    // When there are 2 iframes. it saves 3 times. this is a temporary fix. Should be fixed properly
+    if (recorded_actions.length > 0)
+    browserAppData.storage.local.set({
+        recorded_actions: recorded_actions,
+    }).then(()=>{
+        idx = 0;
+        recorded_actions = [];
+    }); 
+}
+async function fetchAIData(idx, command, value, url, document){
+    if (command === 'go to link'){
+        let go_to_link = {
+            action: 'go to link',
+            data_list: [url],
+            element: "",
+            is_disable: false,
+            name: `Open ${(url.length>25) ? url.slice(0,20) + '...' : url}`,
+            value: url,
+            main: [['go to link', 'selenium action', url]],
+            xpath: "",
+        };
+        recorded_actions[idx] = go_to_link;
+        console.log(recorded_actions);
+        browserAppData.storage.local.set({
+            recorded_actions: recorded_actions,
+        })
+        return;
+    }
+    recorded_actions[idx] = 'empty';
+    browserAppData.storage.local.set({
+        recorded_actions: recorded_actions,
+    })
+    let validate_full_text_by_ai = false
+    if (command === 'validate full text by ai'){
+        command = 'validate full text';
+        validate_full_text_by_ai = true;
+    }
+
+    var dataj = {
+        "page_src": document,
+        "action_name": command,
+        "action_type": "selenium",
+        "action_value": value,
+        "source": "web",
+    };
+    var data = JSON.stringify(dataj);
+
+    const url_ = `${metaData.url}/ai_record_single_action/`
+    const input = {
+        method: "POST",
+        headers: {
+            // "Content-Type": "application/json",
+            "X-Api-Key": metaData.apiKey,
+        },
+        body: data,
+    }
+    var r = await fetch(url_, input)
+    var resp = await r.json();
+    let response = resp.ai_choices;
+
+    if (validate_full_text_by_ai){
+        let text_classifier = await browserAppData.runtime.sendMessage({
+            action: 'content_classify',
+            text: value,
+        });
+
+        console.log("text_classifier", text_classifier);
+        let label = text_classifier[0].label;
+        label = label.charAt(0).toUpperCase() + label.slice(1).toLowerCase();
+        let offset = Number((text_classifier[0].score * 0.9).toFixed(2));
+        offset = Math.max(0.8, offset);
+        response[0].data_set = response[0].data_set.slice(0,-1)
+        .concat([[label, "text classifier offset", offset]])
+        .concat(response[0].data_set.slice(-1))
+        value = '';
+    }
+    else if (command === 'save attribute'){
+        response[0].data_set = response[0].data_set.slice(0,-1)
+        .concat([
+            ["text", "save parameter", "var_name"],
+            ["save attribute", "selenium action", "save attribute"],
+        ])
+        value = '';
+    }
+    else if (['wait', 'wait disable'].includes(command)){
+        value = 10;
+    }
+    response[0].short.value = value;
+    if (value) response[0].data_set[response[0].data_set.length-1][response[0].data_set[0].length-1] = value;
+    recorded_actions[idx] = {
+        action: response[0].short.action,
+        data_list: [response[0].short.value],
+        element: response[0].short.element,
+        is_disable: false,
+        name: response[0].name,
+        value: response[0].short.value,
+        main: response[0].data_set,
+        xpath: response[0].xpath,
+    };
+    console.log(recorded_actions);
+    browserAppData.storage.local.set({
+        recorded_actions: recorded_actions,
+    })
+}
+
+async function record_action(command, value, url, document){
+    if (Object.keys(action_name_convert).includes(command)) command = action_name_convert[command]
+    console.log("... Action recorder start");
+    idx += 1;
+    if (recorded_actions.length === 0 || recorded_actions.length > 0 && recorded_actions[0].action != 'go to link'){
+        let go_to_link = {
+            action: 'go to link',
+            data_list: [url],
+            element: "",
+            is_disable: false,
+            name: `Open ${(url.length>25) ? url.slice(0,20) + '...' : url}`,
+            value: url,
+            main: [['go to link', 'selenium action', url]],
+            xpath: "",
+        };
+        if (recorded_actions.length === 0) recorded_actions[0] = go_to_link;
+        else recorded_actions.unshift(go_to_link);
+        idx += 1;
+    }
+    fetchAIData(idx-1, command, value, url, document);  
+}
+browserAppData.runtime.onMessage.addListener(
+    function(request, sender, sendResponse) {
+        if (request.apiName == 'start_recording') {
+            start_recording();
+        }
+        else if (request.apiName == 'record_action') {
+            record_action(
+                request.command,
+                // request.target,
+                request.value,
+                request.url,
+                request.document,
+            );
+        }
+        else if (request.apiName == 'stop_recording') {
+            stop_recording();
+        }
+    }
+);

Apps/Web/AI_Recorder/background/back_zeuz.js

@@ -1,5 +1,5 @@
 var attachedTabs = {};
-const browser = chrome || browser;
+const browserAppData = chrome || browser;
 /* Fetch the window size */
 function getWindowSize(callback) {
     chrome.storage.local.get('window', function(result) {
@@ -222,10 +222,10 @@ if (chrome.debugger) {
 
 var externalCapabilities = {};
 
-browser.runtime.onMessage.addListener(function(request, sender, sendResponse, type) {
+browserAppData.runtime.onMessage.addListener(function(request, sender, sendResponse, type) {
     if (request.captureEntirePageScreenshot) {
         var windowId = request.captureWindowId || sender.tab.windowId;
-        browser.tabs.captureVisibleTab(windowId, { format: 'png' }).then(function(image) {
+        browserAppData.tabs.captureVisibleTab(windowId, { format: 'png' }).then(function(image) {
             sendResponse({
                 image: image
             });
@@ -288,7 +288,7 @@ chrome.runtime.onMessageExternal.addListener(function(message, sender) {
     }
 });
 
-browser.runtime.onMessage.addListener(function(message, sender, sendResponse) {
+browserAppData.runtime.onMessage.addListener(function(message, sender, sendResponse) {
     if (message.getExternalCapabilities) {
         var now = new Date().getTime();
         Object.keys(externalCapabilities).forEach(function(capabilityGlobalId) {
@@ -302,7 +302,7 @@ browser.runtime.onMessage.addListener(function(message, sender, sendResponse) {
     }
 });
 
-browser.runtime.onMessage.addListener(function(message, sender, sendResponse) {
+browserAppData.runtime.onMessage.addListener(function(message, sender, sendResponse) {
     if (message.checkChromeDebugger) {
         sendResponse({
             status: !!chrome.debugger