Support for running JavaScript as a query ( client-side execution ) (T…

…oolJet#1507)

* Custom js feature

* Disable transformations

* Fix

* Fix

* Fix

* Icon

* Rename variable OthetSources to OtherSources

* Fix

* Fix

Co-authored-by: Sherfin Shamsudeen <sherfin94@gmail.com>

frontend/package.json

@@ -16,6 +16,7 @@
     "@testing-library/user-event": "^7.2.1",
     "@uiw/react-codemirror": "^3.0.6",
     "array-move": "^3.0.1",
+    "axios": "^0.24.0",
     "babel-loader": "^8.0.5",
     "babel-plugin-console-source": "^2.0.5",
     "babel-plugin-import": "^1.13.3",

frontend/src/Editor/DataSourceManager/SourceComponents/Api/Runjs.schema.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json-schema.org/",
+  "$id": "https://tooljet.io/Runjs.schema.json",
+  "title": "Runjs datasource",
+  "description": "A schema defining runjs datasource",
+  "type": "object",
+  "source": {
+    "name": "Run JavaScript",
+    "kind": "runjs",
+    "exposedVariables": {
+      "isLoading": {},
+      "data": {},
+      "rawData": {}
+    },
+    "customTesting": true,
+    "disableTransformations": true
+  }
+}

frontend/src/Editor/DataSourceManager/SourceComponents/Api/Stripe.schema.json

@@ -13,12 +13,17 @@
       "rawData": {}
     },
     "options": {
-      "api_key": { "type": "string", "encrypted": true }
+      "api_key": {
+        "type": "string",
+        "encrypted": true
+      }
     },
     "customTesting": true
   },
   "defaults": {
-    "api_key": { "value": "" }
+    "api_key": {
+      "value": ""
+    }
   },
   "properties": {
     "api_key": {
@@ -28,5 +33,7 @@
       "description": "Api key for stripe"
     }
   },
-  "required": ["api_key"]
-}
+  "required": [
+    "api_key"
+  ]
+}

frontend/src/Editor/DataSourceManager/SourceComponents/index.js

@@ -24,6 +24,9 @@ import MssqlSchema from './Database/Mssql.schema.json';
 import S3Schema from './Database/S3.schema.json';
 import GcsSchema from './Database/Gcs.schema.json';
 
+// Other sources
+import RunjsSchema from './Api/Runjs.schema.json';
+
 const Airtable = ({ ...rest }) => <DynamicForm schema={AirtableSchema} {...rest} />;
 const Restapi = ({ ...rest }) => <DynamicForm schema={RestapiSchema} {...rest} />;
 const Graphql = ({ ...rest }) => <DynamicForm schema={GraphqlSchema} {...rest} />;
@@ -59,8 +62,10 @@ export const ApiSources = [
   GooglesheetSchema.source,
   SlackSchema.source,
 ];
+
+export const OtherSources = [RunjsSchema.source];
 export const CloudStorageSources = [S3Schema.source, GcsSchema.source];
-export const DataSourceTypes = [...DataBaseSources, ...ApiSources, ...CloudStorageSources];
+export const DataSourceTypes = [...DataBaseSources, ...ApiSources, ...CloudStorageSources, ...OtherSources];
 
 export const SourceComponents = {
   Elasticsearch,

frontend/src/Editor/QueryManager/QueryEditors/Runjs.jsx

@@ -0,0 +1,39 @@
+import React from 'react';
+import { CodeHinter } from '../../CodeBuilder/CodeHinter';
+import { changeOption } from './utils';
+import { defaults } from 'lodash';
+
+class Runjs extends React.Component {
+  constructor(props) {
+    super(props);
+    const options = defaults({ ...props.options }, { code: '//Type your JavaScript code here' });
+    this.state = {
+      options,
+    };
+  }
+
+  componentDidMount() {
+  }
+
+  render() {
+    return (
+      <div>
+        <CodeHinter
+          currentState={this.props.currentState}
+          initialValue={this.props.options.code}
+          mode="javascript"
+          theme={this.props.darkMode ? 'monokai' : 'base16-light'}
+          lineNumbers={true}
+          height={400}
+          className="query-hinter"
+          ignoreBraces={true}
+          onChange={(value) => changeOption(this, 'code', value)}
+          isMultiLineJs={false}
+          enablePreview={false}
+        />
+      </div>
+    );
+  }
+}
+
+export { Runjs };

frontend/src/Editor/QueryManager/QueryEditors/index.js

@@ -3,6 +3,7 @@ import React from 'react';
 import DynamicForm from '@/_components/DynamicForm';
 
 import { Restapi } from './Restapi';
+import { Runjs } from './Runjs';
 import { Stripe } from './Stripe';
 
 import MysqlSchema from './Mysql.schema.json';
@@ -37,6 +38,7 @@ const Gcs = ({ ...rest }) => <DynamicForm schema={GcsSchema} {...rest} />;
 
 export const allSources = {
   Restapi,
+  Runjs,
   Stripe,
   Mysql,
   Postgresql,

frontend/src/Editor/QueryManager/QueryManager.jsx

@@ -10,9 +10,15 @@ import ReactJson from 'react-json-view';
 import { previewQuery } from '@/_helpers/appUtils';
 import { EventManager } from '../Inspector/EventManager';
 import { CodeHinter } from '../CodeBuilder/CodeHinter';
+import {
+  DataSourceTypes
+} from '../DataSourceManager/SourceComponents';
 const queryNameRegex = new RegExp('^[A-Za-z0-9_-]*$');
 
-const staticDataSources = [{ kind: 'restapi', id: 'null', name: 'REST API' }];
+const staticDataSources = [
+  { kind: 'restapi', id: 'null', name: 'REST API' },
+  { kind: 'runjs', id: 'runjs', name: 'Run JavaScript code' }
+];
 
 let QueryManager = class QueryManager extends React.Component {
   constructor(props) {
@@ -22,6 +28,7 @@ let QueryManager = class QueryManager extends React.Component {
       options: {},
       selectedQuery: null,
       selectedDataSource: null,
+      dataSourceMeta: {}
     };
 
     this.previewPanelRef = React.createRef();
@@ -31,6 +38,7 @@ let QueryManager = class QueryManager extends React.Component {
     const selectedQuery = props.selectedQuery;
     const dataSourceId = selectedQuery?.data_source_id;
     const source = props.dataSources.find((datasource) => datasource.id === dataSourceId);
+    let dataSourceMeta = DataSourceTypes.find((source) => source.kind === selectedQuery?.kind);
     // const paneHeightChanged = this.state.queryPaneHeight !== props.queryPaneHeight;
 
     this.setState(
@@ -45,6 +53,7 @@ let QueryManager = class QueryManager extends React.Component {
         queryPaneHeight: props.queryPaneHeight,
         currentState: props.currentState,
         selectedSource: source,
+        dataSourceMeta
       },
       () => {
         if (this.props.mode === 'edit') {
@@ -54,6 +63,12 @@ let QueryManager = class QueryManager extends React.Component {
               source = { kind: 'restapi' };
             }
           }
+          if (selectedQuery.kind === 'runjs') {
+            if (!selectedQuery.data_source_id) {
+              source = { kind: 'runjs' };
+            }
+          }
+
           this.setState({
             options: selectedQuery.options,
             selectedDataSource: source,
@@ -83,7 +98,7 @@ let QueryManager = class QueryManager extends React.Component {
   changeDataSource = (sourceId) => {
     const source = [...this.state.dataSources, ...staticDataSources].find((datasource) => datasource.id === sourceId);
 
-    const isSchemaUnavailable = ['restapi', 'stripe'].includes(source.kind);
+    const isSchemaUnavailable = ['restapi', 'stripe', 'runjs'].includes(source.kind);
     const schemaUnavailableOptions = {
       restapi: {
         method: 'get',
@@ -93,6 +108,7 @@ let QueryManager = class QueryManager extends React.Component {
         body: [],
       },
       stripe: {},
+      runjs: {},
     };
 
     this.setState({
@@ -245,6 +261,7 @@ let QueryManager = class QueryManager extends React.Component {
       queryName,
       previewLoading,
       queryPreviewData,
+      dataSourceMeta
     } = this.state;
 
     let ElementToRender = '';
@@ -393,15 +410,19 @@ let QueryManager = class QueryManager extends React.Component {
                       darkMode={this.props.darkMode}
                       isEditMode={this.props.mode === 'edit'}
                     />
-                    <hr></hr>
-                    <div className="mb-3 mt-2">
-                      <Transformation
-                        changeOption={this.optionchanged}
-                        options={this.state.options}
-                        currentState={currentState}
-                        darkMode={this.props.darkMode}
-                      />
-                    </div>
+                    {!dataSourceMeta?.disableTransformations &&
+                      <div>
+                        <hr></hr>
+                        <div className="mb-3 mt-2">
+                          <Transformation
+                            changeOption={this.optionchanged}
+                            options={this.state.options}
+                            currentState={currentState}
+                            darkMode={this.props.darkMode}
+                          />
+                        </div>
+                      </div>
+                    }
                     <div className="row preview-header border-top" ref={this.previewPanelRef}>
                       <div className="py-2">Preview</div>
                     </div>

frontend/src/_helpers/appUtils.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import { toast } from 'react-toastify';
-import { getDynamicVariables, resolveReferences, serializeNestedObjectToQueryParams } from '@/_helpers/utils';
+import { getDynamicVariables, resolveReferences, executeMultilineJS, serializeNestedObjectToQueryParams } from '@/_helpers/utils';
 import { dataqueryService } from '@/_services';
 import _ from 'lodash';
 import moment from 'moment';
@@ -411,8 +411,15 @@ export function previewQuery(_ref, query) {
   _ref.setState({ previewLoading: true });
 
   return new Promise(function (resolve, reject) {
-    dataqueryService
-      .preview(query, options)
+
+    let queryExecutionPromise = null;
+    if (query.kind === 'runjs') {
+      queryExecutionPromise = executeMultilineJS(_ref.state.currentState, query.options.code);
+    } else {
+      queryExecutionPromise = dataqueryService.preview(query, options);
+    }
+
+    queryExecutionPromise
       .then((data) => {
         let finalData = data.data;
 
@@ -497,8 +504,15 @@ export function runQuery(_ref, queryId, queryName, confirmed = undefined, mode)
 
   return new Promise(function (resolve, reject) {
     _self.setState({ currentState: newState }, () => {
-      dataqueryService
-        .run(queryId, options)
+
+      let queryExecutionPromise = null;
+      if (query.kind === 'runjs') {
+        queryExecutionPromise = executeMultilineJS(_self.state.currentState, query.options.code);
+      } else {
+        queryExecutionPromise = dataqueryService.run(queryId, options)
+      }
+
+      queryExecutionPromise
         .then((data) => {
           if (data.status === 'needs_oauth') {
             const url = data.data.auth_url; // Backend generates and return sthe auth url

frontend/src/_helpers/utils.js

@@ -1,5 +1,6 @@
 import moment from 'moment';
 import _ from 'lodash';
+import axios from 'axios';
 
 export function findProp(obj, prop, defval) {
   if (typeof defval === 'undefined') defval = null;
@@ -234,6 +235,24 @@ export function validateWidget({ validationObject, widgetValue, currentState, cu
 }
 
 export function validateEmail(email) {
-  const emailRegex =  /^(([^<>()[\]\.,;:\s@\"]+(\.[^<>()[\]\.,;:\s@\"]+)*)|(\".+\"))@(([^<>()[\]\.,;:\s@\"]+\.)+[^<>()[\]\.,;:\s@\"]{2,})$/i;
+  const emailRegex = /^(([^<>()[\]\.,;:\s@\"]+(\.[^<>()[\]\.,;:\s@\"]+)*)|(\".+\"))@(([^<>()[\]\.,;:\s@\"]+\.)+[^<>()[\]\.,;:\s@\"]{2,})$/i;
   return emailRegex.test(email)
-}
+}
+
+export async function executeMultilineJS(currentState, code) {
+  let result = {}, error = null;
+
+  try {
+
+    const AsyncFunction = new Function(`return Object.getPrototypeOf(async function(){}).constructor`)();
+    var evalFn = new AsyncFunction('moment', '_', 'components', 'queries', 'globals', 'axios', code);
+    result = { status: 'ok', data: await evalFn(moment, _, currentState.components, currentState.queries, currentState.globals, axios) };
+
+  } catch (err) {
+    console.log('JS execution failed: ', err);
+    error = err.stack.split('\n')[0];
+    result = { status: 'failed', data: { message: error, description: error } };
+  }
+
+  return result;
+}