Support lazy scroll and auto complete for PPL parse command

dashboards-observability/common/constants/autocomplete.ts

@@ -10,6 +10,7 @@ export const pipeCommands = [
 { label: 'eval' },
 { label: 'fields' },
 { label: 'head' },
+{ label: 'parse' },
 { label: 'rare' },
 { label: 'rename' },
 { label: 'sort' },
@@ -62,4 +63,4 @@ export type indexItem = {
 export type dataItem = {
     label: string;
     doc_count: any;
-}
+}

dashboards-observability/common/constants/explorer.ts

@@ -28,6 +28,7 @@ export const SAVED_OBJECT_TYPE = 'objectType';
 export const TAB_CREATED_TYPE = 'tabCreatedType';
 export const NEW_TAB = 'newTab';
 export const REDIRECT_TAB = 'redirect_tab';
+export const PAGE_SIZE = 50;
 
 export const DATE_PICKER_FORMAT = 'YYYY-MM-DD HH:mm:ss';
 export const TIME_INTERVAL_OPTIONS = [

dashboards-observability/common/utils/index.ts

@@ -3,5 +3,5 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-export { getIndexPatternFromRawQuery, insertDateRangeToQuery } from './query_utils';
+export { getIndexPatternFromRawQuery, preprocessQuery } from './query_utils';
 export { uiSettingsService } from './settings_service';

dashboards-observability/common/utils/query_utils.ts

@@ -20,7 +20,12 @@ export const getIndexPatternFromRawQuery = (query: string) : string => {
   return '';
 };
 
-export const insertDateRangeToQuery = ({
+const commandExists = (query: string, command: string): boolean => {
+  return new RegExp(`\\|\\s*${command}\\b`).test(query);
+}
+
+// insert time filter command and additional commands based on raw query
+export const preprocessQuery = ({
   rawQuery,
   startTime,
   endTime,
@@ -42,7 +47,13 @@ export const insertDateRangeToQuery = ({
   const tokens = rawQuery.replaceAll(PPL_NEWLINE_REGEX, '').match(PPL_INDEX_INSERT_POINT_REGEX);
 
   if (isEmpty(tokens)) return finalQuery;
-  finalQuery = `${tokens![1]}=${tokens![2]} | where ${timeField} >= '${start}' and ${timeField} <= '${end}'${tokens![3]}`;
+
+  let conditions = `| where ${timeField} >= '${start}' and ${timeField} <= '${end}'`;
+  if (commandExists(rawQuery, 'parse')) {
+    conditions += ` | sort - ${timeField} | head 10000`;
+  }
+
+  finalQuery = `${tokens![1]}=${tokens![2]} ${conditions} ${tokens![3]}`;
 
   return finalQuery;
 };

dashboards-observability/public/components/common/search/autocomplete_logic.tsx

@@ -210,7 +210,7 @@ export const onItemSelect = async ({ setQuery, item }: { setQuery: any, item: an
       else if (splittedModel[splittedModel.length - 2] === 'fields') {
         inFieldsCommaLoop = true;
         return fillSuggestions(str, prefix, fieldsFromBackend);
-      } else if (splittedModel[splittedModel.length - 2] === 'dedup') {
+      } else if (['dedup', 'parse'].includes(splittedModel[splittedModel.length - 2])) {
         return fillSuggestions(str, prefix, fieldsFromBackend);
       } else if (splittedModel[splittedModel.length - 2] === 'where') {
         nextWhere = splittedModel.length;
@@ -265,4 +265,4 @@ export const onItemSelect = async ({ setQuery, item }: { setQuery: any, item: an
       return [];
     }
   };
-
+

dashboards-observability/public/components/explorer/__tests__/__snapshots__/data_grid.test.tsx.snap

@@ -256,7 +256,7 @@ exports[`Datagrid component Renders data grid component 1`] = `
             "timestamp": "2021-05-24 00:00:00",
           }
         }
-        key="doc_view8"
+        key="doc_view11"
         selectedCols={
           Array [
             Object {
@@ -356,7 +356,7 @@ exports[`Datagrid component Renders data grid component 1`] = `
             "timestamp": "2021-05-24 18:27:00",
           }
         }
-        key="doc_view9"
+        key="doc_view12"
         selectedCols={
           Array [
             Object {
@@ -426,5 +426,6 @@ exports[`Datagrid component Renders data grid component 1`] = `
       </DocViewRow>
     </tbody>
   </table>
+  <div />
 </DataGrid>
 `;

dashboards-observability/public/components/explorer/data_grid.scss

@@ -42,7 +42,7 @@
 .osd-table,
 .osdDocTable {
   /**
-  *  Style ES document _source in table view <dt>key:<dt><dd>value</dd>
+  *  Style OpenSearch document _source in table view <dt>key:<dt><dd>value</dd>
   *  Use alpha so this will stand out against non-white backgrounds, e.g. the highlighted
   *  row in the Context Log.
   */

dashboards-observability/public/components/explorer/data_grid.tsx

@@ -5,10 +5,11 @@
 
 import './data_grid.scss';
 
-import React, { useMemo } from 'react';
+import React, { useMemo, useState, useCallback, useEffect, useRef } from 'react';
 import { uniqueId } from 'lodash';
 import { DocViewRow } from './docTable/index';
 import { IExplorerFields, IField } from '../../../common/types/explorer';
+import { PAGE_SIZE } from '../../../common/constants/explorer';
 
 interface DataGridProps {
   rows: Array<any>;
@@ -22,20 +23,49 @@ export function DataGrid(props: DataGridProps) {
     rowsAll,
     explorerFields
   } = props;
+  const [limit, setLimit] = useState(PAGE_SIZE);
+  const loader = useRef<HTMLDivElement>(null);
+
+  const handleObserver = useCallback((entries) => {
+    if (entries[0].isIntersecting) {
+      setLimit((limit) => limit + PAGE_SIZE);
+    }
+  }, []);
+
+  useEffect(() => {
+    if (!loader.current || limit >= rows.length + PAGE_SIZE) return;
+
+    const observer = new IntersectionObserver(handleObserver, {
+      root: null,
+      rootMargin: '500px',
+      threshold: 0,
+    });
+    observer.observe(loader.current);
+
+    return () => observer.disconnect();
+  }, [limit]);
 
   const getTrs = (
     docs: Array<any> = [],
-    explorerFields: Array<IField>
+    explorerFields: Array<IField>,
+    prevTrs: any[] = [],
   ) => {
-    return docs.map((doc) => {
-      return (
+    if (prevTrs.length >= docs.length) return prevTrs;
+
+    // reset limit if no previous table rows
+    if (prevTrs.length === 0 && limit !== PAGE_SIZE) setLimit(PAGE_SIZE);
+
+    const upperLimit = Math.min(prevTrs.length === 0 ? PAGE_SIZE : limit, docs.length);
+    for (let i = prevTrs.length; i < upperLimit; i++) {
+      prevTrs.push(
         <DocViewRow
           key={ uniqueId('doc_view') } 
-          doc={ doc }
+          doc={ docs[i] }
           selectedCols={ explorerFields }
         />
-      );
-    });
+      )
+    }
+    return prevTrs;
   };
 
   const defaultCols = [
@@ -59,7 +89,7 @@ export function DataGrid(props: DataGridProps) {
           </>
       );
     } else {
-      tableHeadContent = fields.map(selField => {
+      tableHeadContent = fields.map((selField: any) => {
         return (
           <th key={ uniqueId('datagrid-header-')}>{ selField.name }</th>
         );
@@ -79,22 +109,26 @@ export function DataGrid(props: DataGridProps) {
     () => getHeaders(explorerFields.queriedFields), 
     [ explorerFields.queriedFields ]
   );
-  const QueriedtableRows = useMemo(
-    () => getTrs(rows, explorerFields.queriedFields),
-    [ rows, explorerFields.queriedFields ]
-  );
+  const [QueriedtableRows, setQueriedtableRows] = useState<any[]>([]);
+  useEffect(() => {
+    setQueriedtableRows(getTrs(rows, explorerFields.queriedFields));
+  }, [ rows, explorerFields.queriedFields ]);
+
   const headers = useMemo(
     () => getHeaders(explorerFields.selectedFields),
     [ explorerFields.selectedFields ]
   );
-  const tableRows = useMemo(
-    () => {
-      const dataToRender = explorerFields?.queriedFields && explorerFields.queriedFields.length > 0 ? rowsAll : rows
-      return getTrs(dataToRender, explorerFields.selectedFields);
-    },
-    [ rows, explorerFields.selectedFields ]
-  );
+  const [tableRows, setTableRows] = useState<any[]>([]);
+  useEffect(() => {
+    const dataToRender = explorerFields?.queriedFields && explorerFields.queriedFields.length > 0 ? rowsAll : rows;
+    setTableRows(getTrs(dataToRender, explorerFields.selectedFields));
+  }, [ rows, explorerFields.selectedFields ]);
 
+  useEffect(() => {
+    setQueriedtableRows((prev) => getTrs(rows, explorerFields.queriedFields, prev));
+    const dataToRender = explorerFields?.queriedFields && explorerFields.queriedFields.length > 0 ? rowsAll : rows;
+    setTableRows((prev) => getTrs(dataToRender, explorerFields.selectedFields, prev));
+  }, [limit]);
 
   return (
     <>
@@ -120,7 +154,7 @@ export function DataGrid(props: DataGridProps) {
           explorerFields?.queriedFields?.length > 0 &&
           explorerFields.selectedFields?.length === 0
         ) ? null : (
-          <table 
+          <table
             className="osd-table table"
             data-test-subj="docTable">
             <thead>
@@ -132,6 +166,7 @@ export function DataGrid(props: DataGridProps) {
           </table>
         )
       }
+      <div ref={loader} />
     </>
   )
 }

dashboards-observability/public/components/explorer/explorer.tsx

@@ -46,7 +46,7 @@ import {
   EVENT_ANALYTICS_DOCUMENTATION_URL,
 } from '../../../common/constants/explorer';
 import { PPL_STATS_REGEX, PPL_NEWLINE_REGEX } from '../../../common/constants/shared';
-import { getIndexPatternFromRawQuery, insertDateRangeToQuery } from '../../../common/utils';
+import { getIndexPatternFromRawQuery, preprocessQuery } from '../../../common/utils';
 import { useFetchEvents, useFetchVisualizations } from './hooks';
 import { changeQuery, changeDateRange, selectQueries } from './slices/query_slice';
 import { selectQueryResult } from './slices/query_result_slice';
@@ -147,7 +147,7 @@ export const Explorer = ({
 
   const composeFinalQuery = (curQuery: any, timeField: string) => {
     if (isEmpty(curQuery![RAW_QUERY])) return '';
-    return insertDateRangeToQuery({
+    return preprocessQuery({
       rawQuery: curQuery![RAW_QUERY],
       startTime: curQuery![SELECTED_DATE_RANGE][0],
       endTime: curQuery![SELECTED_DATE_RANGE][1],

dashboards-observability/test/setup.jest.ts

@@ -10,6 +10,25 @@ configure({ testIdAttribute: 'data-test-subj' });
 
 window.URL.createObjectURL = () => '';
 HTMLCanvasElement.prototype.getContext = () => '';
+window.IntersectionObserver = class IntersectionObserver {
+  constructor() {}
+
+  disconnect() {
+    return null;
+  }
+
+  observe() {
+    return null;
+  }
+
+  takeRecords() {
+    return null;
+  }
+
+  unobserve() {
+    return null;
+  }
+};
 
 jest.mock('@elastic/eui/lib/components/form/form_row/make_id', () => () => 'random-id');