Merge pull request #61 from kavilla/kavilla/sql_with_df

Kavilla/sql with df

package.json

@@ -135,7 +135,7 @@
   },
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
-    "@elastic/datemath": "5.0.3",
+    "@elastic/datemath": "link:packages/opensearch-datemath",
     "@elastic/eui": "npm:@opensearch-project/oui@1.5.1",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0",

packages/opensearch-datemath/index.d.ts

@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

packages/opensearch-datemath/index.js

@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d) => moment.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

src/plugins/data/common/data_frames/utils.ts

@@ -10,6 +10,7 @@
  */
 
 import { SearchResponse } from 'elasticsearch';
+import datemath from '@elastic/datemath';
 import { IDataFrame, IDataFrameWithAggs, PartialDataFrame } from './types';
 import { IFieldType } from './fields';
 
@@ -76,6 +77,31 @@ export const convertResult = (response: IDataFrameResponse): SearchResponse<any>
   return searchResponse;
 };
 
+export const formatFieldValue = (field: IFieldType | Partial<IFieldType>, value: any): any => {
+  return field.format && field.format.convert ? field.format.convert(value) : value;
+};
+
+export const getFieldType = (field: IFieldType | Partial<IFieldType>): string | undefined => {
+  if (field.name) {
+    const fieldName = field.name.toLowerCase();
+    // TODO: feels little biased to check if timestamp.
+    // Has to be a better way to know so to be fair to all data sources
+    if (fieldName.includes('date') || fieldName.includes('timestamp')) {
+      return 'date';
+    }
+  }
+  if (!field.values) return field.type;
+  const firstValue = field.values.filter((value) => value !== null && value !== undefined)[0];
+  if (firstValue instanceof Date || datemath.isDateTime(firstValue)) {
+    return 'date';
+  }
+  return field.type;
+};
+
+export const getTimeField = (data: IDataFrame): IFieldType | undefined => {
+  return data.fields.find((field) => field.type === 'date');
+};
+
 export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => {
   let size = 0;
   const fields = partial.fields.map((field) => {
@@ -84,7 +110,11 @@ export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFr
     } else if (field.values.length > length) {
       size = field.values.length;
     }
+    field.type = getFieldType(field);
     // if (!field.type) {
+    // need to think if this needs to be mapped to OSD field type for example
+    // PPL type for date is TIMESTAMP
+    // OSD is expecting date
     //   field.type = get type
     // }
     // get timeseries field

src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts

@@ -35,7 +35,6 @@ import { buildQueryFromLucene } from './from_lucene';
 import { IIndexPattern } from '../../index_patterns';
 import { Filter } from '../filters';
 import { Query } from '../../query/types';
-import { buildQueryFromSql } from './from_sql';
 
 export interface OpenSearchQueryConfig {
   allowLeadingWildcards: boolean;
@@ -65,21 +64,17 @@ export function buildOpenSearchQuery(
   queries = Array.isArray(queries) ? queries : [queries];
   filters = Array.isArray(filters) ? filters : [filters];
 
-  // TODO: SQL make this combinable. SQL needs to support DSL
-  // console.log('queries', queries);
-  const sqlQueries = queries.filter((query) => query.language === 'SQL');
-  if (sqlQueries.length > 0) {
-    // console.log('sqlQueries', sqlQueries);
-    return buildQueryFromSql(sqlQueries, config.dateFormatTZ);
-  }
-
   const validQueries = queries.filter((query) => has(query, 'query'));
   const queriesByLanguage = groupBy(validQueries, 'language');
   const unsupportedQueries = Object.keys(queriesByLanguage).filter(
     (language) => language !== 'kuery' && language !== 'lucene'
   );
   if (unsupportedQueries.length > 0) {
-    return queries;
+    return {
+      type: 'unsupported',
+      queries,
+      filters,
+    };
   }
 
   const kueryQuery = buildQueryFromKuery(

src/plugins/data/common/search/opensearch_search/types.ts

@@ -57,6 +57,7 @@ export type ISearchRequestParams<T = Record<string, any>> = {
 export interface IOpenSearchSearchRequest
   extends IOpenSearchDashboardsSearchRequest<ISearchRequestParams> {
   indexType?: string;
+  language?: string;
   dataSourceId?: string;
 }
 

src/plugins/data/common/search/search_source/mocks.ts

@@ -47,13 +47,16 @@ export const searchSourceInstanceMock: MockedKeys<ISearchSource> = {
   createChild: jest.fn().mockReturnThis(),
   setParent: jest.fn(),
   getParent: jest.fn().mockReturnThis(),
+  setDataFrame: jest.fn(),
+  getDataFrame: jest.fn().mockReturnThis(),
   fetch: jest.fn().mockResolvedValue({}),
   onRequestStart: jest.fn(),
   getSearchRequestBody: jest.fn(),
   destroy: jest.fn(),
   history: [],
   getSerializedFields: jest.fn(),
   serialize: jest.fn(),
+  flatten: jest.fn().mockReturnThis(),
 };
 
 export const searchSourceCommonMock: jest.Mocked<ISearchStartSearchSource> = {

src/plugins/data/common/search/search_source/search_source.ts

@@ -87,7 +87,7 @@ import { normalizeSortRequest } from './normalize_sort_request';
 import { filterDocvalueFields } from './filter_docvalue_fields';
 import { fieldWildcardFilter } from '../../../../opensearch_dashboards_utils/common';
 import { IIndexPattern } from '../../index_patterns';
-import { IDataFrameResponse, convertResult } from '../../data_frames';
+import { IDataFrame, IDataFrameResponse, convertResult } from '../../data_frames';
 import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..';
 import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types';
 import { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types';
@@ -138,6 +138,7 @@ export class SearchSource {
   private id: string = uniqueId('data_source');
   private searchStrategyId?: string;
   private parent?: SearchSource;
+  private df?: IDataFrame;
   private requestStartHandlers: Array<
     (searchSource: SearchSource, options?: ISearchOptions) => Promise<unknown>
   > = [];
@@ -270,6 +271,24 @@ export class SearchSource {
     return this.parent;
   }
 
+  /**
+   * Set a dataframe to this search source
+   * @param  {IDataFrame} df - dataframe
+   * @return {this} - chainable
+   */
+  setDataFrame(df?: IDataFrame) {
+    this.df = df;
+    return this;
+  }
+
+  /**
+   * Get the data frame of this SearchSource
+   * @return {undefined|IDataFrame}
+   */
+  getDataFrame() {
+    return this.df;
+  }
+
   /**
    * Fetch this source and reject the returned Promise on error
    *
@@ -285,6 +304,13 @@ export class SearchSource {
     let response;
     if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) {
       response = await this.legacyFetch(searchRequest, options);
+    } else if (this.isUnsupportedRequest(searchRequest)) {
+      if (!this.getDataFrame()) {
+        // populate initial dataframe
+        await this.fetchExternalSearch(searchRequest, options);
+      }
+
+      response = await this.fetchExternalSearch(searchRequest, options);
     } else {
       const indexPattern = this.getField('index');
       searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id;
@@ -338,16 +364,41 @@ export class SearchSource {
   private fetchSearch(searchRequest: SearchRequest, options: ISearchOptions) {
     const { search, getConfig, onResponse } = this.dependencies;
 
+    if (this.getDataFrame()) {
+      delete searchRequest.body!.df;
+      this.setDataFrame(undefined);
+    }
+
     const params = getSearchParamsFromRequest(searchRequest, {
       getConfig,
     });
 
     return search(
       { params, indexType: searchRequest.indexType, dataSourceId: searchRequest.dataSourceId },
       options
-    ).then((response: any) => {
+    ).then((response: any) => onResponse(searchRequest, response.rawResponse));
+  }
+
+  /**
+   * Run a non-native search using the search service
+   * @return {Promise<SearchResponse<unknown>>}
+   */
+  private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) {
+    const { search, getConfig, onResponse } = this.dependencies;
+
+    const params = getSearchParamsFromRequest(searchRequest, {
+      getConfig,
+    });
+
+    if (this.getDataFrame()) {
+      params.body!.df = this.getDataFrame();
+    }
+
+    return search({ params }, options).then((response: any) => {
       if (response.hasOwnProperty('type')) {
         if ((response as IDataFrameResponse).type === 'data_frame') {
+          const dataFrameResponse = response as IDataFrameResponse;
+          this.setDataFrame(dataFrameResponse.body as IDataFrame);
           return onResponse(searchRequest, convertResult(response as IDataFrameResponse));
         }
       }
@@ -376,6 +427,10 @@ export class SearchSource {
     );
   }
 
+  private isUnsupportedRequest(request: SearchRequest): boolean {
+    return request.body!.query.hasOwnProperty('type') && request.body!.query.type === 'unsupported';
+  }
+
   /**
    *  Called by requests of this search source when they are started
    *  @param options

src/plugins/data/common/search/search_source/types.ts

@@ -29,7 +29,7 @@
  */
 
 import { NameList } from 'elasticsearch';
-import { Filter, IndexPattern, Query } from '../..';
+import { Filter, IDataFrame, IndexPattern, Query } from '../..';
 import { SearchSource } from './search_source';
 
 /**
@@ -103,6 +103,7 @@ export interface SearchSourceFields {
   searchAfter?: OpenSearchQuerySearchAfter;
   timeout?: string;
   terminate_after?: number;
+  df?: IDataFrame;
 }
 
 export interface SearchSourceOptions {

src/plugins/data/public/ui/query_string_input/language_switcher.tsx

@@ -53,9 +53,6 @@ export function QueryLanguageSwitcher(props: Props) {
   const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', {
     defaultMessage: 'DQL',
   });
-  const sqlLabel = i18n.translate('data.query.queryBar.sqlLanguageName', {
-    defaultMessage: 'SQL',
-  });
 
   const languageOptions: EuiComboBoxOptionOption[] = [
     {
@@ -66,12 +63,6 @@ export function QueryLanguageSwitcher(props: Props) {
       label: dqlLabel,
       value: 'dql',
     },
-    {
-      label: sqlLabel,
-      value: 'sql',
-      // TODO: add option to disable if SQL is not supported
-      // disabled: true,
-    },
   ];
 
   const queryEnhancements = getUiService().queryEnhancements;
@@ -85,10 +76,12 @@ export function QueryLanguageSwitcher(props: Props) {
 
   const setSearchEnhance = (queryLanguage: string) => {
     const queryEnhancement = queryEnhancements.get(queryLanguage);
-    getSearchService().__enhance({
+    const searchService = getSearchService();
+
+    searchService.__enhance({
       searchInterceptor: queryEnhancement
         ? queryEnhancement.search
-        : getSearchService().getDefaultSearchInterceptor(),
+        : searchService.getDefaultSearchInterceptor(),
     });
   };
 

src/plugins/data/public/ui/query_string_input/query_string_input.tsx

@@ -59,6 +59,7 @@ import { QueryLanguageSwitcher } from './language_switcher';
 import { PersistedLog, getQueryLog, matchPairs, toUser, fromUser } from '../../query';
 import { SuggestionsListSize } from '../typeahead/suggestions_component';
 import { SuggestionsComponent } from '..';
+import { getUiService } from '../../services';
 
 export interface QueryStringInputProps {
   indexPatterns: Array<IIndexPattern | string>;
@@ -466,10 +467,16 @@ export default class QueryStringInputUI extends Component<Props, State> {
     });
 
     this.services.storage.set('opensearchDashboards.userQueryLanguage', language);
-
-    const newQuery = { query: '', language };
+    const queryEnhancements = getUiService().queryEnhancements;
+    // TODO: SQL proof you can modify search bar
+    // Will work on adding more to search bar of UI service
+    const input = queryEnhancements.get(language)?.input;
+    const submitOnLanguageSelect = input?.submitOnLanguageSelect ?? true;
+    const newQuery = { query: input?.placeholder ?? '', language };
     this.onChange(newQuery);
-    this.onSubmit(newQuery);
+    if (submitOnLanguageSelect) {
+      this.onSubmit(newQuery);
+    }
   };
 
   private onOutsideClick = () => {

src/plugins/data/public/ui/types.ts

@@ -16,6 +16,10 @@ import { StatefulSearchBarProps } from './search_bar';
 export interface QueryEnhancement {
   language: string;
   search: SearchInterceptor;
+  input?: {
+    placeholder?: string;
+    submitOnLanguageSelect?: boolean;
+  };
 }
 
 export interface UiEnhancements {

src/plugins/data/server/search/search_service.ts

@@ -73,6 +73,7 @@ import {
 } from '../../common/search/aggs/buckets/shard_delay';
 import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn';
 import { ConfigSchema } from '../../config';
+import { IDataFrameResponse } from '../../common';
 
 type StrategyMap = Record<string, ISearchStrategy<any, any>>;
 
@@ -252,7 +253,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   private registerSearchStrategy = <
     SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest,
-    SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse
+    SearchStrategyResponse extends
+      | IOpenSearchDashboardsSearchResponse
+      | IDataFrameResponse = IOpenSearchSearchResponse
   >(
     name: string,
     strategy: ISearchStrategy<SearchStrategyRequest, SearchStrategyResponse>
@@ -262,7 +265,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   private search = <
     SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest,
-    SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse
+    SearchStrategyResponse extends
+      | IOpenSearchDashboardsSearchResponse
+      | IDataFrameResponse = IOpenSearchSearchResponse
   >(
     context: RequestHandlerContext,
     searchRequest: SearchStrategyRequest,
@@ -275,7 +280,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   private getSearchStrategy = <
     SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest,
-    SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse
+    SearchStrategyResponse extends
+      | IOpenSearchDashboardsSearchResponse
+      | IDataFrameResponse = IOpenSearchSearchResponse
   >(
     name: string
   ): ISearchStrategy<SearchStrategyRequest, SearchStrategyResponse> => {