diff --git a/package.json b/package.json index bb058be439fb..97930d03c110 100644 --- a/package.json +++ b/package.json @@ -135,7 +135,7 @@ }, "dependencies": { "@aws-crypto/client-node": "^3.1.1", - "@elastic/datemath": "5.0.3", + "@elastic/datemath": "link:packages/opensearch-datemath", "@elastic/eui": "npm:@opensearch-project/oui@1.5.1", "@elastic/good": "^9.0.1-kibana3", "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0", diff --git a/packages/opensearch-datemath/index.d.ts b/packages/opensearch-datemath/index.d.ts index d5a38f0176ea..0706d7d0dccf 100644 --- a/packages/opensearch-datemath/index.d.ts +++ b/packages/opensearch-datemath/index.d.ts @@ -43,6 +43,8 @@ declare const datemath: { unitsAsc: Unit[]; unitsDesc: Unit[]; + isDateTime(input: any): boolean; + /** * Parses a string into a moment object. The string can be something like "now - 15m". * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this diff --git a/packages/opensearch-datemath/index.js b/packages/opensearch-datemath/index.js index 4367949d7cf0..eba9fd2be5a6 100644 --- a/packages/opensearch-datemath/index.js +++ b/packages/opensearch-datemath/index.js @@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]'; const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf()); +const isDateTime = (d) => moment.isMoment(d); /* * This is a simplified version of opensearch's date parser. * If you pass in a momentjs instance as the third parameter the calculation @@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf()); */ function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) { if (!text) return undefined; - if (momentInstance.isMoment(text)) return text; + if (isDateTime(text)) return text; if (isDate(text)) return momentInstance(text); if (forceNow !== undefined && !isValidDate(forceNow)) { throw new Error('forceNow must be a valid Date'); @@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) { module.exports = { parse: parse, + isDateTime: isDateTime, unitsMap: Object.freeze(unitsMap), units: Object.freeze(units), unitsAsc: Object.freeze(unitsAsc), diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts index 338a17cd387d..59e55d9a1c10 100644 --- a/src/plugins/data/common/data_frames/utils.ts +++ b/src/plugins/data/common/data_frames/utils.ts @@ -10,6 +10,7 @@ */ import { SearchResponse } from 'elasticsearch'; +import datemath from '@elastic/datemath'; import { IDataFrame, IDataFrameWithAggs, PartialDataFrame } from './types'; import { IFieldType } from './fields'; @@ -76,6 +77,31 @@ export const convertResult = (response: IDataFrameResponse): SearchResponse return searchResponse; }; +export const formatFieldValue = (field: IFieldType | Partial, value: any): any => { + return field.format && field.format.convert ? field.format.convert(value) : value; +}; + +export const getFieldType = (field: IFieldType | Partial): string | undefined => { + if (field.name) { + const fieldName = field.name.toLowerCase(); + // TODO: feels little biased to check if timestamp. + // Has to be a better way to know so to be fair to all data sources + if (fieldName.includes('date') || fieldName.includes('timestamp')) { + return 'date'; + } + } + if (!field.values) return field.type; + const firstValue = field.values.filter((value) => value !== null && value !== undefined)[0]; + if (firstValue instanceof Date || datemath.isDateTime(firstValue)) { + return 'date'; + } + return field.type; +}; + +export const getTimeField = (data: IDataFrame): IFieldType | undefined => { + return data.fields.find((field) => field.type === 'date'); +}; + export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => { let size = 0; const fields = partial.fields.map((field) => { @@ -84,7 +110,11 @@ export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFr } else if (field.values.length > length) { size = field.values.length; } + field.type = getFieldType(field); // if (!field.type) { + // need to think if this needs to be mapped to OSD field type for example + // PPL type for date is TIMESTAMP + // OSD is expecting date // field.type = get type // } // get timeseries field diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts index 35a5bdc7a693..5a00faf890f0 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts @@ -35,7 +35,6 @@ import { buildQueryFromLucene } from './from_lucene'; import { IIndexPattern } from '../../index_patterns'; import { Filter } from '../filters'; import { Query } from '../../query/types'; -import { buildQueryFromSql } from './from_sql'; export interface OpenSearchQueryConfig { allowLeadingWildcards: boolean; @@ -65,21 +64,17 @@ export function buildOpenSearchQuery( queries = Array.isArray(queries) ? queries : [queries]; filters = Array.isArray(filters) ? filters : [filters]; - // TODO: SQL make this combinable. SQL needs to support DSL - // console.log('queries', queries); - const sqlQueries = queries.filter((query) => query.language === 'SQL'); - if (sqlQueries.length > 0) { - // console.log('sqlQueries', sqlQueries); - return buildQueryFromSql(sqlQueries, config.dateFormatTZ); - } - const validQueries = queries.filter((query) => has(query, 'query')); const queriesByLanguage = groupBy(validQueries, 'language'); const unsupportedQueries = Object.keys(queriesByLanguage).filter( (language) => language !== 'kuery' && language !== 'lucene' ); if (unsupportedQueries.length > 0) { - return queries; + return { + type: 'unsupported', + queries, + filters, + }; } const kueryQuery = buildQueryFromKuery( diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index 3b93177bf201..f90a3f1de245 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -57,6 +57,7 @@ export type ISearchRequestParams> = { export interface IOpenSearchSearchRequest extends IOpenSearchDashboardsSearchRequest { indexType?: string; + language?: string; dataSourceId?: string; } diff --git a/src/plugins/data/common/search/search_source/mocks.ts b/src/plugins/data/common/search/search_source/mocks.ts index 959d1aebfe53..3f69f14ec688 100644 --- a/src/plugins/data/common/search/search_source/mocks.ts +++ b/src/plugins/data/common/search/search_source/mocks.ts @@ -47,6 +47,8 @@ export const searchSourceInstanceMock: MockedKeys = { createChild: jest.fn().mockReturnThis(), setParent: jest.fn(), getParent: jest.fn().mockReturnThis(), + setDataFrame: jest.fn(), + getDataFrame: jest.fn().mockReturnThis(), fetch: jest.fn().mockResolvedValue({}), onRequestStart: jest.fn(), getSearchRequestBody: jest.fn(), @@ -54,6 +56,7 @@ export const searchSourceInstanceMock: MockedKeys = { history: [], getSerializedFields: jest.fn(), serialize: jest.fn(), + flatten: jest.fn().mockReturnThis(), }; export const searchSourceCommonMock: jest.Mocked = { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 81574482254b..fc0f092ab73c 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -87,7 +87,7 @@ import { normalizeSortRequest } from './normalize_sort_request'; import { filterDocvalueFields } from './filter_docvalue_fields'; import { fieldWildcardFilter } from '../../../../opensearch_dashboards_utils/common'; import { IIndexPattern } from '../../index_patterns'; -import { IDataFrameResponse, convertResult } from '../../data_frames'; +import { IDataFrame, IDataFrameResponse, convertResult } from '../../data_frames'; import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; import { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types'; @@ -138,6 +138,7 @@ export class SearchSource { private id: string = uniqueId('data_source'); private searchStrategyId?: string; private parent?: SearchSource; + private df?: IDataFrame; private requestStartHandlers: Array< (searchSource: SearchSource, options?: ISearchOptions) => Promise > = []; @@ -270,6 +271,24 @@ export class SearchSource { return this.parent; } + /** + * Set a dataframe to this search source + * @param {IDataFrame} df - dataframe + * @return {this} - chainable + */ + setDataFrame(df?: IDataFrame) { + this.df = df; + return this; + } + + /** + * Get the data frame of this SearchSource + * @return {undefined|IDataFrame} + */ + getDataFrame() { + return this.df; + } + /** * Fetch this source and reject the returned Promise on error * @@ -285,6 +304,13 @@ export class SearchSource { let response; if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); + } else if (this.isUnsupportedRequest(searchRequest)) { + if (!this.getDataFrame()) { + // populate initial dataframe + await this.fetchExternalSearch(searchRequest, options); + } + + response = await this.fetchExternalSearch(searchRequest, options); } else { const indexPattern = this.getField('index'); searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id; @@ -338,6 +364,11 @@ export class SearchSource { private fetchSearch(searchRequest: SearchRequest, options: ISearchOptions) { const { search, getConfig, onResponse } = this.dependencies; + if (this.getDataFrame()) { + delete searchRequest.body!.df; + this.setDataFrame(undefined); + } + const params = getSearchParamsFromRequest(searchRequest, { getConfig, }); @@ -345,9 +376,29 @@ export class SearchSource { return search( { params, indexType: searchRequest.indexType, dataSourceId: searchRequest.dataSourceId }, options - ).then((response: any) => { + ).then((response: any) => onResponse(searchRequest, response.rawResponse)); + } + + /** + * Run a non-native search using the search service + * @return {Promise>} + */ + private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) { + const { search, getConfig, onResponse } = this.dependencies; + + const params = getSearchParamsFromRequest(searchRequest, { + getConfig, + }); + + if (this.getDataFrame()) { + params.body!.df = this.getDataFrame(); + } + + return search({ params }, options).then((response: any) => { if (response.hasOwnProperty('type')) { if ((response as IDataFrameResponse).type === 'data_frame') { + const dataFrameResponse = response as IDataFrameResponse; + this.setDataFrame(dataFrameResponse.body as IDataFrame); return onResponse(searchRequest, convertResult(response as IDataFrameResponse)); } } @@ -376,6 +427,10 @@ export class SearchSource { ); } + private isUnsupportedRequest(request: SearchRequest): boolean { + return request.body!.query.hasOwnProperty('type') && request.body!.query.type === 'unsupported'; + } + /** * Called by requests of this search source when they are started * @param options diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts index 13b100aab1a7..9f3fd75e1ce9 100644 --- a/src/plugins/data/common/search/search_source/types.ts +++ b/src/plugins/data/common/search/search_source/types.ts @@ -29,7 +29,7 @@ */ import { NameList } from 'elasticsearch'; -import { Filter, IndexPattern, Query } from '../..'; +import { Filter, IDataFrame, IndexPattern, Query } from '../..'; import { SearchSource } from './search_source'; /** @@ -103,6 +103,7 @@ export interface SearchSourceFields { searchAfter?: OpenSearchQuerySearchAfter; timeout?: string; terminate_after?: number; + df?: IDataFrame; } export interface SearchSourceOptions { diff --git a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx index 98f669c15583..43c76ffb010e 100644 --- a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx +++ b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx @@ -53,9 +53,6 @@ export function QueryLanguageSwitcher(props: Props) { const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', { defaultMessage: 'DQL', }); - const sqlLabel = i18n.translate('data.query.queryBar.sqlLanguageName', { - defaultMessage: 'SQL', - }); const languageOptions: EuiComboBoxOptionOption[] = [ { @@ -66,12 +63,6 @@ export function QueryLanguageSwitcher(props: Props) { label: dqlLabel, value: 'dql', }, - { - label: sqlLabel, - value: 'sql', - // TODO: add option to disable if SQL is not supported - // disabled: true, - }, ]; const queryEnhancements = getUiService().queryEnhancements; @@ -85,10 +76,12 @@ export function QueryLanguageSwitcher(props: Props) { const setSearchEnhance = (queryLanguage: string) => { const queryEnhancement = queryEnhancements.get(queryLanguage); - getSearchService().__enhance({ + const searchService = getSearchService(); + + searchService.__enhance({ searchInterceptor: queryEnhancement ? queryEnhancement.search - : getSearchService().getDefaultSearchInterceptor(), + : searchService.getDefaultSearchInterceptor(), }); }; diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx index 5d071748700c..ed4c67eacc21 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx @@ -59,6 +59,7 @@ import { QueryLanguageSwitcher } from './language_switcher'; import { PersistedLog, getQueryLog, matchPairs, toUser, fromUser } from '../../query'; import { SuggestionsListSize } from '../typeahead/suggestions_component'; import { SuggestionsComponent } from '..'; +import { getUiService } from '../../services'; export interface QueryStringInputProps { indexPatterns: Array; @@ -466,10 +467,16 @@ export default class QueryStringInputUI extends Component { }); this.services.storage.set('opensearchDashboards.userQueryLanguage', language); - - const newQuery = { query: '', language }; + const queryEnhancements = getUiService().queryEnhancements; + // TODO: SQL proof you can modify search bar + // Will work on adding more to search bar of UI service + const input = queryEnhancements.get(language)?.input; + const submitOnLanguageSelect = input?.submitOnLanguageSelect ?? true; + const newQuery = { query: input?.placeholder ?? '', language }; this.onChange(newQuery); - this.onSubmit(newQuery); + if (submitOnLanguageSelect) { + this.onSubmit(newQuery); + } }; private onOutsideClick = () => { diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index e76374c3df38..798c0719c0a3 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -16,6 +16,10 @@ import { StatefulSearchBarProps } from './search_bar'; export interface QueryEnhancement { language: string; search: SearchInterceptor; + input?: { + placeholder?: string; + submitOnLanguageSelect?: boolean; + }; } export interface UiEnhancements { diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 06ed8b36a7fe..f3a78d8f867e 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -73,6 +73,7 @@ import { } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; import { ConfigSchema } from '../../config'; +import { IDataFrameResponse } from '../../common'; type StrategyMap = Record>; @@ -252,7 +253,9 @@ export class SearchService implements Plugin { private registerSearchStrategy = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string, strategy: ISearchStrategy @@ -262,7 +265,9 @@ export class SearchService implements Plugin { private search = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( context: RequestHandlerContext, searchRequest: SearchStrategyRequest, @@ -275,7 +280,9 @@ export class SearchService implements Plugin { private getSearchStrategy = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string ): ISearchStrategy => { diff --git a/src/plugins/data/server/search/types.ts b/src/plugins/data/server/search/types.ts index eb82129f3973..6927d1289673 100644 --- a/src/plugins/data/server/search/types.ts +++ b/src/plugins/data/server/search/types.ts @@ -52,7 +52,9 @@ export interface ISearchSetup { */ registerSearchStrategy: < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string, strategy: ISearchStrategy diff --git a/yarn.lock b/yarn.lock index c9b266d1d1a7..e9d64b9d32c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,12 +1486,9 @@ utility-types "^3.10.0" uuid "^3.3.2" -"@elastic/datemath@5.0.3": - version "5.0.3" - resolved "https://registry.yarnpkg.com/@elastic/datemath/-/datemath-5.0.3.tgz#7baccdab672b9a3ecb7fe8387580670936b58573" - integrity sha512-8Hbr1Uyjm5OcYBfEB60K7sCP6U3IXuWDaLaQmYv3UxgI4jqBWbakoemwWvsqPVUvnwEjuX6z7ghPZbefs8xiaA== - dependencies: - tslib "^1.9.3" +"@elastic/datemath@link:packages/opensearch-datemath": + version "0.0.0" + uid "" "@elastic/ecs-helpers@^1.1.0": version "1.1.0"