[Discover-next] data set picker (opensearch-project#7426)

* update using query manager

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* aggs not working

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* almost there

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* stablish

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* thanks ashwin

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* update ref name

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* fix timefields

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* clean up some console logs

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* go safer route of setting language

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* its working again

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* fix names

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* restore code editor and indices

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* sql df

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

* fix external datasources again

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

---------

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

changelogs/fragments/7368.yml

@@ -0,0 +1,2 @@
+feat:
+- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368))

package.json

@@ -65,7 +65,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",

src/plugins/data/common/data_frames/utils.ts

@@ -45,29 +45,6 @@ export const getRawQueryString = (
   );
 };
 
-/**
- * Parses a raw query string and extracts the query string and data source.
- * @param rawQueryString - The raw query string to parse.
- * @returns An object containing the parsed query string and data source (if found).
- */
-export const parseRawQueryString = (rawQueryString: string) => {
-  const rawDataSource = rawQueryString.match(/::(.*?)::/);
-  return {
-    qs: rawQueryString.replace(/::.*?::/, ''),
-    formattedQs(key: string = '.'): string {
-      const parts = rawQueryString.split('::');
-      if (parts.length > 1) {
-        return (parts.slice(0, 1).join('') + parts.slice(1).join(key)).replace(
-          new RegExp(key + '$'),
-          ''
-        );
-      }
-      return rawQueryString;
-    },
-    ...(rawDataSource && { dataSource: rawDataSource[1] }),
-  };
-};
-
 /**
  * Returns the raw aggregations from the search request.
  *

src/plugins/query_enhancements/public/data_source_connection/utils/index.ts → src/plugins/data/common/data_sets/index.ts

@@ -3,4 +3,4 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-export * from './create_extension';
+export * from './types';

src/plugins/data/common/data_sets/types.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/** @public **/
+export enum SIMPLE_DATA_SOURCE_TYPES {
+  DEFAULT = 'data-source',
+  EXTERNAL = 'external-source',
+}
+
+/** @public **/
+export enum SIMPLE_DATA_SET_TYPES {
+  INDEX_PATTERN = 'index-pattern',
+  TEMPORARY = 'temporary',
+  TEMPORARY_ASYNC = 'temporary-async',
+}
+
+export interface SimpleObject {
+  id: string;
+  title?: string;
+  dataSourceRef?: SimpleDataSource;
+}
+
+export interface SimpleDataSource {
+  id: string;
+  name: string;
+  indices?: SimpleObject[];
+  tables?: SimpleObject[];
+  type: SIMPLE_DATA_SOURCE_TYPES;
+}
+
+export interface SimpleDataSet extends SimpleObject {
+  fields?: any[];
+  timeFieldName?: string;
+  timeFields?: any[];
+  type?: SIMPLE_DATA_SET_TYPES;
+}

src/plugins/data/common/index.ts

@@ -31,6 +31,7 @@
 export * from './constants';
 export * from './opensearch_query';
 export * from './data_frames';
+export * from './data_sets';
 export * from './field_formats';
 export * from './field_mapping';
 export * from './index_patterns';

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -433,11 +433,13 @@ export class IndexPatternsService {
   /**
    * Get an index pattern by id. Cache optimized
    * @param id
+   * @param onlyCheckCache - Only check cache for index pattern if it doesn't exist it will not error out
    */
 
-  get = async (id: string): Promise<IndexPattern> => {
+  get = async (id: string, onlyCheckCache: boolean = false): Promise<IndexPattern> => {
     const cache = indexPatternCache.get(id);
-    if (cache) {
+
+    if (cache || onlyCheckCache) {
       return cache;
     }
 

src/plugins/data/common/search/opensearch_search/types.ts

@@ -48,6 +48,10 @@ export interface ISearchOptions {
    * Use this option to enable support for long numerals.
    */
   withLongNumeralsSupport?: boolean;
+  /**
+   * Use this option to enable support for async.
+   */
+  isAsync?: boolean;
 }
 
 export type ISearchRequestParams<T = Record<string, any>> = {

src/plugins/data/common/search/search_source/search_source.ts

@@ -94,7 +94,6 @@ import {
   convertResult,
   createDataFrame,
   getRawQueryString,
-  parseRawQueryString,
 } from '../../data_frames';
 import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..';
 import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types';
@@ -324,7 +323,12 @@ export class SearchSource {
     const dataFrame = createDataFrame({
       name: searchRequest.index.title || searchRequest.index,
       fields: [],
-      ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }),
+      ...(rawQueryString && {
+        meta: {
+          queryConfig: { qs: rawQueryString },
+          ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }),
+        },
+      }),
     });
     await this.setDataFrame(dataFrame);
     return this.getDataFrame();
@@ -426,7 +430,8 @@ export class SearchSource {
   private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) {
     const { search, getConfig, onResponse } = this.dependencies;
 
-    if (!this.getDataFrame()) {
+    const currentDataframe = this.getDataFrame();
+    if (!currentDataframe || currentDataframe.name !== searchRequest.index?.id) {
       await this.createDataFrame(searchRequest);
     }
 

src/plugins/data/common/types.ts

@@ -35,6 +35,7 @@ export * from './query/types';
 export * from './osd_field_types/types';
 export * from './index_patterns/types';
 export * from './data_frames/types';
+export * from './data_sets/types';
 
 /**
  * If a service is being shared on both the client and the server, and

src/plugins/data/public/antlr/shared/utils.ts

@@ -12,7 +12,7 @@ export interface IDataSourceRequestHandlerParams {
 }
 
 export const getRawSuggestionData$ = (
-  connectionsService,
+  connectionsService: any,
   dataSourceReuqstHandler: ({
     dataSourceId,
     title,
@@ -21,11 +21,11 @@ export const getRawSuggestionData$ = (
 ) =>
   connectionsService.getSelectedConnection$().pipe(
     distinctUntilChanged(),
-    switchMap((connection) => {
+    switchMap((connection: any) => {
       if (connection === undefined) {
         return from(defaultReuqstHandler());
       }
-      const dataSourceId = connection?.id;
+      const dataSourceId = connection?.dataSource?.id;
       const title = connection?.attributes?.title;
       return from(dataSourceReuqstHandler({ dataSourceId, title }));
     })
@@ -34,8 +34,8 @@ export const getRawSuggestionData$ = (
 export const fetchData = (
   tables: string[],
   queryFormatter: (table: string, dataSourceId?: string, title?: string) => any,
-  api,
-  connectionService
+  api: any,
+  connectionService: any
 ): Promise<any[]> => {
   return new Promise((resolve, reject) => {
     getRawSuggestionData$(
@@ -65,16 +65,20 @@ export const fetchData = (
         );
       }
     ).subscribe({
-      next: (dataFrames) => resolve(dataFrames),
-      error: (err) => {
+      next: (dataFrames: any) => resolve(dataFrames),
+      error: (err: any) => {
         // TODO: pipe error to UI
         reject(err);
       },
     });
   });
 };
 
-export const fetchTableSchemas = (tables: string[], api, connectionService): Promise<any[]> => {
+export const fetchTableSchemas = (
+  tables: string[],
+  api: any,
+  connectionService: any
+): Promise<any[]> => {
   return fetchData(
     tables,
     (table, dataSourceId, title) => ({
@@ -96,8 +100,8 @@ export const fetchTableSchemas = (tables: string[], api, connectionService): Pro
 export const fetchColumnValues = (
   tables: string[],
   column: string,
-  api,
-  connectionService
+  api: any,
+  connectionService: any
 ): Promise<any[]> => {
   return fetchData(
     tables,

src/plugins/data/public/index.ts

@@ -445,6 +445,7 @@ export {
   QueryEditorTopRow,
   // for BWC, keeping the old name
   IUiStart as DataPublicPluginStartUi,
+  DataSetNavigator,
 } from './ui';
 
 /**

src/plugins/data/public/query/dataset_manager/dataset_manager.mock.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { DataSetContract } from '.';
+
+const createSetupContractMock = () => {
+  const dataSetManagerMock: jest.Mocked<DataSetContract> = {
+    init: jest.fn(),
+    getDataSet: jest.fn(),
+    setDataSet: jest.fn(),
+    getUpdates$: jest.fn(),
+    getDefaultDataSet: jest.fn(),
+  };
+  return dataSetManagerMock;
+};
+
+export const dataSetManagerMock = {
+  createSetupContract: createSetupContractMock,
+  createStartContract: createSetupContractMock,
+};

src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { DataSetManager } from './dataset_manager';
+import { coreMock } from '../../../../../core/public/mocks';
+import { SimpleDataSet } from '../../../common/data_sets';
+
+describe('DataSetManager', () => {
+  let service: DataSetManager;
+
+  beforeEach(() => {
+    service = new DataSetManager(coreMock.createSetup().uiSettings);
+  });
+
+  test('getUpdates$ is a cold emits only after query changes', () => {
+    const obs$ = service.getUpdates$();
+    const emittedValues: SimpleDataSet[] = [];
+    obs$.subscribe((v) => {
+      emittedValues.push(v!);
+    });
+    expect(emittedValues).toHaveLength(0);
+
+    const newDataSet: SimpleDataSet = { id: 'test_dataset', title: 'Test Dataset' };
+    service.setDataSet(newDataSet);
+    expect(emittedValues).toHaveLength(1);
+    expect(emittedValues[0]).toEqual(newDataSet);
+
+    service.setDataSet({ ...newDataSet });
+    expect(emittedValues).toHaveLength(2);
+  });
+});

src/plugins/data/public/query/dataset_manager/dataset_manager.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { BehaviorSubject } from 'rxjs';
+import { skip } from 'rxjs/operators';
+import { CoreStart } from 'opensearch-dashboards/public';
+import {
+  IndexPatternsService,
+  SIMPLE_DATA_SET_TYPES,
+  SimpleDataSet,
+  SimpleDataSource,
+} from '../../../common';
+
+export class DataSetManager {
+  private dataSet$: BehaviorSubject<SimpleDataSet | undefined>;
+  private indexPatterns?: IndexPatternsService;
+
+  constructor(private readonly uiSettings: CoreStart['uiSettings']) {
+    this.dataSet$ = new BehaviorSubject<SimpleDataSet | undefined>(undefined);
+  }
+
+  public init = (indexPatterns: IndexPatternsService) => {
+    this.indexPatterns = indexPatterns;
+  };
+
+  public getUpdates$ = () => {
+    return this.dataSet$.asObservable().pipe(skip(1));
+  };
+
+  public getDataSet = () => {
+    return this.dataSet$.getValue();
+  };
+
+  /**
+   * Updates the query.
+   * @param {Query} query
+   */
+  public setDataSet = (dataSet: SimpleDataSet | undefined) => {
+    this.dataSet$.next(dataSet);
+  };
+
+  public getDefaultDataSet = async (): Promise<SimpleDataSet | undefined> => {
+    const defaultIndexPatternId = await this.uiSettings.get('defaultIndex');
+    if (!defaultIndexPatternId) {
+      return undefined;
+    }
+
+    const indexPattern = await this.indexPatterns?.get(defaultIndexPatternId);
+    if (!indexPattern) {
+      return undefined;
+    }
+
+    if (!indexPattern.id) {
+      return undefined;
+    }
+
+    return {
+      id: indexPattern.id,
+      title: indexPattern.title,
+      type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN,
+      timeFieldName: indexPattern.timeFieldName,
+      ...(indexPattern.dataSourceRef
+        ? {
+            dataSourceRef: {
+              id: indexPattern.dataSourceRef?.id,
+              name: indexPattern.dataSourceRef?.name,
+              type: indexPattern.dataSourceRef?.type,
+            } as SimpleDataSource,
+          }
+        : {}),
+    };
+  };
+}
+
+export type DataSetContract = PublicMethodsOf<DataSetManager>;

src/plugins/data/public/query/dataset_manager/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export { DataSetContract, DataSetManager } from './dataset_manager';

src/plugins/data/public/query/index.tsx

@@ -32,6 +32,7 @@ export * from './lib';
 
 export * from './query_service';
 export * from './filter_manager';
+export * from './dataset_manager';
 export * from './timefilter';
 export * from './saved_query';
 export * from './persisted_log';