Catalog cache and Session update for async queries (#1500)

* init commit for cache

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

* cache loader init

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

* update cache with accelerations cache, add custom hooks

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

* add session coupling and fix tests for cache

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

* added polling result type, updated version constant

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

---------

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>
(cherry picked from commit 943126f)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

common/constants/data_sources.ts

@@ -22,3 +22,5 @@ export enum DATA_SOURCE_TYPES {
   S3Glue = 's3glue',
 }
 export const ASYNC_POLLING_INTERVAL = 2000;
+
+export const CATALOG_CACHE_VERSION = '1.0';

common/constants/shared.ts

@@ -22,6 +22,7 @@ export const SAVED_OBJECTS = '/saved_objects';
 export const SAVED_QUERY = '/query';
 export const SAVED_VISUALIZATION = '/vis';
 export const CONSOLE_PROXY = '/api/console/proxy';
+export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account';
 
 // Server route
 export const PPL_ENDPOINT = '/_plugins/_ppl';
@@ -254,6 +255,8 @@ export const VISUALIZATION_ERROR = {
 export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE';
 
 export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id';
+export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache';
+export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache';
 
 export const DIRECT_DUMMY_QUERY = 'select 1';
 

common/types/data_connections.ts

@@ -25,3 +25,84 @@ export interface AssociatedObject {
 export type Role = EuiComboBoxOptionOption;
 
 export type DatasourceType = 'S3GLUE' | 'PROMETHEUS';
+
+interface AsyncApiDataResponse {
+  status: string;
+  schema?: Array<{ name: string; type: string }>;
+  datarows?: any;
+  total?: number;
+  size?: number;
+  error?: string;
+}
+
+export interface AsyncApiResponse {
+  data: {
+    ok: boolean;
+    resp: AsyncApiDataResponse;
+  };
+}
+
+export type PollingCallback = (statusObj: AsyncApiResponse) => void;
+
+export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized';
+
+export type LoadCacheType = 'databases' | 'tables' | 'accelerations';
+
+export enum CachedDataSourceStatus {
+  Updated = 'Updated',
+  Failed = 'Failed',
+  Empty = 'Empty',
+}
+
+export interface CachedColumn {
+  name: string;
+  dataType: string;
+}
+
+export interface CachedTable {
+  name: string;
+  columns: CachedColumn[];
+}
+
+export interface CachedDatabase {
+  name: string;
+  tables: CachedTable[];
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+}
+
+export interface CachedDataSource {
+  name: string;
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+  databases: CachedDatabase[];
+}
+
+export interface DataSourceCacheData {
+  version: string;
+  dataSources: CachedDataSource[];
+}
+
+export interface CachedAccelerations {
+  flintIndexName: string;
+  type: AccelerationIndexType;
+  database: string;
+  table: string;
+  indexName: string;
+  autoRefresh: boolean;
+  status: string;
+}
+
+export interface AccelerationsCacheData {
+  version: string;
+  accelerations: CachedAccelerations[];
+  lastUpdated: string; // Assuming date string in UTC format
+  status: CachedDataSourceStatus;
+}
+
+export interface PollingSuccessResult {
+  schema: Array<{ name: string; type: string }>;
+  datarows: Array<Array<string | number | boolean>>;
+}
+
+export type AsyncPollingResult = PollingSuccessResult | null;

common/utils/query_session_utils.ts

@@ -5,12 +5,12 @@
 
 import { ASYNC_QUERY_SESSION_ID } from '../constants/shared';
 
-export const setAsyncSessionId = (value: string | null) => {
+export const setAsyncSessionId = (dataSource: string, value: string | null) => {
   if (value !== null) {
-    sessionStorage.setItem(ASYNC_QUERY_SESSION_ID, value);
+    sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value);
   }
 };
 
-export const getAsyncSessionId = () => {
-  return sessionStorage.getItem(ASYNC_QUERY_SESSION_ID);
+export const getAsyncSessionId = (dataSource: string) => {
+  return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`);
 };

common/utils/shared.ts

@@ -6,3 +6,30 @@
 export function get<T = unknown>(obj: Record<string, any>, path: string, defaultValue?: T): T {
   return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue;
 }
+
+export function addBackticksIfNeeded(input: string): string {
+  // Check if the string already has backticks
+  if (input.startsWith('`') && input.endsWith('`')) {
+    return input; // Return the string as it is
+  } else {
+    // Add backticks to the string
+    return '`' + input + '`';
+  }
+}
+
+export function combineSchemaAndDatarows(
+  schema: Array<{ name: string; type: string }>,
+  datarows: Array<Array<string | number | boolean>>
+): object[] {
+  const combinedData: object[] = [];
+
+  datarows.forEach((row) => {
+    const rowData: { [key: string]: string | number | boolean } = {};
+    schema.forEach((field, index) => {
+      rowData[field.name] = row[index];
+    });
+    combinedData.push(rowData);
+  });
+
+  return combinedData;
+}

public/components/common/search/direct_search.tsx

@@ -28,20 +28,20 @@ import { DirectQueryLoadingStatus, DirectQueryRequest } from '../../../../common
 import { uiSettingsService } from '../../../../common/utils';
 import { getAsyncSessionId, setAsyncSessionId } from '../../../../common/utils/query_session_utils';
 import { get as getObjValue } from '../../../../common/utils/shared';
-import { useFetchEvents } from '../../event_analytics/hooks';
-import { changeQuery } from '../../event_analytics/redux/slices/query_slice';
-import { usePolling } from '../../hooks/use_polling';
 import { coreRefs } from '../../../framework/core_refs';
 import { SQLService } from '../../../services/requests/sql';
 import { SavePanel } from '../../event_analytics/explorer/save_panel';
+import { useFetchEvents } from '../../event_analytics/hooks';
+import { reset as resetResults } from '../../event_analytics/redux/slices/query_result_slice';
+import { changeQuery } from '../../event_analytics/redux/slices/query_slice';
 import {
   selectSearchMetaData,
   update as updateSearchMetaData,
 } from '../../event_analytics/redux/slices/search_meta_data_slice';
-import { reset as resetResults } from '../../event_analytics/redux/slices/query_result_slice';
+import { formatError } from '../../event_analytics/utils';
+import { usePolling } from '../../hooks/use_polling';
 import { PPLReferenceFlyout } from '../helpers';
 import { Autocomplete } from './autocomplete';
-import { formatError } from '../../event_analytics/utils';
 export interface IQueryBarProps {
   query: string;
   tempQuery: string;
@@ -209,7 +209,7 @@ export const DirectSearch = (props: any) => {
         })
       );
     });
-    const sessionId = getAsyncSessionId();
+    const sessionId = getAsyncSessionId(explorerSearchMetadata.datasources[0].label);
     const requestPayload = {
       lang: lang.toLowerCase(),
       query: tempQuery || query,
@@ -223,7 +223,10 @@ export const DirectSearch = (props: any) => {
     sqlService
       .fetch(requestPayload)
       .then((result) => {
-        setAsyncSessionId(getObjValue(result, 'sessionId', null));
+        setAsyncSessionId(
+          explorerSearchMetadata.datasources[0].label,
+          getObjValue(result, 'sessionId', null)
+        );
         if (result.queryId) {
           dispatch(updateSearchMetaData({ tabId, data: { queryId: result.queryId } }));
           startPolling({

public/components/event_analytics/explorer/datasources/datasources_selection.tsx

@@ -3,52 +3,52 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React, { useCallback, useContext, useEffect, useState, useMemo } from 'react';
+import React, { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 import { batch, useDispatch, useSelector } from 'react-redux';
 import { LogExplorerRouterContext } from '../..';
 import {
   DataSourceGroup,
   DataSourceSelectable,
   DataSourceType,
 } from '../../../../../../../src/plugins/data/public';
-import { coreRefs } from '../../../../framework/core_refs';
-import {
-  selectSearchMetaData,
-  update as updateSearchMetaData,
-} from '../../../event_analytics/redux/slices/search_meta_data_slice';
-import { reset as resetCountDistribution } from '../../redux/slices/count_distribution_slice';
-import { reset as resetFields } from '../../redux/slices/field_slice';
-import { reset as resetPatterns } from '../../redux/slices/patterns_slice';
-import { reset as resetQueryResults } from '../../redux/slices/query_result_slice';
-import { changeData, reset as resetQuery } from '../../redux/slices/query_slice';
-import { reset as resetVisualization } from '../../redux/slices/visualization_slice';
-import { reset as resetVisConfig } from '../../redux/slices/viualization_config_slice';
-import { DirectQueryRequest, SelectedDataSource } from '../../../../../common/types/explorer';
-import { ObservabilityDefaultDataSource } from '../../../../framework/datasources/obs_opensearch_datasource';
 import {
-  DATA_SOURCE_TYPE_URL_PARAM_KEY,
   DATA_SOURCE_NAME_URL_PARAM_KEY,
+  DATA_SOURCE_TYPES,
+  DATA_SOURCE_TYPE_URL_PARAM_KEY,
   DEFAULT_DATA_SOURCE_NAME,
+  DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME,
   DEFAULT_DATA_SOURCE_TYPE,
   DEFAULT_DATA_SOURCE_TYPE_NAME,
-  DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME,
-  DATA_SOURCE_TYPES,
-  QUERY_LANGUAGE,
-  OLLY_QUESTION_URL_PARAM_KEY,
   INDEX_URL_PARAM_KEY,
+  OLLY_QUESTION_URL_PARAM_KEY,
+  QUERY_LANGUAGE,
 } from '../../../../../common/constants/data_sources';
-import { SQLService } from '../../../../services/requests/sql';
-import { get as getObjValue } from '../../../../../common/utils/shared';
-import {
-  setAsyncSessionId,
-  getAsyncSessionId,
-} from '../../../../../common/utils/query_session_utils';
-import { DIRECT_DUMMY_QUERY } from '../../../../../common/constants/shared';
 import {
   INDEX,
   OLLY_QUERY_ASSISTANT,
   SELECTED_TIMESTAMP,
 } from '../../../../../common/constants/explorer';
+import { DIRECT_DUMMY_QUERY } from '../../../../../common/constants/shared';
+import { DirectQueryRequest, SelectedDataSource } from '../../../../../common/types/explorer';
+import {
+  getAsyncSessionId,
+  setAsyncSessionId,
+} from '../../../../../common/utils/query_session_utils';
+import { get as getObjValue } from '../../../../../common/utils/shared';
+import { coreRefs } from '../../../../framework/core_refs';
+import { ObservabilityDefaultDataSource } from '../../../../framework/datasources/obs_opensearch_datasource';
+import { SQLService } from '../../../../services/requests/sql';
+import {
+  selectSearchMetaData,
+  update as updateSearchMetaData,
+} from '../../../event_analytics/redux/slices/search_meta_data_slice';
+import { reset as resetCountDistribution } from '../../redux/slices/count_distribution_slice';
+import { reset as resetFields } from '../../redux/slices/field_slice';
+import { reset as resetPatterns } from '../../redux/slices/patterns_slice';
+import { reset as resetQueryResults } from '../../redux/slices/query_result_slice';
+import { changeData, reset as resetQuery } from '../../redux/slices/query_slice';
+import { reset as resetVisualization } from '../../redux/slices/visualization_slice';
+import { reset as resetVisConfig } from '../../redux/slices/viualization_config_slice';
 
 const DATA_SOURCE_SELECTOR_CONFIGS = { customGroupTitleExtension: '' };
 
@@ -141,7 +141,7 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
     sqlService
       .fetch(requestPayload)
       .then((result) => {
-        setAsyncSessionId(getObjValue(result, 'sessionId', null));
+        setAsyncSessionId(dataSource, getObjValue(result, 'sessionId', null));
       })
       .catch((e) => {
         console.error(e);
@@ -226,7 +226,7 @@ export const DataSourceSelection = ({ tabId }: { tabId: string }) => {
     const dsType = explorerSearchMetadata.datasources?.[0]?.type;
     const dsName = explorerSearchMetadata.datasources?.[0]?.label;
     if (
-      !getAsyncSessionId() &&
+      !getAsyncSessionId(dsName) &&
       [DATA_SOURCE_TYPES.SPARK, DATA_SOURCE_TYPES.S3Glue].includes(dsType) &&
       dsName
     ) {

public/framework/catalog_cache/cache_intercept.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  HttpInterceptorResponseError,
+  IHttpInterceptController,
+} from '../../../../../src/core/public';
+import { SECURITY_PLUGIN_ACCOUNT_API } from '../../../common/constants/shared';
+import { CatalogCacheManager } from './cache_manager';
+
+export function catalogCacheInterceptError(): any {
+  return (httpErrorResponse: HttpInterceptorResponseError, _: IHttpInterceptController) => {
+    if (
+      httpErrorResponse.response?.status === 401 &&
+      httpErrorResponse.fetchOptions.path === SECURITY_PLUGIN_ACCOUNT_API
+    ) {
+      // Clears all user catalog cache details
+      CatalogCacheManager.clearDataSourceCache();
+      CatalogCacheManager.clearAccelerationsCache();
+    }
+  };
+}