[data] cached data structures TTL

If the dataset type config passes a meta object with `updateAt`
within the data structure we know that the config opts into the TTL.

If TTL over configuring UI setting then refetch in the case of S3
to get a new session ID in the cache.

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

src/plugins/data/common/constants.ts

@@ -104,6 +104,7 @@ export const UI_SETTINGS = {
   FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault',
   FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues',
   QUERY_ENHANCEMENTS_ENABLED: 'query:enhancements:enabled',
+  QUERY_CACHED_DATA_STRUCTURES_TTL: 'query:cachedDataStructures:ttl',
   QUERY_DATAFRAME_HYDRATION_STRATEGY: 'query:dataframe:hydrationStrategy',
   SEARCH_QUERY_LANGUAGE_BLOCKLIST: 'search:queryLanguageBlocklist',
   NEW_HOME_PAGE: 'home:useNewHomePage',

src/plugins/data/common/datasets/types.ts

@@ -159,6 +159,7 @@ export interface DataStructureCustomMeta {
   type: DATA_STRUCTURE_META_TYPES.CUSTOM;
   icon?: EuiIconProps;
   tooltip?: string;
+  updatedAt?: number;
   [key: string]: any;
 }
 

src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts

@@ -102,7 +102,12 @@ export class DatasetService {
     const cacheKey = `${dataType}.${lastPathItem.id}`;
 
     const cachedDataStructure = this.sessionStorage.get<CachedDataStructure>(cacheKey);
-    if (cachedDataStructure?.children?.length > 0) {
+    if (
+      cachedDataStructure?.children?.length > 0 &&
+      (!cachedDataStructure?.meta?.updatedAt ||
+        Date.now() - cachedDataStructure?.meta?.updatedAt <
+          this.uiSettings.get(UI_SETTINGS.QUERY_CACHED_DATA_STRUCTURES_TTL))
+    ) {
       return this.cacheToDataStructure(dataType, cachedDataStructure);
     }
 

src/plugins/data/server/ui_settings.ts

@@ -720,6 +720,20 @@ export function getUiSettings(): Record<string, UiSettingsParams<unknown>> {
       requiresPageReload: true,
       schema: schema.boolean(),
     },
+    [UI_SETTINGS.QUERY_CACHED_DATA_STRUCTURES_TTL]: {
+      name: i18n.translate('data.advancedSettings.query.cachedDataStructures.ttl', {
+        defaultMessage: 'Cached data structures TTL',
+      }),
+      value: 600000,
+      description: i18n.translate('data.advancedSettings.query.cachedDataStructures.ttl', {
+        defaultMessage: `
+          <strong>Experimental</strong>:
+          Cached data structures TTL before refetching if the data type has it configured.`,
+      }),
+      category: ['search'],
+      requiresPageReload: true,
+      schema: schema.oneOf([schema.number({ min: 0 }), schema.literal('Infinity')]),
+    },
     [UI_SETTINGS.QUERY_DATAFRAME_HYDRATION_STRATEGY]: {
       name: i18n.translate('data.advancedSettings.query.dataFrameHydrationStrategyTitle', {
         defaultMessage: 'Data frame hydration strategy',

src/plugins/query_enhancements/public/datasets/s3_type.ts

@@ -146,6 +146,7 @@ const setMeta = (dataStructure: DataStructure, response: any) => {
     ...dataStructure.meta,
     queryId: response.queryId,
     sessionId: response.sessionId,
+    updatedAt: Date.now(),
   } as DataStructureCustomMeta;
 };