[ML] Fix data visualizer grid failing if one of the fields failed and not updating when refreshed (#115644) (#118267)

* [ML] Initial embed

* [ML] Initial embed props

* [ML] Add top nav link to data viz

* Add visible fields

* Add add data service to register links

* Renames, refactor, use constants

* Renames, refactor, use constants

* Update tests and mocks

* Embeddable

* Update hook to update upon time udpate

* Add filter support to query

* Refactor filter utilities

* Add filter support for embeddable

* Fix saved search data undefined

* Prototype aggregated view/document view switcher

* Prototype flyout

* Prototype save document view option in storage

* Fix filter and query conflict with saved search

* Minor styling edits

* [ML] Initial embed

* [ML] Initial embed props

* Add embeddable 1

* Add visible fields

* Embeddable 2

* Add filter support to query

* Refactor filter utilities

* Add filter support for embeddable

* Fix saved search data undefined

* Prototype aggregated view/document view switcher

* Prototype flyout

* Prototype save document view option in storage

* Fix filter and query conflict with saved search

* Minor styling edits

* Fix missing code after conflicts

* Remove dv locator and flyout

* Make types happy

* Fix types

* Rename toggle option

* Resolve conflicts

* [ML] Reduce size of chart

* [ML] Unbold name, switch icons of show distributions

* [ML] Make size consistent

* [ML] Make page size 25

* [ML] Switch to arrow right and down

* [ML] Make legend font smaller

* [ML] Add user setting

* [ML] Add show preview by default setting

* [ML] Match icon

* Add panels around the subcontent

* Add preference for aggregated vs doc

* Fix types

* Fix types, add constants for adv settings

* Change to data view type

* Temp fix for Kibana/EUI table overflow issue

* Modify line height so text is not cut off, modify widths for varying screen sizes

* Different width padders for different screens

* Fix CI

* Merge latest, move button to the right

* Fix width for bar charts previews

* Fix toggle buttons, fix maps

* Delete unused file

* Fix boolean styling

* Change to enum, discover mode

* Hide field stats

* Hide field stats

* Persist show mini preview/distribution settings

* Remove window size, use size observer instead

* Default to document view

* Remove bold, switch icon

* Set fixed width for top values, reduce font size in table

* Fix custom url tests

* Update width styling for panels

* Fix missing flag for Discover sidebar, jest tests

* Fix max width

* Workaround for sorting

* Fix import

* Fix styling

* Make height uniform, center alignment, fix map and keyword map not same size

Move styling

* Revert "Make height uniform, center alignment, fix map and keyword map not same size"

This reverts commit 8fc42e2

* Revert "Make height uniform, center alignment, fix map and keyword map not same size"

This reverts commit 8fc42e2

* Uniform height, left aligned, flex grid

* Switch top values to have labels

* Center content

* Replace fixed widths with percentage

* Fix table missing field types

* Add dashboard embeddable and filter support

* Fix file data viz styling and tests, lean up imports, remove hard coded pixels

* Add search panel/kql filter bar

* Temporarily fix scrolling

* New kql filters for data visualizer

* Set map height so it will fit the sampler shard size text

* Use eui progress labels

* Fix spacer

* Add beta badge

* Temporarily fix scrolling

* Fix grow for Top Values for

* [ML] Update functional tests to reflect new arrow icons

* [ML] Add filter buttons and KQL bars

* [ML] Update filter bar onChange behavior

* [ML] Update top values filter onChange behavior

* [ML] Update search filters when opening saved search

* [ML] Clean up

* [ML] Remove fit content for height

* [ML] Fix boolean legend

* [ML] Fix header section when browser width is small to large and when index pattern title is too large

* [ML] Hide expander icon when dimension is xs or s & css fixes

* [ML] Delete embeddables because they are not use

* [ML] Rename view mode, refactor to separate hook, add error prompt if can't show, rename wrapper, clean up & fix tests

* [ML] Make doc count 0 for empty fields, update t/f test

* [ML] Add unit testing for search utils

* Fix missing unsubscribe for embeddable output

* Remove redundant onAddFilter for this PR, fix width

* Rename Field Stats to Field stats to match convention

* [ML] Fix expand all/collapse all behavior to override individual setting

* [ML] Fix functional tests should be 0/0%

* [ML] Fix docs content spacing, rename classnames, add filters to Discover, lens, and maps

* [ML] Fix doc count for fields that exists but have no stats

* [ML] Fix icon styling to match Discover but have text/keyword/histogram

* [ML] Fix doc count for fields that exists but have no stats

* [ML] Rename classnames to BEM style

* Resolve latest changes

* Add in place ss

* Refactor helper functions

* Refactor helper functions

* Add error log

* Migrate overall stats to data's search

* Better handle errors

* Fix url so restore session brings back correct view

* Add progress bar

* [ML] Add tests for data viz in Discover

* [ML] Change to combinelatest

* Update tests & dashboard behavior to reflect new advanced settings

* Update telemetry

* Remove workaround after eui bump fix

* Remove dataloader

* Snapshot

* Migrate search to client side

* Consolidate types

* Change back to forkjoin instead of combinelatest for overallstats

* Fix missing bool clause

* Add login

* Fix saved search attributes broken with latest changes

* Update tests

* Fix import

* Match the no results found

* Reset field stats so it reloads when query is refreshed

* Reset field stats so it reloads when query is refreshed

* Add doc stats

* Merge to use hook completely

* Merge to use hook completely

* Fix doc chart doesn't show up when page is first mounted

* Fix Discover auto refresh previously didn't update

* Fix query util to return search source's results right away. Fix texts.

* Refactor documentStats

* Fix doc stats not showing upon page mount

* Fix types

* Delete old files

* Update tests & i18n

* Fix examples, tests

* Remove old files & routes

* Add telemetry, clean up, rename components for clarity

* Fix size of callout message

* Fix texts field

* Consolidate field type

* Consolidate field type, add count to top values

* Clean up

* Update tests

* Remove progress on embedadble

* Update snapshot

* Clean up, consolidate searchOptions

* Fix new es client types

* Fix types

* Fix loading state in Discover

* Remove unused services, Change switchMap to map, mergeMap -> switchMap, update types

* Fix missing filters

* Fix message of table to show searching instead of no items found

* Fix dashboard saved search source persisting time range

* [ML] Fix table message state

* [ML] Fix to not fetch field stats if cardinality is 0

* [ML] Fix locator missing view mode

* [ML] Quit right away if field doesn't exist in docs

* [ML] Change to use batch and only retry with individual field if failed

* [ML] Batch requests for speed and retry failures for resiliency

* No need to fetch field stats if overall stats haven't completed

* Wait on overallStats to complete

* Fix types after merge

* Fix payload size too big 413, num of requests

* Update field icon to using kbn/react-field package

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Author: qn895

src/plugins/discover/public/application/apps/main/components/layout/discover_layout.tsx

@@ -43,8 +43,12 @@ import {
   SavedSearchURLConflictCallout,
   useSavedSearchAliasMatchRedirect,
 } from '../../../../../saved_searches';
-import { DiscoverDataVisualizerGrid } from '../../../../components/data_visualizer_grid';
 import { VIEW_MODE } from '../view_mode_toggle';
+import {
+  DOCUMENTS_VIEW_CLICK,
+  FIELD_STATISTICS_VIEW_CLICK,
+} from '../../../../components/field_stats_table/constants';
+import { FieldStatisticsTable } from '../../../../components/field_stats_table';
 
 /**
  * Local storage key for sidebar persistence state
@@ -54,7 +58,7 @@ export const SIDEBAR_CLOSED_KEY = 'discover:sidebarClosed';
 const SidebarMemoized = React.memo(DiscoverSidebarResponsive);
 const TopNavMemoized = React.memo(DiscoverTopNav);
 const DiscoverChartMemoized = React.memo(DiscoverChart);
-const DataVisualizerGridMemoized = React.memo(DiscoverDataVisualizerGrid);
+const FieldStatisticsTableMemoized = React.memo(FieldStatisticsTable);
 
 export function DiscoverLayout({
   indexPattern,
@@ -95,8 +99,16 @@ export function DiscoverLayout({
   const setDiscoverViewMode = useCallback(
     (mode: VIEW_MODE) => {
       stateContainer.setAppState({ viewMode: mode });
+
+      if (trackUiMetric) {
+        if (mode === VIEW_MODE.AGGREGATED_LEVEL) {
+          trackUiMetric(METRIC_TYPE.CLICK, FIELD_STATISTICS_VIEW_CLICK);
+        } else {
+          trackUiMetric(METRIC_TYPE.CLICK, DOCUMENTS_VIEW_CLICK);
+        }
+      }
     },
-    [stateContainer]
+    [trackUiMetric, stateContainer]
   );
 
   const fetchCounter = useRef<number>(0);
@@ -315,7 +327,7 @@ export function DiscoverLayout({
                       stateContainer={stateContainer}
                     />
                   ) : (
-                    <DataVisualizerGridMemoized
+                    <FieldStatisticsTableMemoized
                       savedSearch={savedSearch}
                       services={services}
                       indexPattern={indexPattern}
@@ -324,6 +336,8 @@ export function DiscoverLayout({
                       columns={columns}
                       stateContainer={stateContainer}
                       onAddFilter={onAddFilter}
+                      trackUiMetric={trackUiMetric}
+                      savedSearchRefetch$={savedSearchRefetch$}
                     />
                   )}
                 </EuiFlexGroup>

src/plugins/discover/public/application/apps/main/components/sidebar/discover_field.tsx

@@ -58,9 +58,11 @@ const FieldInfoIcon: React.FC = memo(() => (
   </EuiToolTip>
 ));
 
-const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => (
-  <FieldIcon type={field.type} label={getFieldTypeName(field.type)} scripted={field.scripted} />
-));
+const DiscoverFieldTypeIcon: React.FC<{ field: IndexPatternField }> = memo(({ field }) => {
+  // If it's a string type, we want to distinguish between keyword and text
+  const tempType = field.type === 'string' && field.esTypes ? field.esTypes[0] : field.type;
+  return <FieldIcon type={tempType} label={getFieldTypeName(tempType)} scripted={field.scripted} />;
+});
 
 const FieldName: React.FC<{ field: IndexPatternField }> = memo(({ field }) => {
   const title =

src/plugins/discover/public/application/apps/main/components/sidebar/discover_sidebar.test.tsx

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { each, cloneDeep } from 'lodash';
+import { cloneDeep, each } from 'lodash';
 import { ReactWrapper } from 'enzyme';
 import { findTestSubject } from '@elastic/eui/lib/test';
 // @ts-expect-error

src/plugins/discover/public/application/apps/main/components/sidebar/lib/get_field_type_name.ts

@@ -51,6 +51,15 @@ export function getFieldTypeName(type: string) {
       return i18n.translate('discover.fieldNameIcons.stringFieldAriaLabel', {
         defaultMessage: 'String field',
       });
+    case 'text':
+      return i18n.translate('discover.fieldNameIcons.textFieldAriaLabel', {
+        defaultMessage: 'Text field',
+      });
+    case 'keyword':
+      return i18n.translate('discover.fieldNameIcons.keywordFieldAriaLabel', {
+        defaultMessage: 'Keyword field',
+      });
+
     case 'nested':
       return i18n.translate('discover.fieldNameIcons.nestedFieldAriaLabel', {
         defaultMessage: 'Nested field',

src/plugins/discover/public/application/apps/main/services/discover_state.ts

@@ -411,5 +411,7 @@ function createUrlGeneratorState({
         }
       : undefined,
     useHash: false,
+    viewMode: appState.viewMode,
+    hideAggregatedPreview: appState.hideAggregatedPreview,
   };
 }

src/plugins/discover/public/application/components/field_stats_table/constants.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/** Telemetry related to field statistics table **/
+export const FIELD_STATISTICS_LOADED = 'field_statistics_loaded';
+export const FIELD_STATISTICS_VIEW_CLICK = 'field_statistics_view_click';
+export const DOCUMENTS_VIEW_CLICK = 'documents_view_click';

src/plugins/discover/public/application/components/data_visualizer_grid/data_visualizer_grid.tsx renamed to src/plugins/discover/public/application/components/field_stats_table/field_stats_table.tsx

@@ -7,18 +7,21 @@
  */
 
 import React, { useEffect, useMemo, useRef, useState } from 'react';
-import { Filter } from '@kbn/es-query';
+import type { Filter } from '@kbn/es-query';
+import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics';
 import { IndexPatternField, IndexPattern, DataView, Query } from '../../../../../data/common';
-import { DiscoverServices } from '../../../build_services';
+import type { DiscoverServices } from '../../../build_services';
 import {
   EmbeddableInput,
   EmbeddableOutput,
   ErrorEmbeddable,
   IEmbeddable,
   isErrorEmbeddable,
 } from '../../../../../embeddable/public';
+import { FIELD_STATISTICS_LOADED } from './constants';
 import { SavedSearch } from '../../../saved_searches';
 import { GetStateReturn } from '../../apps/main/services/discover_state';
+import { DataRefetch$ } from '../../apps/main/services/use_saved_search';
 
 export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   indexPattern: IndexPattern;
@@ -36,7 +39,7 @@ export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput {
   showDistributions?: boolean;
 }
 
-export interface DiscoverDataVisualizerGridProps {
+export interface FieldStatisticsTableProps {
   /**
    * Determines which columns are displayed
    */
@@ -69,14 +72,24 @@ export interface DiscoverDataVisualizerGridProps {
    * Filters query to update the table content
    */
   filters?: Filter[];
+  /**
+   * State container with persisted settings
+   */
   stateContainer?: GetStateReturn;
   /**
    * Callback to add a filter to filter bar
    */
   onAddFilter?: (field: IndexPatternField | string, value: string, type: '+' | '-') => void;
+  /**
+   * Metric tracking function
+   * @param metricType
+   * @param eventName
+   */
+  trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void;
+  savedSearchRefetch$?: DataRefetch$;
 }
 
-export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProps) => {
+export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
   const {
     services,
     indexPattern,
@@ -86,9 +99,10 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp
     filters,
     stateContainer,
     onAddFilter,
+    trackUiMetric,
+    savedSearchRefetch$,
   } = props;
   const { uiSettings } = services;
-
   const [embeddable, setEmbeddable] = useState<
     | ErrorEmbeddable
     | IEmbeddable<DataVisualizerGridEmbeddableInput, DataVisualizerGridEmbeddableOutput>
@@ -109,10 +123,16 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp
       }
     });
 
+    const refetch = savedSearchRefetch$?.subscribe(() => {
+      if (embeddable && !isErrorEmbeddable(embeddable)) {
+        embeddable.updateInput({ lastReloadRequestTime: Date.now() });
+      }
+    });
     return () => {
       sub?.unsubscribe();
+      refetch?.unsubscribe();
     };
-  }, [embeddable, stateContainer]);
+  }, [embeddable, stateContainer, savedSearchRefetch$]);
 
   useEffect(() => {
     if (embeddable && !isErrorEmbeddable(embeddable)) {
@@ -135,17 +155,11 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp
       embeddable.updateInput({
         showPreviewByDefault,
       });
+
       embeddable.reload();
     }
   }, [showPreviewByDefault, uiSettings, embeddable]);
 
-  useEffect(() => {
-    return () => {
-      // Clean up embeddable upon unmounting
-      embeddable?.destroy();
-    };
-  }, [embeddable]);
-
   useEffect(() => {
     let unmounted = false;
     const loadEmbeddable = async () => {
@@ -181,8 +195,15 @@ export const DiscoverDataVisualizerGrid = (props: DiscoverDataVisualizerGridProp
   useEffect(() => {
     if (embeddableRoot.current && embeddable) {
       embeddable.render(embeddableRoot.current);
+
+      trackUiMetric?.(METRIC_TYPE.LOADED, FIELD_STATISTICS_LOADED);
     }
-  }, [embeddable, embeddableRoot, uiSettings]);
+
+    return () => {
+      // Clean up embeddable upon unmounting
+      embeddable?.destroy();
+    };
+  }, [embeddable, embeddableRoot, uiSettings, trackUiMetric]);
 
   return (
     <div

src/plugins/discover/public/application/components/data_visualizer_grid/field_stats_table_embeddable.tsx renamed to src/plugins/discover/public/application/components/field_stats_table/field_stats_table_saved_search_embeddable.tsx

@@ -8,15 +8,12 @@
 
 import React from 'react';
 import { I18nProvider } from '@kbn/i18n/react';
-import {
-  DiscoverDataVisualizerGrid,
-  DiscoverDataVisualizerGridProps,
-} from './data_visualizer_grid';
+import { FieldStatisticsTable, FieldStatisticsTableProps } from './field_stats_table';
 
-export function FieldStatsTableEmbeddable(renderProps: DiscoverDataVisualizerGridProps) {
+export function FieldStatsTableSavedSearchEmbeddable(renderProps: FieldStatisticsTableProps) {
   return (
     <I18nProvider>
-      <DiscoverDataVisualizerGrid
+      <FieldStatisticsTable
         savedSearch={renderProps.savedSearch}
         services={renderProps.services}
         indexPattern={renderProps.indexPattern}

src/plugins/discover/public/application/components/data_visualizer_grid/index.ts renamed to src/plugins/discover/public/application/components/field_stats_table/index.ts

@@ -6,4 +6,5 @@
  * Side Public License, v 1.
  */
 
-export { DiscoverDataVisualizerGrid } from './data_visualizer_grid';
+export { FieldStatisticsTable } from './field_stats_table';
+export { FieldStatsTableSavedSearchEmbeddable } from './field_stats_table_saved_search_embeddable';

src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx

@@ -47,7 +47,7 @@ import { getDefaultSort } from '../apps/main/components/doc_table';
 import { SortOrder } from '../apps/main/components/doc_table/components/table_header/helpers';
 import { updateSearchSource } from './helpers/update_search_source';
 import { VIEW_MODE } from '../apps/main/components/view_mode_toggle';
-import { FieldStatsTableEmbeddable } from '../components/data_visualizer_grid/field_stats_table_embeddable';
+import { FieldStatsTableSavedSearchEmbeddable } from '../components/field_stats_table';
 
 export type SearchProps = Partial<DiscoverGridProps> &
   Partial<DocTableProps> & {
@@ -391,7 +391,7 @@ export class SavedSearchEmbeddable
       Array.isArray(searchProps.columns)
     ) {
       ReactDOM.render(
-        <FieldStatsTableEmbeddable
+        <FieldStatsTableSavedSearchEmbeddable
           services={searchProps.services}
           indexPattern={searchProps.indexPattern}
           columns={searchProps.columns}