[ML] Data Frames: Fix to avoid queries for indices with more than 1024 fields. (#37183) (#37227)

- 7.0 introduced indices.query.bool.max_clause_count which defaults to 1024. This can break certain queries (e.g. simple_query) for indices which have more than 1024 fields (e.g. certain beats indices). The optional data frames query uses simple_query and could therefor break the source index preview as well the pivot preview and pivot job itself given these conditions.
- Originally the default query (* used for simple_query) was always applied for source index previews and pivot previews. A new check isDefaultQuery() will now allow a) the source index preview to use a more efficient match_all query and b) avoid adding the query to the pivot config. This avoids triggering the max_clause_count when no optional query is set.
- If an index has more than 1024 fields, the input form for an optional query will be hidden. A helper text explains the reasoning. This avoids triggering max_clause_count related errors from within the UI. A user can still copy a UI created config to the clipboard and add an optional query in Kibana dev console.
- Additionally, this PR adds a fix to format date fields in the source index preview table using moment-timezone and formatHumanReadableDateTimeSeconds to display dates with the correct timezone.

Author: walterra

x-pack/plugins/ml/public/data_frame/common/request.test.ts

@@ -11,9 +11,32 @@ import { DefinePivotExposedState } from '../components/define_pivot/define_pivot
 
 import { PIVOT_SUPPORTED_GROUP_BY_AGGS } from './pivot_group_by';
 import { PivotAggsConfig, PIVOT_SUPPORTED_AGGS } from './pivot_aggs';
-import { getDataFramePreviewRequest, getDataFrameRequest, getPivotQuery } from './request';
+import {
+  getDataFramePreviewRequest,
+  getDataFrameRequest,
+  getPivotQuery,
+  isDefaultQuery,
+  isSimpleQuery,
+  PivotQuery,
+} from './request';
+
+const defaultQuery: PivotQuery = { query_string: { query: '*' } };
+const matchAllQuery: PivotQuery = { match_all: {} };
+const simpleQuery: PivotQuery = { query_string: { query: 'airline:AAL' } };
 
 describe('Data Frame: Common', () => {
+  test('isSimpleQuery()', () => {
+    expect(isSimpleQuery(defaultQuery)).toBe(true);
+    expect(isSimpleQuery(matchAllQuery)).toBe(false);
+    expect(isSimpleQuery(simpleQuery)).toBe(true);
+  });
+
+  test('isDefaultQuery()', () => {
+    expect(isDefaultQuery(defaultQuery)).toBe(true);
+    expect(isDefaultQuery(matchAllQuery)).toBe(false);
+    expect(isDefaultQuery(simpleQuery)).toBe(false);
+  });
+
   test('getPivotQuery()', () => {
     const query = getPivotQuery('the-query');
 

x-pack/plugins/ml/public/data_frame/common/request.ts

@@ -68,6 +68,14 @@ export function getPivotQuery(search: string | SavedSearchQuery): PivotQuery {
   return search;
 }
 
+export function isSimpleQuery(arg: any): arg is SimpleQuery {
+  return arg.query_string !== undefined;
+}
+
+export function isDefaultQuery(query: PivotQuery): boolean {
+  return isSimpleQuery(query) && query.query_string.query === '*';
+}
+
 export function getDataFramePreviewRequest(
   indexPatternTitle: IndexPattern['title'],
   query: PivotQuery,
@@ -77,14 +85,17 @@ export function getDataFramePreviewRequest(
   const request: DataFramePreviewRequest = {
     source: {
       index: indexPatternTitle,
-      query,
     },
     pivot: {
       group_by: {},
       aggregations: {},
     },
   };
 
+  if (!isDefaultQuery(query)) {
+    request.source.query = query;
+  }
+
   groupBy.forEach(g => {
     if (g.agg === PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS) {
       const termsAgg: TermsAgg = {

x-pack/plugins/ml/public/data_frame/components/define_pivot/common.test.ts

@@ -128,13 +128,7 @@ describe('Data Frame: Define Pivot Common', () => {
     expect(pivotPreviewDevConsoleStatement).toBe(`POST _data_frame/transforms/_preview
 {
   "source": {
-    "index": "the-index-pattern-title",
-    "query": {
-      "query_string": {
-        "query": "*",
-        "default_operator": "AND"
-      }
-    }
+    "index": "the-index-pattern-title"
   },
   "pivot": {
     "group_by": {

x-pack/plugins/ml/public/data_frame/components/define_pivot/define_pivot_form.tsx

@@ -191,6 +191,11 @@ export const DefinePivotForm: SFC<Props> = React.memo(({ overrides = {}, onChang
     ]
   );
 
+  // TODO This should use the actual value of `indices.query.bool.max_clause_count`
+  const maxIndexFields = 1024;
+  const numIndexFields = indexPattern.fields.length;
+  const disabledQuery = numIndexFields > maxIndexFields;
+
   return (
     <EuiFlexGroup>
       <EuiFlexItem grow={false} style={{ minWidth: '420px' }}>
@@ -201,28 +206,42 @@ export const DefinePivotForm: SFC<Props> = React.memo(({ overrides = {}, onChang
                 label={i18n.translate('xpack.ml.dataframe.definePivotForm.indexPatternLabel', {
                   defaultMessage: 'Index pattern',
                 })}
+                helpText={
+                  disabledQuery
+                    ? i18n.translate('xpack.ml.dataframe.definePivotForm.indexPatternHelpText', {
+                        defaultMessage:
+                          'An optional query for this index pattern is not supported. The number of supported index fields is {maxIndexFields} whereas this index has {numIndexFields} fields.',
+                        values: {
+                          maxIndexFields,
+                          numIndexFields,
+                        },
+                      })
+                    : ''
+                }
               >
                 <span>{kibanaContext.currentIndexPattern.title}</span>
               </EuiFormRow>
-              <EuiFormRow
-                label={i18n.translate('xpack.ml.dataframe.definePivotForm.queryLabel', {
-                  defaultMessage: 'Query',
-                })}
-                helpText={i18n.translate('xpack.ml.dataframe.definePivotForm.queryHelpText', {
-                  defaultMessage: 'Use a query string to filter the source data (optional).',
-                })}
-              >
-                <EuiFieldSearch
-                  placeholder={i18n.translate(
-                    'xpack.ml.dataframe.definePivotForm.queryPlaceholder',
-                    {
-                      defaultMessage: 'Search...',
-                    }
-                  )}
-                  onChange={searchHandler}
-                  value={search === defaultSearch ? emptySearch : search}
-                />
-              </EuiFormRow>
+              {!disabledQuery && (
+                <EuiFormRow
+                  label={i18n.translate('xpack.ml.dataframe.definePivotForm.queryLabel', {
+                    defaultMessage: 'Query',
+                  })}
+                  helpText={i18n.translate('xpack.ml.dataframe.definePivotForm.queryHelpText', {
+                    defaultMessage: 'Use a query string to filter the source data (optional).',
+                  })}
+                >
+                  <EuiFieldSearch
+                    placeholder={i18n.translate(
+                      'xpack.ml.dataframe.definePivotForm.queryPlaceholder',
+                      {
+                        defaultMessage: 'Search...',
+                      }
+                    )}
+                    onChange={searchHandler}
+                    value={search === defaultSearch ? emptySearch : search}
+                  />
+                </EuiFormRow>
+              )}
             </Fragment>
           )}
 

x-pack/plugins/ml/public/data_frame/components/source_index_preview/source_index_preview.tsx

@@ -5,6 +5,7 @@
  */
 
 import React, { FunctionComponent, useContext, useState } from 'react';
+import moment from 'moment-timezone';
 
 import { i18n } from '@kbn/i18n';
 
@@ -36,7 +37,9 @@ interface ExpandableTableProps extends EuiInMemoryTableProps {
 
 const ExpandableTable = (EuiInMemoryTable as any) as FunctionComponent<ExpandableTableProps>;
 
+import { KBN_FIELD_TYPES } from '../../../../common/constants/field_types';
 import { Dictionary } from '../../../../common/types/common';
+import { formatHumanReadableDateTimeSeconds } from '../../../util/date_utils';
 
 import { isKibanaContext, KibanaContext, PivotQuery } from '../../common';
 
@@ -208,15 +211,23 @@ export const SourceIndexPreview: React.SFC<Props> = React.memo(({ cellClick, que
     const column = {
       field: `_source.${k}`,
       name: k,
-      render: undefined,
       sortable: true,
       truncateText: true,
     } as Dictionary<any>;
 
+    const field = indexPattern.fields.find(f => f.name === k);
+    const render = (d: string) => {
+      return field !== undefined && field.type === KBN_FIELD_TYPES.DATE
+        ? formatHumanReadableDateTimeSeconds(moment(d).unix() * 1000)
+        : d;
+    };
+
+    column.render = render;
+
     if (CELL_CLICK_ENABLED && cellClick) {
       column.render = (d: string) => (
         <EuiButtonEmpty size="xs" onClick={() => cellClick(`${k}:(${d})`)}>
-          {d}
+          {render(d)}
         </EuiButtonEmpty>
       );
     }

x-pack/plugins/ml/public/data_frame/components/source_index_preview/use_source_index_data.ts

@@ -12,7 +12,7 @@ import { IndexPattern } from 'ui/index_patterns';
 
 import { ml } from '../../../services/ml_api_service';
 
-import { PivotQuery } from '../../common';
+import { isDefaultQuery, PivotQuery } from '../../common';
 import { EsDoc, EsFieldName, getDefaultSelectableFields } from './common';
 
 const SEARCH_SIZE = 1000;
@@ -48,7 +48,8 @@ export const useSourceIndexData = (
       const resp: SearchResponse<any> = await ml.esSearch({
         index: indexPattern.title,
         size: SEARCH_SIZE,
-        body: { query },
+        // Instead of using the default query (`*`), fall back to a more efficient `match_all` query.
+        body: { query: isDefaultQuery(query) ? { match_all: {} } : query },
       });
 
       const docs = resp.hits.hits;