[ML] Use indices options in anomaly detection job wizards (#91830) (#94755)

* [ML] WIP Datafeed preview refactor

* adding indices options to ad job creation searches

* update datafeed preview schema

* updating types

* recalculating wizard time range on JSON edit save

* updating endpoint docs

* fixing types

* more type fixes

* fixing missing runtime fields

* using isPopulatedObject

* adding indices options schema

* fixing test

* fixing schema

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: James Gowdy <jgowdy@elastic.co>

Author: kibanamachine

x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts

@@ -45,7 +45,7 @@ export type Aggregation = Record<
   }
 >;
 
-interface IndicesOptions {
+export interface IndicesOptions {
   expand_wildcards?: 'all' | 'open' | 'closed' | 'hidden' | 'none';
   ignore_unavailable?: boolean;
   allow_no_indices?: boolean;

x-pack/plugins/ml/common/types/job_service.ts

@@ -5,7 +5,9 @@
  * 2.0.
  */
 
-import { Job, JobStats } from './anomaly_detection_jobs';
+import { Job, JobStats, IndicesOptions } from './anomaly_detection_jobs';
+import { RuntimeMappings } from './fields';
+import { ES_AGGREGATION } from '../constants/aggregation_types';
 
 export interface MlJobsResponse {
   jobs: Job[];
@@ -23,3 +25,18 @@ export interface JobsExistResponse {
     isGroup: boolean;
   };
 }
+
+export interface BucketSpanEstimatorData {
+  aggTypes: Array<ES_AGGREGATION | null>;
+  duration: {
+    start: number;
+    end: number;
+  };
+  fields: Array<string | null>;
+  index: string;
+  query: any;
+  splitField: string | undefined;
+  timeField: string | undefined;
+  runtimeMappings: RuntimeMappings | undefined;
+  indicesOptions: IndicesOptions | undefined;
+}

x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/chart_loader.ts

@@ -29,7 +29,8 @@ export function chartLoaderProvider(mlResultsService: MlResultsService) {
       job.data_description.time_field,
       job.data_counts.earliest_record_timestamp,
       job.data_counts.latest_record_timestamp,
-      intervalMs
+      intervalMs,
+      job.datafeed_config.indices_options
     );
     if (resp.error !== undefined) {
       throw resp.error;

x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts

@@ -8,6 +8,7 @@
 import memoizeOne from 'memoize-one';
 import { isEqual } from 'lodash';
 import { IndexPatternTitle } from '../../../../../../common/types/kibana';
+import { IndicesOptions } from '../../../../../../common/types/anomaly_detection_jobs';
 import {
   Field,
   SplitField,
@@ -56,7 +57,8 @@ export class ChartLoader {
     splitField: SplitField,
     splitFieldValue: SplitFieldValue,
     intervalMs: number,
-    runtimeMappings: RuntimeMappings | null
+    runtimeMappings: RuntimeMappings | null,
+    indicesOptions?: IndicesOptions
   ): Promise<LineChartData> {
     if (this._timeFieldName !== '') {
       if (aggFieldPairsCanBeCharted(aggFieldPairs) === false) {
@@ -77,7 +79,8 @@ export class ChartLoader {
         aggFieldPairNames,
         splitFieldName,
         splitFieldValue,
-        runtimeMappings ?? undefined
+        runtimeMappings ?? undefined,
+        indicesOptions
       );
 
       return resp.results;
@@ -91,7 +94,8 @@ export class ChartLoader {
     aggFieldPairs: AggFieldPair[],
     splitField: SplitField,
     intervalMs: number,
-    runtimeMappings: RuntimeMappings | null
+    runtimeMappings: RuntimeMappings | null,
+    indicesOptions?: IndicesOptions
   ): Promise<LineChartData> {
     if (this._timeFieldName !== '') {
       if (aggFieldPairsCanBeCharted(aggFieldPairs) === false) {
@@ -111,7 +115,8 @@ export class ChartLoader {
         this._query,
         aggFieldPairNames,
         splitFieldName,
-        runtimeMappings ?? undefined
+        runtimeMappings ?? undefined,
+        indicesOptions
       );
 
       return resp.results;
@@ -122,7 +127,8 @@ export class ChartLoader {
   async loadEventRateChart(
     start: number,
     end: number,
-    intervalMs: number
+    intervalMs: number,
+    indicesOptions?: IndicesOptions
   ): Promise<LineChartPoint[]> {
     if (this._timeFieldName !== '') {
       const resp = await getEventRateData(
@@ -131,7 +137,8 @@ export class ChartLoader {
         this._timeFieldName,
         start,
         end,
-        intervalMs * 3
+        intervalMs * 3,
+        indicesOptions
       );
       if (resp.error !== undefined) {
         throw resp.error;
@@ -147,14 +154,16 @@ export class ChartLoader {
 
   async loadFieldExampleValues(
     field: Field,
-    runtimeMappings: RuntimeMappings | null
+    runtimeMappings: RuntimeMappings | null,
+    indicesOptions?: IndicesOptions
   ): Promise<string[]> {
     const { results } = await getCategoryFields(
       this._indexPatternTitle,
       field.name,
       10,
       this._query,
-      runtimeMappings ?? undefined
+      runtimeMappings ?? undefined,
+      indicesOptions
     );
     return results;
   }

x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/searches.ts

@@ -9,6 +9,7 @@ import { get } from 'lodash';
 
 import { ml } from '../../../../services/ml_api_service';
 import { RuntimeMappings } from '../../../../../../common/types/fields';
+import { IndicesOptions } from '../../../../../../common/types/anomaly_detection_jobs';
 
 interface CategoryResults {
   success: boolean;
@@ -20,7 +21,8 @@ export function getCategoryFields(
   fieldName: string,
   size: number,
   query: any,
-  runtimeMappings?: RuntimeMappings
+  runtimeMappings?: RuntimeMappings,
+  indicesOptions?: IndicesOptions
 ): Promise<CategoryResults> {
   return new Promise((resolve, reject) => {
     ml.esSearch({
@@ -38,6 +40,7 @@ export function getCategoryFields(
         },
         ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}),
       },
+      ...(indicesOptions ?? {}),
     })
       .then((resp: any) => {
         const catFields = get(resp, ['aggregations', 'catFields', 'buckets'], []);

x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts

@@ -177,16 +177,13 @@ export class AdvancedJobCreator extends JobCreator {
   // load the start and end times for the selected index
   // and apply them to the job creator
   public async autoSetTimeRange() {
-    try {
-      const { start, end } = await ml.getTimeFieldRange({
-        index: this._indexPatternTitle,
-        timeFieldName: this.timeFieldName,
-        query: this.query,
-      });
-      this.setTimeRange(start.epoch, end.epoch);
-    } catch (error) {
-      throw Error(error);
-    }
+    const { start, end } = await ml.getTimeFieldRange({
+      index: this._indexPatternTitle,
+      timeFieldName: this.timeFieldName,
+      query: this.query,
+      indicesOptions: this.datafeedConfig.indices_options,
+    });
+    this.setTimeRange(start.epoch, end.epoch);
   }
 
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {

x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts

@@ -51,7 +51,8 @@ export class CategorizationExamplesLoader {
       this._jobCreator.start,
       this._jobCreator.end,
       analyzer,
-      this._jobCreator.runtimeMappings ?? undefined
+      this._jobCreator.runtimeMappings ?? undefined,
+      this._jobCreator.datafeedConfig.indices_options
     );
     return resp;
   }

x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/results_loader.ts

@@ -256,7 +256,8 @@ export class ResultsLoader {
       if (this._jobCreator.splitField !== null) {
         const fieldValues = await this._chartLoader.loadFieldExampleValues(
           this._jobCreator.splitField,
-          this._jobCreator.runtimeMappings
+          this._jobCreator.runtimeMappings,
+          this._jobCreator.datafeedConfig.indices_options
         );
         if (fieldValues.length > 0) {
           this._detectorSplitFieldFilters = {

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx

@@ -25,7 +25,9 @@ import { CombinedJob, Datafeed } from '../../../../../../../../common/types/anom
 import { ML_EDITOR_MODE, MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor';
 import { isValidJson } from '../../../../../../../../common/util/validation_utils';
 import { JobCreatorContext } from '../../job_creator_context';
+import { isAdvancedJobCreator } from '../../../../common/job_creator';
 import { DatafeedPreview } from '../datafeed_preview_flyout';
+import { useToastNotificationService } from '../../../../../../services/toast_notification_service';
 
 export enum EDITOR_MODE {
   HIDDEN,
@@ -40,6 +42,7 @@ interface Props {
 }
 export const JsonEditorFlyout: FC<Props> = ({ isDisabled, jobEditorMode, datafeedEditorMode }) => {
   const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext);
+  const { displayErrorToast } = useToastNotificationService();
   const [showJsonFlyout, setShowJsonFlyout] = useState(false);
   const [showChangedIndicesWarning, setShowChangedIndicesWarning] = useState(false);
 
@@ -120,10 +123,23 @@ export const JsonEditorFlyout: FC<Props> = ({ isDisabled, jobEditorMode, datafee
     setSaveable(valid);
   }
 
-  function onSave() {
+  async function onSave() {
     const jobConfig = JSON.parse(jobConfigString);
     const datafeedConfig = JSON.parse(collapseLiteralStrings(datafeedConfigString));
     jobCreator.cloneFromExistingJob(jobConfig, datafeedConfig);
+    if (isAdvancedJobCreator(jobCreator)) {
+      try {
+        await jobCreator.autoSetTimeRange();
+      } catch (error) {
+        const title = i18n.translate(
+          'xpack.ml.newJob.wizard.jsonFlyout.autoSetJobCreatorTimeRange.error',
+          {
+            defaultMessage: `Error retrieving beginning and end times of index`,
+          }
+        );
+        displayErrorToast(error, title);
+      }
+    }
     jobCreatorUpdate();
     setShowJsonFlyout(false);
   }

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts

@@ -9,12 +9,13 @@ import { useContext, useState } from 'react';
 
 import { JobCreatorContext } from '../../../job_creator_context';
 import { EVENT_RATE_FIELD_ID } from '../../../../../../../../../common/types/fields';
+import { BucketSpanEstimatorData } from '../../../../../../../../../common/types/job_service';
 import {
   isMultiMetricJobCreator,
   isPopulationJobCreator,
   isAdvancedJobCreator,
 } from '../../../../../common/job_creator';
-import { ml, BucketSpanEstimatorData } from '../../../../../../../services/ml_api_service';
+import { ml } from '../../../../../../../services/ml_api_service';
 import { useMlContext } from '../../../../../../../contexts/ml';
 import { getToastNotificationService } from '../../../../../../../services/toast_notification_service';
 
@@ -41,6 +42,7 @@ export function useEstimateBucketSpan() {
     splitField: undefined,
     timeField: mlContext.currentIndexPattern.timeFieldName,
     runtimeMappings: jobCreator.runtimeMappings ?? undefined,
+    indicesOptions: jobCreator.datafeedConfig.indices_options,
   };
 
   if (