[ML] Replace use of rest_total_hits_as_int with track_total_hits (#78423) (#78909)

* [ML] Replace use of rest_total_hits_as_int with track_total_hits

* [ML] Replace use of lodash get for obtaining hits total

* [ML] Update mocks to use new total hits format

Author: peteharverson

x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.js

@@ -297,11 +297,10 @@ export function getTestUrl(job, customUrl) {
   return new Promise((resolve, reject) => {
     ml.results
       .anomalySearch({
-        rest_total_hits_as_int: true,
         body,
       })
       .then((resp) => {
-        if (resp.hits.total > 0) {
+        if (resp.hits.total.value > 0) {
           const record = resp.hits.hits[0]._source;
           testUrl = replaceTokensInUrlValue(customUrl, bucketSpanSecs, record, 'timestamp');
           resolve(testUrl);

x-pack/plugins/ml/public/application/services/forecast_service.js

@@ -52,7 +52,6 @@ function getForecastsSummary(job, query, earliestMs, maxResults) {
     ml.results
       .anomalySearch({
         size: maxResults,
-        rest_total_hits_as_int: true,
         body: {
           query: {
             bool: {
@@ -63,7 +62,7 @@ function getForecastsSummary(job, query, earliestMs, maxResults) {
         },
       })
       .then((resp) => {
-        if (resp.hits.total !== 0) {
+        if (resp.hits.total.value > 0) {
           obj.forecasts = resp.hits.hits.map((hit) => hit._source);
         }
 
@@ -346,7 +345,6 @@ function getForecastRequestStats(job, forecastId) {
     ml.results
       .anomalySearch({
         size: 1,
-        rest_total_hits_as_int: true,
         body: {
           query: {
             bool: {
@@ -356,7 +354,7 @@ function getForecastRequestStats(job, forecastId) {
         },
       })
       .then((resp) => {
-        if (resp.hits.total !== 0) {
+        if (resp.hits.total.value > 0) {
           obj.stats = resp.hits.hits[0]._source;
         }
         resolve(obj);

x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts

@@ -403,7 +403,6 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
       return mlApiServices.results
         .anomalySearch$({
           index: ML_RESULTS_INDEX_PATTERN,
-          rest_total_hits_as_int: true,
           size: maxResults !== undefined ? maxResults : 100,
           body: {
             query: {
@@ -428,7 +427,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
         })
         .pipe(
           map((resp) => {
-            if (resp.hits.total !== 0) {
+            if (resp.hits.total.value > 0) {
               each(resp.hits.hits, (hit: any) => {
                 obj.records.push(hit._source);
               });

x-pack/plugins/ml/public/application/services/results_service/results_service.js

@@ -726,7 +726,6 @@ export function resultsServiceProvider(mlApiServices) {
         mlApiServices.results
           .anomalySearch({
             size: maxResults !== undefined ? maxResults : 100,
-            rest_total_hits_as_int: true,
             body: {
               _source: ['job_id', 'detector_index', 'influencers', 'record_score'],
               query: {
@@ -750,7 +749,7 @@ export function resultsServiceProvider(mlApiServices) {
             },
           })
           .then((resp) => {
-            if (resp.hits.total !== 0) {
+            if (resp.hits.total.value > 0) {
               each(resp.hits.hits, (hit) => {
                 obj.records.push(hit._source);
               });
@@ -858,7 +857,6 @@ export function resultsServiceProvider(mlApiServices) {
         mlApiServices.results
           .anomalySearch({
             size: maxResults !== undefined ? maxResults : 100,
-            rest_total_hits_as_int: true,
             body: {
               query: {
                 bool: {
@@ -881,7 +879,7 @@ export function resultsServiceProvider(mlApiServices) {
             },
           })
           .then((resp) => {
-            if (resp.hits.total !== 0) {
+            if (resp.hits.total.value > 0) {
               each(resp.hits.hits, (hit) => {
                 obj.records.push(hit._source);
               });
@@ -983,7 +981,6 @@ export function resultsServiceProvider(mlApiServices) {
         mlApiServices.results
           .anomalySearch({
             size: maxResults !== undefined ? maxResults : 100,
-            rest_total_hits_as_int: true,
             body: {
               query: {
                 bool: {
@@ -1006,7 +1003,7 @@ export function resultsServiceProvider(mlApiServices) {
             },
           })
           .then((resp) => {
-            if (resp.hits.total !== 0) {
+            if (resp.hits.total.value > 0) {
               each(resp.hits.hits, (hit) => {
                 obj.records.push(hit._source);
               });
@@ -1059,7 +1056,6 @@ export function resultsServiceProvider(mlApiServices) {
         mlApiServices
           .esSearch({
             index,
-            rest_total_hits_as_int: true,
             size: 0,
             body: {
               query: {
@@ -1091,7 +1087,7 @@ export function resultsServiceProvider(mlApiServices) {
               const time = dataForTime.key;
               obj.results[time] = dataForTime.doc_count;
             });
-            obj.total = resp.hits.total;
+            obj.total = resp.hits.total.value;
 
             resolve(obj);
           })
@@ -1228,13 +1224,13 @@ export function resultsServiceProvider(mlApiServices) {
           .esSearch({
             index,
             body,
-            rest_total_hits_as_int: true,
+            track_total_hits: true,
           })
           .then((resp) => {
             // Because of the sampling, results of metricFunctions which use sum or count
             // can be significantly skewed. Taking into account totalHits we calculate a
             // a factor to normalize results for these metricFunctions.
-            const totalHits = get(resp, ['hits', 'total'], 0);
+            const totalHits = resp.hits.total.value;
             const successfulShards = get(resp, ['_shards', 'successful'], 0);
 
             let normalizeFactor = 1;

x-pack/plugins/ml/server/models/data_frame_analytics/analytics_audit_messages.ts

@@ -71,7 +71,6 @@ export function analyticsAuditMessagesProvider({ asInternalUser }: IScopedCluste
     const { body } = await asInternalUser.search({
       index: ML_NOTIFICATION_INDEX_PATTERN,
       ignore_unavailable: true,
-      rest_total_hits_as_int: true,
       size: SIZE,
       body: {
         sort: [{ timestamp: { order: 'desc' } }, { job_id: { order: 'asc' } }],
@@ -80,7 +79,7 @@ export function analyticsAuditMessagesProvider({ asInternalUser }: IScopedCluste
     });
 
     let messages = [];
-    if (body.hits.total !== 0) {
+    if (body.hits.total.value > 0) {
       messages = body.hits.hits.map((hit: Message) => hit._source);
       messages.reverse();
     }

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts

@@ -248,12 +248,11 @@ export class DataRecognizer {
 
     const { body } = await this._asCurrentUser.search({
       index,
-      rest_total_hits_as_int: true,
       size,
       body: searchBody,
     });
 
-    return body.hits.total !== 0;
+    return body.hits.total.value > 0;
   }
 
   async listModules() {

x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts

@@ -630,12 +630,12 @@ export class DataVisualizer {
 
     const { body } = await this._asCurrentUser.search({
       index,
-      rest_total_hits_as_int: true,
+      track_total_hits: true,
       size,
       body: searchBody,
     });
     const aggregations = body.aggregations;
-    const totalCount = get(body, ['hits', 'total'], 0);
+    const totalCount = body.hits.total.value;
     const stats = {
       totalCount,
       aggregatableExistsFields: [] as FieldData[],
@@ -697,11 +697,10 @@ export class DataVisualizer {
 
     const { body } = await this._asCurrentUser.search({
       index,
-      rest_total_hits_as_int: true,
       size,
       body: searchBody,
     });
-    return body.hits.total > 0;
+    return body.hits.total.value > 0;
   }
 
   async getDocumentCountStats(
@@ -1167,15 +1166,14 @@ export class DataVisualizer {
 
     const { body } = await this._asCurrentUser.search({
       index,
-      rest_total_hits_as_int: true,
       size,
       body: searchBody,
     });
     const stats = {
       fieldName: field,
       examples: [] as any[],
     };
-    if (body.hits.total !== 0) {
+    if (body.hits.total.value > 0) {
       const hits = body.hits.hits;
       for (let i = 0; i < hits.length; i++) {
         // Look in the _source for the field value.

x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js

@@ -102,7 +102,6 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
     const { body } = await asInternalUser.search({
       index: ML_NOTIFICATION_INDEX_PATTERN,
       ignore_unavailable: true,
-      rest_total_hits_as_int: true,
       size: SIZE,
       body: {
         sort: [{ timestamp: { order: 'desc' } }, { job_id: { order: 'asc' } }],
@@ -111,7 +110,7 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
     });
 
     let messages = [];
-    if (body.hits.total !== 0) {
+    if (body.hits.total.value > 0) {
       messages = body.hits.hits.map((hit) => hit._source);
     }
     return messages;
@@ -153,7 +152,6 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
     const { body } = await asInternalUser.search({
       index: ML_NOTIFICATION_INDEX_PATTERN,
       ignore_unavailable: true,
-      rest_total_hits_as_int: true,
       size: 0,
       body: {
         query,
@@ -196,7 +194,7 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
     let messagesPerJob = [];
     const jobMessages = [];
     if (
-      body.hits.total !== 0 &&
+      body.hits.total.value > 0 &&
       body.aggregations &&
       body.aggregations.levelsPerJob &&
       body.aggregations.levelsPerJob.buckets &&

x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts

@@ -89,7 +89,7 @@ function processSearchResults(resp: any, fields: string[]): ProcessedResults {
   return {
     success: true,
     results: tempResults,
-    totalResults: resp.hits.total,
+    totalResults: resp.hits.total.value,
   };
 }
 
@@ -107,7 +107,7 @@ function getSearchJsonFromConfig(
   const json = {
     index: indexPatternTitle,
     size: 0,
-    rest_total_hits_as_int: true,
+    track_total_hits: true,
     body: {
       query: {},
       aggs: {

x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts

@@ -118,7 +118,7 @@ function processSearchResults(resp: any, fields: string[]): ProcessedResults {
   return {
     success: true,
     results: tempResults,
-    totalResults: resp.hits.total,
+    totalResults: resp.hits.total.value,
   };
 }
 
@@ -135,7 +135,7 @@ function getPopulationSearchJsonFromConfig(
   const json = {
     index: indexPatternTitle,
     size: 0,
-    rest_total_hits_as_int: true,
+    track_total_hits: true,
     body: {
       query: {},
       aggs: {