[Maps] do not track total hits for elasticsearch search requests (#91754) (#95455)

nreese · kibanamachine · web-flow · commit 6b0063fc18a9 · 2021-03-25T17:11:46.000-04:00
* [Maps] do not track total hits for elasticsearch search requests

* set track_total_hits for es_search_source tooltip fetch

* tslint

* searchSource doc updates, set track_total_hits in MVT requests

* revert changes made to searchsourcefields docs

* tslint

* review feedback

* tslint

* remove Hits (Total) from functional tests

* remove sleep in functional test

* tslint

* fix method name

Co-authored-by: Kibana Machine &lt;42973632+kibanamachine@users.noreply.github.com&gt;

Co-authored-by: Kibana Machine &lt;42973632+kibanamachine@users.noreply.github.com&gt;
diff --git a/x-pack/plugins/maps/common/elasticsearch_util/index.ts b/x-pack/plugins/maps/common/elasticsearch_util/index.ts
@@ -8,3 +8,4 @@
 export * from './es_agg_utils';
 export * from './convert_to_geojson';
 export * from './elasticsearch_geo_utils';
+export { isTotalHitsGreaterThan, TotalHits } from './total_hits';
diff --git a/x-pack/plugins/maps/common/elasticsearch_util/total_hits.test.ts b/x-pack/plugins/maps/common/elasticsearch_util/total_hits.test.ts
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isTotalHitsGreaterThan, TotalHits } from './total_hits';
+
+describe('total.relation: eq', () => {
+  const totalHits = {
+    value: 100,
+    relation: 'eq' as TotalHits['relation'],
+  };
+
+  test('total.value: 100 should be more than 90', () => {
+    expect(isTotalHitsGreaterThan(totalHits, 90)).toBe(true);
+  });
+
+  test('total.value: 100 should not be more than 100', () => {
+    expect(isTotalHitsGreaterThan(totalHits, 100)).toBe(false);
+  });
+
+  test('total.value: 100 should not be more than 110', () => {
+    expect(isTotalHitsGreaterThan(totalHits, 110)).toBe(false);
+  });
+});
+
+describe('total.relation: gte', () => {
+  const totalHits = {
+    value: 100,
+    relation: 'gte' as TotalHits['relation'],
+  };
+
+  test('total.value: 100 should be more than 90', () => {
+    expect(isTotalHitsGreaterThan(totalHits, 90)).toBe(true);
+  });
+
+  test('total.value: 100 should be more than 100', () => {
+    expect(isTotalHitsGreaterThan(totalHits, 100)).toBe(true);
+  });
+
+  test('total.value: 100 should throw error when value is more than 100', () => {
+    expect(() => {
+      isTotalHitsGreaterThan(totalHits, 110);
+    }).toThrow();
+  });
+});
diff --git a/x-pack/plugins/maps/common/elasticsearch_util/total_hits.ts b/x-pack/plugins/maps/common/elasticsearch_util/total_hits.ts
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export interface TotalHits {
+  value: number;
+  relation: 'eq' | 'gte';
+}
+
+export function isTotalHitsGreaterThan(totalHits: TotalHits, value: number) {
+  if (totalHits.relation === 'eq') {
+    return totalHits.value > value;
+  }
+
+  if (value > totalHits.value) {
+    throw new Error(
+      i18n.translate('xpack.maps.totalHits.lowerBoundPrecisionExceeded', {
+        defaultMessage: `Unable to determine if total hits is greater than value. Total hits precision is lower then value. Total hits: {totalHitsString}, value: {value}. Ensure _search.body.track_total_hits is at least as large as value.`,
+        values: {
+          totalHitsString: JSON.stringify(totalHits, null, ''),
+          value,
+        },
+      })
+    );
+  }
+
+  return true;
+}
diff --git a/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts b/x-pack/plugins/maps/public/classes/layers/blended_vector_layer/blended_vector_layer.ts
@@ -22,6 +22,7 @@ import {
   LAYER_STYLE_TYPE,
   FIELD_ORIGIN,
 } from '../../../../common/constants';
+import { isTotalHitsGreaterThan, TotalHits } from '../../../../common/elasticsearch_util';
 import { ESGeoGridSource } from '../../sources/es_geo_grid_source/es_geo_grid_source';
 import { canSkipSourceUpdate } from '../../util/can_skip_fetch';
 import { IESSource } from '../../sources/es_source';
@@ -323,13 +324,18 @@ export class BlendedVectorLayer extends VectorLayer implements IVectorLayer {
         syncContext.startLoading(dataRequestId, requestToken, searchFilters);
         const abortController = new AbortController();
         syncContext.registerCancelCallback(requestToken, () => abortController.abort());
+        const maxResultWindow = await this._documentSource.getMaxResultWindow();
         const searchSource = await this._documentSource.makeSearchSource(searchFilters, 0);
+        searchSource.setField('trackTotalHits', maxResultWindow + 1);
         const resp = await searchSource.fetch({
           abortSignal: abortController.signal,
           sessionId: syncContext.dataFilters.searchSessionId,
+          legacyHitsTotal: false,
         });
-        const maxResultWindow = await this._documentSource.getMaxResultWindow();
-        isSyncClustered = resp.hits.total > maxResultWindow;
+        isSyncClustered = isTotalHitsGreaterThan(
+          (resp.hits.total as unknown) as TotalHits,
+          maxResultWindow
+        );
         const countData = { isSyncClustered } as CountData;
         syncContext.stopLoading(dataRequestId, requestToken, countData, searchFilters);
       } catch (error) {
diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx
@@ -368,6 +368,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements ITiledSingle
   ): Promise<GeoJsonWithMeta> {
     const indexPattern: IndexPattern = await this.getIndexPattern();
     const searchSource: ISearchSource = await this.makeSearchSource(searchFilters, 0);
+    searchSource.setField('trackTotalHits', false);
 
     let bucketsPerGrid = 1;
     this.getMetricFields().forEach((metricField) => {
diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx
@@ -190,6 +190,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
     // Fetch entities
     //
     const entitySearchSource = await this.makeSearchSource(searchFilters, 0);
+    entitySearchSource.setField('trackTotalHits', false);
     const splitField = getField(indexPattern, this._descriptor.splitField);
     const cardinalityAgg = { precision_threshold: 1 };
     const termsAgg = { size: MAX_TRACKS };
@@ -250,6 +251,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
     const tracksSearchFilters = { ...searchFilters };
     delete tracksSearchFilters.buffer;
     const tracksSearchSource = await this.makeSearchSource(tracksSearchFilters, 0);
+    tracksSearchSource.setField('trackTotalHits', false);
     tracksSearchSource.setField('aggs', {
       tracks: {
         filters: {
diff --git a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js
@@ -109,6 +109,7 @@ export class ESPewPewSource extends AbstractESAggSource {
   async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
     const indexPattern = await this.getIndexPattern();
     const searchSource = await this.makeSearchSource(searchFilters, 0);
+    searchSource.setField('trackTotalHits', false);
     searchSource.setField('aggs', {
       destSplit: {
         terms: {
@@ -168,6 +169,7 @@ export class ESPewPewSource extends AbstractESAggSource {
 
   async getBoundsForFilters(boundsFilters, registerCancelCallback) {
     const searchSource = await this.makeSearchSource(boundsFilters, 0);
+    searchSource.setField('trackTotalHits', false);
     searchSource.setField('aggs', {
       destFitToBounds: {
         geo_bounds: {
@@ -185,7 +187,10 @@ export class ESPewPewSource extends AbstractESAggSource {
     try {
       const abortController = new AbortController();
       registerCancelCallback(() => abortController.abort());
-      const esResp = await searchSource.fetch({ abortSignal: abortController.signal });
+      const esResp = await searchSource.fetch({
+        abortSignal: abortController.signal,
+        legacyHitsTotal: false,
+      });
       if (esResp.aggregations.destFitToBounds.bounds) {
         corners.push([
           esResp.aggregations.destFitToBounds.bounds.top_left.lon,
diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx
@@ -18,6 +18,7 @@ import {
   addFieldToDSL,
   getField,
   hitsToGeoJson,
+  isTotalHitsGreaterThan,
   PreIndexedShape,
 } from '../../../../common/elasticsearch_util';
 // @ts-expect-error
@@ -313,6 +314,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
     };
 
     const searchSource = await this.makeSearchSource(searchFilters, 0);
+    searchSource.setField('trackTotalHits', false);
     searchSource.setField('aggs', {
       totalEntities: {
         cardinality: addFieldToDSL(cardinalityAgg, topHitsSplitField),
@@ -343,11 +345,10 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
     const areEntitiesTrimmed = entityBuckets.length >= DEFAULT_MAX_BUCKETS_LIMIT;
     let areTopHitsTrimmed = false;
     entityBuckets.forEach((entityBucket: any) => {
-      const total = _.get(entityBucket, 'entityHits.hits.total', 0);
       const hits = _.get(entityBucket, 'entityHits.hits.hits', []);
       // Reverse hits list so top documents by sort are drawn on top
       allHits.push(...hits.reverse());
-      if (total > hits.length) {
+      if (isTotalHitsGreaterThan(entityBucket.entityHits.hits.total, hits.length)) {
         areTopHitsTrimmed = true;
       }
     });
@@ -385,6 +386,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
       maxResultWindow,
       initialSearchContext
     );
+    searchSource.setField('trackTotalHits', maxResultWindow + 1);
     searchSource.setField('fieldsFromSource', searchFilters.fieldNames); // Setting "fields" filters out unused scripted fields
     if (sourceOnlyFields.length === 0) {
       searchSource.setField('source', false); // do not need anything from _source
@@ -408,7 +410,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
       hits: resp.hits.hits.reverse(), // Reverse hits so top documents by sort are drawn on top
       meta: {
         resultsCount: resp.hits.hits.length,
-        areResultsTrimmed: resp.hits.total > resp.hits.hits.length,
+        areResultsTrimmed: isTotalHitsGreaterThan(resp.hits.total, resp.hits.hits.length),
       },
     };
   }
@@ -508,6 +510,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
     const initialSearchContext = { docvalue_fields: docValueFields }; // Request fields in docvalue_fields insted of _source
     const searchService = getSearchService();
     const searchSource = await searchService.searchSource.create(initialSearchContext as object);
+    searchSource.setField('trackTotalHits', false);
 
     searchSource.setField('index', indexPattern);
     searchSource.setField('size', 1);
@@ -520,7 +523,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye
     searchSource.setField('query', query);
     searchSource.setField('fieldsFromSource', this._getTooltipPropertyNames());
 
-    const resp = await searchSource.fetch();
+    const resp = await searchSource.fetch({ legacyHitsTotal: false });
 
     const hit = _.get(resp, 'hits.hits[0]');
     if (!hit) {
diff --git a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts
@@ -195,6 +195,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource
       resp = await searchSource.fetch({
         abortSignal: abortController.signal,
         sessionId: searchSessionId,
+        legacyHitsTotal: false,
       });
       if (inspectorRequest) {
         const responseStats = search.getResponseInspectorStats(resp, searchSource);
@@ -247,6 +248,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource
       }
     }
     const searchService = getSearchService();
+
     const searchSource = await searchService.searchSource.create(initialSearchContext);
 
     searchSource.setField('index', indexPattern);
@@ -272,6 +274,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource
     registerCancelCallback: (callback: () => void) => void
   ): Promise<MapExtent | null> {
     const searchSource = await this.makeSearchSource(boundsFilters, 0);
+    searchSource.setField('trackTotalHits', false);
     searchSource.setField('aggs', {
       fitToBounds: {
         geo_bounds: {
@@ -284,7 +287,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource
     try {
       const abortController = new AbortController();
       registerCancelCallback(() => abortController.abort());
-      const esResp = await searchSource.fetch({ abortSignal: abortController.signal });
+      const esResp = await searchSource.fetch({
+        abortSignal: abortController.signal,
+        legacyHitsTotal: false,
+      });
 
       if (!esResp.aggregations) {
         return null;
diff --git a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts
@@ -127,6 +127,7 @@ export class ESTermSource extends AbstractESAggSource implements ITermJoinSource
 
     const indexPattern = await this.getIndexPattern();
     const searchSource: ISearchSource = await this.makeSearchSource(searchFilters, 0);
+    searchSource.setField('trackTotalHits', false);
     const termsField = getField(indexPattern, this._termField.getName());
     const termsAgg = {
       size: this._descriptor.size !== undefined ? this._descriptor.size : DEFAULT_MAX_BUCKETS_LIMIT,
diff --git a/x-pack/plugins/maps/server/mvt/get_tile.ts b/x-pack/plugins/maps/server/mvt/get_tile.ts
@@ -23,7 +23,12 @@ import {
   SUPER_FINE_ZOOM_DELTA,
 } from '../../common/constants';
 
-import { convertRegularRespToGeoJson, hitsToGeoJson } from '../../common/elasticsearch_util';
+import {
+  convertRegularRespToGeoJson,
+  hitsToGeoJson,
+  isTotalHitsGreaterThan,
+  TotalHits,
+} from '../../common/elasticsearch_util';
 import { flattenHit } from './util';
 import { ESBounds, tileToESBbox } from '../../common/geo_tile_utils';
 import { getCentroidFeatures } from '../../common/get_centroid_features';
@@ -67,6 +72,7 @@ export async function getGridTile({
       MAX_ZOOM
     );
     requestBody.aggs[GEOTILE_GRID_AGG_NAME].geotile_grid.bounds = tileBounds;
+    requestBody.track_total_hits = false;
 
     const response = await context
       .search!.search(
@@ -78,6 +84,7 @@ export async function getGridTile({
         },
         {
           sessionId: searchSessionId,
+          legacyHitsTotal: false,
           abortSignal,
         }
       )
@@ -130,6 +137,7 @@ export async function getTile({
 
     const searchOptions = {
       sessionId: searchSessionId,
+      legacyHitsTotal: false,
       abortSignal,
     };
 
@@ -141,14 +149,20 @@ export async function getTile({
             body: {
               size: 0,
               query: requestBody.query,
+              track_total_hits: requestBody.size + 1,
             },
           },
         },
         searchOptions
       )
       .toPromise();
 
-    if (countResponse.rawResponse.hits.total > requestBody.size) {
+    if (
+      isTotalHitsGreaterThan(
+        (countResponse.rawResponse.hits.total as unknown) as TotalHits,
+        requestBody.size
+      )
+    ) {
       // Generate "too many features"-bounds
       const bboxResponse = await context
         .search!.search(
@@ -165,6 +179,7 @@ export async function getTile({
                     },
                   },
                 },
+                track_total_hits: false,
               },
             },
           },
@@ -191,7 +206,10 @@ export async function getTile({
           {
             params: {
               index,
-              body: requestBody,
+              body: {
+                ...requestBody,
+                track_total_hits: false,
+              },
             },
           },
           searchOptions
diff --git a/x-pack/test/functional/apps/maps/blended_vector_layer.js b/x-pack/test/functional/apps/maps/blended_vector_layer.js
@@ -27,28 +27,21 @@ export default function ({ getPageObjects, getService }) {
     });
 
     it('should request documents when zoomed to smaller regions showing less data', async () => {
-      const hits = await PageObjects.maps.getHits();
+      const response = await PageObjects.maps.getResponse();
       // Allow a range of hits to account for variances in browser window size.
-      expect(parseInt(hits)).to.be.within(30, 40);
+      expect(response.hits.hits.length).to.be.within(30, 40);
     });
 
     it('should request clusters when zoomed to larger regions showing lots of data', async () => {
       await PageObjects.maps.setView(20, -90, 2);
-      await inspector.open();
-      await inspector.openInspectorRequestsView();
-      const requestStats = await inspector.getTableData();
-      const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits');
-      const totalHits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits (total)');
-      await inspector.close();
-
-      expect(hits).to.equal('0');
-      expect(totalHits).to.equal('14000');
+      const response = await PageObjects.maps.getResponse();
+      expect(response.aggregations.gridSplit.buckets.length).to.equal(17);
     });
 
     it('should request documents when query narrows data', async () => {
       await PageObjects.maps.setAndSubmitQuery('bytes > 19000');
-      const hits = await PageObjects.maps.getHits();
-      expect(hits).to.equal('75');
+      const response = await PageObjects.maps.getResponse();
+      expect(response.hits.hits.length).to.equal(75);
     });
   });
 }
diff --git a/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js b/x-pack/test/functional/apps/maps/documents_source/docvalue_fields.js
diff --git a/x-pack/test/functional/apps/maps/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/embeddable/dashboard.js
diff --git a/x-pack/test/functional/apps/maps/es_geo_grid_source.js b/x-pack/test/functional/apps/maps/es_geo_grid_source.js
diff --git a/x-pack/test/functional/apps/maps/es_pew_pew_source.js b/x-pack/test/functional/apps/maps/es_pew_pew_source.js
diff --git a/x-pack/test/functional/apps/maps/joins.js b/x-pack/test/functional/apps/maps/joins.js
diff --git a/x-pack/test/functional/page_objects/gis_page.ts b/x-pack/test/functional/page_objects/gis_page.ts
