[Maps] Add mvt support for ES doc sources (#75698)

Author: thomasneirynck

x-pack/package.json

@@ -313,6 +313,7 @@
     "file-type": "^10.9.0",
     "font-awesome": "4.7.0",
     "fp-ts": "^2.3.1",
+    "geojson-vt": "^3.2.1",
     "get-port": "^5.0.0",
     "getos": "^3.1.0",
     "git-url-parse": "11.1.2",
@@ -384,6 +385,7 @@
     "ui-select": "0.19.8",
     "uuid": "3.3.2",
     "vscode-languageserver": "^5.2.1",
+    "vt-pbf": "^3.1.1",
     "webpack": "^4.41.5",
     "wellknown": "^0.5.0",
     "xml2js": "^0.4.22",

x-pack/plugins/maps/common/constants.ts

@@ -33,6 +33,12 @@ export const MAP_PATH = 'map';
 export const GIS_API_PATH = `api/${APP_ID}`;
 export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`;
 export const FONTS_API_PATH = `${GIS_API_PATH}/fonts`;
+export const API_ROOT_PATH = `/${GIS_API_PATH}`;
+
+export const MVT_GETTILE_API_PATH = 'mvt/getTile';
+export const MVT_SOURCE_LAYER_NAME = 'source_layer';
+export const KBN_TOO_MANY_FEATURES_PROPERTY = '__kbn_too_many_features__';
+export const KBN_TOO_MANY_FEATURES_IMAGE_ID = '__kbn_too_many_features_image_id__';
 
 const MAP_BASE_URL = `/${MAPS_APP_PATH}/${MAP_PATH}`;
 export function getNewMapPath() {
@@ -220,6 +226,7 @@ export enum SCALING_TYPES {
   LIMIT = 'LIMIT',
   CLUSTERS = 'CLUSTERS',
   TOP_HITS = 'TOP_HITS',
+  MVT = 'MVT',
 }
 
 export const RGBA_0000 = 'rgba(0,0,0,0)';

x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts

@@ -18,7 +18,6 @@ export type MapFilters = {
   refreshTimerLastTriggeredAt?: string;
   timeFilters: TimeRange;
   zoom: number;
-  geogridPrecision?: number;
 };
 
 type ESSearchSourceSyncMeta = {

x-pack/plugins/maps/common/elasticsearch_geo_utils.d.ts

@@ -4,7 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { FeatureCollection, GeoJsonProperties } from 'geojson';
 import { MapExtent } from './descriptor_types';
+import { ES_GEO_FIELD_TYPE } from './constants';
 
 export function scaleBounds(bounds: MapExtent, scaleFactor: number): MapExtent;
 
@@ -13,3 +15,11 @@ export function turfBboxToBounds(turfBbox: unknown): MapExtent;
 export function clampToLatBounds(lat: number): number;
 
 export function clampToLonBounds(lon: number): number;
+
+export function hitsToGeoJson(
+  hits: Array<Record<string, unknown>>,
+  flattenHit: (elasticSearchHit: Record<string, unknown>) => GeoJsonProperties,
+  geoFieldName: string,
+  geoFieldType: ES_GEO_FIELD_TYPE,
+  epochMillisFields: string[]
+): FeatureCollection;

x-pack/plugins/maps/public/classes/fields/es_doc_field.ts

@@ -15,18 +15,22 @@ import { IVectorSource } from '../sources/vector_source';
 
 export class ESDocField extends AbstractField implements IField {
   private readonly _source: IESSource;
+  private readonly _canReadFromGeoJson: boolean;
 
   constructor({
     fieldName,
     source,
     origin,
+    canReadFromGeoJson = true,
   }: {
     fieldName: string;
     source: IESSource;
     origin: FIELD_ORIGIN;
+    canReadFromGeoJson?: boolean;
   }) {
     super({ fieldName, origin });
     this._source = source;
+    this._canReadFromGeoJson = canReadFromGeoJson;
   }
 
   canValueBeFormatted(): boolean {
@@ -60,6 +64,10 @@ export class ESDocField extends AbstractField implements IField {
     return true;
   }
 
+  canReadFromGeoJson(): boolean {
+    return this._canReadFromGeoJson;
+  }
+
   async getOrdinalFieldMetaRequest(): Promise<unknown> {
     const indexPatternField = await this._getIndexPatternField();
 

x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx

@@ -128,32 +128,41 @@ describe('syncData', () => {
     sinon.assert.notCalled(syncContext2.stopLoading);
   });
 
-  it('Should resync when changes to source params', async () => {
-    const layer1: TiledVectorLayer = createLayer({}, {});
-    const syncContext1 = new MockSyncContext({ dataFilters: {} });
-
-    await layer1.syncData(syncContext1);
-
-    const dataRequestDescriptor: DataRequestDescriptor = {
-      data: defaultConfig,
-      dataId: 'source',
-    };
-    const layer2: TiledVectorLayer = createLayer(
-      {
-        __dataRequests: [dataRequestDescriptor],
-      },
-      { layerName: 'barfoo' }
-    );
-    const syncContext2 = new MockSyncContext({ dataFilters: {} });
-    await layer2.syncData(syncContext2);
-
-    // @ts-expect-error
-    sinon.assert.calledOnce(syncContext2.startLoading);
-    // @ts-expect-error
-    sinon.assert.calledOnce(syncContext2.stopLoading);
-
-    // @ts-expect-error
-    const call = syncContext2.stopLoading.getCall(0);
-    expect(call.args[2]).toEqual({ ...defaultConfig, layerName: 'barfoo' });
+  describe('Should resync when changes to source params: ', () => {
+    [
+      { layerName: 'barfoo' },
+      { urlTemplate: 'https://sub.example.com/{z}/{x}/{y}.pbf' },
+      { minSourceZoom: 1 },
+      { maxSourceZoom: 12 },
+    ].forEach((changes) => {
+      it(`change in ${Object.keys(changes).join(',')}`, async () => {
+        const layer1: TiledVectorLayer = createLayer({}, {});
+        const syncContext1 = new MockSyncContext({ dataFilters: {} });
+
+        await layer1.syncData(syncContext1);
+
+        const dataRequestDescriptor: DataRequestDescriptor = {
+          data: defaultConfig,
+          dataId: 'source',
+        };
+        const layer2: TiledVectorLayer = createLayer(
+          {
+            __dataRequests: [dataRequestDescriptor],
+          },
+          changes
+        );
+        const syncContext2 = new MockSyncContext({ dataFilters: {} });
+        await layer2.syncData(syncContext2);
+
+        // @ts-expect-error
+        sinon.assert.calledOnce(syncContext2.startLoading);
+        // @ts-expect-error
+        sinon.assert.calledOnce(syncContext2.stopLoading);
+
+        // @ts-expect-error
+        const call = syncContext2.stopLoading.getCall(0);
+        expect(call.args[2]).toEqual({ ...defaultConfig, ...changes });
+      });
+    });
   });
 });

x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx

@@ -63,21 +63,24 @@ export class TiledVectorLayer extends VectorLayer {
     );
     const prevDataRequest = this.getSourceDataRequest();
 
+    const templateWithMeta = await this._source.getUrlTemplateWithMeta(searchFilters);
     if (prevDataRequest) {
       const data: MVTSingleLayerVectorSourceConfig = prevDataRequest.getData() as MVTSingleLayerVectorSourceConfig;
-      const canSkipBecauseNoChanges =
-        data.layerName === this._source.getLayerName() &&
-        data.minSourceZoom === this._source.getMinZoom() &&
-        data.maxSourceZoom === this._source.getMaxZoom();
-
-      if (canSkipBecauseNoChanges) {
-        return null;
+      if (data) {
+        const canSkipBecauseNoChanges =
+          data.layerName === this._source.getLayerName() &&
+          data.minSourceZoom === this._source.getMinZoom() &&
+          data.maxSourceZoom === this._source.getMaxZoom() &&
+          data.urlTemplate === templateWithMeta.urlTemplate;
+
+        if (canSkipBecauseNoChanges) {
+          return null;
+        }
       }
     }
 
     startLoading(SOURCE_DATA_REQUEST_ID, requestToken, searchFilters);
     try {
-      const templateWithMeta = await this._source.getUrlTemplateWithMeta();
       stopLoading(SOURCE_DATA_REQUEST_ID, requestToken, templateWithMeta, {});
     } catch (error) {
       onLoadError(SOURCE_DATA_REQUEST_ID, requestToken, error.message);
@@ -160,6 +163,11 @@ export class TiledVectorLayer extends VectorLayer {
       return false;
     }
 
+    if (!mbTileSource.tiles) {
+      // Expected source is not compatible, so remove.
+      return true;
+    }
+
     const isSourceDifferent =
       mbTileSource.tiles[0] !== tiledSourceMeta.urlTemplate ||
       mbTileSource.minzoom !== tiledSourceMeta.minSourceZoom ||

x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js

@@ -15,9 +15,11 @@ import {
   SOURCE_BOUNDS_DATA_REQUEST_ID,
   FEATURE_VISIBLE_PROPERTY_NAME,
   EMPTY_FEATURE_COLLECTION,
+  KBN_TOO_MANY_FEATURES_PROPERTY,
   LAYER_TYPE,
   FIELD_ORIGIN,
   LAYER_STYLE_TYPE,
+  KBN_TOO_MANY_FEATURES_IMAGE_ID,
 } from '../../../../common/constants';
 import _ from 'lodash';
 import { JoinTooltipProperty } from '../../tooltips/join_tooltip_property';
@@ -777,6 +779,8 @@ export class VectorLayer extends AbstractLayer {
     const sourceId = this.getId();
     const fillLayerId = this._getMbPolygonLayerId();
     const lineLayerId = this._getMbLineLayerId();
+    const tooManyFeaturesLayerId = this._getMbTooManyFeaturesLayerId();
+
     const hasJoins = this.hasJoins();
     if (!mbMap.getLayer(fillLayerId)) {
       const mbLayer = {
@@ -802,6 +806,30 @@ export class VectorLayer extends AbstractLayer {
       }
       mbMap.addLayer(mbLayer);
     }
+    if (!mbMap.getLayer(tooManyFeaturesLayerId)) {
+      const mbLayer = {
+        id: tooManyFeaturesLayerId,
+        type: 'fill',
+        source: sourceId,
+        paint: {},
+      };
+      if (mvtSourceLayer) {
+        mbLayer['source-layer'] = mvtSourceLayer;
+      }
+      mbMap.addLayer(mbLayer);
+      mbMap.setFilter(tooManyFeaturesLayerId, [
+        '==',
+        ['get', KBN_TOO_MANY_FEATURES_PROPERTY],
+        true,
+      ]);
+      mbMap.setPaintProperty(
+        tooManyFeaturesLayerId,
+        'fill-pattern',
+        KBN_TOO_MANY_FEATURES_IMAGE_ID
+      );
+      mbMap.setPaintProperty(tooManyFeaturesLayerId, 'fill-opacity', this.getAlpha());
+    }
+
     this.getCurrentStyle().setMBPaintProperties({
       alpha: this.getAlpha(),
       mbMap,
@@ -822,6 +850,9 @@ export class VectorLayer extends AbstractLayer {
     if (lineFilterExpr !== mbMap.getFilter(lineLayerId)) {
       mbMap.setFilter(lineLayerId, lineFilterExpr);
     }
+
+    this.syncVisibilityWithMb(mbMap, tooManyFeaturesLayerId);
+    mbMap.setLayerZoomRange(tooManyFeaturesLayerId, this.getMinZoom(), this.getMaxZoom());
   }
 
   _syncStylePropertiesWithMb(mbMap) {
@@ -836,6 +867,19 @@ export class VectorLayer extends AbstractLayer {
         type: 'geojson',
         data: EMPTY_FEATURE_COLLECTION,
       });
+    } else if (mbSource.type !== 'geojson') {
+      // Recreate source when existing source is not geojson. This can occur when layer changes from tile layer to vector layer.
+      this.getMbLayerIds().forEach((mbLayerId) => {
+        if (mbMap.getLayer(mbLayerId)) {
+          mbMap.removeLayer(mbLayerId);
+        }
+      });
+
+      mbMap.removeSource(this._getMbSourceId());
+      mbMap.addSource(this._getMbSourceId(), {
+        type: 'geojson',
+        data: EMPTY_FEATURE_COLLECTION,
+      });
     }
   }
 
@@ -865,13 +909,18 @@ export class VectorLayer extends AbstractLayer {
     return this.makeMbLayerId('fill');
   }
 
+  _getMbTooManyFeaturesLayerId() {
+    return this.makeMbLayerId('toomanyfeatures');
+  }
+
   getMbLayerIds() {
     return [
       this._getMbPointLayerId(),
       this._getMbTextLayerId(),
       this._getMbSymbolLayerId(),
       this._getMbLineLayerId(),
       this._getMbPolygonLayerId(),
+      this._getMbTooManyFeaturesLayerId(),
     ];
   }
 

x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { MapExtent, MapFilters } from '../../../../common/descriptor_types';
+import { MapExtent, VectorSourceRequestMeta } from '../../../../common/descriptor_types';
 
 jest.mock('../../../kibana_services');
 
@@ -19,6 +19,7 @@ import { SearchSource } from '../../../../../../../src/plugins/data/public/searc
 
 export class MockSearchSource {
   setField = jest.fn();
+  setParent() {}
 }
 
 describe('ESGeoGridSource', () => {
@@ -104,6 +105,9 @@ describe('ESGeoGridSource', () => {
           async create() {
             return mockSearchSource as SearchSource;
           },
+          createEmpty() {
+            return mockSearchSource as SearchSource;
+          },
         },
       };
 
@@ -120,16 +124,24 @@ describe('ESGeoGridSource', () => {
       maxLat: 80,
     };
 
-    const mapFilters: MapFilters = {
+    const mapFilters: VectorSourceRequestMeta = {
       geogridPrecision: 4,
       filters: [],
       timeFilters: {
         from: 'now',
         to: '15m',
         mode: 'relative',
       },
-      // extent,
+      extent,
+      applyGlobalQuery: true,
+      fieldNames: [],
       buffer: extent,
+      sourceQuery: {
+        query: '',
+        language: 'KQL',
+        queryLastTriggeredAt: '2019-04-25T20:53:22.331Z',
+      },
+      sourceMeta: null,
       zoom: 0,
     };