[SIEM] Adds support for apm-* to the network map (#54876) (#55142)

## Summary

Resolves #52297, #52565

To improve the display of APM data within SIEM (specifically the `HTTP Table` and `Network Map`), this PR adds `apm-*-transcation*` to `siem:defaultIndex`, and additional support for showing `client`/`server` layers on the `Network Map` when a matching `apm-*` index pattern is present.

The map now supports pattern matching when checking for available Kibana Index Patterns, and so matches `apm-*-transcation*` -> `apm-*` (if exists). Additionally, the map config was updated to generate layers for client/server geo fields (instead of the usual source/dest) since these are the fields Transactions use.

![image](https://user-images.githubusercontent.com/2946766/72573225-2a038880-3882-11ea-9590-a545d726dbf9.png)

<img width="1214" alt="Screen Shot 2020-01-14 at 18 22 11" src="https://user-images.githubusercontent.com/2946766/72407120-bcd5e300-371b-11ea-90cc-a0714320a59c.png">


### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

- [ ] ~This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~
- [X] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)
- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials
  - Will work with @benskelker on updating the maps docs
- [X] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
- [ ] ~This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~

### For maintainers

- [ ] ~This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~
- [ ] ~This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~

Author: spong

x-pack/legacy/plugins/siem/default_index_pattern.ts

@@ -6,6 +6,7 @@
 
 /** The comma-delimited list of Elasticsearch indices from which the SIEM app collects events */
 export const defaultIndexPattern = [
+  'apm-*-transaction*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',

x-pack/legacy/plugins/siem/public/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap

@@ -5,11 +5,16 @@
  */
 
 import { IndexPatternMapping } from '../types';
+import { IndexPatternSavedObject } from '../../ml_popover/types';
 
 export const mockIndexPatternIds: IndexPatternMapping[] = [
   { title: 'filebeat-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
 ];
 
+export const mockAPMIndexPatternIds: IndexPatternMapping[] = [
+  { title: 'apm-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
+];
+
 export const mockSourceLayer = {
   sourceDescriptor: {
     id: 'uuid.v4()',
@@ -113,6 +118,109 @@ export const mockDestinationLayer = {
   query: { query: '', language: 'kuery' },
 };
 
+export const mockClientLayer = {
+  sourceDescriptor: {
+    id: 'uuid.v4()',
+    type: 'ES_SEARCH',
+    applyGlobalQuery: true,
+    geoField: 'client.geo.location',
+    filterByMapBounds: false,
+    tooltipProperties: [
+      'host.name',
+      'client.ip',
+      'client.domain',
+      'client.geo.country_iso_code',
+      'client.as.organization.name',
+    ],
+    useTopHits: false,
+    topHitsTimeField: '@timestamp',
+    topHitsSize: 1,
+    indexPatternId: '8c7323ac-97ad-4b53-ac0a-40f8f691a918',
+  },
+  style: {
+    type: 'VECTOR',
+    properties: {
+      fillColor: {
+        type: 'STATIC',
+        options: { color: '#6092C0' },
+      },
+      lineColor: {
+        type: 'STATIC',
+        options: { color: '#FFFFFF' },
+      },
+      lineWidth: { type: 'STATIC', options: { size: 2 } },
+      iconSize: { type: 'STATIC', options: { size: 8 } },
+      iconOrientation: {
+        type: 'STATIC',
+        options: { orientation: 0 },
+      },
+      symbol: {
+        options: { symbolizeAs: 'icon', symbolId: 'home' },
+      },
+    },
+  },
+  id: 'uuid.v4()',
+  label: `apm-* | Client Point`,
+  minZoom: 0,
+  maxZoom: 24,
+  alpha: 1,
+  visible: true,
+  type: 'VECTOR',
+  query: { query: '', language: 'kuery' },
+  joins: [],
+};
+
+export const mockServerLayer = {
+  sourceDescriptor: {
+    id: 'uuid.v4()',
+    type: 'ES_SEARCH',
+    applyGlobalQuery: true,
+    geoField: 'server.geo.location',
+    filterByMapBounds: true,
+    tooltipProperties: [
+      'host.name',
+      'server.ip',
+      'server.domain',
+      'server.geo.country_iso_code',
+      'server.as.organization.name',
+    ],
+    useTopHits: false,
+    topHitsTimeField: '@timestamp',
+    topHitsSize: 1,
+    indexPatternId: '8c7323ac-97ad-4b53-ac0a-40f8f691a918',
+  },
+  style: {
+    type: 'VECTOR',
+    properties: {
+      fillColor: {
+        type: 'STATIC',
+        options: { color: '#D36086' },
+      },
+      lineColor: {
+        type: 'STATIC',
+        options: { color: '#FFFFFF' },
+      },
+      lineWidth: { type: 'STATIC', options: { size: 2 } },
+      iconSize: { type: 'STATIC', options: { size: 8 } },
+      iconOrientation: {
+        type: 'STATIC',
+        options: { orientation: 0 },
+      },
+      symbol: {
+        options: { symbolizeAs: 'icon', symbolId: 'marker' },
+      },
+    },
+  },
+  id: 'uuid.v4()',
+  label: `apm-* | Server Point`,
+  minZoom: 0,
+  maxZoom: 24,
+  alpha: 1,
+  visible: true,
+  type: 'VECTOR',
+  query: { query: '', language: 'kuery' },
+};
+
 export const mockLineLayer = {
   sourceDescriptor: {
     type: 'ES_PEW_PEW',
@@ -173,6 +281,66 @@ export const mockLineLayer = {
   query: { query: '', language: 'kuery' },
 };
 
+export const mockClientServerLineLayer = {
+  sourceDescriptor: {
+    type: 'ES_PEW_PEW',
+    applyGlobalQuery: true,
+    id: 'uuid.v4()',
+    indexPatternId: '8c7323ac-97ad-4b53-ac0a-40f8f691a918',
+    sourceGeoField: 'client.geo.location',
+    destGeoField: 'server.geo.location',
+    metrics: [
+      { type: 'sum', field: 'client.bytes', label: 'client.bytes' },
+      { type: 'sum', field: 'server.bytes', label: 'server.bytes' },
+    ],
+  },
+  style: {
+    type: 'VECTOR',
+    properties: {
+      fillColor: {
+        type: 'STATIC',
+        options: { color: '#1EA593' },
+      },
+      lineColor: {
+        type: 'STATIC',
+        options: { color: '#6092C0' },
+      },
+      lineWidth: {
+        type: 'DYNAMIC',
+        options: {
+          field: {
+            label: 'count',
+            name: 'doc_count',
+            origin: 'source',
+          },
+          minSize: 1,
+          maxSize: 8,
+          fieldMetaOptions: {
+            isEnabled: true,
+            sigma: 3,
+          },
+        },
+      },
+      iconSize: { type: 'STATIC', options: { size: 10 } },
+      iconOrientation: {
+        type: 'STATIC',
+        options: { orientation: 0 },
+      },
+      symbol: {
+        options: { symbolizeAs: 'circle', symbolId: 'airfield' },
+      },
+    },
+  },
+  id: 'uuid.v4()',
+  label: `apm-* | Line`,
+  minZoom: 0,
+  maxZoom: 24,
+  alpha: 0.5,
+  visible: true,
+  type: 'VECTOR',
+  query: { query: '', language: 'kuery' },
+};
+
 export const mockLayerList = [
   {
     sourceDescriptor: { type: 'EMS_TMS', isAutoSelect: true },
@@ -209,3 +377,83 @@ export const mockLayerListDouble = [
   mockDestinationLayer,
   mockSourceLayer,
 ];
+
+export const mockLayerListMixed = [
+  {
+    sourceDescriptor: { type: 'EMS_TMS', isAutoSelect: true },
+    id: 'uuid.v4()',
+    label: null,
+    minZoom: 0,
+    maxZoom: 24,
+    alpha: 1,
+    visible: true,
+    style: null,
+    type: 'VECTOR_TILE',
+  },
+  mockLineLayer,
+  mockDestinationLayer,
+  mockSourceLayer,
+  mockClientServerLineLayer,
+  mockServerLayer,
+  mockClientLayer,
+];
+
+export const mockAPMIndexPattern: IndexPatternSavedObject = {
+  id: 'apm-*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: 'apm-*',
+  },
+};
+
+export const mockAPMRegexIndexPattern: IndexPatternSavedObject = {
+  id: 'apm-7.*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: 'apm-7.*',
+  },
+};
+
+export const mockFilebeatIndexPattern: IndexPatternSavedObject = {
+  id: 'filebeat-*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: 'filebeat-*',
+  },
+};
+
+export const mockAuditbeatIndexPattern: IndexPatternSavedObject = {
+  id: 'auditbeat-*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: 'auditbeat-*',
+  },
+};
+
+export const mockAPMTransactionIndexPattern: IndexPatternSavedObject = {
+  id: 'apm-*-transaction*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: 'apm-*-transaction*',
+  },
+};
+
+export const mockGlobIndexPattern: IndexPatternSavedObject = {
+  id: '*',
+  type: 'index-pattern',
+  updated_at: '',
+  version: 'abc',
+  attributes: {
+    title: '*',
+  },
+};

x-pack/legacy/plugins/siem/public/components/embeddables/__mocks__/mock.ts

@@ -18,7 +18,7 @@ import { Loader } from '../loader';
 import { displayErrorToast, useStateToaster } from '../toasters';
 import { Embeddable } from './embeddable';
 import { EmbeddableHeader } from './embeddable_header';
-import { createEmbeddable } from './embedded_map_helpers';
+import { createEmbeddable, findMatchingIndexPatterns } from './embedded_map_helpers';
 import { IndexPatternsMissingPrompt } from './index_patterns_missing_prompt';
 import { MapToolTip } from './map_tool_tip/map_tool_tip';
 import * as i18n from './translations';
@@ -107,10 +107,12 @@ export const EmbeddedMapComponent = ({
   useEffect(() => {
     let isSubscribed = true;
     async function setupEmbeddable() {
-      // Ensure at least one `siem:defaultIndex` index pattern exists before trying to import
-      const matchingIndexPatterns = kibanaIndexPatterns.filter(ip =>
-        siemDefaultIndices.includes(ip.attributes.title)
-      );
+      // Ensure at least one `siem:defaultIndex` kibana index pattern exists before creating embeddable
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns,
+        siemDefaultIndices,
+      });
+
       if (matchingIndexPatterns.length === 0 && isSubscribed) {
         setIsLoading(false);
         setIsIndexError(true);

x-pack/legacy/plugins/siem/public/components/embeddables/embedded_map.tsx

@@ -4,9 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { createEmbeddable } from './embedded_map_helpers';
+import { createEmbeddable, findMatchingIndexPatterns } from './embedded_map_helpers';
 import { createUiNewPlatformMock } from 'ui/new_platform/__mocks__/helpers';
 import { createPortalNode } from 'react-reverse-portal';
+import {
+  mockAPMIndexPattern,
+  mockAPMRegexIndexPattern,
+  mockAPMTransactionIndexPattern,
+  mockAuditbeatIndexPattern,
+  mockFilebeatIndexPattern,
+  mockGlobIndexPattern,
+} from './__mocks__/mock';
 
 jest.mock('ui/new_platform');
 
@@ -51,4 +59,58 @@ describe('embedded_map_helpers', () => {
       expect(embeddable.reload).toHaveBeenCalledTimes(1);
     });
   });
+
+  describe('findMatchingIndexPatterns', () => {
+    const siemDefaultIndices = [
+      'apm-*-transaction*',
+      'auditbeat-*',
+      'endgame-*',
+      'filebeat-*',
+      'packetbeat-*',
+      'winlogbeat-*',
+    ];
+
+    test('finds exact matching index patterns ', () => {
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns: [mockFilebeatIndexPattern, mockAuditbeatIndexPattern],
+        siemDefaultIndices,
+      });
+      expect(matchingIndexPatterns).toEqual([mockFilebeatIndexPattern, mockAuditbeatIndexPattern]);
+    });
+
+    test('finds glob-matched index patterns ', () => {
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns: [mockAPMIndexPattern, mockFilebeatIndexPattern],
+        siemDefaultIndices,
+      });
+      expect(matchingIndexPatterns).toEqual([mockAPMIndexPattern, mockFilebeatIndexPattern]);
+    });
+
+    test('does not find glob-matched index pattern containing regex', () => {
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns: [mockAPMRegexIndexPattern, mockFilebeatIndexPattern],
+        siemDefaultIndices,
+      });
+      expect(matchingIndexPatterns).toEqual([mockFilebeatIndexPattern]);
+    });
+
+    test('finds exact glob-matched index patterns ', () => {
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns: [mockAPMTransactionIndexPattern, mockFilebeatIndexPattern],
+        siemDefaultIndices,
+      });
+      expect(matchingIndexPatterns).toEqual([
+        mockAPMTransactionIndexPattern,
+        mockFilebeatIndexPattern,
+      ]);
+    });
+
+    test('finds glob-only index patterns ', () => {
+      const matchingIndexPatterns = findMatchingIndexPatterns({
+        kibanaIndexPatterns: [mockGlobIndexPattern, mockFilebeatIndexPattern],
+        siemDefaultIndices,
+      });
+      expect(matchingIndexPatterns).toEqual([mockGlobIndexPattern, mockFilebeatIndexPattern]);
+    });
+  });
 });