[Security Solution][Detection Engine] Adds threat matching to the rul…

…e creator (#78955) (#79230)

## Summary

This adds threat matching rule type to the rule creator.

Screen shot of creating a threat match

<img width="1023" alt="Screen Shot 2020-09-30 at 3 31 09 PM" src="https://user-images.githubusercontent.com/1151048/94742158-791b1c00-0332-11eb-9d79-78ab431322f0.png">

---

Screen shot of the description after creating one

<img width="1128" alt="Screen Shot 2020-09-30 at 3 29 32 PM" src="https://user-images.githubusercontent.com/1151048/94742203-8b955580-0332-11eb-837f-5b4383044a13.png">

---

Screen shot of first creating a threat match without values filled out

<img width="1017" alt="Screen Shot 2020-09-30 at 3 27 29 PM" src="https://user-images.githubusercontent.com/1151048/94742222-95b75400-0332-11eb-9872-e7670e917941.png">

Additions and bug fixes:
* Changes the threat index to be an array
* Adds a threat_language to the REST schema so that we can use KQL, Lucene, (others in the future)
* Adds plumbing for threat_list to work with the other REST endpoints such as PUT, PATCH, etc...
* Adds the AND, OR dialog and user interface

**Usage**
If you are a team member using the team servers you can skip this usage section of creating threat index. Otherwise if you want to know how to create a mock threat index, instructions are below.

Go to the folder:
```ts
/kibana/x-pack/plugins/security_solution/server/lib/detection_engine/scripts
```

And post a small ECS threat mapping to the index called `mock-threat-list`:
```ts
./create_threat_mapping.sh
```

Then to post a small number of threats that represent simple port numbers you can run:
```ts
./create_threat_data.sh
```

However, feel free to also manually create them directly in your dev tools like so:

```ts
# Posts a threat list item called some-name with an IP but change these out for valid data in your system
PUT mock-threat-list-1/_doc/9999
{
  "@timestamp": "2020-09-09T20:30:45.725Z",
  "host": {
    "name": "some-name",
    "ip": "127.0.0.1"
  }
}
```

```ts
# Posts a destination port number to watch
PUT mock-threat-list-1/_doc/10000
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "destination": {
    "port": "443"
  }
}
```

```ts
# Posts a source port number to watch
PUT mock-threat-list-1/_doc/10001
{
  "@timestamp": "2020-09-08T20:30:45.725Z",
  "source": {
    "port": "443"
  }
}
```

### Checklist

- [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)
- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)
- [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)
- [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts

@@ -60,7 +60,7 @@ export const getAddPrepackagedThreatMatchRulesSchemaMock = (): AddPrepackagedRul
   rule_id: 'rule-1',
   version: 1,
   threat_query: '*:*',
-  threat_index: 'list-index',
+  threat_index: ['list-index'],
   threat_mapping: [
     {
       entries: [
@@ -118,7 +118,7 @@ export const getAddPrepackagedThreatMatchRulesSchemaDecodedMock = (): AddPrepack
   exceptions_list: [],
   rule_id: 'rule-1',
   threat_query: '*:*',
-  threat_index: 'list-index',
+  threat_index: ['list-index'],
   threat_mapping: [
     {
       entries: [

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -51,6 +51,7 @@ import {
   threat_query,
   threat_filters,
   threat_mapping,
+  threat_language,
 } from '../types/threat_mapping';
 
 import {
@@ -128,6 +129,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       threat_mapping, // defaults to "undefined" if not set during decode
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
+      threat_language, // defaults "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts

@@ -66,7 +66,7 @@ export const getCreateThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): CreateRu
   language: 'kuery',
   rule_id: ruleId,
   threat_query: '*:*',
-  threat_index: 'list-index',
+  threat_index: ['list-index'],
   threat_mapping: [
     {
       entries: [
@@ -124,7 +124,7 @@ export const getCreateThreatMatchRulesSchemaDecodedMock = (): CreateRulesSchemaD
   exceptions_list: [],
   rule_id: 'rule-1',
   threat_query: '*:*',
-  threat_index: 'list-index',
+  threat_index: ['list-index'],
   threat_mapping: [
     {
       entries: [

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts

@@ -52,6 +52,7 @@ import {
   threat_query,
   threat_filters,
   threat_mapping,
+  threat_language,
 } from '../types/threat_mapping';
 
 import {
@@ -124,6 +125,7 @@ export const createRulesSchema = t.intersection([
       threat_query, // defaults to "undefined" if not set during decode
       threat_filters, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
+      threat_language, // defaults "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts

@@ -86,7 +86,7 @@ export const getImportThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): ImportRu
   risk_score: 55,
   language: 'kuery',
   rule_id: ruleId,
-  threat_index: 'index-123',
+  threat_index: ['index-123'],
   threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }],
   threat_query: '*:*',
   threat_filters: [
@@ -136,7 +136,7 @@ export const getImportThreatMatchRulesSchemaDecodedMock = (): ImportRulesSchemaD
   rule_id: 'rule-1',
   immutable: false,
   threat_query: '*:*',
-  threat_index: 'index-123',
+  threat_index: ['index-123'],
   threat_mapping: [
     {
       entries: [

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -58,6 +58,7 @@ import {
   threat_query,
   threat_filters,
   threat_mapping,
+  threat_language,
 } from '../types/threat_mapping';
 
 import {
@@ -147,6 +148,7 @@ export const importRulesSchema = t.intersection([
       threat_mapping, // defaults to "undefined" if not set during decode
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
+      threat_language, // defaults "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -48,6 +48,13 @@ import {
   severity_mapping,
   event_category_override,
 } from '../common/schemas';
+import {
+  threat_index,
+  threat_query,
+  threat_filters,
+  threat_mapping,
+  threat_language,
+} from '../types/threat_mapping';
 import { listArrayOrUndefined } from '../types/lists';
 
 /**
@@ -97,6 +104,11 @@ export const patchRulesSchema = t.exact(
     note,
     version,
     exceptions_list: listArrayOrUndefined,
+    threat_index,
+    threat_query,
+    threat_filters,
+    threat_mapping,
+    threat_language,
   })
 );
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts

@@ -49,6 +49,13 @@ import {
   SeverityMapping,
   event_category_override,
 } from '../common/schemas';
+import {
+  threat_index,
+  threat_query,
+  threat_filters,
+  threat_mapping,
+  threat_language,
+} from '../types/threat_mapping';
 
 import {
   DefaultStringArray,
@@ -122,6 +129,11 @@ export const updateRulesSchema = t.intersection([
       note, // defaults to "undefined" if not set during decode
       version, // defaults to "undefined" if not set during decode
       exceptions_list: DefaultListArray, // defaults to empty array if not set during decode
+      threat_mapping, // defaults to "undefined" if not set during decode
+      threat_query, // defaults to "undefined" if not set during decode
+      threat_filters, // defaults to "undefined" if not set during decode
+      threat_index, // defaults to "undefined" if not set during decode
+      threat_language, // defaults "undefined" if not set during decode
     })
   ),
 ]);

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -87,7 +87,7 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R
   return {
     ...getRulesSchemaMock(anchorDate),
     type: 'threat_match',
-    threat_index: 'index-123',
+    threat_index: ['index-123'],
     threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }],
     threat_query: '*:*',
     threat_filters: [

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts

@@ -626,7 +626,7 @@ describe('rules_schema', () => {
       const message = pipe(checked, foldLeftRight);
 
       expect(getPaths(left(message.errors))).toEqual([
-        'invalid keys "threat_index,threat_mapping,[{"entries":[{"field":"host.name","type":"mapping","value":"host.name"}]}],threat_query,threat_filters,[{"bool":{"must":[{"query_string":{"query":"host.name: linux","analyze_wildcard":true,"time_zone":"Zulu"}}],"filter":[],"should":[],"must_not":[]}}]"',
+        'invalid keys "threat_index,["index-123"],threat_mapping,[{"entries":[{"field":"host.name","type":"mapping","value":"host.name"}]}],threat_query,threat_filters,[{"bool":{"must":[{"query_string":{"query":"host.name: linux","analyze_wildcard":true,"time_zone":"Zulu"}}],"filter":[],"should":[],"must_not":[]}}]"',
       ]);
       expect(message.schema).toEqual({});
     });
@@ -764,7 +764,7 @@ describe('rules_schema', () => {
 
     test('should return 5 fields for a rule of type "threat_match"', () => {
       const fields = addThreatMatchFields({ type: 'threat_match' });
-      expect(fields.length).toEqual(5);
+      expect(fields.length).toEqual(6);
     });
   });
 

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts

@@ -66,6 +66,7 @@ import {
   threat_query,
   threat_filters,
   threat_mapping,
+  threat_language,
 } from '../types/threat_mapping';
 
 import { DefaultListArray } from '../types/lists_default_array';
@@ -144,6 +145,7 @@ export const dependentRulesSchema = t.partial({
   threat_index,
   threat_query,
   threat_mapping,
+  threat_language,
 });
 
 /**
@@ -277,6 +279,7 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly):
       t.exact(t.type({ threat_query: dependentRulesSchema.props.threat_query })),
       t.exact(t.type({ threat_index: dependentRulesSchema.props.threat_index })),
       t.exact(t.type({ threat_mapping: dependentRulesSchema.props.threat_mapping })),
+      t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })),
       t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })),
       t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })),
     ];

x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts

@@ -33,4 +33,5 @@ export * from './positive_integer';
 export * from './positive_integer_greater_than_zero';
 export * from './references_default_array';
 export * from './risk_score';
+export * from './threat_mapping';
 export * from './uuid';

x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts

@@ -7,6 +7,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 
 import * as t from 'io-ts';
+import { language } from '../common/schemas';
 import { NonEmptyString } from './non_empty_string';
 
 export const threat_query = t.string;
@@ -19,29 +20,38 @@ export type ThreatFilters = t.TypeOf<typeof threat_filters>;
 export const threatFiltersOrUndefined = t.union([threat_filters, t.undefined]);
 export type ThreatFiltersOrUndefined = t.TypeOf<typeof threatFiltersOrUndefined>;
 
-export const threatMappingEntries = t.array(
-  t.exact(
-    t.type({
-      field: NonEmptyString,
-      type: t.keyof({ mapping: null }),
-      value: NonEmptyString,
-    })
-  )
+export const threatMapEntry = t.exact(
+  t.type({
+    field: NonEmptyString,
+    type: t.keyof({ mapping: null }),
+    value: NonEmptyString,
+  })
 );
+
+export type ThreatMapEntry = t.TypeOf<typeof threatMapEntry>;
+
+export const threatMappingEntries = t.array(threatMapEntry);
 export type ThreatMappingEntries = t.TypeOf<typeof threatMappingEntries>;
 
-export const threat_mapping = t.array(
-  t.exact(
-    t.type({
-      entries: threatMappingEntries,
-    })
-  )
+export const threatMap = t.exact(
+  t.type({
+    entries: threatMappingEntries,
+  })
 );
+export type ThreatMap = t.TypeOf<typeof threatMap>;
+
+export const threat_mapping = t.array(threatMap);
 export type ThreatMapping = t.TypeOf<typeof threat_mapping>;
 
 export const threatMappingOrUndefined = t.union([threat_mapping, t.undefined]);
 export type ThreatMappingOrUndefined = t.TypeOf<typeof threatMappingOrUndefined>;
 
-export const threat_index = t.string;
+export const threat_index = t.array(t.string);
+export type ThreatIndex = t.TypeOf<typeof threat_index>;
 export const threatIndexOrUndefined = t.union([threat_index, t.undefined]);
 export type ThreatIndexOrUndefined = t.TypeOf<typeof threatIndexOrUndefined>;
+
+export const threat_language = t.union([language, t.undefined]);
+export type ThreatLanguage = t.TypeOf<typeof threat_language>;
+export const threatLanguageOrUndefined = t.union([threat_language, t.undefined]);
+export type ThreatLanguageOrUndefined = t.TypeOf<typeof threatLanguageOrUndefined>;

x-pack/plugins/security_solution/public/common/components/threat_match/and_badge.test.tsx

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { ThemeProvider } from 'styled-components';
+import { mount } from 'enzyme';
+import euiLightVars from '@elastic/eui/dist/eui_theme_light.json';
+
+import { AndBadgeComponent } from './and_badge';
+
+describe('AndBadgeComponent', () => {
+  test('it renders entryItemIndexItemEntryFirstRowAndBadge for very first item', () => {
+    const wrapper = mount(
+      <ThemeProvider theme={() => ({ eui: euiLightVars, darkMode: false })}>
+        <AndBadgeComponent entriesLength={2} entryItemIndex={0} />
+      </ThemeProvider>
+    );
+
+    expect(wrapper.find('[data-test-subj="entryItemEntryFirstRowAndBadge"]').exists()).toBeTruthy();
+  });
+
+  test('it renders entryItemEntryInvisibleAndBadge if "entriesLength" is 1 or less', () => {
+    const wrapper = mount(
+      <ThemeProvider theme={() => ({ eui: euiLightVars, darkMode: false })}>
+        <AndBadgeComponent entriesLength={1} entryItemIndex={0} />
+      </ThemeProvider>
+    );
+
+    expect(
+      wrapper.find('[data-test-subj="entryItemEntryInvisibleAndBadge"]').exists()
+    ).toBeTruthy();
+  });
+
+  test('it renders regular "and" badge if item is not the first one and includes more than one entry', () => {
+    const wrapper = mount(
+      <ThemeProvider theme={() => ({ eui: euiLightVars, darkMode: false })}>
+        <AndBadgeComponent entriesLength={2} entryItemIndex={1} />
+      </ThemeProvider>
+    );
+
+    expect(wrapper.find('[data-test-subj="entryItemEntryAndBadge"]').exists()).toBeTruthy();
+  });
+});

x-pack/plugins/security_solution/public/common/components/threat_match/and_badge.tsx

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiFlexItem } from '@elastic/eui';
+import styled from 'styled-components';
+
+import { AndOrBadge } from '../and_or_badge';
+
+const MyInvisibleAndBadge = styled(EuiFlexItem)`
+  visibility: hidden;
+`;
+
+const MyFirstRowContainer = styled(EuiFlexItem)`
+  padding-top: 20px;
+`;
+
+interface AndBadgeProps {
+  entriesLength: number;
+  entryItemIndex: number;
+}
+
+export const AndBadgeComponent = React.memo<AndBadgeProps>(({ entriesLength, entryItemIndex }) => {
+  const badge = <AndOrBadge includeAntennas type="and" />;
+
+  if (entriesLength > 1 && entryItemIndex === 0) {
+    return (
+      <MyFirstRowContainer grow={false} data-test-subj="entryItemEntryFirstRowAndBadge">
+        {badge}
+      </MyFirstRowContainer>
+    );
+  } else if (entriesLength <= 1) {
+    return (
+      <MyInvisibleAndBadge grow={false} data-test-subj="entryItemEntryInvisibleAndBadge">
+        {badge}
+      </MyInvisibleAndBadge>
+    );
+  } else {
+    return (
+      <EuiFlexItem grow={false} data-test-subj="entryItemEntryAndBadge">
+        {badge}
+      </EuiFlexItem>
+    );
+  }
+});
+
+AndBadgeComponent.displayName = 'AndBadge';