Skip to content

Commit faf4b04

Browse files
marshallmainelasticmachine
marshallmain
and
elasticmachine
authored
[Detections][EQL] EQL rule execution in detection engine (#77419)
* First draft of EQL rules in detection engine * Reorganize functions to separate files * Start adding eventCategoryOverride option for EQL rules * Add building block alerts for each event within sequence * Use eql instead of eql_query for rule type * Remove unused imports * Fix tests * Add basic tests for buildEqlSearchRequest * Add rulesSchema tests for eql * Add buildSignalFromSequence test * Add threat rule fields to buildRuleWithoutOverrides * Fix buildSignalFromSequence typecheck error * Add more tests * Add tests for wrapBuildingBlock and generateSignalId * Use isEqlRule function and fix import error * delete frank * Move sequence interface to types.ts * Fix import * Remove EQL execution placeholder, add back language to eql rule type * allow no indices for eql search * Fix unit tests for language update * Fix buildEqlSearchRequest tests * Replace signal.child with signal.group * remove unused import * Move sequence signal group building to separate testable function * Unbork the merge conflict resolution Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
1 parent 500ad8b commit faf4b04

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1337
-108
lines changed

‎x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts‎

Lines changed: 135 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { getQueryFilter, buildExceptionFilter } from './get_query_filter';
7+
import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
88
import { Filter, EsQueryConfig } from 'src/plugins/data/public';
99
import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
1010

@@ -1085,4 +1085,138 @@ describe('get_filter', () => {
10851085
});
10861086
});
10871087
});
1088+
1089+
describe('buildEqlSearchRequest', () => {
1090+
test('should build a basic request with time range', () => {
1091+
const request = buildEqlSearchRequest(
1092+
'process where true',
1093+
['testindex1', 'testindex2'],
1094+
'now-5m',
1095+
'now',
1096+
100,
1097+
undefined,
1098+
[],
1099+
undefined
1100+
);
1101+
expect(request).toEqual({
1102+
method: 'POST',
1103+
path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
1104+
body: {
1105+
size: 100,
1106+
query: 'process where true',
1107+
filter: {
1108+
range: {
1109+
'@timestamp': {
1110+
gte: 'now-5m',
1111+
lte: 'now',
1112+
},
1113+
},
1114+
},
1115+
},
1116+
});
1117+
});
1118+
1119+
test('should build a request with timestamp and event category overrides', () => {
1120+
const request = buildEqlSearchRequest(
1121+
'process where true',
1122+
['testindex1', 'testindex2'],
1123+
'now-5m',
1124+
'now',
1125+
100,
1126+
'event.ingested',
1127+
[],
1128+
'event.other_category'
1129+
);
1130+
expect(request).toEqual({
1131+
method: 'POST',
1132+
path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
1133+
event_category_field: 'event.other_category',
1134+
body: {
1135+
size: 100,
1136+
query: 'process where true',
1137+
filter: {
1138+
range: {
1139+
'event.ingested': {
1140+
gte: 'now-5m',
1141+
lte: 'now',
1142+
},
1143+
},
1144+
},
1145+
},
1146+
});
1147+
});
1148+
1149+
test('should build a request with exceptions', () => {
1150+
const request = buildEqlSearchRequest(
1151+
'process where true',
1152+
['testindex1', 'testindex2'],
1153+
'now-5m',
1154+
'now',
1155+
100,
1156+
undefined,
1157+
[getExceptionListItemSchemaMock()],
1158+
undefined
1159+
);
1160+
expect(request).toEqual({
1161+
method: 'POST',
1162+
path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`,
1163+
body: {
1164+
size: 100,
1165+
query: 'process where true',
1166+
filter: {
1167+
range: {
1168+
'@timestamp': {
1169+
gte: 'now-5m',
1170+
lte: 'now',
1171+
},
1172+
},
1173+
bool: {
1174+
must_not: {
1175+
bool: {
1176+
should: [
1177+
{
1178+
bool: {
1179+
filter: [
1180+
{
1181+
nested: {
1182+
path: 'some.parentField',
1183+
query: {
1184+
bool: {
1185+
minimum_should_match: 1,
1186+
should: [
1187+
{
1188+
match_phrase: {
1189+
'some.parentField.nested.field': 'some value',
1190+
},
1191+
},
1192+
],
1193+
},
1194+
},
1195+
score_mode: 'none',
1196+
},
1197+
},
1198+
{
1199+
bool: {
1200+
minimum_should_match: 1,
1201+
should: [
1202+
{
1203+
match_phrase: {
1204+
'some.not.nested.field': 'some value',
1205+
},
1206+
},
1207+
],
1208+
},
1209+
},
1210+
],
1211+
},
1212+
},
1213+
],
1214+
},
1215+
},
1216+
},
1217+
},
1218+
},
1219+
});
1220+
});
1221+
});
10881222
});

‎x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@ import {
1717
CreateExceptionListItemSchema,
1818
} from '../../../lists/common/schemas';
1919
import { buildExceptionListQueries } from './build_exceptions_query';
20-
import { Query as QueryString, Language, Index } from './schemas/common/schemas';
20+
import {
21+
Query as QueryString,
22+
Language,
23+
Index,
24+
TimestampOverrideOrUndefined,
25+
} from './schemas/common/schemas';
2126

2227
export const getQueryFilter = (
2328
query: QueryString,
@@ -67,6 +72,78 @@ export const getQueryFilter = (
6772
return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
6873
};
6974

75+
interface EqlSearchRequest {
76+
method: string;
77+
path: string;
78+
body: object;
79+
event_category_field?: string;
80+
}
81+
82+
export const buildEqlSearchRequest = (
83+
query: string,
84+
index: string[],
85+
from: string,
86+
to: string,
87+
size: number,
88+
timestampOverride: TimestampOverrideOrUndefined,
89+
exceptionLists: ExceptionListItemSchema[],
90+
eventCategoryOverride: string | undefined
91+
): EqlSearchRequest => {
92+
const timestamp = timestampOverride ?? '@timestamp';
93+
const indexPattern: IIndexPattern = {
94+
fields: [],
95+
title: index.join(),
96+
};
97+
const config: EsQueryConfig = {
98+
allowLeadingWildcards: true,
99+
queryStringOptions: { analyze_wildcard: true },
100+
ignoreFilterIfFieldNotInIndex: false,
101+
dateFormatTZ: 'Zulu',
102+
};
103+
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
104+
let exceptionFilter: Filter | undefined;
105+
if (exceptionQueries.length > 0) {
106+
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
107+
// allowing us to make 1024-item chunks of exception list items.
108+
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
109+
// very conservative value.
110+
exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
111+
}
112+
const indexString = index.join();
113+
const baseRequest = {
114+
method: 'POST',
115+
path: `/${indexString}/_eql/search?allow_no_indices=true`,
116+
body: {
117+
size,
118+
query,
119+
filter: {
120+
range: {
121+
[timestamp]: {
122+
gte: from,
123+
lte: to,
124+
},
125+
},
126+
bool:
127+
exceptionFilter !== undefined
128+
? {
129+
must_not: {
130+
bool: exceptionFilter?.query.bool,
131+
},
132+
}
133+
: undefined,
134+
},
135+
},
136+
};
137+
if (eventCategoryOverride) {
138+
return {
139+
...baseRequest,
140+
event_category_field: eventCategoryOverride,
141+
};
142+
} else {
143+
return baseRequest;
144+
}
145+
};
146+
70147
export const buildExceptionFilter = (
71148
exceptionQueries: Query[],
72149
indexPattern: IIndexPattern,

‎x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ export type Enabled = t.TypeOf<typeof enabled>;
4040
export const enabledOrUndefined = t.union([enabled, t.undefined]);
4141
export type EnabledOrUndefined = t.TypeOf<typeof enabledOrUndefined>;
4242

43+
export const event_category_override = t.string;
44+
export type EventCategoryOverride = t.TypeOf<typeof event_category_override>;
45+
46+
export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]);
47+
export type EventCategoryOverrideOrUndefined = t.TypeOf<typeof eventCategoryOverrideOrUndefined>;
48+
4349
export const false_positives = t.array(t.string);
4450
export type FalsePositives = t.TypeOf<typeof false_positives>;
4551

‎x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import {
4444
Author,
4545
RiskScoreMapping,
4646
SeverityMapping,
47+
event_category_override,
4748
} from '../common/schemas';
4849
import {
4950
threat_index,
@@ -96,6 +97,7 @@ export const addPrepackagedRulesSchema = t.intersection([
9697
author: DefaultStringArray, // defaults to empty array of strings if not set during decode
9798
building_block_type, // defaults to undefined if not set during decode
9899
enabled: DefaultBooleanFalse, // defaults to false if not set during decode
100+
event_category_override, // defaults to "undefined" if not set during decode
99101
false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
100102
filters, // defaults to undefined if not set during decode
101103
from: DefaultFromString, // defaults to "now-6m" if not set during decode

‎x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import {
4545
Author,
4646
RiskScoreMapping,
4747
SeverityMapping,
48+
event_category_override,
4849
} from '../common/schemas';
4950
import {
5051
threat_index,
@@ -88,6 +89,7 @@ export const createRulesSchema = t.intersection([
8889
author: DefaultStringArray, // defaults to empty array of strings if not set during decode
8990
building_block_type, // defaults to undefined if not set during decode
9091
enabled: DefaultBooleanTrue, // defaults to true if not set during decode
92+
event_category_override, // defaults to "undefined" if not set during decode
9193
false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
9294
filters, // defaults to undefined if not set during decode
9395
from: DefaultFromString, // defaults to "now-6m" if not set during decode

‎x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import {
5151
Author,
5252
RiskScoreMapping,
5353
SeverityMapping,
54+
event_category_override,
5455
} from '../common/schemas';
5556
import {
5657
threat_index,
@@ -107,6 +108,7 @@ export const importRulesSchema = t.intersection([
107108
author: DefaultStringArray, // defaults to empty array of strings if not set during decode
108109
building_block_type, // defaults to undefined if not set during decode
109110
enabled: DefaultBooleanTrue, // defaults to true if not set during decode
111+
event_category_override, // defaults to "undefined" if not set during decode
110112
false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
111113
filters, // defaults to undefined if not set during decode
112114
from: DefaultFromString, // defaults to "now-6m" if not set during decode

‎x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import {
4646
timestamp_override,
4747
risk_score_mapping,
4848
severity_mapping,
49+
event_category_override,
4950
} from '../common/schemas';
5051
import { listArrayOrUndefined } from '../types/lists';
5152

@@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact(
6566
actions,
6667
anomaly_threshold,
6768
enabled,
69+
event_category_override,
6870
false_positives,
6971
filters,
7072
from,

‎x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import {
4747
Author,
4848
RiskScoreMapping,
4949
SeverityMapping,
50+
event_category_override,
5051
} from '../common/schemas';
5152

5253
import {
@@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([
9091
author: DefaultStringArray, // defaults to empty array of strings if not set during decode
9192
building_block_type, // defaults to undefined if not set during decode
9293
enabled: DefaultBooleanTrue, // defaults to true if not set during decode
94+
event_category_override,
9395
false_positives: DefaultStringArray, // defaults to empty string array if not set during decode
9496
filters, // defaults to undefined if not set during decode
9597
from: DefaultFromString, // defaults to "now-6m" if not set during decode

‎x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
5252
severity: 'high',
5353
severity_mapping: [],
5454
updated_by: 'elastic_kibana',
55-
tags: [],
55+
tags: ['some fake tag 1', 'some fake tag 2'],
5656
to: 'now',
5757
type: 'query',
5858
threat: [],
@@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem
6161
status_date: '2020-02-22T16:47:50.047Z',
6262
last_success_at: '2020-02-22T16:47:50.047Z',
6363
last_success_message: 'succeeded',
64-
output_index: '.siem-signals-hassanabad-frank-default',
64+
output_index: '.siem-signals-default',
6565
max_signals: 100,
6666
risk_score: 55,
6767
risk_score_mapping: [],
@@ -110,3 +110,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R
110110
],
111111
};
112112
};
113+
114+
export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => {
115+
return {
116+
...getRulesSchemaMock(anchorDate),
117+
language: 'eql',
118+
type: 'eql',
119+
query: 'process where true',
120+
};
121+
};

0 commit comments

Comments
 (0)