Skip to content

Commit c80c143

Browse files
jgowdyelasticjgowdyelastic
committed
[ML] Fixing module datafeed overrides (elastic#78925)
1 parent b891f9b commit c80c143

27 files changed

+756
-589
lines changed

‎x-pack/plugins/ml/common/types/modules.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export interface ModuleJob {
1111
config: Omit<Job, 'job_id'>;
1212
}
1313

14-
export interface ModuleDataFeed {
14+
export interface ModuleDatafeed {
1515
id: string;
1616
config: Omit<Datafeed, 'datafeed_id'>;
1717
}
@@ -49,7 +49,7 @@ export interface Module {
4949
defaultIndexPattern: string;
5050
query: any;
5151
jobs: ModuleJob[];
52-
datafeeds: ModuleDataFeed[];
52+
datafeeds: ModuleDatafeed[];
5353
kibana: KibanaObjects;
5454
}
5555

‎x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { MlInfoResponse } from '../../../common/types/ml_server_info';
1717
import {
1818
KibanaObjects,
1919
KibanaObjectConfig,
20-
ModuleDataFeed,
20+
ModuleDatafeed,
2121
ModuleJob,
2222
Module,
2323
JobOverride,
@@ -283,7 +283,7 @@ export class DataRecognizer {
283283
}
284284

285285
const jobs: ModuleJob[] = [];
286-
const datafeeds: ModuleDataFeed[] = [];
286+
const datafeeds: ModuleDatafeed[] = [];
287287
const kibana: KibanaObjects = {};
288288
// load all of the job configs
289289
await Promise.all(
@@ -710,7 +710,7 @@ export class DataRecognizer {
710710
// save the datafeeds.
711711
// if any fail (e.g. it already exists), catch the error and mark the result
712712
// as success: false
713-
async saveDatafeeds(datafeeds: ModuleDataFeed[]) {
713+
async saveDatafeeds(datafeeds: ModuleDatafeed[]) {
714714
return await Promise.all(
715715
datafeeds.map(async (datafeed) => {
716716
try {
@@ -723,7 +723,7 @@ export class DataRecognizer {
723723
);
724724
}
725725

726-
async saveDatafeed(datafeed: ModuleDataFeed) {
726+
async saveDatafeed(datafeed: ModuleDatafeed) {
727727
return this._asInternalUser.ml.putDatafeed(
728728
{
729729
datafeed_id: datafeed.id,
@@ -734,7 +734,7 @@ export class DataRecognizer {
734734
}
735735

736736
async startDatafeeds(
737-
datafeeds: ModuleDataFeed[],
737+
datafeeds: ModuleDatafeed[],
738738
start?: number,
739739
end?: number
740740
): Promise<{ [key: string]: DatafeedResponse }> {
@@ -746,7 +746,7 @@ export class DataRecognizer {
746746
}
747747

748748
async startDatafeed(
749-
datafeed: ModuleDataFeed,
749+
datafeed: ModuleDatafeed,
750750
start: number | undefined,
751751
end: number | undefined
752752
): Promise<DatafeedResponse> {
@@ -1229,6 +1229,25 @@ export class DataRecognizer {
12291229
const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides];
12301230
const { datafeeds } = moduleConfig;
12311231

1232+
// for some items in the datafeed, we should not merge.
1233+
// we should instead use the whole override object
1234+
function overwriteObjects(source: ModuleDatafeed['config'], update: DatafeedOverride) {
1235+
Object.entries(update).forEach(([key, val]) => {
1236+
if (typeof val === 'object') {
1237+
switch (key) {
1238+
case 'query':
1239+
case 'aggregations':
1240+
case 'aggs':
1241+
case 'script_fields':
1242+
source[key] = val as any;
1243+
break;
1244+
default:
1245+
break;
1246+
}
1247+
}
1248+
});
1249+
}
1250+
12321251
// separate all the overrides.
12331252
// the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module
12341253
const generalOverrides: GeneralDatafeedsOverride[] = [];
@@ -1244,6 +1263,7 @@ export class DataRecognizer {
12441263
generalOverrides.forEach((o) => {
12451264
datafeeds.forEach(({ config }) => {
12461265
merge(config, o);
1266+
overwriteObjects(config, o);
12471267
});
12481268
});
12491269

@@ -1259,6 +1279,7 @@ export class DataRecognizer {
12591279
delete o.job_id;
12601280
delete o.datafeed_id;
12611281
merge(datafeed.config, o);
1282+
overwriteObjects(datafeed.config, o);
12621283
}
12631284
});
12641285
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
{
22
"job_type": "anomaly_detector",
33
"description": "Logs UI: Detects anomalies in count of log entries by category",
4-
"groups": ["logs-ui"],
4+
"groups": [
5+
"logs-ui"
6+
],
57
"analysis_config": {
68
"bucket_span": "15m",
79
"categorization_field_name": "message",
@@ -14,7 +16,10 @@
1416
"use_null": true
1517
}
1618
],
17-
"influencers": ["event.dataset", "mlcategory"],
19+
"influencers": [
20+
"event.dataset",
21+
"mlcategory"
22+
],
1823
"per_partition_categorization": {
1924
"enabled": true,
2025
"stop_on_warn": false

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"icon": "metricbeatApp"
2+
"icon": "metricbeatApp"
33
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@
88
"query": {
99
"bool": {
1010
"filter": {
11-
"terms" : { "event.dataset" : ["system.cpu", "system.filesystem"]}
11+
"terms": {
12+
"event.dataset": [
13+
"system.cpu",
14+
"system.filesystem"
15+
]
16+
}
1217
}
1318
}
1419
},

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,14 @@
66
"query": {
77
"bool": {
88
"filter": {
9-
"term": { "event.dataset": "system.cpu" }
9+
"term": {
10+
"event.dataset": "system.cpu"
11+
}
1012
},
1113
"must": {
12-
"exists": { "field": "system.cpu.iowait.pct" }
14+
"exists": {
15+
"field": "system.cpu.iowait.pct"
16+
}
1317
}
1418
}
1519
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
{
2-
"job_id": "JOB_ID",
3-
"indices": [
4-
"INDEX_PATTERN_NAME"
5-
],
6-
"query": {
7-
"bool": {
8-
"filter": {
9-
"term": { "event.dataset": "system.filesystem" }
10-
},
11-
"must": {
12-
"exists": { "field": "system.filesystem.used.pct" }
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"query": {
7+
"bool": {
8+
"filter": {
9+
"term": {
10+
"event.dataset": "system.filesystem"
11+
}
12+
},
13+
"must": {
14+
"exists": {
15+
"field": "system.filesystem.used.pct"
1316
}
1417
}
1518
}
1619
}
20+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
{
2-
"job_id": "JOB_ID",
3-
"indices": [
4-
"INDEX_PATTERN_NAME"
5-
],
6-
"query": {
7-
"bool": {
8-
"must": {
9-
"exists": { "field": "event.dataset" }
2+
"job_id": "JOB_ID",
3+
"indices": [
4+
"INDEX_PATTERN_NAME"
5+
],
6+
"query": {
7+
"bool": {
8+
"must": {
9+
"exists": {
10+
"field": "event.dataset"
1011
}
1112
}
1213
}
1314
}
15+
}

‎x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json‎

Lines changed: 54 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,56 @@
11
{
2-
"job_type": "anomaly_detector",
3-
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
4-
"groups": ["metricbeat"],
5-
"analysis_config": {
6-
"bucket_span": "10m",
7-
"detectors": [
8-
{
9-
"detector_description": "high mean system.cpu.iowait.pct",
10-
"function": "high_mean",
11-
"field_name": "system.cpu.iowait.pct",
12-
"partition_field_name": "host.name",
13-
"custom_rules": [
14-
{
15-
"actions": [
16-
"skip_result"
17-
],
18-
"conditions": [
19-
{
20-
"applies_to": "actual",
21-
"operator": "lt",
22-
"value": 0.25
23-
}
24-
]
25-
}
26-
]
27-
}
28-
],
29-
"influencers": [
30-
"host.name"
31-
]
32-
},
33-
"analysis_limits": {
34-
"model_memory_limit": "25mb"
35-
},
36-
"data_description": {
37-
"time_field": "@timestamp",
38-
"time_format": "epoch_ms"
39-
},
40-
"custom_settings": {
41-
"created_by": "ml-module-metricbeat-system",
42-
"custom_urls": [
43-
{
44-
"url_name": "Host overview",
45-
"time_range": "3h",
46-
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
47-
},
48-
{
49-
"url_name": "Raw data",
50-
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
51-
}
52-
]
53-
}
2+
"job_type": "anomaly_detector",
3+
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
4+
"groups": [
5+
"metricbeat"
6+
],
7+
"analysis_config": {
8+
"bucket_span": "10m",
9+
"detectors": [
10+
{
11+
"detector_description": "high mean system.cpu.iowait.pct",
12+
"function": "high_mean",
13+
"field_name": "system.cpu.iowait.pct",
14+
"partition_field_name": "host.name",
15+
"custom_rules": [
16+
{
17+
"actions": [
18+
"skip_result"
19+
],
20+
"conditions": [
21+
{
22+
"applies_to": "actual",
23+
"operator": "lt",
24+
"value": 0.25
25+
}
26+
]
27+
}
28+
]
29+
}
30+
],
31+
"influencers": [
32+
"host.name"
33+
]
34+
},
35+
"analysis_limits": {
36+
"model_memory_limit": "25mb"
37+
},
38+
"data_description": {
39+
"time_field": "@timestamp",
40+
"time_format": "epoch_ms"
41+
},
42+
"custom_settings": {
43+
"created_by": "ml-module-metricbeat-system",
44+
"custom_urls": [
45+
{
46+
"url_name": "Host overview",
47+
"time_range": "3h",
48+
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
49+
},
50+
{
51+
"url_name": "Raw data",
52+
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
53+
}
54+
]
5455
}
56+
}

0 commit comments

Comments
 (0)