Skip to content

Commit a3ede09

Browse files
phillipbblaklaybulelasticmachinesimianhacker
authored
[Metrics UI] Anomaly Detection setup flow for Metrics (#76787) (#78432)
* adds metrics ml integration * Add ability to create ml jobs from inventory * Fix i18n stuff * Fix typecheck * renames jobs, updates datafeeds * adds allow_no_indices: true for datafeeds * Revert "[Metrics UI] Replace Snapshot API with Metrics API (#76253)" This reverts commit 0ca6472. * Add ability to fetch anomalies * Fix typecheck * Fix typecheck * Fix i18n * Fix lint, use the right partition field * Delete log files * Fix merge * Fix merge issues * Update name of jobs * Remove CPU job * [Metrics UI] Replace Snapshot API with Metrics API (#76253) - Remove server/lib/snapshot - Replace backend for /api/infra/snapshot with data from Metrics API - Fixing tests with updates to the snapshot node Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> * Add links back to ML for anomalies and manage jobs * Fix typecheck * Remove unecessary validation Co-authored-by: Michael Hirsch <michaelahirsch@gmail.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Chris Cowan <chris@chriscowan.us> Co-authored-by: Michael Hirsch <michaelahirsch@gmail.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Chris Cowan <chris@chriscowan.us>
1 parent 0d01637 commit a3ede09

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+5392
-100
lines changed

‎x-pack/plugins/infra/common/http_api/infra_ml/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export * from './results';

‎x-pack/plugins/infra/common/http_api/infra_ml/results/common.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import * as rt from 'io-ts';
8+
9+
// [Sort field value, tiebreaker value]
10+
export const paginationCursorRT = rt.tuple([
11+
rt.union([rt.string, rt.number]),
12+
rt.union([rt.string, rt.number]),
13+
]);
14+
15+
export type PaginationCursor = rt.TypeOf<typeof paginationCursorRT>;
16+
17+
export const anomalyTypeRT = rt.keyof({
18+
metrics_hosts: null,
19+
metrics_k8s: null,
20+
});
21+
22+
export type AnomalyType = rt.TypeOf<typeof anomalyTypeRT>;
23+
24+
const sortOptionsRT = rt.keyof({
25+
anomalyScore: null,
26+
dataset: null,
27+
startTime: null,
28+
});
29+
30+
const sortDirectionsRT = rt.keyof({
31+
asc: null,
32+
desc: null,
33+
});
34+
35+
const paginationPreviousPageCursorRT = rt.type({
36+
searchBefore: paginationCursorRT,
37+
});
38+
39+
const paginationNextPageCursorRT = rt.type({
40+
searchAfter: paginationCursorRT,
41+
});
42+
43+
export const paginationRT = rt.intersection([
44+
rt.type({
45+
pageSize: rt.number,
46+
}),
47+
rt.partial({
48+
cursor: rt.union([paginationPreviousPageCursorRT, paginationNextPageCursorRT]),
49+
}),
50+
]);
51+
52+
export type Pagination = rt.TypeOf<typeof paginationRT>;
53+
54+
export const sortRT = rt.type({
55+
field: sortOptionsRT,
56+
direction: sortDirectionsRT,
57+
});
58+
59+
export type Sort = rt.TypeOf<typeof sortRT>;

‎x-pack/plugins/infra/common/http_api/infra_ml/results/index.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export * from './metrics_hosts_anomalies';
8+
export * from './metrics_k8s_anomalies';
9+
export * from './common';

‎x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_hosts_anomalies.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import * as rt from 'io-ts';
8+
9+
import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
10+
import { anomalyTypeRT, paginationCursorRT, sortRT, paginationRT } from './common';
11+
12+
export const INFA_ML_GET_METRICS_HOSTS_ANOMALIES_PATH =
13+
'/api/infra/infra_ml/results/metrics_hosts_anomalies';
14+
15+
const metricsHostAnomalyCommonFieldsRT = rt.type({
16+
id: rt.string,
17+
anomalyScore: rt.number,
18+
typical: rt.number,
19+
actual: rt.number,
20+
type: anomalyTypeRT,
21+
duration: rt.number,
22+
startTime: rt.number,
23+
jobId: rt.string,
24+
});
25+
const metricsHostsAnomalyRT = metricsHostAnomalyCommonFieldsRT;
26+
27+
export type MetricsHostsAnomaly = rt.TypeOf<typeof metricsHostsAnomalyRT>;
28+
29+
export const getMetricsHostsAnomaliesSuccessReponsePayloadRT = rt.intersection([
30+
rt.type({
31+
data: rt.intersection([
32+
rt.type({
33+
anomalies: rt.array(metricsHostsAnomalyRT),
34+
// Signifies there are more entries backwards or forwards. If this was a request
35+
// for a previous page, there are more previous pages, if this was a request for a next page,
36+
// there are more next pages.
37+
hasMoreEntries: rt.boolean,
38+
}),
39+
rt.partial({
40+
paginationCursors: rt.type({
41+
// The cursor to use to fetch the previous page
42+
previousPageCursor: paginationCursorRT,
43+
// The cursor to use to fetch the next page
44+
nextPageCursor: paginationCursorRT,
45+
}),
46+
}),
47+
]),
48+
}),
49+
rt.partial({
50+
timing: routeTimingMetadataRT,
51+
}),
52+
]);
53+
54+
export type GetMetricsHostsAnomaliesSuccessResponsePayload = rt.TypeOf<
55+
typeof getMetricsHostsAnomaliesSuccessReponsePayloadRT
56+
>;
57+
58+
export const getMetricsHostsAnomaliesRequestPayloadRT = rt.type({
59+
data: rt.intersection([
60+
rt.type({
61+
// the ID of the source configuration
62+
sourceId: rt.string,
63+
// the time range to fetch the log entry anomalies from
64+
timeRange: timeRangeRT,
65+
}),
66+
rt.partial({
67+
// Pagination properties
68+
pagination: paginationRT,
69+
// Sort properties
70+
sort: sortRT,
71+
// // Dataset filters
72+
// datasets: rt.array(rt.string),
73+
}),
74+
]),
75+
});
76+
77+
export type GetMetricsHostsAnomaliesRequestPayload = rt.TypeOf<
78+
typeof getMetricsHostsAnomaliesRequestPayloadRT
79+
>;

‎x-pack/plugins/infra/common/http_api/infra_ml/results/metrics_k8s_anomalies.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import * as rt from 'io-ts';
8+
9+
import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
10+
import { paginationCursorRT, anomalyTypeRT, sortRT, paginationRT } from './common';
11+
12+
export const INFA_ML_GET_METRICS_K8S_ANOMALIES_PATH =
13+
'/api/infra/infra_ml/results/metrics_k8s_anomalies';
14+
15+
const metricsK8sAnomalyCommonFieldsRT = rt.type({
16+
id: rt.string,
17+
anomalyScore: rt.number,
18+
typical: rt.number,
19+
actual: rt.number,
20+
type: anomalyTypeRT,
21+
duration: rt.number,
22+
startTime: rt.number,
23+
jobId: rt.string,
24+
});
25+
const metricsK8sAnomalyRT = metricsK8sAnomalyCommonFieldsRT;
26+
27+
export type MetricsK8sAnomaly = rt.TypeOf<typeof metricsK8sAnomalyRT>;
28+
29+
export const getMetricsK8sAnomaliesSuccessReponsePayloadRT = rt.intersection([
30+
rt.type({
31+
data: rt.intersection([
32+
rt.type({
33+
anomalies: rt.array(metricsK8sAnomalyRT),
34+
// Signifies there are more entries backwards or forwards. If this was a request
35+
// for a previous page, there are more previous pages, if this was a request for a next page,
36+
// there are more next pages.
37+
hasMoreEntries: rt.boolean,
38+
}),
39+
rt.partial({
40+
paginationCursors: rt.type({
41+
// The cursor to use to fetch the previous page
42+
previousPageCursor: paginationCursorRT,
43+
// The cursor to use to fetch the next page
44+
nextPageCursor: paginationCursorRT,
45+
}),
46+
}),
47+
]),
48+
}),
49+
rt.partial({
50+
timing: routeTimingMetadataRT,
51+
}),
52+
]);
53+
54+
export type GetMetricsK8sAnomaliesSuccessResponsePayload = rt.TypeOf<
55+
typeof getMetricsK8sAnomaliesSuccessReponsePayloadRT
56+
>;
57+
58+
export const getMetricsK8sAnomaliesRequestPayloadRT = rt.type({
59+
data: rt.intersection([
60+
rt.type({
61+
// the ID of the source configuration
62+
sourceId: rt.string,
63+
// the time range to fetch the log entry anomalies from
64+
timeRange: timeRangeRT,
65+
}),
66+
rt.partial({
67+
// Pagination properties
68+
pagination: paginationRT,
69+
// Sort properties
70+
sort: sortRT,
71+
// Dataset filters
72+
datasets: rt.array(rt.string),
73+
}),
74+
]),
75+
});
76+
77+
export type GetMetricsK8sAnomaliesRequestPayload = rt.TypeOf<
78+
typeof getMetricsK8sAnomaliesRequestPayloadRT
79+
>;

‎x-pack/plugins/infra/common/infra_ml/anomaly_results.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export const ML_SEVERITY_SCORES = {
8+
warning: 3,
9+
minor: 25,
10+
major: 50,
11+
critical: 75,
12+
};
13+
14+
export type MLSeverityScoreCategories = keyof typeof ML_SEVERITY_SCORES;
15+
16+
export const ML_SEVERITY_COLORS = {
17+
critical: 'rgb(228, 72, 72)',
18+
major: 'rgb(229, 113, 0)',
19+
minor: 'rgb(255, 221, 0)',
20+
warning: 'rgb(125, 180, 226)',
21+
};
22+
23+
export const getSeverityCategoryForScore = (
24+
score: number
25+
): MLSeverityScoreCategories | undefined => {
26+
if (score >= ML_SEVERITY_SCORES.critical) {
27+
return 'critical';
28+
} else if (score >= ML_SEVERITY_SCORES.major) {
29+
return 'major';
30+
} else if (score >= ML_SEVERITY_SCORES.minor) {
31+
return 'minor';
32+
} else if (score >= ML_SEVERITY_SCORES.warning) {
33+
return 'warning';
34+
} else {
35+
// Category is too low to include
36+
return undefined;
37+
}
38+
};
39+
40+
export const formatAnomalyScore = (score: number) => {
41+
return Math.round(score);
42+
};
43+
44+
export const formatOneDecimalPlace = (number: number) => {
45+
return Math.round(number * 10) / 10;
46+
};
47+
48+
export const getFriendlyNameForPartitionId = (partitionId: string) => {
49+
return partitionId !== '' ? partitionId : 'unknown';
50+
};
51+
52+
export const compareDatasetsByMaximumAnomalyScore = <
53+
Dataset extends { maximumAnomalyScore: number }
54+
>(
55+
firstDataset: Dataset,
56+
secondDataset: Dataset
57+
) => firstDataset.maximumAnomalyScore - secondDataset.maximumAnomalyScore;

‎x-pack/plugins/infra/common/infra_ml/index.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export * from './infra_ml';
8+
export * from './anomaly_results';
9+
export * from './job_parameters';
10+
export * from './metrics_hosts_ml';
11+
export * from './metrics_k8s_ml';

‎x-pack/plugins/infra/common/infra_ml/infra_ml.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
// combines and abstracts job and datafeed status
8+
export type JobStatus =
9+
| 'unknown'
10+
| 'missing'
11+
| 'initializing'
12+
| 'stopped'
13+
| 'started'
14+
| 'finished'
15+
| 'failed';
16+
17+
export type SetupStatus =
18+
| { type: 'initializing' } // acquiring job statuses to determine setup status
19+
| { type: 'unknown' } // job status could not be acquired (failed request etc)
20+
| { type: 'required' } // setup required
21+
| { type: 'pending' } // In the process of setting up the module for the first time or retrying, waiting for response
22+
| { type: 'succeeded' } // setup succeeded, notifying user
23+
| {
24+
type: 'failed';
25+
reasons: string[];
26+
} // setup failed, notifying user
27+
| {
28+
type: 'skipped';
29+
newlyCreated?: boolean;
30+
}; // setup is not necessary
31+
32+
/**
33+
* Maps a job status to the possibility that results have already been produced
34+
* before this state was reached.
35+
*/
36+
export const isJobStatusWithResults = (jobStatus: JobStatus) =>
37+
['started', 'finished', 'stopped', 'failed'].includes(jobStatus);
38+
39+
export const isHealthyJobStatus = (jobStatus: JobStatus) =>
40+
['started', 'finished'].includes(jobStatus);
41+
42+
/**
43+
* Maps a setup status to the possibility that results have already been
44+
* produced before this state was reached.
45+
*/
46+
export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
47+
setupStatus.type === 'skipped';
48+
49+
const KIBANA_SAMPLE_DATA_INDICES = ['kibana_sample_data_logs*'];
50+
51+
export const isExampleDataIndex = (indexName: string) =>
52+
KIBANA_SAMPLE_DATA_INDICES.includes(indexName);

0 commit comments

Comments
 (0)