Skip to content

Commit

Permalink
[APM] Move ML anomaly transformation to backend
Browse files Browse the repository at this point in the history
  • Loading branch information
@sorenlouv
sorenlouv committed Nov 22, 2018
1 parent c9904de commit 126948c
Show file tree
Hide file tree
Showing 33 changed files with 877 additions and 829 deletions.
8 changes: 5 additions & 3 deletions x-pack/plugins/apm/public/components/shared/charts/CustomPlot/StaticPlot.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import {
YAxis,
HorizontalGridLines,
LineSeries,
AreaSeries
AreaSeries,
VerticalRectSeries
} from 'react-vis';
import PropTypes from 'prop-types';
import React, { PureComponent } from 'react';
Expand Down Expand Up @@ -56,13 +57,14 @@ class StaticPlot extends PureComponent {
case 'areaMaxHeight':
const yMax = last(plotValues.yTickValues);
const data = serie.data.map(p => ({
x0: p.x0,
x: p.x,
y0: 0,
y: p.y ? yMax : null
y: yMax
}));

return (
<AreaSeries
<VerticalRectSeries
getNull={d => d.y !== null}
key={serie.title}
xType="time"
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/apm/public/services/rest/apm.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { ServiceAPIResponse } from 'x-pack/plugins/apm/server/lib/services/get_s
import { ServiceListAPIResponse } from 'x-pack/plugins/apm/server/lib/services/get_services';
import { TraceListAPIResponse } from 'x-pack/plugins/apm/server/lib/traces/get_top_traces';
import { TraceAPIResponse } from 'x-pack/plugins/apm/server/lib/traces/get_trace';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data';
import { ITransactionDistributionAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/distribution';
import { TransactionListAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/get_top_transactions';
import { TransactionAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/get_transaction';
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/apm/public/store/reactReduxRequest/transactionDetailsCharts.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import React from 'react';
import { Request, RRRRender } from 'react-redux-request';
import { createSelector } from 'reselect';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data';
import { loadCharts } from '../../services/rest/apm';
import { IReduxState } from '../rootReducer';
import { getCharts } from '../selectors/chartSelectors';
Expand Down
11 changes: 4 additions & 7 deletions x-pack/plugins/apm/public/store/reactReduxRequest/transactionOverviewCharts.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { get, isEmpty } from 'lodash';
import { get } from 'lodash';
import React from 'react';
import { Request, RRRRender } from 'react-redux-request';
import { createSelector } from 'reselect';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data';
import { loadCharts } from '../../services/rest/apm';
import { IReduxState } from '../rootReducer';
import { getCharts } from '../selectors/chartSelectors';
Expand Down Expand Up @@ -39,11 +39,8 @@ export const getTransactionOverviewCharts = createSelector(
);

export function hasDynamicBaseline(state: IReduxState) {
return !isEmpty(
get(
state,
`reactReduxRequest[${ID}].data.responseTimes.avgAnomalies.buckets`
)
return (
get(state, `reactReduxRequest[${ID}].data.anomalyTimeSeries`) !== undefined
);
}

Expand Down
82 changes: 2 additions & 80 deletions x-pack/plugins/apm/public/store/selectors/__tests__/chartSelectors.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,55 +4,10 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { AvgAnomalyBucket } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_avg_response_time_anomalies/get_anomaly_aggs/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/transform';
import {
getAnomalyBoundaryValues,
getAnomalyScoreValues,
getResponseTimeSeries,
getTpmSeries
} from '../chartSelectors';
import { anomalyData } from './mockData/anomalyData';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data';
import { getResponseTimeSeries, getTpmSeries } from '../chartSelectors';

describe('chartSelectors', () => {
describe('getAnomalyScoreValues', () => {
it('should return anomaly score series', () => {
const dates = [0, 1000, 2000, 3000, 4000, 5000, 6000];
const buckets = [
{
anomalyScore: null
},
{
anomalyScore: 80
},
{
anomalyScore: 0
},
{
anomalyScore: 0
},
{
anomalyScore: 70
},
{
anomalyScore: 80
},
{
anomalyScore: 0
}
] as AvgAnomalyBucket[];

expect(getAnomalyScoreValues(dates, buckets, 1000)).toEqual([
{ x: 1000, y: 1 },
{ x: 2000, y: 1 },
{ x: 3000 },
{ x: 5000, y: 1 },
{ x: 6000, y: 1 },
{ x: 7000 }
]);
});
});

describe('getResponseTimeSeries', () => {
const chartsData = {
dates: [0, 1000, 2000, 3000, 4000, 5000],
Expand Down Expand Up @@ -101,37 +56,4 @@ describe('chartSelectors', () => {
expect(getTpmSeries(chartsData, transactionType)).toMatchSnapshot();
});
});

describe('getAnomalyBoundaryValues', () => {
const { dates, buckets } = anomalyData;
const bucketSize = 240000;

it('should return correct buckets', () => {
expect(getAnomalyBoundaryValues(dates, buckets, bucketSize)).toEqual([
{ x: 1530614880000, y: 54799, y0: 15669 },
{ x: 1530615060000, y: 49874, y0: 17808 },
{ x: 1530615300000, y: 49421, y0: 18012 },
{ x: 1530615540000, y: 49654, y0: 17889 },
{ x: 1530615780000, y: 50026, y0: 17713 },
{ x: 1530616020000, y: 49371, y0: 18044 },
{ x: 1530616260000, y: 50110, y0: 17713 },
{ x: 1530616500000, y: 50419, y0: 17582 },
{ x: 1530616620000, y: 50419, y0: 17582 }
]);
});

it('should extend the last bucket with a size of bucketSize', () => {
const [lastBucket, secondLastBuckets] = getAnomalyBoundaryValues(
dates,
buckets,
bucketSize
).reverse();

expect(secondLastBuckets.y).toBe(lastBucket.y);
expect(secondLastBuckets.y0).toBe(lastBucket.y0);
expect(
(lastBucket.x as number) - (secondLastBuckets.x as number)
).toBeLessThanOrEqual(bucketSize);
});
});
});
119 changes: 8 additions & 111 deletions x-pack/plugins/apm/public/store/selectors/chartSelectors.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@
*/

import d3 from 'd3';
import { difference, last, memoize, zipObject } from 'lodash';
import { rgba } from 'polished';
import { difference, memoize, zipObject } from 'lodash';
import { colors } from 'x-pack/plugins/apm/common/variables';
import { AvgAnomalyBucket } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_avg_response_time_anomalies/get_anomaly_aggs/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/transform';
import { TimeSeriesAPIResponse } from 'x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data';
import { StringMap } from 'x-pack/plugins/apm/typings/common';
import { asDecimal, asMillis, tpmUnit } from '../../utils/formatters';
import { IUrlParams } from '../urlParams';
Expand All @@ -19,10 +17,6 @@ interface Coordinate {
y?: number | null;
}

interface BoundaryCoordinate extends Coordinate {
y0: number | null;
}

export const getEmptySerie = memoize(
(start = Date.now() - 3600000, end = Date.now()) => {
const dates = d3.time
Expand Down Expand Up @@ -76,8 +70,8 @@ interface TimeSerie {
}

export function getResponseTimeSeries(chartsData: TimeSeriesAPIResponse) {
const { dates, overallAvgDuration } = chartsData;
const { avg, p95, p99, avgAnomalies } = chartsData.responseTimes;
const { dates, overallAvgDuration, anomalyTimeSeries } = chartsData;
const { avg, p95, p99 } = chartsData.responseTimes;

const series: TimeSerie[] = [
{
Expand All @@ -103,35 +97,10 @@ export function getResponseTimeSeries(chartsData: TimeSeriesAPIResponse) {
}
];

if (avgAnomalies) {
// insert after Avg. serie
series.splice(1, 0, {
title: 'Anomaly Boundaries',
hideLegend: true,
hideTooltipValue: true,
data: getAnomalyBoundaryValues(
dates,
avgAnomalies.buckets,
avgAnomalies.bucketSizeAsMillis
),
type: 'area',
color: 'none',
areaColor: rgba(colors.apmBlue, 0.1)
});

series.splice(1, 0, {
title: 'Anomaly score',
hideLegend: true,
hideTooltipValue: true,
data: getAnomalyScoreValues(
dates,
avgAnomalies.buckets,
avgAnomalies.bucketSizeAsMillis
),
type: 'areaMaxHeight',
color: 'none',
areaColor: rgba(colors.apmRed, 0.1)
});
if (anomalyTimeSeries) {
// insert after Avg. series
series.splice(1, 0, anomalyTimeSeries.anomalyBoundariesSeries);
series.splice(1, 0, anomalyTimeSeries.anomalyScoreSeries);
}

return series;
Expand Down Expand Up @@ -194,75 +163,3 @@ function getChartValues(
y: buckets[i]
}));
}

export function getAnomalyScoreValues(
dates: number[] = [],
buckets: AvgAnomalyBucket[] = [],
bucketSizeAsMillis: number
) {
const ANOMALY_THRESHOLD = 75;
const getX = (currentX: number, i: number) =>
currentX + bucketSizeAsMillis * i;

return dates
.map((date, i) => {
const { anomalyScore } = buckets[i];
return {
x: date,
anomalyScore
};
})
.filter(p => {
const res =
p && p.anomalyScore != null && p.anomalyScore > ANOMALY_THRESHOLD;
return res;
})
.reduce<Coordinate[]>((acc, p, i, points) => {
const nextPoint = points[i + 1] || {};
const endX = getX(p.x, 1);
acc.push({ x: p.x, y: 1 });
if (nextPoint.x == null || nextPoint.x > endX) {
acc.push(
{
x: endX,
y: 1
},
{
x: getX(p.x, 2)
}
);
}

return acc;
}, []);
}

export function getAnomalyBoundaryValues(
dates: number[] = [],
buckets: AvgAnomalyBucket[] = [],
bucketSizeAsMillis: number
) {
const lastX = last(dates);
return dates
.map((date, i) => {
const bucket = buckets[i];
return {
x: date,
y0: bucket.lower,
y: bucket.upper
};
})
.filter(p => p.y != null)
.reduce<BoundaryCoordinate[]>((acc, p, i, points) => {
const isLast = last(points) === p;
acc.push(p);

if (isLast) {
acc.push({
...p,
x: Math.min(p.x + bucketSizeAsMillis, lastX) // avoid going beyond the last date
});
}
return acc;
}, []);
}
26 changes: 9 additions & 17 deletions ...y_aggs/__snapshots__/fetcher.test.ts.snap → ...series/__snapshots__/fetcher.test.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,14 @@ Array [
},
"date_histogram": Object {
"extended_bounds": Object {
"max": 1,
"min": 0,
"max": 200000,
"min": 90000,
},
"field": "timestamp",
"interval": "myInterval",
"min_doc_count": 0,
},
},
"top_hits": Object {
"top_hits": Object {
"_source": Object {
"includes": Array [
"bucket_span",
],
},
"size": 1,
"sort": Array [
"bucket_span",
],
},
},
},
"query": Object {
"bool": Object {
Expand All @@ -56,12 +43,17 @@ Array [
"range": Object {
"timestamp": Object {
"format": "epoch_millis",
"gte": 0,
"lte": 1,
"gte": 90000,
"lte": 200000,
},
},
},
],
"must": Object {
"exists": Object {
"field": "bucket_span",
},
},
},
},
"size": 0,
Expand Down
Loading

0 comments on commit 126948c

Please sign in to comment.