Skip to content

Commit 56162d7

Browse files
dgieselaardgieselaar
authored
[7.x] [APM] Script for creating test archive (#76926) (#77026)
* [APM] Script for creating functional test archive * Lock down variables; add documentation * Update tests
1 parent c5ea18d commit 56162d7

File tree

15 files changed

+127018
-45
lines changed

15 files changed

+127018
-45
lines changed

‎x-pack/plugins/apm/dev_docs/updating_functional_tests_archives.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
### Updating functional tests archives
2+
3+
Some of our API tests use an archive generated by the [`esarchiver`](https://www.elastic.co/guide/en/kibana/current/development-functional-tests.html) script. Updating the main archive (`apm_8.0.0`) is a scripted process, where a 30m snapshot is downloaded from a cluster running the [APM Integration Testing server](https://github.com/elastic/apm-integration-testing). The script will copy the generated archives into the `fixtures/es_archiver` folders of our test suites (currently `basic` and `trial`). It will also generate a file that contains metadata about the archive, that can be imported to get the time range of the snapshot.
4+
5+
Usage:
6+
`node x-pack/plugins/apm/scripts/create-functional-tests-archive --es-url=https://admin:changeme@localhost:9200 --kibana-url=https://localhost:5601`
7+
8+

‎x-pack/plugins/apm/scripts/aggregate-latency-metrics/index.ts‎

Lines changed: 11 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,12 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { Client } from '@elastic/elasticsearch';
87
import { argv } from 'yargs';
98
import pLimit from 'p-limit';
109
import pRetry from 'p-retry';
11-
import { parse, format } from 'url';
1210
import { set } from '@elastic/safer-lodash-set';
1311
import { uniq, without, merge, flatten } from 'lodash';
1412
import * as histogram from 'hdr-histogram-js';
15-
import { ESSearchResponse } from '../../typings/elasticsearch';
1613
import {
1714
HOST_NAME,
1815
SERVICE_NAME,
@@ -28,6 +25,8 @@ import {
2825
} from '../../common/elasticsearch_fieldnames';
2926
import { stampLogger } from '../shared/stamp-logger';
3027
import { createOrUpdateIndex } from '../shared/create-or-update-index';
28+
import { parseIndexUrl } from '../shared/parse_index_url';
29+
import { ESClient, getEsClient } from '../shared/get_es_client';
3130

3231
// This script will try to estimate how many latency metric documents
3332
// will be created based on the available transaction documents.
@@ -125,41 +124,18 @@ export async function aggregateLatencyMetrics() {
125124
const source = String(argv.source ?? '');
126125
const dest = String(argv.dest ?? '');
127126

128-
function getClientOptionsFromIndexUrl(
129-
url: string
130-
): { node: string; index: string } {
131-
const parsed = parse(url);
132-
const { pathname, ...rest } = parsed;
127+
const sourceOptions = parseIndexUrl(source);
133128

134-
return {
135-
node: format(rest),
136-
index: pathname!.replace('/', ''),
137-
};
138-
}
139-
140-
const sourceOptions = getClientOptionsFromIndexUrl(source);
141-
142-
const sourceClient = new Client({
143-
node: sourceOptions.node,
144-
ssl: {
145-
rejectUnauthorized: false,
146-
},
147-
requestTimeout: 120000,
148-
});
129+
const sourceClient = getEsClient({ node: sourceOptions.node });
149130

150-
let destClient: Client | undefined;
131+
let destClient: ESClient | undefined;
151132
let destOptions: { node: string; index: string } | undefined;
152133

153134
const uploadMetrics = !!dest;
154135

155136
if (uploadMetrics) {
156-
destOptions = getClientOptionsFromIndexUrl(dest);
157-
destClient = new Client({
158-
node: destOptions.node,
159-
ssl: {
160-
rejectUnauthorized: false,
161-
},
162-
});
137+
destOptions = parseIndexUrl(dest);
138+
destClient = getEsClient({ node: destOptions.node });
163139

164140
const mappings = (
165141
await sourceClient.indices.getMapping({
@@ -298,10 +274,9 @@ export async function aggregateLatencyMetrics() {
298274
},
299275
};
300276

301-
const response = (await sourceClient.search(params))
302-
.body as ESSearchResponse<unknown, typeof params>;
277+
const response = await sourceClient.search(params);
303278

304-
const { aggregations } = response;
279+
const { aggregations } = response.body;
305280

306281
if (!aggregations) {
307282
return buckets;
@@ -333,10 +308,9 @@ export async function aggregateLatencyMetrics() {
333308
},
334309
};
335310

336-
const response = (await sourceClient.search(params))
337-
.body as ESSearchResponse<unknown, typeof params>;
311+
const response = await sourceClient.search(params);
338312

339-
return response.hits.total.value;
313+
return response.body.hits.total.value;
340314
}
341315

342316
const [buckets, numberOfTransactionDocuments] = await Promise.all([

‎x-pack/plugins/apm/scripts/create-functional-tests-archive.js‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
// compile typescript on the fly
8+
// eslint-disable-next-line import/no-extraneous-dependencies
9+
require('@babel/register')({
10+
extensions: ['.js', '.ts'],
11+
plugins: ['@babel/plugin-proposal-optional-chaining'],
12+
presets: [
13+
'@babel/typescript',
14+
['@babel/preset-env', { targets: { node: 'current' } }],
15+
],
16+
});
17+
18+
require('./create-functional-tests-archive/index.ts');

‎x-pack/plugins/apm/scripts/create-functional-tests-archive/index.ts‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import { argv } from 'yargs';
8+
import { execSync } from 'child_process';
9+
import moment from 'moment';
10+
import path from 'path';
11+
import fs from 'fs';
12+
import { stampLogger } from '../shared/stamp-logger';
13+
14+
async function run() {
15+
stampLogger();
16+
17+
const archiveName = 'apm_8.0.0';
18+
19+
// include important APM data and ML data
20+
const indices =
21+
'apm-*-transaction,apm-*-span,apm-*-error,apm-*-metric,.ml-anomalies*,.ml-config';
22+
23+
const esUrl = argv['es-url'] as string | undefined;
24+
25+
if (!esUrl) {
26+
throw new Error('--es-url is not set');
27+
}
28+
const kibanaUrl = argv['kibana-url'] as string | undefined;
29+
30+
if (!kibanaUrl) {
31+
throw new Error('--kibana-url is not set');
32+
}
33+
const gte = moment().subtract(1, 'hour').toISOString();
34+
const lt = moment(gte).add(30, 'minutes').toISOString();
35+
36+
// eslint-disable-next-line no-console
37+
console.log(`Archiving from ${gte} to ${lt}...`);
38+
39+
// APM data uses '@timestamp' (ECS), ML data uses 'timestamp'
40+
41+
const rangeQueries = [
42+
{
43+
range: {
44+
'@timestamp': {
45+
gte,
46+
lt,
47+
},
48+
},
49+
},
50+
{
51+
range: {
52+
timestamp: {
53+
gte,
54+
lt,
55+
},
56+
},
57+
},
58+
];
59+
60+
// some of the data is timeless/content
61+
const query = {
62+
bool: {
63+
should: [
64+
...rangeQueries,
65+
{
66+
bool: {
67+
must_not: [
68+
{
69+
exists: {
70+
field: '@timestamp',
71+
},
72+
},
73+
{
74+
exists: {
75+
field: 'timestamp',
76+
},
77+
},
78+
],
79+
},
80+
},
81+
],
82+
minimum_should_match: 1,
83+
},
84+
};
85+
86+
const archivesDir = path.join(__dirname, '.archives');
87+
const root = path.join(__dirname, '../../../../..');
88+
89+
// create the archive
90+
91+
execSync(
92+
`node scripts/es_archiver save ${archiveName} ${indices} --dir=${archivesDir} --kibana-url=${kibanaUrl} --es-url=${esUrl} --query='${JSON.stringify(
93+
query
94+
)}'`,
95+
{
96+
cwd: root,
97+
stdio: 'inherit',
98+
}
99+
);
100+
101+
const targetDirs = ['trial', 'basic'];
102+
103+
// copy the archives to the test fixtures
104+
105+
await Promise.all(
106+
targetDirs.map(async (target) => {
107+
const targetPath = path.resolve(
108+
__dirname,
109+
'../../../../test/apm_api_integration/',
110+
target
111+
);
112+
const targetArchivesPath = path.resolve(
113+
targetPath,
114+
'fixtures/es_archiver',
115+
archiveName
116+
);
117+
118+
if (!fs.existsSync(targetArchivesPath)) {
119+
fs.mkdirSync(targetArchivesPath);
120+
}
121+
122+
fs.copyFileSync(
123+
path.join(archivesDir, archiveName, 'data.json.gz'),
124+
path.join(targetArchivesPath, 'data.json.gz')
125+
);
126+
fs.copyFileSync(
127+
path.join(archivesDir, archiveName, 'mappings.json'),
128+
path.join(targetArchivesPath, 'mappings.json')
129+
);
130+
131+
const currentConfig = {};
132+
133+
// get the current metadata and extend/override metadata for the new archive
134+
const configFilePath = path.join(targetPath, 'archives_metadata.ts');
135+
136+
try {
137+
Object.assign(currentConfig, (await import(configFilePath)).default);
138+
} catch (error) {
139+
// do nothing
140+
}
141+
142+
const newConfig = {
143+
...currentConfig,
144+
[archiveName]: {
145+
start: gte,
146+
end: lt,
147+
},
148+
};
149+
150+
fs.writeFileSync(
151+
configFilePath,
152+
`export default ${JSON.stringify(newConfig, null, 2)}`,
153+
{ encoding: 'utf-8' }
154+
);
155+
})
156+
);
157+
158+
fs.unlinkSync(path.join(archivesDir, archiveName, 'data.json.gz'));
159+
fs.unlinkSync(path.join(archivesDir, archiveName, 'mappings.json'));
160+
fs.rmdirSync(path.join(archivesDir, archiveName));
161+
fs.rmdirSync(archivesDir);
162+
163+
// run ESLint on the generated metadata files
164+
165+
execSync('node scripts/eslint **/*/archives_metadata.ts --fix', {
166+
cwd: root,
167+
stdio: 'inherit',
168+
});
169+
}
170+
171+
run()
172+
.then(() => {
173+
process.exit(0);
174+
})
175+
.catch((err) => {
176+
// eslint-disable-next-line no-console
177+
console.log(err);
178+
process.exit(1);
179+
});

‎x-pack/plugins/apm/scripts/shared/create-or-update-index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { Client } from '@elastic/elasticsearch';
7+
import { ESClient } from './get_es_client';
88

99
export async function createOrUpdateIndex({
1010
client,
1111
clear,
1212
indexName,
1313
template,
1414
}: {
15-
client: Client;
15+
client: ESClient;
1616
clear: boolean;
1717
indexName: string;
1818
template: any;

‎x-pack/plugins/apm/scripts/shared/get_es_client.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import { Client } from '@elastic/elasticsearch';
8+
import { ApiKeyAuth, BasicAuth } from '@elastic/elasticsearch/lib/pool';
9+
import { ESSearchResponse, ESSearchRequest } from '../../typings/elasticsearch';
10+
11+
export type ESClient = ReturnType<typeof getEsClient>;
12+
13+
export function getEsClient({
14+
node,
15+
auth,
16+
}: {
17+
node: string;
18+
auth?: BasicAuth | ApiKeyAuth;
19+
}) {
20+
const client = new Client({
21+
node,
22+
ssl: {
23+
rejectUnauthorized: false,
24+
},
25+
requestTimeout: 120000,
26+
auth,
27+
});
28+
29+
return {
30+
...client,
31+
async search<TDocument, TSearchRequest extends ESSearchRequest>(
32+
request: TSearchRequest
33+
) {
34+
const response = await client.search(request as any);
35+
36+
return {
37+
...response,
38+
body: response.body as ESSearchResponse<TDocument, TSearchRequest>,
39+
};
40+
},
41+
};
42+
}

‎x-pack/plugins/apm/scripts/shared/parse_index_url.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import { parse, format } from 'url';
8+
9+
export function parseIndexUrl(url: string): { node: string; index: string } {
10+
const parsed = parse(url);
11+
const { pathname, ...rest } = parsed;
12+
13+
return {
14+
node: format(rest),
15+
index: pathname!.replace('/', ''),
16+
};
17+
}

‎x-pack/test/apm_api_integration/basic/archives_metadata.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export default {
8+
'apm_8.0.0': {
9+
start: '2020-09-09T06:11:22.998Z',
10+
end: '2020-09-09T06:41:22.998Z',
11+
},
12+
};

‎x-pack/test/apm_api_integration/basic/fixtures/es_archiver/apm_8.0.0/data.json.gz‎

1.21 MB
Binary file not shown.

0 commit comments

Comments
 (0)