Skip to content

Commit 7fea0ad

Browse files
tsullivantsullivan
authored
[Reporting] Restore the "csv by savedobject" endpoint for 7.17 (#148030)
## Summary This restores an endpoint that was added in 7.3 in [this PR](#34571), and was removed in 7.9 in [this PR](#71031). The changes are re-done on top of 7.17, but still has a mostly-compatible with the one that existed in 7.3-7.8. This serves 3rd parties that relied on the earlier experimental code. Supports: * Saved searches with filters * Saved searches with custom sorting * Saved searches with or without selected columns * Exports based on Index Patterns with or without a "time field" * Requests can have an [optional POST body](https://github.com/elastic/kibana/pull/148030/files#diff-0f565e26f3309c257fa919c5db227c3b7a78237015940c3d3677cbb1132a6701R27-R37) with extra time range filters and/or specify a custom time zone. LIMITATIONS: * This endpoint is currently not supported in 8.x at this time. * Saved Search objects created in older versions of Kibana may not work. * Searching across hundreds of shards in the query could cause Elasticsearch instability. * Some minor bugs in the output of the CSV may exist, such as fields not being formatted exactly as in the Discover table. * This code may be forward-ported to `main` in a way that uses a different API that is not compatible with this change. * Does not allow "raw state" to be merged with the Search object, as in the previous code. Otherwise, the API is compatible with the previous code. * This feature remains in "experimental" status, and is not ready to be documented at this time. ## Testing Since there is not a UI for this endpoint, there are a few options for testing: 1. Run the functional test: ```sh node scripts/functional_tests.js \ --config x-pack/test/reporting_api_integration/reporting_and_security.config.ts \ --grep 'CSV Generation from Saved Search ID' ``` 2. Create a saved search in Kibana, and use a script to send a request ```sh POST_URL="${HOST}/api/reporting/v1/generate/csv/saved-object/search:"$SAVED_SEARCH_ID ## Run transaction to generate a report, wait for execution completion, download the report, and send the # report as an email attachment # 1. Send a request to generate a report DOWNLOAD_PATH=$(curl --silent -XPOST "$POST_URL" -H "kbn-xsrf: kibana-reporting" -H "${AUTH_HEADER}" | jq -e -r ".payload.path | values") if [ -z "$DOWNLOAD_PATH" ]; then echo "Something went wrong! Could not send the request to generate a report!" 1>&2 # TEST curl --silent -XPOST "$POST_URL" -H "kbn-xsrf: kibana-reporting" -H "${AUTH_HEADER}" exit 1 fi # 2. Log the path used to download the report DOWNLOAD_PATH=${HOST}$DOWNLOAD_PATH echo Download path: $DOWNLOAD_PATH # 3. Wait for report execution to finish echo While the report is executing in the Kibana server, the reporting service will return a 503 status code response. STATUS='' while [[ -z $STATUS || $STATUS =~ .*503.* ]] do echo Waiting 5 seconds... sleep 5 STATUS=$(curl --silent --head "$DOWNLOAD_PATH" -H "${AUTH_HEADER}" | head -1) if [[ -z "$STATUS" || $STATUS =~ .*500.* ]]; then echo "Something went wrong! Could not request the report execution status!" 1>&2 curl "$DOWNLOAD_PATH" -H "${AUTH_HEADER}" 1>&2 exit 1 fi echo $STATUS done # 4. Download final report and show the contents in the console curl -v "$DOWNLOAD_PATH" -H "$AUTH_HEADER" ``` 3. Test that the above script from (2) works in 7.8, and continues to work after migrating to 7.17.
1 parent 336bdb6 commit 7fea0ad

File tree

31 files changed

+3448
-4
lines changed

31 files changed

+3448
-4
lines changed

‎x-pack/plugins/reporting/common/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ export const DEFAULT_VIEWPORT = {
6666
};
6767

6868
// Export Type Definitions
69+
export const CSV_SAVED_OBJECT_JOB_TYPE = 'csv_saved_object';
70+
6971
export const CSV_REPORT_TYPE = 'CSV';
7072
export const CSV_JOB_TYPE = 'csv_searchsource';
7173

‎x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import type { BaseParams, BasePayload } from '../base';
9+
10+
interface CsvFromSavedObjectBase {
11+
objectType: 'saved search';
12+
timerange?: {
13+
timezone?: string;
14+
min?: string | number;
15+
max?: string | number;
16+
};
17+
savedObjectId: string;
18+
}
19+
20+
export type JobParamsCsvFromSavedObject = CsvFromSavedObjectBase & BaseParams;
21+
export type TaskPayloadCsvFromSavedObject = CsvFromSavedObjectBase & BasePayload;

‎x-pack/plugins/reporting/common/types/export_types/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
export * from './csv';
99
export * from './csv_searchsource';
1010
export * from './csv_searchsource_immediate';
11+
export * from './csv_saved_object';
1112
export * from './png';
1213
export * from './png_v2';
1314
export * from './printable_pdf';

‎x-pack/plugins/reporting/server/core.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ export class ReportingCore {
259259
return this.pluginSetupDeps;
260260
}
261261

262-
private async getSavedObjectsClient(request: KibanaRequest) {
262+
public async getSavedObjectsClient(request: KibanaRequest) {
263263
const { savedObjects } = await this.getPluginStartDeps();
264264
return savedObjects.getScopedClient(request) as SavedObjectsClientContract;
265265
}

‎x-pack/plugins/reporting/server/export_types/csv_saved_object/create_job.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { CreateJobFn, CreateJobFnFactory } from '../../types';
9+
import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from './types';
10+
11+
type CreateJobFnType = CreateJobFn<JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject>;
12+
13+
export const createJobFnFactory: CreateJobFnFactory<CreateJobFnType> =
14+
function createJobFactoryFn() {
15+
return async function createJob(jobParams) {
16+
// params have been finalized in server/routes/generate_from_savedobject.ts
17+
return jobParams;
18+
};
19+
};

‎x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
jest.mock('../csv_searchsource/generate_csv', () => ({
9+
CsvGenerator: class CsvGeneratorMock {
10+
generateData() {
11+
return {
12+
size: 123,
13+
content_type: 'text/csv',
14+
};
15+
}
16+
},
17+
}));
18+
19+
jest.mock('./lib/get_sharing_data', () => ({
20+
getSharingData: jest.fn(() => ({ columns: [], searchSource: {} })),
21+
}));
22+
23+
import { Writable } from 'stream';
24+
import nodeCrypto from '@elastic/node-crypto';
25+
import { ReportingCore } from '../../';
26+
import { CancellationToken } from '../../../common';
27+
import {
28+
createMockConfigSchema,
29+
createMockLevelLogger,
30+
createMockReportingCore,
31+
} from '../../test_helpers';
32+
import { runTaskFnFactory } from './execute_job';
33+
34+
const logger = createMockLevelLogger();
35+
const encryptionKey = 'tetkey';
36+
const headers = { sid: 'cooltestheaders' };
37+
let encryptedHeaders: string;
38+
let reportingCore: ReportingCore;
39+
let stream: jest.Mocked<Writable>;
40+
41+
beforeAll(async () => {
42+
const crypto = nodeCrypto({ encryptionKey });
43+
encryptedHeaders = await crypto.encrypt(headers);
44+
});
45+
46+
beforeEach(async () => {
47+
stream = {} as typeof stream;
48+
reportingCore = await createMockReportingCore(createMockConfigSchema({ encryptionKey }));
49+
});
50+
51+
test('recognized saved search', async () => {
52+
reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({
53+
get: () => ({
54+
attributes: {
55+
kibanaSavedObjectMeta: {
56+
searchSourceJSON: '{"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}',
57+
},
58+
},
59+
references: [
60+
{
61+
id: 'logstash-yes-*',
62+
name: 'kibanaSavedObjectMeta.searchSourceJSON.index',
63+
type: 'index-pattern',
64+
},
65+
],
66+
}),
67+
});
68+
69+
const runTask = runTaskFnFactory(reportingCore, logger);
70+
const payload = await runTask(
71+
'cool-job-id',
72+
{
73+
headers: encryptedHeaders,
74+
browserTimezone: 'US/Alaska',
75+
savedObjectId: '123-456-abc-defgh',
76+
objectType: 'saved search',
77+
title: 'Test Search',
78+
version: '7.17.0',
79+
},
80+
new CancellationToken(),
81+
stream
82+
);
83+
84+
expect(payload).toMatchInlineSnapshot(`
85+
Object {
86+
"content_type": "text/csv",
87+
"size": 123,
88+
}
89+
`);
90+
});
91+
92+
test('saved search object is missing references', async () => {
93+
reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({
94+
get: () => ({
95+
attributes: {
96+
kibanaSavedObjectMeta: {
97+
searchSourceJSON: '{"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}',
98+
},
99+
},
100+
}),
101+
});
102+
103+
const runTask = runTaskFnFactory(reportingCore, logger);
104+
const runTest = async () => {
105+
await runTask(
106+
'cool-job-id',
107+
{
108+
headers: encryptedHeaders,
109+
browserTimezone: 'US/Alaska',
110+
savedObjectId: '123-456-abc-defgh',
111+
objectType: 'saved search',
112+
title: 'Test Search',
113+
version: '7.17.0',
114+
},
115+
new CancellationToken(),
116+
stream
117+
);
118+
};
119+
120+
await expect(runTest).rejects.toEqual(
121+
new Error('Could not find reference for kibanaSavedObjectMeta.searchSourceJSON.index')
122+
);
123+
});
124+
125+
test('invalid saved search', async () => {
126+
reportingCore.getSavedObjectsClient = jest.fn().mockResolvedValue({ get: jest.fn() });
127+
const runTask = runTaskFnFactory(reportingCore, logger);
128+
const runTest = async () => {
129+
await runTask(
130+
'cool-job-id',
131+
{
132+
headers: encryptedHeaders,
133+
browserTimezone: 'US/Alaska',
134+
savedObjectId: '123-456-abc-defgh',
135+
objectType: 'saved search',
136+
title: 'Test Search',
137+
version: '7.17.0',
138+
},
139+
new CancellationToken(),
140+
stream
141+
);
142+
};
143+
144+
await expect(runTest).rejects.toEqual(new Error('Saved search object is not valid'));
145+
});

‎x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { SavedObject } from 'kibana/server';
9+
import type { SearchSourceFields } from 'src/plugins/data/common';
10+
import type { VisualizationSavedObjectAttributes } from 'src/plugins/visualizations/common';
11+
import { DeepPartial } from 'utility-types';
12+
import { JobParamsCSV } from '../..';
13+
import { injectReferences, parseSearchSourceJSON } from '../../../../../../src/plugins/data/common';
14+
import { CSV_JOB_TYPE } from '../../../common/constants';
15+
import { getFieldFormats } from '../../services';
16+
import type { RunTaskFn, RunTaskFnFactory } from '../../types';
17+
import { decryptJobHeaders } from '../common';
18+
import { CsvGenerator } from '../csv_searchsource/generate_csv';
19+
import { getSharingData } from './lib';
20+
import type { TaskPayloadCsvFromSavedObject } from './types';
21+
22+
type RunTaskFnType = RunTaskFn<TaskPayloadCsvFromSavedObject>;
23+
type SavedSearchObjectType = SavedObject<
24+
VisualizationSavedObjectAttributes & { columns?: string[]; sort: Array<[string, string]> }
25+
>;
26+
type ParsedSearchSourceJSON = SearchSourceFields & { indexRefName?: string };
27+
28+
function isSavedObject(
29+
savedSearch: SavedSearchObjectType | unknown
30+
): savedSearch is SavedSearchObjectType {
31+
return (
32+
(savedSearch as DeepPartial<SavedSearchObjectType> | undefined)?.attributes
33+
?.kibanaSavedObjectMeta?.searchSourceJSON != null
34+
);
35+
}
36+
37+
export const runTaskFnFactory: RunTaskFnFactory<RunTaskFnType> = (reporting, _logger) => {
38+
const config = reporting.getConfig();
39+
40+
return async function runTask(jobId, job, cancellationToken, stream) {
41+
const logger = _logger.clone([CSV_JOB_TYPE, 'execute-job', jobId]);
42+
43+
const encryptionKey = config.get('encryptionKey');
44+
const headers = await decryptJobHeaders(encryptionKey, job.headers, logger);
45+
const fakeRequest = reporting.getFakeRequest({ headers }, job.spaceId, logger);
46+
const uiSettings = await reporting.getUiSettingsClient(fakeRequest, logger);
47+
const savedObjects = await reporting.getSavedObjectsClient(fakeRequest);
48+
const dataPluginStart = await reporting.getDataService();
49+
const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(uiSettings);
50+
51+
const [es, searchSourceStart] = await Promise.all([
52+
(await reporting.getEsClient()).asScoped(fakeRequest),
53+
await dataPluginStart.search.searchSource.asScoped(fakeRequest),
54+
]);
55+
56+
const clients = {
57+
uiSettings,
58+
data: dataPluginStart.search.asScoped(fakeRequest),
59+
es,
60+
};
61+
const dependencies = {
62+
searchSourceStart,
63+
fieldFormatsRegistry,
64+
};
65+
66+
// Get the Saved Search Fields object from ID
67+
const savedSearch = await savedObjects.get('search', job.savedObjectId);
68+
69+
if (!isSavedObject(savedSearch)) {
70+
throw new Error(`Saved search object is not valid`);
71+
}
72+
73+
// allowed to throw an Invalid JSON error if the JSON is not parseable.
74+
const searchSourceFields: ParsedSearchSourceJSON = parseSearchSourceJSON(
75+
savedSearch.attributes.kibanaSavedObjectMeta.searchSourceJSON
76+
);
77+
78+
const indexRefName = searchSourceFields.indexRefName;
79+
if (!indexRefName) {
80+
throw new Error(`Saved Search data is missing a reference to an Index Pattern!`);
81+
}
82+
83+
// Inject references into the Saved Search Fields
84+
const searchSourceFieldsWithRefs = injectReferences(
85+
{ ...searchSourceFields, indexRefName },
86+
savedSearch.references ?? []
87+
);
88+
89+
// Form the Saved Search attributes and SearchSource into a config that's compatible with CsvGenerator
90+
const { columns, searchSource } = await getSharingData(
91+
{ uiSettings },
92+
await searchSourceStart.create(searchSourceFieldsWithRefs),
93+
savedSearch,
94+
job.timerange
95+
);
96+
97+
const jobParamsCsv: JobParamsCSV = { ...job, columns, searchSource };
98+
const csv = new CsvGenerator(
99+
jobParamsCsv,
100+
config,
101+
clients,
102+
dependencies,
103+
cancellationToken,
104+
logger,
105+
stream
106+
);
107+
return await csv.generateData();
108+
};
109+
};

‎x-pack/plugins/reporting/server/export_types/csv_saved_object/index.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import {
9+
CSV_SAVED_OBJECT_JOB_TYPE as CSV_JOB_TYPE,
10+
LICENSE_TYPE_BASIC,
11+
LICENSE_TYPE_ENTERPRISE,
12+
LICENSE_TYPE_GOLD,
13+
LICENSE_TYPE_PLATINUM,
14+
LICENSE_TYPE_CLOUD_STANDARD,
15+
LICENSE_TYPE_TRIAL,
16+
} from '../../../common/constants';
17+
import { CreateJobFn, ExportTypeDefinition, RunTaskFn } from '../../types';
18+
import { createJobFnFactory } from './create_job';
19+
import { runTaskFnFactory } from './execute_job';
20+
import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from './types';
21+
22+
export const getExportType = (): ExportTypeDefinition<
23+
CreateJobFn<JobParamsCsvFromSavedObject>,
24+
RunTaskFn<TaskPayloadCsvFromSavedObject>
25+
> => ({
26+
id: CSV_JOB_TYPE,
27+
name: CSV_JOB_TYPE,
28+
jobType: CSV_JOB_TYPE,
29+
jobContentExtension: 'csv',
30+
createJobFnFactory,
31+
runTaskFnFactory,
32+
validLicenses: [
33+
LICENSE_TYPE_TRIAL,
34+
LICENSE_TYPE_BASIC,
35+
LICENSE_TYPE_CLOUD_STANDARD,
36+
LICENSE_TYPE_GOLD,
37+
LICENSE_TYPE_PLATINUM,
38+
LICENSE_TYPE_ENTERPRISE,
39+
],
40+
});

0 commit comments

Comments
 (0)