Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix csv missing fields issue and empty csv on _source fields #206

Merged
merged 1 commit into from
Nov 3, 2021
Merged

fix csv missing fields issue and empty csv on _source fields #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/dashboards-reports-test-and-build-workflow.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: [pull_request, push]
env:
PLUGIN_NAME: reportsDashboards
ARTIFACT_NAME: reports-dashboards
OPENSEARCH_VERSION: '1.x'
OPENSEARCH_VERSION: '1.1'
OPENSEARCH_PLUGIN_VERSION: 1.1.0.0

jobs:
Expand Down
6 changes: 4 additions & 2 deletions dashboards-reports/server/routes/lib/createReport.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ export const createReport = async (
// @ts-ignore
const timezone = request.query.timezone;
// @ts-ignore
const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
const dateFormat =
request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
const {
basePath,
serverInfo: { protocol, port, hostname },
Expand Down Expand Up @@ -97,7 +98,8 @@ export const createReport = async (
report,
opensearchClient,
dateFormat,
isScheduledTask
isScheduledTask,
logger
);
} else {
// report source can only be one of [saved search, visualization, dashboard, notebook]
Expand Down
144 changes: 129 additions & 15 deletions dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import 'regenerator-runtime/runtime';
import { createSavedSearchReport } from '../savedSearchReportHelper';
import { reportSchema } from '../../../model';
import { mockLogger } from '../../../../test/__mocks__/loggerMock';
import _ from 'lodash';

/**
* The mock and sample input for saved search export function.
Expand Down Expand Up @@ -54,14 +56,16 @@ const input = {
configIds: [],
title: 'title',
textDescription: 'text description',
htmlDescription: 'html description'
htmlDescription: 'html description',
},
trigger: {
trigger_type: 'On demand',
},
},
};

const mockDateFormat = 'date_hour_minute_second_fraction';

/**
* Max result window size in OpenSearch index settings.
*/
Expand All @@ -78,30 +82,45 @@ describe('test create saved search report', () => {
const client = mockOpenSearchClient(hits);
const { timeCreated, fileName } = await createSavedSearchReport(
input,
client
client,
mockDateFormat,
undefined,
mockLogger
);
expect(fileName).toContain(`test report table order_`);
}, 20000);

test('create report with expected file name extension', async () => {
const csvReport = await createSavedSearchReport(
input,
mockOpenSearchClient([])
mockOpenSearchClient([]),
mockDateFormat,
undefined,
mockLogger
);
expect(csvReport.fileName).toContain('.csv');

input.report_definition.report_params.core_params.report_format = 'xlsx';
const xlsxReport = await createSavedSearchReport(
input,
mockOpenSearchClient([])
mockOpenSearchClient([]),
mockDateFormat,
undefined,
mockLogger
);
expect(xlsxReport.fileName).toContain('.xlsx');
}, 20000);

test('create report for empty data set', async () => {
const hits: Array<{ _source: any }> = [];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);
expect(dataUrl).toEqual('');
}, 20000);

Expand All @@ -114,7 +133,13 @@ describe('test create saved search report', () => {
hit({ category: 'c5', customer_gender: 'Male' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -141,7 +166,13 @@ describe('test create saved search report', () => {
hit({ category: 'c11', customer_gender: 'Male' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand Down Expand Up @@ -171,7 +202,13 @@ describe('test create saved search report', () => {
hit({ category: 'c5', customer_gender: 'Male' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male');
}, 20000);
Expand All @@ -193,7 +230,13 @@ describe('test create saved search report', () => {
hit({ category: 'c10', customer_gender: 'Female' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -219,7 +262,13 @@ describe('test create saved search report', () => {
hit({ category: 'c6', customer_gender: 'Female' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -239,7 +288,13 @@ describe('test create saved search report', () => {
hit({ category: ',,c3', customer_gender: 'Male,,,' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -265,7 +320,13 @@ describe('test create saved search report', () => {
hits,
'"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' +
Expand All @@ -283,7 +344,13 @@ describe('test create saved search report', () => {
hit({ category: ',,,@c5', customer_gender: 'Male' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -307,7 +374,13 @@ describe('test create saved search report', () => {
hit({ category: ',,,@c5', customer_gender: 'Male' }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' +
Expand All @@ -327,13 +400,54 @@ test('create report for data set contains null field value', async () => {
hit({ category: 'c3', customer_gender: null }),
];
const client = mockOpenSearchClient(hits);
const { dataUrl } = await createSavedSearchReport(input, client);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, '
);
}, 20000);

test('create report for data set with metadata fields', async () => {
const metadataFields = { _index: 'nameofindex', _id: 'someid' };
let hits = [
hit({ category: 'c1', customer_gender: 'Male' }),
hit({ category: 'c2', customer_gender: 'Male' }),
hit({ category: 'c3', customer_gender: 'Male' }),
hit({ category: 'c4', customer_gender: 'Male' }),
hit({ category: 'c5', customer_gender: 'Male' }),
];
hits.forEach((i) => {
_.merge(i, metadataFields);
});

const client = mockOpenSearchClient(
hits,
'"category", "customer_gender","_index","_id"'
);
const { dataUrl } = await createSavedSearchReport(
input,
client,
mockDateFormat,
undefined,
mockLogger
);

expect(dataUrl).toEqual(
'category,customer_gender,_index,_id\n' +
'c1,Male,nameofindex,someid\n' +
'c2,Male,nameofindex,someid\n' +
'c3,Male,nameofindex,someid\n' +
'c4,Male,nameofindex,someid\n' +
'c5,Male,nameofindex,someid'
);
}, 20000);

/**
* Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
*/
Expand Down
19 changes: 6 additions & 13 deletions dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,9 @@ import 'regenerator-runtime/runtime';
import { createVisualReport } from '../visual_report/visualReportHelper';
import { Logger } from '../../../../../../src/core/server';
import { ReportParamsSchemaType, reportSchema } from '../../../model';
import { mockLogger } from '../../../../test/__mocks__/loggerMock';

const mockLogger: Logger = {
info: jest.fn(),
trace: jest.fn(),
warn: jest.fn(),
debug: jest.fn(),
error: jest.fn(),
fatal: jest.fn(),
log: jest.fn(),
get: jest.fn(),
};

const mockHeader = { mockKey: 'mockValue' };
const input = {
query_url: '/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d',
time_from: 1343576635300,
Expand Down Expand Up @@ -84,7 +75,8 @@ describe('test create visual report', () => {
const { dataUrl, fileName } = await createVisualReport(
reportParams as ReportParamsSchemaType,
mockHtmlPath,
mockLogger
mockLogger,
mockHeader
);
expect(fileName).toContain(`${reportParams.report_name}`);
expect(fileName).toContain('.png');
Expand All @@ -99,7 +91,8 @@ describe('test create visual report', () => {
const { dataUrl, fileName } = await createVisualReport(
reportParams as ReportParamsSchemaType,
mockHtmlPath,
mockLogger
mockLogger,
mockHeader
);
expect(fileName).toContain(`${reportParams.report_name}`);
expect(fileName).toContain('.pdf');
Expand Down
9 changes: 6 additions & 3 deletions dashboards-reports/server/routes/utils/dataReportHelpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,17 @@ export var metaData = {
// Get the selected columns by the user.
export const getSelectedFields = async (columns) => {
const selectedFields = [];
let fields_exist = false;
for (let column of columns) {
if (column !== '_source') {
metaData.fields_exist = true;
fields_exist = true;
selectedFields.push(column);
} else {
fields_exist = false;
selectedFields.push('_source');
}
}
metaData.fields_exist = fields_exist;
metaData.selectedFields = selectedFields;
};

Expand Down Expand Up @@ -191,7 +194,7 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string)
}
delete data['fields'];
if (report._source.fields_exist === true) {
let result = traverse(data._source, report._source.selectedFields);
let result = traverse(data, report._source.selectedFields);
hits.push(params.excel ? sanitize(result) : result);
} else {
hits.push(params.excel ? sanitize(data) : data);
Expand Down Expand Up @@ -229,7 +232,7 @@ function flattenHits(hits, result = {}, prefix = '') {
) {
flattenHits(value, result, prefix + key + '.');
} else {
result[prefix + key] = value;
result[prefix.replace(/^_source\./, '') + key] = value;
}
}
return result;
Expand Down
Loading