diff --git a/.github/workflows/dashboards-reports-test-and-build-workflow.yml b/.github/workflows/dashboards-reports-test-and-build-workflow.yml index ede2e62c..6dac8119 100644 --- a/.github/workflows/dashboards-reports-test-and-build-workflow.yml +++ b/.github/workflows/dashboards-reports-test-and-build-workflow.yml @@ -5,7 +5,7 @@ on: [pull_request, push] env: PLUGIN_NAME: reportsDashboards ARTIFACT_NAME: reports-dashboards - OPENSEARCH_VERSION: '1.x' + OPENSEARCH_VERSION: '1.1' OPENSEARCH_PLUGIN_VERSION: 1.1.0.0 jobs: diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts index 36709343..70058a87 100644 --- a/dashboards-reports/server/routes/lib/createReport.ts +++ b/dashboards-reports/server/routes/lib/createReport.ts @@ -69,7 +69,8 @@ export const createReport = async ( // @ts-ignore const timezone = request.query.timezone; // @ts-ignore - const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; + const dateFormat = + request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; const { basePath, serverInfo: { protocol, port, hostname }, @@ -97,7 +98,8 @@ export const createReport = async ( report, opensearchClient, dateFormat, - isScheduledTask + isScheduledTask, + logger ); } else { // report source can only be one of [saved search, visualization, dashboard, notebook] diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 6d3b7377..536bdbaf 100644 --- a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -27,6 +27,8 @@ import 'regenerator-runtime/runtime'; import { createSavedSearchReport } from '../savedSearchReportHelper'; import { reportSchema } from '../../../model'; +import { mockLogger } from '../../../../test/__mocks__/loggerMock'; +import _ from 'lodash'; /** * The mock and sample input for saved search export function. @@ -54,7 +56,7 @@ const input = { configIds: [], title: 'title', textDescription: 'text description', - htmlDescription: 'html description' + htmlDescription: 'html description', }, trigger: { trigger_type: 'On demand', @@ -62,6 +64,8 @@ const input = { }, }; +const mockDateFormat = 'date_hour_minute_second_fraction'; + /** * Max result window size in OpenSearch index settings. */ @@ -78,7 +82,10 @@ describe('test create saved search report', () => { const client = mockOpenSearchClient(hits); const { timeCreated, fileName } = await createSavedSearchReport( input, - client + client, + mockDateFormat, + undefined, + mockLogger ); expect(fileName).toContain(`test report table order_`); }, 20000); @@ -86,14 +93,20 @@ describe('test create saved search report', () => { test('create report with expected file name extension', async () => { const csvReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + mockDateFormat, + undefined, + mockLogger ); expect(csvReport.fileName).toContain('.csv'); input.report_definition.report_params.core_params.report_format = 'xlsx'; const xlsxReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + mockDateFormat, + undefined, + mockLogger ); expect(xlsxReport.fileName).toContain('.xlsx'); }, 20000); @@ -101,7 +114,13 @@ describe('test create saved search report', () => { test('create report for empty data set', async () => { const hits: Array<{ _source: any }> = []; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual(''); }, 20000); @@ -114,7 +133,13 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -141,7 +166,13 @@ describe('test create saved search report', () => { hit({ category: 'c11', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -171,7 +202,13 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male'); }, 20000); @@ -193,7 +230,13 @@ describe('test create saved search report', () => { hit({ category: 'c10', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -219,7 +262,13 @@ describe('test create saved search report', () => { hit({ category: 'c6', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -239,7 +288,13 @@ describe('test create saved search report', () => { hit({ category: ',,c3', customer_gender: 'Male,,,' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -265,7 +320,13 @@ describe('test create saved search report', () => { hits, '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' ); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' + @@ -283,7 +344,13 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -307,7 +374,13 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -327,13 +400,54 @@ test('create report for data set contains null field value', async () => { hit({ category: 'c3', customer_gender: null }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, ' ); }, 20000); +test('create report for data set with metadata fields', async () => { + const metadataFields = { _index: 'nameofindex', _id: 'someid' }; + let hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + ]; + hits.forEach((i) => { + _.merge(i, metadataFields); + }); + + const client = mockOpenSearchClient( + hits, + '"category", "customer_gender","_index","_id"' + ); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); + + expect(dataUrl).toEqual( + 'category,customer_gender,_index,_id\n' + + 'c1,Male,nameofindex,someid\n' + + 'c2,Male,nameofindex,someid\n' + + 'c3,Male,nameofindex,someid\n' + + 'c4,Male,nameofindex,someid\n' + + 'c5,Male,nameofindex,someid' + ); +}, 20000); + /** * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ diff --git a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts index ff2aca0b..fa3808b7 100644 --- a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts @@ -28,18 +28,9 @@ import 'regenerator-runtime/runtime'; import { createVisualReport } from '../visual_report/visualReportHelper'; import { Logger } from '../../../../../../src/core/server'; import { ReportParamsSchemaType, reportSchema } from '../../../model'; +import { mockLogger } from '../../../../test/__mocks__/loggerMock'; -const mockLogger: Logger = { - info: jest.fn(), - trace: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - get: jest.fn(), -}; - +const mockHeader = { mockKey: 'mockValue' }; const input = { query_url: '/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d', time_from: 1343576635300, @@ -84,7 +75,8 @@ describe('test create visual report', () => { const { dataUrl, fileName } = await createVisualReport( reportParams as ReportParamsSchemaType, mockHtmlPath, - mockLogger + mockLogger, + mockHeader ); expect(fileName).toContain(`${reportParams.report_name}`); expect(fileName).toContain('.png'); @@ -99,7 +91,8 @@ describe('test create visual report', () => { const { dataUrl, fileName } = await createVisualReport( reportParams as ReportParamsSchemaType, mockHtmlPath, - mockLogger + mockLogger, + mockHeader ); expect(fileName).toContain(`${reportParams.report_name}`); expect(fileName).toContain('.pdf'); diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index 7a3b6d19..a5ede76c 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -49,14 +49,17 @@ export var metaData = { // Get the selected columns by the user. export const getSelectedFields = async (columns) => { const selectedFields = []; + let fields_exist = false; for (let column of columns) { if (column !== '_source') { - metaData.fields_exist = true; + fields_exist = true; selectedFields.push(column); } else { + fields_exist = false; selectedFields.push('_source'); } } + metaData.fields_exist = fields_exist; metaData.selectedFields = selectedFields; }; @@ -191,7 +194,7 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) } delete data['fields']; if (report._source.fields_exist === true) { - let result = traverse(data._source, report._source.selectedFields); + let result = traverse(data, report._source.selectedFields); hits.push(params.excel ? sanitize(result) : result); } else { hits.push(params.excel ? sanitize(data) : data); @@ -229,7 +232,7 @@ function flattenHits(hits, result = {}, prefix = '') { ) { flattenHits(value, result, prefix + key + '.'); } else { - result[prefix + key] = value; + result[prefix.replace(/^_source\./, '') + key] = value; } } return result; diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index 76a5a9cf..f42836f9 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -34,6 +34,7 @@ import { import { ILegacyClusterClient, ILegacyScopedClusterClient, + Logger, } from '../../../../../src/core/server'; import { getFileName, callCluster } from './helpers'; import { CreateReportResultType } from './types'; @@ -49,18 +50,20 @@ export async function createSavedSearchReport( report: any, client: ILegacyClusterClient | ILegacyScopedClusterClient, dateFormat: string, - isScheduledTask: boolean = true + isScheduledTask: boolean = true, + logger: Logger ): Promise { const params = report.report_definition.report_params; const reportFormat = params.core_params.report_format; const reportName = params.report_name; - await populateMetaData(client, report, isScheduledTask); + await populateMetaData(client, report, isScheduledTask, logger); const data = await generateReportData( client, params.core_params, dateFormat, - isScheduledTask + isScheduledTask, + logger ); const curTime = new Date(); @@ -81,7 +84,8 @@ export async function createSavedSearchReport( async function populateMetaData( client: ILegacyClusterClient | ILegacyScopedClusterClient, report: any, - isScheduledTask: boolean + isScheduledTask: boolean, + logger: Logger ) { metaData.saved_search_id = report.report_definition.report_params.core_params.saved_search_id; @@ -144,7 +148,8 @@ async function generateReportData( client: ILegacyClusterClient | ILegacyScopedClusterClient, params: any, dateFormat: string, - isScheduledTask: boolean + isScheduledTask: boolean, + logger: Logger ) { let opensearchData: any = {}; const arrayHits: any = []; @@ -159,6 +164,9 @@ async function generateReportData( } const reqBody = buildRequestBody(buildQuery(report, 0)); + logger.info( + `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` + ); if (total > maxResultSize) { await getOpenSearchDataByScroll(); } else { diff --git a/dashboards-reports/test/__mocks__/loggerMock.ts b/dashboards-reports/test/__mocks__/loggerMock.ts new file mode 100644 index 00000000..bb798679 --- /dev/null +++ b/dashboards-reports/test/__mocks__/loggerMock.ts @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +const mockLogger = { + info: jest.fn(), + trace: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + get: jest.fn(), +}; + +export { mockLogger };