Skip to content

Commit

Permalink
Fix csv parsing function (opensearch-project#53)
Browse files Browse the repository at this point in the history
* Fix csv parsing function

Signed-off-by: Joshua Li <joshuali925@gmail.com>

* Flatten nested results for csv

Signed-off-by: Joshua Li <joshuali925@gmail.com>

* Add test case for nested fields

Signed-off-by: Joshua Li <joshuali925@gmail.com>
  • Loading branch information
joshuali925 authored May 25, 2021
1 parent 2067dd3 commit b80ad2a
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 22 deletions.
41 changes: 33 additions & 8 deletions server/routes/utils/__tests__/savedSearchReportHelper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,31 @@ describe('test create saved search report', () => {
);
}, 20000);

test('create report for data set with nested fields', async () => {
const hits = [
hit({
'geoip.country_iso_code': 'GB',
'geoip.location': { lon: -0.1, lat: 51.5 },
}),
hit({
'geoip.country_iso_code': 'US',
'geoip.city_name': 'New York',
'geoip.location': { lon: -74, lat: 40.8 },
}),
];
const client = mockOpenSearchClient(
hits,
'"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
);
const { dataUrl } = await createSavedSearchReport(input, client);

expect(dataUrl).toEqual(
'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' +
'GB,-0.1,51.5, \n' +
'US,-74,40.8,New York'
);
}, 20000);

test('create report by sanitizing data set for Excel', async () => {
const hits = [
hit({ category: 'c1', customer_gender: '=Male' }),
Expand Down Expand Up @@ -312,7 +337,10 @@ test('create report for data set contains null field value', async () => {
/**
* Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
*/
function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
function mockOpenSearchClient(
mockHits: Array<{ _source: any }>,
columns = '"category", "customer_gender"'
) {
let call = 0;
const client = jest.fn();
client.callAsInternalUser = jest
Expand All @@ -323,7 +351,7 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
return {
_source: params.id.startsWith('index-pattern:')
? mockIndexPattern()
: mockSavedSearch(),
: mockSavedSearch(columns),
};
case 'indices.getSettings':
return mockIndexSettings();
Expand Down Expand Up @@ -357,9 +385,9 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) {
}

/**
* Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 selected fields: category and customer_gender.
* Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 default selected fields: category and customer_gender.
*/
function mockSavedSearch() {
function mockSavedSearch(columns = '"category", "customer_gender"') {
return JSON.parse(`
{
"type": "search",
Expand All @@ -368,10 +396,7 @@ function mockSavedSearch() {
"title": "Show category and gender",
"description": "",
"hits": 0,
"columns": [
"category",
"customer_gender"
],
"columns": [ ${columns} ],
"sort": [],
"version": 1,
"opensearchDashboardsSavedObjectMeta": {
Expand Down
42 changes: 28 additions & 14 deletions server/routes/utils/dataReportHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { DATA_REPORT_CONFIG } from './constants';
import esb from 'elastic-builder';
import moment from 'moment';
import converter from 'json-2-csv';
import _ from 'lodash';

export var metaData = {
saved_search_id: <string>null,
Expand Down Expand Up @@ -181,7 +182,7 @@ export const getOpenSearchData = (arrayHits, report, params) => {
}
delete data['fields'];
if (report._source.fields_exist === true) {
let result = traverse(data, report._source.selectedFields);
let result = traverse(data._source, report._source.selectedFields);
hits.push(params.excel ? sanitize(result) : result);
} else {
hits.push(params.excel ? sanitize(data) : data);
Expand All @@ -208,26 +209,39 @@ export const convertToCSV = async (dataset) => {
return convertedData;
};

//Return only the selected fields
function traverse(data, keys, result = {}) {
for (let k of Object.keys(data)) {
if (keys.includes(k)) {
result = Object.assign({}, result, {
[k]: data[k],
});
continue;
}
function flattenHits(hits, result = {}, prefix = '') {
for (const [key, value] of Object.entries(hits)) {
if (!hits.hasOwnProperty(key)) continue;
if (
data[k] &&
typeof data[k] === 'object' &&
Object.keys(data[k]).length > 0
value != null &&
typeof value === 'object' &&
!Array.isArray(value) &&
Object.keys(value).length > 0
) {
result = traverse(data[k], keys, result);
flattenHits(value, result, prefix + key + '.');
} else {
result[prefix + key] = value;
}
}
return result;
}

//Return only the selected fields
function traverse(data, keys, result = {}) {
data = flattenHits(data);
const sourceKeys = Object.keys(data);
keys.forEach((key) => {
const value = _.get(data, key, undefined);
if (value !== undefined) result[key] = value;
else {
Object.keys(data)
.filter((sourceKey) => sourceKey.startsWith(key + '.'))
.forEach((sourceKey) => (result[sourceKey] = data[sourceKey]));
}
});
return result;
}

/**
* Escape special characters if field value prefixed with.
* This is intend to avoid CSV injection in Microsoft Excel.
Expand Down

0 comments on commit b80ad2a

Please sign in to comment.