Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve xlsx import #1271

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
+211 −135
Open

Improve xlsx import #1271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3c7c7f4
use xlsx
alex-Arc Oct 18, 2024
7f67e72
add existing custom fields to import map
alex-Arc Oct 18, 2024
8b7a5a8
keep colours of existing custom fields
alex-Arc Oct 18, 2024
cf48825
use unknown
alex-Arc Oct 18, 2024
3ff60ce
fix
alex-Arc Oct 18, 2024
705858e
revert
alex-Arc Oct 18, 2024
6757a4c
allow space and upper case
alex-Arc Oct 18, 2024
4856fb1
fix test
alex-Arc Oct 18, 2024
d60856a
define empty sheet
alex-Arc Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ import { useFieldArray, useForm } from 'react-hook-form';
import { Button, IconButton, Input, Select, Tooltip } from '@chakra-ui/react';
import { IoAdd } from '@react-icons/all-files/io5/IoAdd';
import { IoTrash } from '@react-icons/all-files/io5/IoTrash';
import { ImportMap } from 'ontime-utils';
import { ImportMap, isAlphanumericWithSpace } from 'ontime-utils';

import { isAlphanumeric } from '../../../../../common/utils/regex';
import * as Panel from '../../PanelUtils';
import useGoogleSheet from '../useGoogleSheet';
import { useSheetStore } from '../useSheetStore';
Expand Down Expand Up @@ -190,9 +189,10 @@ export default function ImportMapForm(props: ImportMapFormProps) {
defaultValue={ontimeName}
placeholder='Name of the field as shown in Ontime'
{...register(`custom.${index}.ontimeName`, {
pattern: {
value: isAlphanumeric,
message: 'Custom field name must be alphanumeric',
validate: (value) => {
if (!isAlphanumericWithSpace(value))
return 'Only alphanumeric characters and space are allowed';
return true;
},
})}
/>
Expand Down
9 changes: 5 additions & 4 deletions apps/client/src/features/app-settings/panel/sources-panel/preview/PreviewRundown.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ export default function PreviewRundown(props: PreviewRundownProps) {
// we only count Ontime Events which are 1 based in client
let eventIndex = 0;

const fieldHeaders = Object.keys(customFields);
const fieldKeys = Object.keys(customFields);
const fieldLabels = fieldKeys.map((key) => customFields[key].label);

return (
<Panel.Table>
Expand All @@ -44,8 +45,8 @@ export default function PreviewRundown(props: PreviewRundownProps) {
<th>Colour</th>
<th>Timer Type</th>
<th>End Action</th>
{fieldHeaders.map((field) => (
<th key={field}>{field}</th>
{fieldLabels.map((label) => (
<th key={label}>{label}</th>
))}
</tr>
</thead>
Expand Down Expand Up @@ -102,7 +103,7 @@ export default function PreviewRundown(props: PreviewRundownProps) {
<Tag>{event.endAction}</Tag>
</td>
{isOntimeEvent(event) &&
fieldHeaders.map((field) => {
fieldKeys.map((field) => {
let value = '';
if (field in event.custom) {
value = event.custom[field];
Expand Down
4 changes: 2 additions & 2 deletions apps/server/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
"lowdb": "^7.0.1",
"multer": "^1.4.5-lts.1",
"node-osc": "^9.0.2",
"node-xlsx": "^0.23.0",
"ontime-utils": "workspace:*",
"sanitize-filename": "^1.6.3",
"steno": "^4.0.2",
"ws": "^8.13.0"
"ws": "^8.13.0",
"xlsx": "^0.18.5"
},
"devDependencies": {
"@types/cors": "^2.8.17",
Expand Down
19 changes: 11 additions & 8 deletions apps/server/src/api-data/excel/excel.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ import { ImportMap } from 'ontime-utils';

import { extname } from 'path';
import { existsSync } from 'fs';
import xlsx from 'node-xlsx';
import xlsx from 'xlsx';
import type { WorkBook } from 'xlsx';

import { parseExcel } from '../../utils/parser.js';
import { parseRundown } from '../../utils/parserFunctions.js';
import { deleteFile } from '../../utils/parserUtils.js';
import { getCustomFields } from '../../services/rundown-service/rundownCache.js';

let excelData: { name: string; data: unknown[][] }[] = [];
let excelData: WorkBook = xlsx.utils.book_new();

export async function saveExcelFile(filePath: string) {
if (!existsSync(filePath)) {
Expand All @@ -23,32 +25,33 @@ export async function saveExcelFile(filePath: string) {
if (extname(filePath) != '.xlsx') {
throw new Error('Wrong file format');
}
excelData = xlsx.parse(filePath, { cellDates: true });
excelData = xlsx.readFile(filePath, { cellDates: true, cellFormula: false });
alex-Arc marked this conversation as resolved.
Show resolved Hide resolved
alex-Arc marked this conversation as resolved.
Show resolved Hide resolved

await deleteFile(filePath);
}

export function listWorksheets() {
return excelData.map((value) => value.name);
export function listWorksheets(): string[] {
return excelData.SheetNames;
alex-Arc marked this conversation as resolved.
Show resolved Hide resolved
}

export function generateRundownPreview(options: ImportMap): { rundown: OntimeRundown; customFields: CustomFields } {
const data = excelData.find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase())?.data;
const data = excelData.Sheets[options.worksheet];
alex-Arc marked this conversation as resolved.
Show resolved Hide resolved

if (!data) {
throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`);
}

const dataFromExcel = parseExcel(data, options);
const arrayOfData: unknown[][] = xlsx.utils.sheet_to_json(data, { header: 1, blankrows: false, raw: false });

const dataFromExcel = parseExcel(arrayOfData, getCustomFields(), options);
// we run the parsed data through an extra step to ensure the objects shape
const { rundown, customFields } = parseRundown(dataFromExcel);
if (rundown.length === 0) {
throw new Error(`Could not find data to import in the worksheet: ${options.worksheet}`);
}

// clear the data
excelData = [];
excelData = undefined;

return { rundown, customFields };
}
5 changes: 3 additions & 2 deletions apps/server/src/services/sheet-service/SheetService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { parseExcel } from '../../utils/parser.js';
import { logger } from '../../classes/Logger.js';
import { parseRundown } from '../../utils/parserFunctions.js';
import { getRundown } from '../rundown-service/rundownUtils.js';
import { getCustomFields } from '../rundown-service/rundownCache.js';

const sheetScope = 'https://www.googleapis.com/auth/spreadsheets';
const codesUrl = 'https://oauth2.googleapis.com/device/code';
Expand Down Expand Up @@ -288,7 +289,7 @@ export async function upload(sheetId: string, options: ImportMap) {
throw new Error(`Sheet read failed: ${readResponse.statusText}`);
}

const { rundownMetadata } = parseExcel(readResponse.data.values, options);
const { rundownMetadata } = parseExcel(readResponse.data.values, getCustomFields(), options);
const rundown = getRundown();
const titleRow = Object.values(rundownMetadata)[0]['row'];
const updateRundown = Array<sheets_v4.Schema$Request>();
Expand Down Expand Up @@ -365,7 +366,7 @@ export async function download(
throw new Error(`Sheet read failed: ${googleResponse.statusText}`);
}

const dataFromSheet = parseExcel(googleResponse.data.values, options);
const dataFromSheet = parseExcel(googleResponse.data.values, getCustomFields(), options);
const { customFields, rundown } = parseRundown(dataFromSheet);
if (rundown.length < 1) {
throw new Error('Sheet: Could not find data to import in the worksheet');
Expand Down
94 changes: 78 additions & 16 deletions apps/server/src/utils/__tests__/parser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { assertType, vi } from 'vitest';

import {
CustomFields,
DatabaseModel,
EndAction,
OntimeEvent,
Expand Down Expand Up @@ -788,7 +789,7 @@ describe('getCustomFieldData()', () => {
},
} as ImportMap;

const result = getCustomFieldData(importMap);
const result = getCustomFieldData(importMap, {});
expect(result.customFields).toStrictEqual({
lighting: {
type: 'string',
Expand All @@ -807,6 +808,61 @@ describe('getCustomFieldData()', () => {
},
});

// it is an inverted record of <importKey, ontimeKey>
expect(result.customFieldImportKeys).toStrictEqual({
lx: 'lighting',
sound: 'sound',
av: 'video',
});
});
it('keeps colour information from existing fields', () => {
const importMap = {
worksheet: 'event schedule',
timeStart: 'time start',
linkStart: 'link start',
timeEnd: 'time end',
duration: 'duration',
cue: 'cue',
title: 'title',
isPublic: 'public',
skip: 'skip',
note: 'notes',
colour: 'colour',
endAction: 'end action',
timerType: 'timer type',
timeWarning: 'warning time',
timeDanger: 'danger time',
custom: {
lighting: 'lx',
sound: 'sound',
video: 'av',
},
} as ImportMap;

const customFields: CustomFields = {
lighting: { label: 'lx', type: 'string', colour: 'red' },
sound: { label: 'sound', type: 'string', colour: 'green' },
};

const result = getCustomFieldData(importMap, customFields);
expect(result.customFields).toStrictEqual({
lighting: {
type: 'string',
colour: 'red',
label: 'lighting',
},
sound: {
type: 'string',
colour: 'green',
label: 'sound',
},
video: {
type: 'string',
colour: '',
label: 'video',
},
});

// it is an inverted record of <importKey, ontimeKey>
expect(result.customFieldImportKeys).toStrictEqual({
lx: 'lighting',
Expand Down Expand Up @@ -919,7 +975,7 @@ describe('parseExcel()', () => {
note: 'Ballyhoo',
custom: {
user0: 'a0',
user1: 'a1',
User1: 'a1',
user2: 'a2',
user3: 'a3',
user4: 'a4',
Expand Down Expand Up @@ -952,21 +1008,27 @@ describe('parseExcel()', () => {
},
];

const parsedData = parseExcel(testdata, importMap);
const existingCustomFields: CustomFields = {
user0: { type: 'string', colour: 'red', label: 'user0' },
User1: { type: 'string', colour: 'green', label: 'user1' },
user2: { type: 'string', colour: 'blue', label: 'user2' },
};

const parsedData = parseExcel(testdata, existingCustomFields, importMap);
expect(parsedData.customFields).toStrictEqual({
user0: {
type: 'string',
colour: '',
colour: 'red',
label: 'user0',
},
user1: {
User1: {
type: 'string',
colour: '',
colour: 'green',
label: 'User1',
},
user2: {
type: 'string',
colour: '',
colour: 'blue',
label: 'user2',
},
user3: {
Expand Down Expand Up @@ -1123,7 +1185,7 @@ describe('parseExcel()', () => {
},
];

const parsedData = parseExcel(testdata, importMap);
const parsedData = parseExcel(testdata, {}, importMap);
expect(parsedData.customFields).toStrictEqual({
niu1: {
type: 'string',
Expand Down Expand Up @@ -1229,7 +1291,7 @@ describe('parseExcel()', () => {
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testdata, importMap);
const result = parseExcel(testdata, {}, importMap);
expect(result.rundown.length).toBe(1);
expect((result.rundown.at(0) as OntimeEvent).title).toBe('A song from the hearth');
});
Expand Down Expand Up @@ -1322,7 +1384,7 @@ describe('parseExcel()', () => {
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testdata, importMap);
const result = parseExcel(testdata, {}, importMap);
expect(result.rundown.length).toBe(2);
expect((result.rundown.at(0) as OntimeEvent).type).toBe(SupportedEvent.Block);
});
Expand Down Expand Up @@ -1392,7 +1454,7 @@ describe('parseExcel()', () => {
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testdata, importMap);
const result = parseExcel(testdata, {}, importMap);
expect(result.rundown.length).toBe(2);
expect((result.rundown.at(0) as OntimeEvent).type).toBe(SupportedEvent.Event);
expect((result.rundown.at(0) as OntimeEvent).timerType).toBe(TimerType.CountDown);
Expand Down Expand Up @@ -1510,7 +1572,7 @@ describe('parseExcel()', () => {
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testdata, importMap);
const result = parseExcel(testdata, {}, importMap);
expect(result.rundown.length).toBe(3);
expect((result.rundown.at(0) as OntimeEvent).type).toBe(SupportedEvent.Event);
expect((result.rundown.at(0) as OntimeEvent).timerType).toBe(TimerType.CountDown);
Expand Down Expand Up @@ -1551,7 +1613,7 @@ describe('parseExcel()', () => {
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testData, importMap);
const result = parseExcel(testData, {}, importMap);
const { rundown } = parseRundown(result);
const events = rundown.filter((e) => e.type === SupportedEvent.Event) as OntimeEvent[];
expect((events.at(0) as OntimeEvent).timeStart).toEqual(16200000);
Expand Down Expand Up @@ -1588,7 +1650,7 @@ describe('parseExcel()', () => {
custom: {},
};

const result = parseExcel(testData, importMap);
const result = parseExcel(testData, {}, importMap);
const { rundown } = parseRundown(result);
const events = rundown.filter((e) => e.type === SupportedEvent.Event) as OntimeEvent[];
expect((events.at(0) as OntimeEvent).timeStart).toEqual(16200000); //<--leading white space in MAP
Expand Down Expand Up @@ -1640,7 +1702,7 @@ describe('parseExcel()', () => {
custom: {},
};

const result = parseExcel(testData, importMap);
const result = parseExcel(testData, {}, importMap);
const parseResult = parseRundown(result);

cache.init(parseResult.rundown, parseResult.customFields);
Expand Down Expand Up @@ -1774,7 +1836,7 @@ describe('parseExcel()', () => {
['MEET4', '#779BE7', '', '', 30, true, 'Meeting 4', '', 'count-up', 'none', 11, '00:05:00', 'TRUE', 'FALSE'],
];

const parsedData = parseExcel(testData);
const parsedData = parseExcel(testData, {});
alex-Arc marked this conversation as resolved.
Show resolved Hide resolved
const { rundown } = parsedData;

// elements in bug report
Expand Down
Loading