Skip to content

Commit 298f170

Browse files
fix: don't fail exportData calls on empty datasets (apify#3115)
CSV exports on empty datasets caused errors because of reading the first item for the CSV header. Additionally, the changes also include the fixes from apify#2734 for `BasicCrawler.exportData`. --------- Co-authored-by: Jindřich Bär <jindrichbar@gmail.com>
1 parent a35376d commit 298f170

File tree

2 files changed

+29
-1
lines changed

2 files changed

+29
-1
lines changed

packages/basic-crawler/src/internals/basic-crawler.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1240,7 +1240,20 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
12401240
const items = await dataset.export(options);
12411241

12421242
if (format === 'csv') {
1243-
const value = stringify([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);
1243+
let value: string;
1244+
if (items.length === 0) {
1245+
value = '';
1246+
} else {
1247+
const keys = Object.keys(items[0]);
1248+
1249+
value = stringify([
1250+
keys,
1251+
...items.map((item) => {
1252+
return keys.map((k) => item[k]);
1253+
}),
1254+
]);
1255+
}
1256+
12441257
await ensureDir(dirname(path));
12451258
await writeFile(path, value);
12461259
this.log.info(`Export to ${path} finished!`);

test/core/crawlers/basic_crawler.test.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1716,6 +1716,21 @@ describe('BasicCrawler', () => {
17161716
await rm(`${tmpDir}/result.json`);
17171717
});
17181718

1719+
test('exports do not fail on empty dataset', async () => {
1720+
const crawler = new BasicCrawler();
1721+
1722+
await crawler.exportData(`${tmpDir}/result.csv`);
1723+
await crawler.exportData(`${tmpDir}/result.json`);
1724+
1725+
const csv = await readFile(`${tmpDir}/result.csv`);
1726+
expect(csv.toString()).toBe('');
1727+
const json = await readFile(`${tmpDir}/result.json`);
1728+
expect(json.toString()).toBe('[]\n');
1729+
1730+
await rm(`${tmpDir}/result.csv`);
1731+
await rm(`${tmpDir}/result.json`);
1732+
});
1733+
17191734
test('should expose pushData helper', async () => {
17201735
const crawler = new BasicCrawler({
17211736
requestHandler: async ({ pushData }) => pushData(payload),

0 commit comments

Comments
 (0)