From 9510facfa1aa8a1d987003a3919b8ec23a2d4824 Mon Sep 17 00:00:00 2001 From: Akos Balasko Date: Sat, 18 Mar 2023 19:23:08 +0100 Subject: [PATCH] fix: truncate if the filepath exceeds the OS limit (#377) --- package-lock.json | 5 + package.json | 1 + src/models/InternalLink.ts | 1 + src/runtime-properties.ts | 21 ++++- src/utils/apply-links.ts | 27 +++++- src/utils/filename-utils.ts | 5 + src/utils/folder-utils.ts | 35 ++++++- .../turndown-rules/internal-links-rule.ts | 8 +- test/data/test-long-linked-notes-NoteB.md | 9 ++ test/data/test-long-linked-notes.enex | 6 ++ test/data/test-long-note.enex | 5 + test/yarle-special-cases.spec.ts | 92 +++++++++++++++++-- test/yarle.spec.ts | 1 + 13 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 test/data/test-long-linked-notes-NoteB.md create mode 100644 test/data/test-long-linked-notes.enex create mode 100644 test/data/test-long-note.enex diff --git a/package-lock.json b/package-lock.json index 2e43bc8b..b241b8d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23586,6 +23586,11 @@ "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", "dev": true }, + "nanoid": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", diff --git a/package.json b/package.json index 9d2a1c63..0f1fe526 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "mime-types": "2.1.28", "minimist": "1.2.6", "moment": "2.29.4", + "nanoid": "3.3.4", "proxyquire": "2.1.3", "rimraf": "3.0.2", "sanitize-filename": "1.6.3", diff --git a/src/models/InternalLink.ts b/src/models/InternalLink.ts index 41480b2b..687970a5 100644 --- a/src/models/InternalLink.ts +++ b/src/models/InternalLink.ts @@ -1,4 +1,5 @@ export interface InternalLink { url: string; title: string; + uniqueEnd?: string; } diff --git a/src/runtime-properties.ts b/src/runtime-properties.ts index 88d7ff22..2c83c466 100644 --- a/src/runtime-properties.ts +++ b/src/runtime-properties.ts @@ -1,11 +1,20 @@ import { InternalLink, NoteData } from './models'; +export interface NoteIdNameEntry { + title: string; + noteName: string; + notebookName: string; + uniqueEnd: string; +} +export interface NoteIdNames { + [key: string]: NoteIdNameEntry; +} export class RuntimePropertiesSingleton { static instance: RuntimePropertiesSingleton; - noteIdNameMap: any; - noteIdNameTOCMap: any; // Table of Contents map - the trusted source + noteIdNameMap: NoteIdNames; + noteIdNameTOCMap: NoteIdNames; // Table of Contents map - the trusted source currentNoteName: string; currentNotebookName: string; currentNotePath: string; @@ -30,6 +39,7 @@ export class RuntimePropertiesSingleton { title: linkItem.title, noteName: this.currentNoteName, notebookName: this.currentNotebookName, + uniqueEnd: linkItem.uniqueEnd, }; } addItemToTOCMap(linkItem: InternalLink): void { @@ -38,6 +48,7 @@ export class RuntimePropertiesSingleton { title: linkItem.title, noteName: this.currentNoteName, notebookName: this.currentNotebookName, + uniqueEnd: linkItem.uniqueEnd, }; } @@ -49,13 +60,17 @@ export class RuntimePropertiesSingleton { return this.noteIdNameTOCMap; } - getAllNoteIdNameMap(): any { + getAllNoteIdNameMap(): NoteIdNames { return { ...this.noteIdNameMap, ...this.noteIdNameTOCMap, }; } + getNoteIdNameMapByNoteTitle(noteTitle: string): any { + return Object.values(this.getAllNoteIdNameMap()).filter(noteIdName => noteIdName.title === noteTitle); + } + setCurrentNotebookName(currentNotebookName: string): void { this.currentNotebookName = currentNotebookName; } diff --git a/src/utils/apply-links.ts b/src/utils/apply-links.ts index 1d926d04..b9f22a74 100644 --- a/src/utils/apply-links.ts +++ b/src/utils/apply-links.ts @@ -5,13 +5,23 @@ import * as path from 'path'; import { YarleOptions } from './../YarleOptions'; import { RuntimePropertiesSingleton } from './../runtime-properties'; +import { truncatFileName } from './folder-utils'; import { escapeStringRegexp } from './escape-string-regexp'; export const applyLinks = (options: YarleOptions, outputNotebookFolders: Array): void => { const linkNameMap = RuntimePropertiesSingleton.getInstance(); const allLinks = linkNameMap.getAllNoteIdNameMap(); + const allconvertedFiles: Array = []; + for (const outputFolder of outputNotebookFolders){ + getAllFiles(outputFolder, allconvertedFiles); + } for (const [linkName, linkProps] of Object.entries(allLinks)) { - const fileName: string = (linkProps as any)['title']; + const uniqueId = linkProps.uniqueEnd; + let fileName = (linkProps as any)['title']; + if (allconvertedFiles.find(fn => fn.includes(uniqueId))) { + fileName = truncatFileName(fileName, uniqueId); + } + const notebookName: string = (linkProps as any)['notebookName']; const encodedFileName = options.urlEncodeFileNamesAndLinks ? encodeURI(fileName as string) : fileName as string; @@ -44,3 +54,18 @@ export const applyLinks = (options: YarleOptions, outputNotebookFolders: Array): Array => { + const files = fs.readdirSync(dirPath); + + arrayOfFiles = arrayOfFiles || []; + files.forEach(file => { + if (fs.statSync(`${dirPath}${path.sep}${file}`).isDirectory()) { + arrayOfFiles = getAllFiles(`${dirPath}${path.sep}${file}`, arrayOfFiles); + } else { + arrayOfFiles.push(path.join(__dirname, dirPath, '/', file)); + } + }); + + return arrayOfFiles; + }; diff --git a/src/utils/filename-utils.ts b/src/utils/filename-utils.ts index c2a4c6cb..199a1e90 100644 --- a/src/utils/filename-utils.ts +++ b/src/utils/filename-utils.ts @@ -4,6 +4,7 @@ import * as fs from 'fs'; import Moment from 'moment'; import * as path from 'path'; import * as mime from 'mime-types'; +import { nanoid } from 'nanoid'; import { yarleOptions } from '../yarle'; @@ -100,6 +101,10 @@ export const getZettelKastelId = (note: any, dstPath: string): string => { }; +export const getUniqueId = (): string => { + return nanoid(5); +}; + export const getNoteName = (dstPath: string, note: any): string => { let noteName; diff --git a/src/utils/folder-utils.ts b/src/utils/folder-utils.ts index 5357c004..2bdf6370 100644 --- a/src/utils/folder-utils.ts +++ b/src/utils/folder-utils.ts @@ -5,19 +5,48 @@ import * as path from 'path'; import { Path } from '../paths'; import { yarleOptions } from '../yarle'; -import { getNoteFileName, getNoteName } from './filename-utils'; +import { getNoteFileName, getNoteName, getUniqueId, normalizeTitle } from './filename-utils'; import { loggerInfo } from './loggerInfo'; -import { logger } from './logger'; import { OutputFormat } from './../output-format'; +import { RuntimePropertiesSingleton } from './../runtime-properties'; export const paths: Path = {}; +const MAX_PATH = 249; export const getResourceDir = (dstPath: string, note: any): string => { return getNoteName(dstPath, note).replace(/\s/g, '_'); }; +export const truncatFileName = (fileName: string, uniqueId: string): string => { + + if (fileName.length <= 11) { + throw Error('FATAL: note folder directory path exceeds the OS limitation. Please pick a destination closer to the root folder.'); + } + + const fullPath = `${getNotesPath()}${path.sep}${fileName}`; + + return fullPath.length <  MAX_PATH ? fileName : `${fileName.slice(0, MAX_PATH - 11)}_${uniqueId}.md`; +}; + +const truncateFilePath = (note: any, fileName: string, fullFilePath: string): string => { + const noteIdNameMap = RuntimePropertiesSingleton.getInstance(); + + const noteIdMap = noteIdNameMap.getNoteIdNameMapByNoteTitle(normalizeTitle(note.title))[0] || {uniqueEnd: getUniqueId()}; + + + if (fileName.length <= 11) { + throw Error('FATAL: note folder directory path exceeds the OS limitation. Please pick a destination closer to the root folder.'); + } + + return `${fullFilePath.slice(0, MAX_PATH - 11)}_${noteIdMap.uniqueEnd}.md`; + // -11 is the nanoid 5 char +_+ the max possible extension of the note (.md vs .html) +}; + const getFilePath = (dstPath: string, note: any): string => { - return `${dstPath}${path.sep}${getNoteFileName(dstPath, note)}`; + const fileName = getNoteFileName(dstPath, note); + const fullFilePath = `${dstPath}${path.sep}${normalizeTitle(fileName)}`; + + return fullFilePath.length <  MAX_PATH ? fullFilePath : truncateFilePath(note, fileName, fullFilePath); }; export const getMdFilePath = (note: any): string => { diff --git a/src/utils/turndown-rules/internal-links-rule.ts b/src/utils/turndown-rules/internal-links-rule.ts index 43dbd252..b102362e 100644 --- a/src/utils/turndown-rules/internal-links-rule.ts +++ b/src/utils/turndown-rules/internal-links-rule.ts @@ -1,7 +1,7 @@ import marked, { Token } from 'marked'; import * as _ from 'lodash'; -import { normalizeTitle } from '../filename-utils'; +import { getUniqueId, normalizeTitle } from '../filename-utils'; import { OutputFormat } from '../../output-format'; import { yarleOptions } from '../../yarle'; import { getTurndownService } from '../turndown-service'; @@ -25,7 +25,6 @@ export const wikiStyleLinksRule = { if (!nodeProxy.href) { return ''; } - let internalTurndownedContent = getTurndownService(yarleOptions).turndown(removeBrackets(node.innerHTML)); internalTurndownedContent = removeDoubleBackSlashes(internalTurndownedContent); @@ -64,10 +63,11 @@ export const wikiStyleLinksRule = { if (value.startsWith('evernote://')) { const fileName = normalizeTitle(token['text']); const noteIdNameMap = RuntimePropertiesSingleton.getInstance(); + const uniqueId = getUniqueId(); if (isTOC(noteIdNameMap.getCurrentNoteName())) { - noteIdNameMap.addItemToTOCMap({ url: value, title: fileName }); + noteIdNameMap.addItemToTOCMap({ url: value, title: fileName, uniqueEnd: uniqueId }); } else { - noteIdNameMap.addItemToMap({ url: value, title: fileName }); + noteIdNameMap.addItemToMap({ url: value, title: fileName, uniqueEnd: uniqueId }); } const linkedNoteId = value; diff --git a/test/data/test-long-linked-notes-NoteB.md b/test/data/test-long-linked-notes-NoteB.md new file mode 100644 index 00000000..eaef1e57 --- /dev/null +++ b/test/data/test-long-linked-notes-NoteB.md @@ -0,0 +1,9 @@ +# NoteB + + + +This is the content of NoteB, and a reference to [[This is going to be a really reallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreall_7Kcx7.md|This is going to be a really reallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreally note title]] + + Created at: 2020-05-28T15:14:04+01:00 + Updated at: 2020-05-28T15:14:42+01:00 + Notebook: test-long-linked-notes diff --git a/test/data/test-long-linked-notes.enex b/test/data/test-long-linked-notes.enex new file mode 100644 index 00000000..4b6a354b --- /dev/null +++ b/test/data/test-long-linked-notes.enex @@ -0,0 +1,6 @@ + + + +NoteB]]>20200528T141404Z20200528T141442Z46.3757019042968818.13928805659798128.9876251220703akosdesktop.mac0 +This is going to be a really reallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreally note title
This is the content of NoteA
]]>20200528T141349Z20200528T141402Z46.3757019042968818.13928805659798128.9876251220703akosdesktop.mac0 + diff --git a/test/data/test-long-note.enex b/test/data/test-long-note.enex new file mode 100644 index 00000000..073d8091 --- /dev/null +++ b/test/data/test-long-note.enex @@ -0,0 +1,5 @@ + + + +This is going to be a really reallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreally note title
text1

]]>19691006T084413Z20181006T101436Zakosdesktop.mac0 + diff --git a/test/yarle-special-cases.spec.ts b/test/yarle-special-cases.spec.ts index 9e7ca376..e9b073b7 100644 --- a/test/yarle-special-cases.spec.ts +++ b/test/yarle-special-cases.spec.ts @@ -3,12 +3,12 @@ import fs from 'fs'; import eol from 'eol'; import mockTimezone from 'timezone-mock'; import * as path from 'path'; + import { OutputFormat } from './../src/output-format'; import * as utils from './../src/utils'; import * as yarle from './../src/yarle'; import * as dropTheRopeRunner from './../src/dropTheRopeRunner'; import { YarleOptions } from './../src/YarleOptions'; -import { LOGFILE } from './../src/utils'; import { TaskOutputFormat } from '../src/task-output-format'; const testDataFolder = `.${path.sep}test${path.sep}data${path.sep}`; @@ -42,7 +42,6 @@ dateFormat: undefined, assert.equal(true, errorHappened); }); - it('Enex file with note containing a picture', async () => { const options: YarleOptions = { dateFormat: undefined, @@ -51,7 +50,8 @@ dateFormat: undefined, isMetadataNeeded: true, }; await yarle.dropTheRope(options); - console.log(`conversion log: ${fs.readFileSync(LOGFILE)}`); + // tslint:disable-next-line:no-console + console.log(`conversion log: ${fs.readFileSync(utils.LOGFILE)}`); assert.equal( fs.existsSync( `${__dirname}/../out/notes/test-withPicture/test - note with picture.md`, @@ -83,7 +83,8 @@ dateFormat: undefined, isMetadataNeeded: true, }; await yarle.dropTheRope(options); - console.log(`conversion log: ${fs.readFileSync(LOGFILE)}`); + // tslint:disable-next-line:no-console + console.log(`conversion log: ${fs.readFileSync(utils.LOGFILE)}`); assert.equal( fs.existsSync( `${__dirname}/../out/notes/test-withPicture/test - note with picture.md`, @@ -192,20 +193,20 @@ dateFormat: undefined, await yarle.dropTheRope(options); assert.equal( fs.existsSync( - `/tmp/out/notes/test-textWithImage/Untitled.md`, + '/tmp/out/notes/test-textWithImage/Untitled.md', ), true, ); assert.equal( fs.existsSync( - `/tmp/out/notes/test-textWithImage/_resources/Untitled.resources`, + '/tmp/out/notes/test-textWithImage/_resources/Untitled.resources', ), true, ); assert.equal( eol.auto(fs.readFileSync( - `/tmp/out/notes//test-textWithImage/Untitled.md`, + '/tmp/out/notes//test-textWithImage/Untitled.md', 'utf8', )), fs.readFileSync(`${__dirname}/data/test-textWithImage.md`, 'utf8'), @@ -1060,6 +1061,51 @@ dateFormat: undefined, fs.readFileSync(`${__dirname}/data/test-old-note.md`, 'utf8'), ); }); + it('really long filename', async () => { + const options: YarleOptions = { + enexSources: [ `${testDataFolder}test-long-note.enex` ], + outputDir: 'out', + templateFile: `${testDataFolder}full_template.templ`, + isMetadataNeeded: true, + outputFormat: OutputFormat.ObsidianMD, + skipEnexFileNameFromOutputPath: false, + + }; + await yarle.dropTheRope(options); + const expectedFileNamePrefix = 'This is going to be a really reallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreallyreally'; + + const fileList = fs.readdirSync(`${__dirname}/../out/notes/test-long-note`); + assert.equal(fileList.filter(fileName => fileName.startsWith(expectedFileNamePrefix)).length, 1); + + }); + + it('really long filename - with link', async () => { + const options: YarleOptions = { + enexSources: [ `${testDataFolder}test-long-linked-notes.enex` ], + outputDir: 'out', + templateFile: `${testDataFolder}full_template.templ`, + isMetadataNeeded: true, + outputFormat: OutputFormat.ObsidianMD, + obsidianSettings: {omitLinkDisplayName: false}, + skipEnexFileNameFromOutputPath: false, + + }; + await dropTheRopeRunner.run(options); + const expectedFileNamePrefix = 'This is going to be a really really'; + + const fileList = fs.readdirSync(`${__dirname}/../out/notes/test-long-linked-notes`); + assert.equal(fileList.filter(fileName => fileName.startsWith(expectedFileNamePrefix)).length, 1); + + assert.equal( + eol.auto(fs.readFileSync( + `${__dirname}/../out/notes/test-long-linked-notes/NoteB.md`, + 'utf8', + )).includes('evernote://'), + false, + ); + + }); + it('yaml tags list', async () => { const options: YarleOptions = { dateFormat: undefined, @@ -1121,3 +1167,35 @@ dateFormat: undefined, }); }); + +describe('Yarle error cases', async () => { + before(() => { + mockTimezone.register('Europe/London'); + + }); + + after(() => { + mockTimezone.unregister(); + + }); + + it('really long filePath', async () => { + const options: YarleOptions = { + enexSources: [ `${testDataFolder}test-justText.enex` ], + outputDir: 'out/longlonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglonglongfolderName', + templateFile: `${testDataFolder}full_template.templ`, + isMetadataNeeded: true, + outputFormat: OutputFormat.ObsidianMD, + skipEnexFileNameFromOutputPath: false, + + }; + try { + await yarle.dropTheRope(options); + assert.equal(true, false); + } catch (e: any) { + + assert.equal(e.message.startsWith('ENAMETOOLONG: name too long') || e.message.startsWith('EINVAL: invalid argument, mkdir'), true); + } + }); + +}); diff --git a/test/yarle.spec.ts b/test/yarle.spec.ts index 32b80ad7..b3a39957 100644 --- a/test/yarle.spec.ts +++ b/test/yarle.spec.ts @@ -24,6 +24,7 @@ describe('Yarle simple cases', async () => { afterEach(async () => { utils.clearMdNotesDistDir(); + }); const tests: Array = yarleTests;