From 07b724d65b86ef0d6c1e47fba608a6b5b6e24699 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Tue, 20 Nov 2018 00:42:21 +0000 Subject: [PATCH] All: Fixes #992: Allow non-ASCII chars when exporting MD and handle duplicate filenames --- CliClient/package-lock.json | 5 -- CliClient/package.json | 1 - CliClient/run_test.sh | 1 + CliClient/tests/pathUtils.js | 38 ++++++++++++++ CliClient/tests/services_InteropService.js | 26 ++++++++++ ElectronClient/app/package-lock.json | 5 -- ElectronClient/app/package.json | 1 - ReactNativeClient/lib/fs-driver-base.js | 19 +++++++ ReactNativeClient/lib/path-utils.js | 49 +++++++++++++++++-- .../services/InteropService_Exporter_Md.js | 10 ++-- 10 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 CliClient/tests/pathUtils.js diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index 7bfe810be41..d990616e94b 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -2890,11 +2890,6 @@ "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=" }, - "unidecode": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/unidecode/-/unidecode-0.1.8.tgz", - "integrity": "sha1-77swFTi8RSRqmsjFWdcvAVMFBT4=" - }, "uniq": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index df7d8cecb89..1ef37f8fab0 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -68,7 +68,6 @@ "tar": "^4.4.0", "tcp-port-used": "^0.1.2", "tkwidgets": "^0.5.26", - "unidecode": "^0.1.8", "url-parse": "^1.2.0", "uuid": "^3.0.1", "valid-url": "^1.0.9", diff --git a/CliClient/run_test.sh b/CliClient/run_test.sh index 2dfa8a3e78c..2b62bbe73d2 100755 --- a/CliClient/run_test.sh +++ b/CliClient/run_test.sh @@ -30,6 +30,7 @@ npm test tests-build/models_Folder.js npm test tests-build/models_Note.js npm test tests-build/models_Tag.js npm test tests-build/models_Setting.js +npm test tests-build/pathUtils.js npm test tests-build/services_InteropService.js npm test tests-build/services_ResourceService.js npm test tests-build/urlUtils.js diff --git a/CliClient/tests/pathUtils.js b/CliClient/tests/pathUtils.js new file mode 100644 index 00000000000..91378c40438 --- /dev/null +++ b/CliClient/tests/pathUtils.js @@ -0,0 +1,38 @@ +require('app-module-path').addPath(__dirname); + +const { friendlySafeFilename } = require('lib/path-utils.js'); +const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js'); + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + +describe('pathUtils', function() { + + beforeEach(async (done) => { + done(); + }); + + it('should create friendly safe filename', async (done) => { + const testCases = [ + ['生活', '生活'], + ['not/good', 'not_good'], + ['really/not/good', 'really_not_good'], + ['con', '___'], + ['no space at the end ', 'no space at the end'], + ['nor dots...', 'nor dots'], + ['thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong', 'thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong'], + ]; + + for (let i = 0; i < testCases.length; i++) { + const t = testCases[i]; + expect(friendlySafeFilename(t[0])).toBe(t[1]); + } + + expect(!!friendlySafeFilename('')).toBe(true); + expect(!!friendlySafeFilename('...')).toBe(true); + + done(); + }); + +}); \ No newline at end of file diff --git a/CliClient/tests/services_InteropService.js b/CliClient/tests/services_InteropService.js index d53f8f62813..bb9bd88f1e0 100644 --- a/CliClient/tests/services_InteropService.js +++ b/CliClient/tests/services_InteropService.js @@ -331,4 +331,30 @@ describe('services_InteropService', function() { expect(obj.body).toBe(items[i].body); } })); + + it('should export MD with unicode filenames', asyncTest(async () => { + const service = new InteropService(); + let folder1 = await Folder.save({ title: 'folder1' }); + let folder2 = await Folder.save({ title: 'ジョプリン' }); + let note1 = await Note.save({ title: '生活', parent_id: folder1.id }); + let note2 = await Note.save({ title: '生活', parent_id: folder1.id }); + let note2b = await Note.save({ title: '生活', parent_id: folder1.id }); + let note3 = await Note.save({ title: '', parent_id: folder1.id }); + let note4 = await Note.save({ title: '', parent_id: folder1.id }); + let note5 = await Note.save({ title: 'salut, ça roule ?', parent_id: folder1.id }); + let note6 = await Note.save({ title: 'ジョプリン', parent_id: folder2.id }); + + const outDir = exportDir(); + + await service.export({ path: outDir, format: 'md' }); + + expect(await shim.fsDriver().exists(outDir + '/folder1/生活.md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (1).md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (2).md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled.md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled (1).md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/folder1/salut, ça roule _.md')).toBe(true); + expect(await shim.fsDriver().exists(outDir + '/ジョプリン/ジョプリン.md')).toBe(true); + })); + }); \ No newline at end of file diff --git a/ElectronClient/app/package-lock.json b/ElectronClient/app/package-lock.json index ed11ff7415a..c7e2bcd28e7 100644 --- a/ElectronClient/app/package-lock.json +++ b/ElectronClient/app/package-lock.json @@ -6981,11 +6981,6 @@ "random-bytes": "~1.0.0" } }, - "unidecode": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/unidecode/-/unidecode-0.1.8.tgz", - "integrity": "sha1-77swFTi8RSRqmsjFWdcvAVMFBT4=" - }, "union-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", diff --git a/ElectronClient/app/package.json b/ElectronClient/app/package.json index 2cf945c5159..740c10eb978 100644 --- a/ElectronClient/app/package.json +++ b/ElectronClient/app/package.json @@ -127,7 +127,6 @@ "syswide-cas": "^5.1.0", "tar": "^4.4.4", "tcp-port-used": "^0.1.2", - "unidecode": "^0.1.8", "url-parse": "^1.4.1", "uuid": "^3.2.1", "valid-url": "^1.0.9", diff --git a/ReactNativeClient/lib/fs-driver-base.js b/ReactNativeClient/lib/fs-driver-base.js index 3f661bcf8c9..cbbb78dcd91 100644 --- a/ReactNativeClient/lib/fs-driver-base.js +++ b/ReactNativeClient/lib/fs-driver-base.js @@ -1,3 +1,5 @@ +const { filename, fileExtension } = require('lib/path-utils'); + class FsDriverBase { async isDirectory(path) { @@ -19,6 +21,23 @@ class FsDriverBase { return output; } + async findUniqueFilename(name) { + let counter = 1; + + let nameNoExt = filename(name, true); + let extension = fileExtension(name); + if (extension) extension = '.' + extension; + let nameToTry = nameNoExt + extension; + while (true) { + const exists = await this.exists(nameToTry); + if (!exists) return nameToTry; + nameToTry = nameNoExt + ' (' + counter + ')' + extension; + counter++; + if (counter >= 1000) nameToTry = nameNoExt + ' (' + ((new Date()).getTime()) + ')' + extension; + if (counter >= 10000) throw new Error('Cannot find unique title'); + } + } + } module.exports = FsDriverBase; \ No newline at end of file diff --git a/ReactNativeClient/lib/path-utils.js b/ReactNativeClient/lib/path-utils.js index 72dff2554e2..e9d67c124f9 100644 --- a/ReactNativeClient/lib/path-utils.js +++ b/ReactNativeClient/lib/path-utils.js @@ -1,3 +1,5 @@ +const { _ } = require('lib/locale'); + function dirname(path) { if (!path) throw new Error('Path is empty'); let s = path.split(/\/|\\/); @@ -11,9 +13,9 @@ function basename(path) { return s[s.length - 1]; } -function filename(path) { +function filename(path, includeDir = false) { if (!path) throw new Error('Path is empty'); - let output = basename(path); + let output = includeDir ? path : basename(path); if (output.indexOf('.') < 0) return output; output = output.split('.'); @@ -48,6 +50,47 @@ function safeFilename(e, maxLength = null, allowSpaces = false) { return output.substr(0, maxLength); } +let friendlySafeFilename_blackListChars = '/<>:\'"\\|?*'; +for (let i = 0; i < 32; i++) { + friendlySafeFilename_blackListChars += String.fromCharCode(i); +} + +const friendlySafeFilename_blackListNames = [".", "..", "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"]; + +function friendlySafeFilename(e, maxLength = null) { + if (maxLength === null) maxLength = 255; + if (!e || !e.replace) return _('Untitled'); + + let output = ''; + for (let i = 0; i < e.length; i++) { + const c = e[i]; + if (friendlySafeFilename_blackListChars.indexOf(c) >= 0) { + output += '_'; + } else { + output += c; + } + } + + if (output.length <= 4) { + if (friendlySafeFilename_blackListNames.indexOf(output.toUpperCase()) >= 0) { + output = '___'; + } + } + + while (output.length) { + const c = output[output.length - 1]; + if (c === ' ' || c === '.') { + output = output.substr(0, output.length - 1); + } else { + break; + } + } + + if (!output) return _('Untitled'); + + return output.substr(0, maxLength); +} + function toSystemSlashes(path, os = null) { if (os === null) os = process.platform; if (os === 'win32') return path.replace(/\//g, "\\"); @@ -62,4 +105,4 @@ function ltrimSlashes(path) { return path.replace(/^\/+/, ''); } -module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes }; \ No newline at end of file +module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, friendlySafeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes }; \ No newline at end of file diff --git a/ReactNativeClient/lib/services/InteropService_Exporter_Md.js b/ReactNativeClient/lib/services/InteropService_Exporter_Md.js index 2947ff5cd6c..13d22162b66 100644 --- a/ReactNativeClient/lib/services/InteropService_Exporter_Md.js +++ b/ReactNativeClient/lib/services/InteropService_Exporter_Md.js @@ -1,10 +1,9 @@ const InteropService_Exporter_Base = require('lib/services/InteropService_Exporter_Base'); -const { basename, filename, safeFilename } = require('lib/path-utils.js'); +const { basename, filename, friendlySafeFilename } = require('lib/path-utils.js'); const BaseModel = require('lib/BaseModel'); const Folder = require('lib/models/Folder'); const Note = require('lib/models/Note'); const { shim } = require('lib/shim'); -const unidecode = require('unidecode'); class InteropService_Exporter_Md extends InteropService_Exporter_Base { @@ -21,7 +20,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base { let output = ''; while (true) { if (item.type_ === BaseModel.TYPE_FOLDER) { - output = safeFilename(item.title, null, true) + '/' + output; + output = friendlySafeFilename(item.title, null, true) + '/' + output; + output = await shim.fsDriver().findUniqueFilename(output); } if (!item.parent_id) return output; item = await Folder.load(item.parent_id); @@ -32,7 +32,6 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base { async processItem(ItemClass, item) { if ([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER].indexOf(item.type_) < 0) return; - const filename = safeFilename(item.title, null, true); const dirPath = this.destDir_ + '/' + (await this.makeDirPath_(item)); if (this.createdDirs_.indexOf(dirPath) < 0) { @@ -41,7 +40,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base { } if (item.type_ === BaseModel.TYPE_NOTE) { - const noteFilePath = dirPath + '/' + safeFilename(unidecode(item.title), null, true) + '.md'; + let noteFilePath = dirPath + '/' + friendlySafeFilename(item.title, null, true) + '.md'; + noteFilePath = await shim.fsDriver().findUniqueFilename(noteFilePath); const noteContent = await Note.serializeForEdit(item); await shim.fsDriver().writeFile(noteFilePath, noteContent, 'utf-8'); }