All: Fixes #992: Allow non-ASCII chars when exporting MD and handle d…

…uplicate filenames

CliClient/package.json

@@ -68,7 +68,6 @@
     "tar": "^4.4.0",
     "tcp-port-used": "^0.1.2",
     "tkwidgets": "^0.5.26",
-    "unidecode": "^0.1.8",
     "url-parse": "^1.2.0",
     "uuid": "^3.0.1",
     "valid-url": "^1.0.9",

CliClient/run_test.sh

@@ -30,6 +30,7 @@ npm test tests-build/models_Folder.js
 npm test tests-build/models_Note.js
 npm test tests-build/models_Tag.js
 npm test tests-build/models_Setting.js
+npm test tests-build/pathUtils.js
 npm test tests-build/services_InteropService.js
 npm test tests-build/services_ResourceService.js
 npm test tests-build/urlUtils.js

CliClient/tests/pathUtils.js

@@ -0,0 +1,38 @@
+require('app-module-path').addPath(__dirname);
+
+const { friendlySafeFilename } = require('lib/path-utils.js');
+const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
+
+process.on('unhandledRejection', (reason, p) => {
+	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+describe('pathUtils', function() {
+
+	beforeEach(async (done) => {
+		done();
+	});
+
+	it('should create friendly safe filename', async (done) => {
+		const testCases = [
+			['生活', '生活'],
+			['not/good', 'not_good'],
+			['really/not/good', 'really_not_good'],
+			['con', '___'],
+			['no space at the end ', 'no space at the end'],
+			['nor dots...', 'nor dots'],
+			['thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong', 'thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong'],
+		];
+
+		for (let i = 0; i < testCases.length; i++) {
+			const t = testCases[i];
+			expect(friendlySafeFilename(t[0])).toBe(t[1]);
+		}
+
+		expect(!!friendlySafeFilename('')).toBe(true);
+		expect(!!friendlySafeFilename('...')).toBe(true);
+
+		done();
+	});
+
+});

CliClient/tests/services_InteropService.js

@@ -331,4 +331,30 @@ describe('services_InteropService', function() {
 			expect(obj.body).toBe(items[i].body);
 		}
 	}));
+
+	it('should export MD with unicode filenames', asyncTest(async () => {
+		const service = new InteropService();
+		let folder1 = await Folder.save({ title: 'folder1' });
+		let folder2 = await Folder.save({ title: 'ジョプリン' });
+		let note1 = await Note.save({ title: '生活', parent_id: folder1.id });
+		let note2 = await Note.save({ title: '生活', parent_id: folder1.id });
+		let note2b = await Note.save({ title: '生活', parent_id: folder1.id });
+		let note3 = await Note.save({ title: '', parent_id: folder1.id });
+		let note4 = await Note.save({ title: '', parent_id: folder1.id });
+		let note5 = await Note.save({ title: 'salut, ça roule ?', parent_id: folder1.id });
+		let note6 = await Note.save({ title: 'ジョプリン', parent_id: folder2.id });
+
+		const outDir = exportDir();
+
+		await service.export({ path: outDir, format: 'md' });
+
+		expect(await shim.fsDriver().exists(outDir + '/folder1/生活.md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (1).md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (2).md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled.md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled (1).md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/folder1/salut, ça roule _.md')).toBe(true);
+		expect(await shim.fsDriver().exists(outDir + '/ジョプリン/ジョプリン.md')).toBe(true);
+	}));
+
 });

ElectronClient/app/package.json

@@ -127,7 +127,6 @@
     "syswide-cas": "^5.1.0",
     "tar": "^4.4.4",
     "tcp-port-used": "^0.1.2",
-    "unidecode": "^0.1.8",
     "url-parse": "^1.4.1",
     "uuid": "^3.2.1",
     "valid-url": "^1.0.9",

ReactNativeClient/lib/fs-driver-base.js

@@ -1,3 +1,5 @@
+const { filename, fileExtension } = require('lib/path-utils');
+
 class FsDriverBase {
 
 	async isDirectory(path) {
@@ -19,6 +21,23 @@ class FsDriverBase {
 		return output;
 	}
 
+	async findUniqueFilename(name) {
+		let counter = 1;
+
+		let nameNoExt = filename(name, true);
+		let extension = fileExtension(name);
+		if (extension) extension = '.' + extension;
+		let nameToTry = nameNoExt + extension;
+		while (true) {
+			const exists = await this.exists(nameToTry);
+			if (!exists) return nameToTry;
+			nameToTry = nameNoExt + ' (' + counter + ')' + extension;
+			counter++;
+			if (counter >= 1000) nameToTry = nameNoExt + ' (' + ((new Date()).getTime()) + ')' + extension;
+			if (counter >= 10000) throw new Error('Cannot find unique title');
+		}
+	}
+
 }
 
 module.exports = FsDriverBase;

ReactNativeClient/lib/path-utils.js

@@ -1,3 +1,5 @@
+const { _ } = require('lib/locale');
+
 function dirname(path) {
 	if (!path) throw new Error('Path is empty');
 	let s = path.split(/\/|\\/);
@@ -11,9 +13,9 @@ function basename(path) {
 	return s[s.length - 1];
 }
 
-function filename(path) {
+function filename(path, includeDir = false) {
 	if (!path) throw new Error('Path is empty');
-	let output = basename(path);
+	let output = includeDir ? path : basename(path);
 	if (output.indexOf('.') < 0) return output;
 
 	output = output.split('.');
@@ -48,6 +50,47 @@ function safeFilename(e, maxLength = null, allowSpaces = false) {
 	return output.substr(0, maxLength);
 }
 
+let friendlySafeFilename_blackListChars = '/<>:\'"\\|?*';
+for (let i = 0; i < 32; i++) {
+	friendlySafeFilename_blackListChars += String.fromCharCode(i);
+}
+
+const friendlySafeFilename_blackListNames = [".", "..", "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"];
+
+function friendlySafeFilename(e, maxLength = null) {
+	if (maxLength === null) maxLength = 255;
+	if (!e || !e.replace) return _('Untitled');
+
+	let output = '';
+	for (let i = 0; i < e.length; i++) {
+		const c = e[i];
+		if (friendlySafeFilename_blackListChars.indexOf(c) >= 0) {
+			output += '_';
+		} else {
+			output += c;
+		}
+	}
+
+	if (output.length <= 4) {
+		if (friendlySafeFilename_blackListNames.indexOf(output.toUpperCase()) >= 0) {
+			output = '___';
+		}
+	}
+
+	while (output.length) {
+		const c = output[output.length - 1];
+		if (c === ' ' || c === '.') {
+			output = output.substr(0, output.length - 1);
+		} else {
+			break;
+		}
+	}
+
+	if (!output) return _('Untitled'); 
+
+	return output.substr(0, maxLength);
+}
+
 function toSystemSlashes(path, os = null) {
 	if (os === null) os = process.platform;
 	if (os === 'win32') return path.replace(/\//g, "\\");
@@ -62,4 +105,4 @@ function ltrimSlashes(path) {
 	return path.replace(/^\/+/, '');
 }
 
-module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes };
+module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, friendlySafeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes };

ReactNativeClient/lib/services/InteropService_Exporter_Md.js

@@ -1,10 +1,9 @@
 const InteropService_Exporter_Base = require('lib/services/InteropService_Exporter_Base');
-const { basename, filename, safeFilename } = require('lib/path-utils.js');
+const { basename, filename, friendlySafeFilename } = require('lib/path-utils.js');
 const BaseModel = require('lib/BaseModel');
 const Folder = require('lib/models/Folder');
 const Note = require('lib/models/Note');
 const { shim } = require('lib/shim');
-const unidecode = require('unidecode');
 
 class InteropService_Exporter_Md extends InteropService_Exporter_Base {
 
@@ -21,7 +20,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
 		let output = '';
 		while (true) {
 			if (item.type_ === BaseModel.TYPE_FOLDER) {
-				output = safeFilename(item.title, null, true) + '/' + output;
+				output = friendlySafeFilename(item.title, null, true) + '/' + output;
+				output = await shim.fsDriver().findUniqueFilename(output);
 			}
 			if (!item.parent_id) return output;
 			item = await Folder.load(item.parent_id);
@@ -32,7 +32,6 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
 	async processItem(ItemClass, item) {
 		if ([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER].indexOf(item.type_) < 0) return;
 
-		const filename = safeFilename(item.title, null, true);
 		const dirPath = this.destDir_ + '/' + (await this.makeDirPath_(item));
 
 		if (this.createdDirs_.indexOf(dirPath) < 0) {
@@ -41,7 +40,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
 		}
 
 		if (item.type_ === BaseModel.TYPE_NOTE) {
-			const noteFilePath = dirPath + '/' + safeFilename(unidecode(item.title), null, true) + '.md';
+			let noteFilePath = dirPath + '/' + friendlySafeFilename(item.title, null, true) + '.md';
+			noteFilePath = await shim.fsDriver().findUniqueFilename(noteFilePath);
 			const noteContent = await Note.serializeForEdit(item);
 			await shim.fsDriver().writeFile(noteFilePath, noteContent, 'utf-8');
 		}