Added html encode option

Author: mallocator

index.js

@@ -6,6 +6,7 @@ var path = require('path');
 var _ = require('lodash');
 var flat = require('flat');
 var gutil = require('gulp-util');
+var he = require('he');
 var through = require('through2');
 
 
@@ -38,49 +39,67 @@ var defaults = {
   ignoreErrors: false,
   dryRun: false,
   includeOriginal: false,
-  ignoreTokens: false
+  ignoreTokens: false,
+  encodeEntities: true
 };
 
+/**
+ * A helper function to test whether an option was set to true or the value matches the options regular expression.
+ * @param {boolean|string|RegExp} needle
+ * @param {String} haystack
+ * @returns {boolean}
+ */
+function trueOrMatch(needle, haystack) {
+  if (needle === true) {
+    return true;
+  }
+  if (needle instanceof RegExp && needle.test(haystack)) {
+    return true;
+  }
+  return !!(needle instanceof String && haystack.indexOf(needle) !== -1);
+
+}
+
+
 /**
  * Loads the dictionaries from the locale directory.
  * @param {Object} options
  */
 function load(options) {
   if (cache[options.locales]) {
-    dictionaries = cache[options.locales];
-  } else {
-    try {
-      var files = fs.readdirSync(options.locales);
-      for (var i in files) {
-        var file = files[i];
-        switch (path.extname(file)) {
-          case '.json':
-          case '.js':
-            dictionaries[path.basename(file, path.extname(file))] = flat(require(path.join(process.cwd(), options.locales, file)));
-            break;
-          case '.ini':
-            var iniData = fs.readFileSync(path.join(process.cwd(), options.locales, file));
-            dictionaries[path.basename(file, path.extname(file))] = flat(ini2json(iniData));
-            break;
-          case '.csv':
-            var csvData = fs.readFileSync(path.join(process.cwd(), options.locales, file));
-            dictionaries[path.basename(file, path.extname(file))] = csv2json(csvData);
-            break;
-        }
-      }
-      if (options.cache) {
-        cache[options.locales] = dictionaries;
+    return dictionaries = cache[options.locales];
+  }
+  try {
+    var files = fs.readdirSync(options.locales);
+    for (var i in files) {
+      var file = files[i];
+      switch (path.extname(file)) {
+        case '.json':
+        case '.js':
+          dictionaries[path.basename(file, path.extname(file))] = flat(require(path.join(process.cwd(), options.locales, file)));
+          break;
+        case '.ini':
+          var iniData = fs.readFileSync(path.join(process.cwd(), options.locales, file));
+          dictionaries[path.basename(file, path.extname(file))] = flat(ini2json(iniData));
+          break;
+        case '.csv':
+          var csvData = fs.readFileSync(path.join(process.cwd(), options.locales, file));
+          dictionaries[path.basename(file, path.extname(file))] = csv2json(csvData);
+          break;
       }
-    } catch (e) {
-      throw new Error('No translation dictionaries have been found!');
     }
+    if (options.cache) {
+      cache[options.locales] = dictionaries;
+    }
+  } catch (e) {
+    throw new Error('No translation dictionaries have been found!');
   }
 }
 
 /**
  * Splits a line from an ini file into 2. Any subsequent '=' are ignored.
- * @param {string} line
- * @returns {string[]}
+ * @param {String} line
+ * @returns {String[]}
  */
 function splitIniLine(line) {
   var separator = line.indexOf('=');
@@ -95,7 +114,7 @@ function splitIniLine(line) {
 
 /**
  * Simple conversion helper to get a json file from an ini file.
- * @param {string} iniData
+ * @param {String} iniData
  * @returns {{}}
  */
 function ini2json(iniData) {
@@ -127,8 +146,8 @@ function ini2json(iniData) {
 
 /**
  * Converts a line of a CSV file to an array of strings, omitting empty fields.
- * @param {string} line
- * @returns {string[]}
+ * @param {String} line
+ * @returns {String[]}
  */
 function splitCsvLine(line) {
   if (!line.trim().length) {
@@ -163,7 +182,7 @@ function splitCsvLine(line) {
 
 /**
  * Simple conversion helper to get a json file from a csv file.
- * @param {string} csvData
+ * @param {String} csvData
  * @returns {Object}
  */
 function csv2json(csvData) {
@@ -187,9 +206,9 @@ function csv2json(csvData) {
 /**
  * Performs the actual translation from a tokenized source to the final content.
  * @param {Object} options
- * @param {string} contents
+ * @param {String} contents
  * @param {number} copied
- * @param {string} filePath
+ * @param {String} filePath
  * @returns {Object}
  */
 function translate(options, contents, copied, filePath) {
@@ -205,7 +224,7 @@ function translate(options, contents, copied, filePath) {
     throw new Error('No translation dictionaries available to create any files!');
   }
   var i = contents.indexOf(options.delimiter.prefix);
-  if (!(options.ignoreTokens === true || options.ignoreTokens instanceof RegExp && options.ignoreTokens.test(filePath))) {
+  if (!trueOrMatch(options.ignoreTokens, filePath)) {
     while ((i !== -1)) {
       var endMatch, length, token, key;
       var tail = contents.substr(i);
@@ -226,8 +245,12 @@ function translate(options, contents, copied, filePath) {
       for (var lang in processed) {
         processed[lang] += contents.substring(copied, i);
         if (dictionaries[lang][key] !== undefined) {
-          processed[lang] += dictionaries[lang][key];
-        } else if (options.warn) {
+          if (trueOrMatch(options.encodeEntities, filePath)) {
+            processed[lang] += he.encode(dictionaries[lang][key], { useNamedReferences: true });
+          } else {
+            processed[lang] += dictionaries[lang][key];
+          }
+        } else if (trueOrMatch(options.warn, filePath)) {
           gutil.log('Missing translation of language', lang, 'for key', key, 'in file', filePath);
         }
         processed[lang] += contents.substring(i + length, next == -1 ? contents.length : next);
@@ -315,18 +338,18 @@ module.exports = function(options) {
 
     try {
       var files = replace(file, options);
-      if (options.dryRun === true || options.dryRun instanceof RegExp && options.dryRun.test(file.path)) {
+      if (trueOrMatch(options.dryRun, file.path)) {
         this.push(file);
       } else {
-        if (options.includeOriginal === true || options.includeOriginal instanceof RegExp && options.includeOriginal.test(file.path)) {
+        if (trueOrMatch(options.includeOriginal, file.path)) {
           this.push(file);
         }
         for (var i in files) {
           this.push(files[i]);
         }
       }
     } catch (err) {
-      if (!options.ignoreErrors) {
+      if (!trueOrMatch(options.ignoreErrors, file.path)) {
         this.emit('error', new gutil.PluginError('gulp-international', err));
       }
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gulp-international",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "A gulp plugin that creates multi language versions of your source files",
   "license": "Apache-2.0",
   "homepage": "http://github.com/mallocator/gulp-international",
@@ -46,6 +46,7 @@
   "dependencies": {
     "flat": "^2.0.0",
     "gulp-util": "^3.0.1",
+    "he": "^0.5.0",
     "lodash": "^4.6.1",
     "through2": "^2.0.1"
   },

readme.md

@@ -155,12 +155,23 @@ The opposite of the whitelist. Any language specified here will be ignored durin
 with a string or multiple with an array of strings.
 
 
+### encodeEntities
+
+Type: boolean|RegExp|String  
+Default: ```true```
+
+Any non utf8 characters that do not map to html are replaced with html entities if this option is enabled. A translation such as "über"
+would be replaced with "über". If the setting is a regular expression then only files with a matching (original) filename will be 
+escaped. If the option is instead a string then the filename is searched for this substring.
+
+
 ### warn
 
-Type: boolean  
+Type: boolean|RegExp|String  
 Default: ```true```
 
-This enables warnings to be printed out if any tokens are missing.
+This enables warnings to be printed out if any tokens are missing. If the setting is a regular expression then only files with a matching 
+(original) filename will throw warnings. If the option is instead a string then the filename is searched for this substring.
 
 
 ### cache
@@ -174,20 +185,22 @@ on your configuration you might want to disable caching based on how your livere
 
 ### ignoreErrors
 
-Type: boolean  
+Type: boolean|RegExp|String  
 Default: ```false```
 
-Allows to disable throwing of any errors that might occur so that the pipe will continue executing.
+Allows to disable throwing of any errors that might occur so that the pipe will continue executing. If the setting is a regular expression 
+then only files with a matching (original) filename will ignore errors. If the option is instead a string then the filename is searched for 
+this substring.
 
 
 ### dryRun
 
-Type: boolean|RegExp  
+Type: boolean|RegExp|String  
 Default: ```false```
 
 When set to true the plugin will perform all operations for a translation, but will pass on the original file along the pipe instead
 of the newly generated ones. If the setting is a regular expression then only files with a matching (original) filename will be 
-ignored.
+ignored. If the option is instead a string then the filename is searched for this substring.
 
 Flow graph:
 ```
@@ -202,7 +215,7 @@ Default: ```false```
 
 When set to true the plugin will ignore all tokens, but still create new files as if they were different for each language. This
 differs from a dryRun, which would instead pass on the original file. If the setting is a regular expression then only files 
-with a matching (original) filename will be ignored.
+with a matching (original) filename will be ignored. If the option is instead a string then the filename is searched for this substring.
 
 Flow graph:
 ```
@@ -215,8 +228,9 @@ source.file -> source-lang1.file -> original content
 Type: boolean|RegExp  
 Default: ```false```
 
-When set to true the original file is passed along the pipe along with all translated files. If the setting is a 
-regular expression then only files with a matching (original) filename will be included.
+When set to true the original file is passed along the pipe along with all translated files. If the setting is a regular expression then 
+only files with a matching (original) filename will be included. If the option is instead a string then the filename is searched for this 
+substring.
 
 Flow graph:
 ```

test/index.test.js

@@ -53,7 +53,7 @@ describe('gulp-international', () => {
 
 
     it('should not process languages on the blacklist', done => {
-      var options = { locales: 'test/locales', blacklist: 'en_US' };
+      var options = { locales: 'test/locales', blacklist: 'en_US', encodeEntities: false };
       helper(options, files => {
         expect(files.length).to.equal(3);
         expect(files[0].contents.toString('utf8')).to.equal('<html><body><h1>Inhalt1</h1></body></html>');
@@ -173,6 +173,16 @@ describe('gulp-international', () => {
         done();
       });
     });
+
+
+    it('should translate html entities if the option is enabled (default)', done => {
+      var content = '<html><body><h1>R.entity</h1></body></html>';
+      var options = { locales: 'test/locales', whitelist: 'de_DE' };
+      helper(options, content, files => {
+        expect(files[0].contents.toString()).to.equal('<html><body><h1>S&uuml;p&auml;r Sp&auml;&szlig;&ouml;!</h1></body></html>');
+        done();
+      });
+    });
   });
 
   describe('Error cases', () => {
@@ -345,7 +355,8 @@ describe('gulp-international', () => {
 `;
       var options = {
         locales: 'test/locales',
-        whitelist: ['en_US', 'pt-BR']
+        whitelist: ['en_US', 'pt-BR'],
+        encodeEntities: false
       };
       helper(options, content, files => {
         expect(files.length).to.equal(2);

test/locales/de_DE.ini

@@ -1,4 +1,5 @@
 token1=Inhalt1
+entity=Süpär Späßö!
 
 [section1]
 token2=Inhalt2