Merge branch 'develop' of github.com:postmanlabs/newman into greenkee…

…per/xmlbuilder-11.0.0
NikolozCh · Feb 20, 2019 · f15703d · f15703d
2 parents d51e293 + 1d47aa2
commit f15703d
Show file tree

Hide file tree

Showing 15 changed files with 1,976 additions and 253 deletions.
diff --git a/CHANGELOG.yaml b/CHANGELOG.yaml
@@ -1,4 +1,9 @@
-# Newman Changelog
+master:
+  fixed bugs:
+    - '#1871 Trimmed incoming JSON string to remove byte order marks and other such artefacts.'
+  new features:
+    - 'Added automatic (best guess) encoding detection of UTF-16LE and ISO-8859-1'
+
 4.3.1:
   date: 2018-12-26
   fixed bugs:

diff --git a/README.md b/README.md
@@ -533,13 +533,22 @@ To use Newman in Docker check our [docker documentation](https://github.com/post
 
 ## Compatibility
 
+### NodeJS
+
 |      Newman       |    Node    |
 |:-----------------:|:----------:|
 |       v3.x        |  >= v4.x   |
 |       v4.x        |  >= v6.x   |
 
 The current Node version compatibility can also be seen from the `engines.node` property in [package.json](https://github.com/postmanlabs/newman/blob/develop/package.json)
 
+### File Encoding
+
+Newman attempts to detect file encoding for files that are provided as
+command line input. However, it mostly relies on NodeJS and the underlying
+operating system to do the heavy lifting. Currently, `ASCII`, `UTF-8`, `UTF-16LE`
+and `ISO-8859-1` are the only ones that are detection assisted.
+
 [back to top](#table-of-contents)
 
 ## Contributing

diff --git a/lib/config/rc-file.js b/lib/config/rc-file.js
@@ -3,7 +3,8 @@ var _ = require('lodash'),
     fs = require('fs'),
     join = require('path').join,
     async = require('async'),
-    parseJson = require('parse-json'),
+    util = require('../util'),
+    liquidJSON = require('liquid-json'),
 
     /**
      * Name of the directory that contains the file denoted by FILE_NAME.
@@ -40,9 +41,9 @@ module.exports.load = (callback) => {
             if (err) {
                 return cb(null, {}); // err masked to avoid overpopulating terminal with missing .newmanrc messages
             }
-            data && data.toString && (data = data.toString());
+            data && data.toString && (data = data.toString(util.detectEncoding(data)).trim());
             try {
-                return cb(null, parseJson(data));
+                return cb(null, liquidJSON.parse(data));
             }
             catch (e) {
                 return cb(_.set(e, 'help', `The file at ${path} contains invalid data.`));

diff --git a/lib/reporters/cli/index.js b/lib/reporters/cli/index.js
@@ -14,6 +14,8 @@ var _ = require('lodash'),
     DOT = '.',
     E = '',
 
+    CACHED_TIMING_PHASE = '(cache)',
+
     PostmanCLIReporter,
     extractSNR;
 
@@ -160,15 +162,17 @@ PostmanCLIReporter = function (emitter, reporterOptions, options) {
         }
 
         var timingTable,
+            timingPhases,
             size = o.response && o.response.size(),
-            timingPhases = o.response && util.beautifyTime(sdk.Response.timingPhases(o.response.timings)),
             timingHeaders = {
+                prepare: 'prepare',
                 wait: 'wait',
                 dns: 'dns-lookup',
                 tcp: 'tcp-handshake',
                 secureHandshake: 'ssl-handshake',
                 firstByte: 'transfer-start',
                 download: 'download',
+                process: 'process',
                 total: 'total'
             };
 
@@ -178,17 +182,19 @@ PostmanCLIReporter = function (emitter, reporterOptions, options) {
             util.filesize(size), util.prettyms(o.response.responseTime));
 
         // print timing info of the request
-        if (options.verbose && timingPhases) {
+        if (options.verbose && o.response && o.response.timings) {
+            timingPhases = util.beautifyTime(sdk.Response.timingPhases(o.response.timings));
+
             timingTable = new Table({
                 chars: _.defaults({ mid: '', middle: '' }, cliUtils.cliTableTemplate_Blank),
                 colAligns: _.fill(Array(_.size(timingPhases)), 'left'),
                 style: { 'padding-left': 2 }
             });
 
-            timingPhases = _.transform(timingHeaders, (result, value, key) => {
-                if (timingPhases[key]) {
-                    result.headers.push(colors.white(value));
-                    result.values.push(colors.log(timingPhases[key]));
+            timingPhases = _.transform(timingHeaders, (result, header, key) => {
+                if (_.has(timingPhases, key)) {
+                    result.headers.push(colors.white(header));
+                    result.values.push(colors.log(timingPhases[key] || CACHED_TIMING_PHASE));
                 }
             }, { headers: [], values: [] });
 
@@ -198,7 +204,7 @@ PostmanCLIReporter = function (emitter, reporterOptions, options) {
             // add time of phases in the table
             timingTable.push(timingPhases.values);
 
-            print(LF + timingTable + LF);
+            print(LF + timingTable + LF + LF);
         }
     });
 

diff --git a/lib/run/options.js b/lib/run/options.js
@@ -3,7 +3,7 @@ var _ = require('lodash'),
     Collection = require('postman-collection').Collection,
     VariableScope = require('postman-collection').VariableScope,
     transformer = require('postman-collection-transformer'),
-    parseJson = require('parse-json'),
+    liquidJSON = require('liquid-json'),
     parseCsv = require('csv-parse'),
     util = require('../util'),
     config = require('../config'),
@@ -270,7 +270,7 @@ var _ = require('lodash'),
                 async.waterfall([
                     (cb) => {
                         try {
-                            return cb(null, parseJson(data.trim()));
+                            return cb(null, liquidJSON.parse(data.trim()));
                         }
                         catch (e) {
                             return cb(null, undefined); // e masked to avoid displaying JSON parse errors for CSV files

diff --git a/lib/util.js b/lib/util.js
@@ -2,10 +2,11 @@ var fs = require('fs'),
     url = require('url'),
 
     _ = require('lodash'),
-    prettyms = require('pretty-ms'),
+    chardet = require('chardet'),
     filesize = require('filesize'),
+    prettyms = require('pretty-ms'),
+    liquidJSON = require('liquid-json'),
     request = require('postman-request'),
-    parseJson = require('parse-json'),
 
     util,
     version = require('../package.json').version,
@@ -19,6 +20,19 @@ var fs = require('fs'),
      */
     FILESIZE_OPTIONS = { spacer: '' },
 
+    /**
+     * Maps the charset returned by chardet to node buffer ones
+     *
+     * @constant
+     * @type {Object}
+     */
+    CHARDET_BUFF_MAP = {
+        ASCII: 'ascii',
+        'UTF-8': 'utf8',
+        'UTF-16LE': 'utf16le',
+        'ISO-8859-1': 'latin1'
+    },
+
     POSTMAN_API_HOST = 'api.getpostman.com',
 
     POSTMAN_API_URL = 'https://' + POSTMAN_API_HOST,
@@ -78,33 +92,10 @@ util = {
      * @returns {Object} - {event1: time1, event2: time2, ...} (time in string with appropriate unit)
      */
     beautifyTime: function (obj) {
-        // bail out if object is not given
-        if (!obj) { return; }
-
-        let i,
-            intervals = {
-                m: 6e7,
-                s: 1e6,
-                ms: 1e3,
-                ns: 1
-            };
-
-        return _.reduce(obj, (beautified, value, key) => {
-            const nanoSeconds = Math.floor(value * 1000);
-            var calc;
-
-            for (i in intervals) {
-                calc = Math.floor(nanoSeconds / intervals[i]);
-
-                if (calc > 0) {
-                    beautified[key] = `${calc}${i}`;
-
-                    return beautified;
-                }
-            }
-
-            return beautified;
-        }, {});
+        return _.forEach(obj, (value, key) => {
+            // convert only non-zero values
+            value && (obj[key] = this.prettyms(value));
+        });
     },
 
     /**
@@ -137,6 +128,16 @@ util = {
         return chain.join(_.isString(separator) ? separator : SEP);
     },
 
+    /**
+     * Given a buffer, it tries to match relevant encoding of the buffer.
+     *
+     * @param {Buffer} buff - Buffer for which encoding needs to be determined
+     * @returns {String|undefined} - Detected encoding of the given buffer
+     */
+    detectEncoding: function (buff) {
+        return CHARDET_BUFF_MAP[chardet.detect(buff)];
+    },
+
     /**
      * Loads JSON data from the given location.
      *
@@ -173,7 +174,7 @@ util = {
                 }
 
                 try {
-                    _.isString(body) && (body = parseJson(body.trim()));
+                    _.isString(body) && (body = liquidJSON.parse(body.trim()));
                 }
                 catch (e) {
                     return callback(_.set(e, 'help', `the url "${location}" did not provide valid JSON data`));
@@ -206,7 +207,7 @@ util = {
                 }
 
                 try {
-                    value = parseJson(value.toString());
+                    value = liquidJSON.parse(value.toString(util.detectEncoding(value)).trim());
                 }
                 catch (e) {
                     return callback(_.set(e, 'help', `the file at ${location} does not contain valid JSON data`));
@@ -241,7 +242,7 @@ util = {
                     return callback(err);
                 }
 
-                return callback(null, value.toString());
+                return callback(null, value.toString(util.detectEncoding(value)));
             });
     },