diff --git a/src/tokenize.js b/src/tokenize.js index 460d0ae..4888e69 100644 --- a/src/tokenize.js +++ b/src/tokenize.js @@ -46,7 +46,7 @@ exports.tokenizeBufferIter = function*(buffer, {encoding = null} = {}) { // Guess encoding - let detectedEncoding = jschardet.detect(buffer.slice(0, 100)).encoding + let detectedEncoding = jschardet.detect(buffer.slice(0, 1000)).encoding let contents = iconv.decode(buffer, detectedEncoding) let tokens = exports.tokenizeIter(contents)