Increasing size of buffer for first encoding guess

Addresses #7
SabakiHQ · Mar 12, 2020 · 3766246 · 3766246
1 parent ba02cce
commit 3766246
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/tokenize.js b/src/tokenize.js
@@ -46,7 +46,7 @@ exports.tokenizeBufferIter = function*(buffer, {encoding = null} = {}) {
 
   // Guess encoding
 
-  let detectedEncoding = jschardet.detect(buffer.slice(0, 100)).encoding
+  let detectedEncoding = jschardet.detect(buffer.slice(0, 1000)).encoding
   let contents = iconv.decode(buffer, detectedEncoding)
   let tokens = exports.tokenizeIter(contents)