Skip to content

Commit

Permalink
Remove test buffers idea
Browse files Browse the repository at this point in the history
Close #7
  • Loading branch information
yishn committed Mar 15, 2020
1 parent 69b1260 commit 5c22007
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 34 deletions.
22 changes: 2 additions & 20 deletions src/jschardet.js
Original file line number Diff line number Diff line change
@@ -1,30 +1,12 @@
let jschardet = (() => {
module.exports = (() => {
try {
let m = require('jschardet')
if (m == null) throw new Error()

return m
} catch (err) {
return {
detect: () => ({encoding: 'UTF-8'}),
detectBuffers() {
return this.detect()
}
detect: () => ({encoding: 'UTF-8'})
}
}
})()

module.exports = {
detectBuffers(buffers) {
let u = new jschardet.UniversalDetector()
u.reset()

for (let buf of buffers) {
u.feed(buf.toString('binary'))
}

u.close()
return u.result
},
...jschardet
}
14 changes: 0 additions & 14 deletions src/tokenize.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ exports.tokenizeBufferIter = function*(buffer, {encoding = null} = {}) {
// Search for encoding

let prelude = []
let testBuffers = []

while (true) {
let next = tokens.next()
Expand All @@ -72,22 +71,9 @@ exports.tokenizeBufferIter = function*(buffer, {encoding = null} = {}) {
) {
encoding = unescapeString(value.slice(1, -1))
break
} else if (
type === 'c_value_type' &&
lastToken != null &&
lastToken.type === 'prop_ident' &&
encodingDetectionProps.includes(lastToken.value)
) {
testBuffers.push(iconv.encode(value.slice(1, -1), detectedEncoding))

if (testBuffers.reduce((sum, buf) => sum + buf.length, 0) > 100) break
}
}

if (encoding == null && testBuffers.length > 0) {
encoding = jschardet.detectBuffers(testBuffers).encoding
}

if (
encoding != null &&
encoding != detectedEncoding &&
Expand Down

0 comments on commit 5c22007

Please sign in to comment.