update chardet

Author: gonejack

chardet/chardet.charamel.go

@@ -0,0 +1,98 @@
+//go:build charamel
+
+package chardet
+
+import (
+	"errors"
+
+	"github.com/gonejack/charamel"
+)
+
+func init() {
+	prefer(DetectEncodingByCharamel)
+}
+
+var encodings = []charamel.Encoding{
+	// UTF编码
+	charamel.UTF8,
+	charamel.UTF16,
+	charamel.UTF16BE,
+	charamel.UTF16LE,
+	charamel.UTF32,
+	charamel.UTF32BE,
+	charamel.UTF32LE,
+
+	// ASCII
+	charamel.ASCII,
+
+	// 西欧编码 (ISO-8859)
+	charamel.LATIN1,    // ISO-8859-1
+	charamel.ISO88592,  // ISO-8859-2
+	charamel.ISO88593,  // ISO-8859-3
+	charamel.ISO88594,  // ISO-8859-4
+	charamel.ISO88595,  // ISO-8859-5
+	charamel.ISO88596,  // ISO-8859-6
+	charamel.ISO88597,  // ISO-8859-7
+	charamel.ISO88598,  // ISO-8859-8
+	charamel.ISO88599,  // ISO-8859-9
+	charamel.ISO885910, // ISO-8859-10
+	charamel.ISO885911, // ISO-8859-11
+	charamel.ISO885913, // ISO-8859-13
+	charamel.ISO885914, // ISO-8859-14
+	charamel.ISO885915, // ISO-8859-15
+	charamel.ISO885916, // ISO-8859-16
+
+	// Windows编码 (CP1250-1258)
+	charamel.CP1250,
+	charamel.CP1251,
+	charamel.CP1252,
+	charamel.CP1253,
+	charamel.CP1254,
+	charamel.CP1255,
+	charamel.CP1256,
+	charamel.CP1257,
+	charamel.CP1258,
+
+	// 中文编码
+	charamel.GB2312,
+	charamel.GBK,
+	charamel.GB18030,
+	charamel.BIG5,
+	charamel.BIG5HKSCS,
+	charamel.HZ, // hz-gb-2312
+
+	// 日文编码
+	charamel.EUCJP,     // euc-jp
+	charamel.SHIFTJIS,  // shift_jis
+	charamel.ISO2022JP, // iso-2022-jp
+
+	// 韩文编码
+	charamel.EUCKR,     // euc-kr
+	charamel.CP949,     // windows-949
+	charamel.ISO2022KR, // iso-2022-kr
+
+	// 俄文编码
+	charamel.KOI8R, // koi8-r
+	charamel.KOI8U, // koi8-u
+	charamel.CP866, // cp866
+
+	// 泰文编码
+	charamel.TIS620, // tis-620
+	charamel.CP874,  // windows-874
+
+	// Mac编码
+	charamel.MACROMAN,    // macintosh
+	charamel.MACCYRILLIC, // x-mac-cyrillic
+}
+
+func DetectEncodingByCharamel(dat []byte) (string, error) {
+	d, err := charamel.NewDetector(encodings, 0.9)
+	if err != nil {
+		return "", err
+	}
+	v := d.Detect(dat)
+	if v == nil {
+		return "", errors.New("detect failed by github.com/gonejack/charamel")
+	}
+	return v.String(), nil
+}

go.mod

@@ -8,11 +8,13 @@ require (
 	github.com/alecthomas/kong v0.8.1
 	github.com/endeveit/enca v0.0.0-20160315071803-00fe968221ab
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
+	github.com/gonejack/charamel v1.0.1
 	github.com/wlynxg/chardet v1.0.0
 	golang.org/x/text v0.14.0
 )
 
 require (
+	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 )

go.sum

@@ -10,6 +10,8 @@ github.com/endeveit/enca v0.0.0-20160315071803-00fe968221ab h1:8sh8Pynho3gYrdzdb
 github.com/endeveit/enca v0.0.0-20160315071803-00fe968221ab/go.mod h1:p9sYlSrwy19GJyed1EXDwdZeL4rVBd1tPoPgDvs7U1Q=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/gonejack/charamel v1.0.1 h1:TKBCkAl0PoI+0s6jJJMRoLl2rLhuHL+BPcXvCCsa6qo=
+github.com/gonejack/charamel v1.0.1/go.mod h1:RQJBTqDLll8x8xAJvJAFhQSoOphw8NKZ+paIcDS0aLk=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -18,6 +20,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/wlynxg/chardet v1.0.0 h1:2gEgdmy/at4xIC+mOfNf1OFsb4LtnG9IcumfRXii/d0=
 github.com/wlynxg/chardet v1.0.0/go.mod h1:DgEUcneT6QieJ9qEhtRFOHWOjSNLPAo8lwUhjNopcFE=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=