better charset detection

Author: gonejack

chardet/chardet.go

@@ -0,0 +1,18 @@
+package chardet
+
+import (
+	gogschardet "github.com/gogs/chardet"
+	"github.com/wlynxg/chardet"
+)
+
+func detectEncoding(dat []byte) (string, error) {
+	r1 := chardet.Detect(dat)
+	if r1.Encoding > "" {
+		return r1.Encoding, nil
+	}
+	r2, err := gogschardet.NewTextDetector().DetectBest(dat)
+	if err != nil {
+		return "", err
+	}
+	return r2.Charset, nil
+}

chardet/chardet_golang.go

@@ -0,0 +1,5 @@
+package chardet
+
+func DetectEncoding(dat []byte) (string, error) {
+	return detectEncoding(dat)
+}

chardet/chardet_uchardet.go

@@ -0,0 +1,18 @@
+//go:build uchardet
+
+package chardet
+
+import (
+	"github.com/gonejack/transcode/chardet/uchardet"
+)
+
+func DetectEncoding(dat []byte) (string, error) {
+	dec := uchardet.NewChardet()
+	defer dec.Release()
+	if dec.Handle(dat) == 0 {
+		if v := dec.End(); v > "" {
+			return v, nil
+		}
+	}
+	return detectEncoding(dat)
+}

chardet/uchardet/uchardet.go

@@ -0,0 +1,68 @@
+package uchardet
+
+// https://github.com/centny/uchardet
+
+/*
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <uchardet/uchardet.h>
+// Apple Silicon (M1/M2/M3)
+#cgo darwin,arm64 CPPFLAGS: -I/opt/homebrew/opt/uchardet/include
+#cgo darwin,arm64 LDFLAGS: -L/opt/homebrew/opt/uchardet/lib -luchardet
+
+// Intel macOS (x86_64)
+#cgo darwin,amd64 CPPFLAGS: -I/usr/local/include
+#cgo darwin,amd64 LDFLAGS: -L/usr/local/lib -luchardet
+
+// Linux (assumes installed in /usr/local)
+#cgo linux CPPFLAGS: -I/usr/local/include
+#cgo linux LDFLAGS: -L/usr/local/lib -luchardet
+*/
+import "C"
+import "unsafe"
+
+// Chardet is the binding uchardet_t on libuchardet
+type Chardet struct {
+	det C.uchardet_t
+}
+
+// NewChardet is the default creator to create Chardet
+func NewChardet() *Chardet {
+	return &Chardet{
+		det: C.uchardet_new(),
+	}
+}
+
+// Release will free the Chardet
+func (c *Chardet) Release() {
+	C.uchardet_delete(c.det)
+}
+
+// Handle will process the data slice
+func (c *Chardet) Handle(buf []byte) int {
+	var data = (*C.char)(unsafe.Pointer(&buf[0]))
+	var dlen = C.size_t(len(buf))
+	return int(C.uchardet_handle_data(c.det, data, dlen))
+}
+
+// Reset encoding detector.
+func (c *Chardet) Reset() {
+	C.uchardet_reset(c.det)
+}
+
+// End is ending the process and return the encoding name
+func (c *Chardet) End() string {
+	C.uchardet_data_end(c.det)
+	return cstring(C.uchardet_get_charset(c.det))
+}
+
+func cstring(cs *C.char) string {
+	clen := C.strlen(cs)
+	if clen < 1 {
+		return ""
+	}
+	buf := make([]byte, clen+1)
+	C.strcpy((*C.char)(unsafe.Pointer(&buf[0])), cs)
+	return string(buf[:clen])
+}

go.mod

@@ -1,9 +1,17 @@
 module github.com/gonejack/transcode
 
-go 1.21
+go 1.24.0
+
+toolchain go1.24.3
 
 require (
 	github.com/alecthomas/kong v0.8.1
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
+	github.com/wlynxg/chardet v1.0.0
 	golang.org/x/text v0.14.0
 )
+
+require (
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+)

go.sum

@@ -1,14 +1,28 @@
 github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
 github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA=
-github.com/alecthomas/kong v0.8.0 h1:ryDCzutfIqJPnNn0omnrgHLbAggDQM2VWHikE1xqK7s=
-github.com/alecthomas/kong v0.8.0/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
 github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY=
 github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
 github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
 github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/wlynxg/chardet v1.0.0 h1:2gEgdmy/at4xIC+mOfNf1OFsb4LtnG9IcumfRXii/d0=
+github.com/wlynxg/chardet v1.0.0/go.mod h1:DgEUcneT6QieJ9qEhtRFOHWOjSNLPAo8lwUhjNopcFE=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=