extract: Add zstd support

Testing[1] suggests that zstd compression might be better than xz, with
decompression being much faster than xz, so it's worth considering it
for our bundles.

There is no native support for zstd in golang, so this makes use of
https://pkg.go.dev/github.com/klauspost/compress/zstd for handling this
format. An alternative would be https://github.com/DataDog/zstd which
provides performance numbers, but which is not a pure go implementation,
which is why I picked the other one.

github.com/h2non/filetype does not support zstd type detection, so
a bit of additional code had to be added for that. Before registering a
zstd type, we check if it's supported, this will avoid potential issues
after h2non/filetype#98 is merged.

When compressed with `zstd --ultra -22 -T0`, the 4.6.9 bundle is 2.2G in
size, and uncompresses in about 1 minute. The xz-compressed bundle is
2.6G in size, and takes 3 minutes to uncompress.

An existing bundle can be converted to zstd for testing purposes with:
xzcat $bundle |zstd  -  -o crc_libvirt_xxx.crcbundle

This fixes crc-org#1340 for now.

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/h2non/filetype v1.1.0
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
 	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/klauspost/compress v1.11.7
 	github.com/libvirt/libvirt-go-xml v6.10.0+incompatible
 	github.com/magiconair/properties v1.8.4 // indirect
 	github.com/mattn/go-colorable v0.1.8

go.sum

@@ -548,6 +548,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.7.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
+github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=

pkg/extract/extract.go

@@ -3,6 +3,7 @@ package extract
 import (
 	"archive/tar"
 	"archive/zip"
+	"bytes"
 	"compress/gzip"
 	"fmt"
 	"io"
@@ -13,6 +14,7 @@ import (
 	"github.com/cheggaaa/pb/v3"
 	"github.com/code-ready/crc/pkg/crc/logging"
 	"github.com/h2non/filetype"
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 	"github.com/xi2/xz"
 	terminal "golang.org/x/term"
@@ -28,6 +30,25 @@ func Uncompress(tarball, targetDir string, showProgress bool) ([]string, error)
 	return uncompress(tarball, targetDir, nil, showProgress && terminal.IsTerminal(int(os.Stdout.Fd())))
 }
 
+func zstdMatcher(data []byte) bool {
+	/* magic value from shared-mime-info */
+	var zstdMagic = []byte{0x28, 0xB5, 0x2F, 0xFD}
+
+	return bytes.HasPrefix(data, zstdMagic)
+}
+
+func registerZstdMatcher() {
+	if filetype.IsSupported("zst") {
+		return
+	}
+	zstdType := filetype.AddType("zst", "application/zstd")
+	filetype.AddMatcher(zstdType, zstdMatcher)
+}
+
+func init() {
+	registerZstdMatcher()
+}
+
 func uncompress(tarball, targetDir string, fileFilter func(string) bool, showProgress bool) ([]string, error) {
 	logging.Debugf("Uncompressing %s to %s", tarball, targetDir)
 
@@ -60,6 +81,12 @@ func uncompress(tarball, targetDir string, fileFilter func(string) bool, showPro
 			return nil, err
 		}
 		return untar(reader, targetDir, fileFilter, showProgress)
+	case filetype.Is(header, "zst"):
+		reader, err := zstd.NewReader(file)
+		if err != nil {
+			return nil, err
+		}
+		return untar(reader, targetDir, fileFilter, showProgress)
 	case filetype.Is(header, "gz"):
 		reader, err := gzip.NewReader(file)
 		if err != nil {

pkg/extract/extract_test.go

@@ -28,6 +28,7 @@ var (
 		"test.tar.gz",
 		"test.zip",
 		"test.tar.xz",
+		"test.tar.zst",
 	}
 )