zstd: Skip entropy on random data

zstd/blockenc.go

@@ -444,9 +444,9 @@ func fuzzFseEncoder(data []byte) int {
 }
 
 // encode will encode the block and append the output in b.output.
-func (b *blockEnc) encode(raw bool) error {
+func (b *blockEnc) encode(raw, rawAllLits bool) error {
 	if len(b.sequences) == 0 {
-		return b.encodeLits(raw)
+		return b.encodeLits(rawAllLits)
 	}
 	// We want some difference
 	if len(b.literals) > (b.size - (b.size >> 5)) {

zstd/encoder.go

@@ -280,7 +280,7 @@ func (e *Encoder) nextBlock(final bool) error {
 			// If we got the exact same number of literals as input,
 			// assume the literals cannot be compressed.
 			if len(src) != len(blk.literals) || len(src) != e.o.blockSize {
-				err = blk.encode(e.o.noEntropy)
+				err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy)
 			}
 			switch err {
 			case errIncompressible:
@@ -491,7 +491,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
 		if len(blk.literals) != len(src) || len(src) != e.o.blockSize {
 			// Output directly to dst
 			blk.output = dst
-			err = blk.encode(e.o.noEntropy)
+			err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy)
 		}
 
 		switch err {
@@ -528,7 +528,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
 			// If we got the exact same number of literals as input,
 			// assume the literals cannot be compressed.
 			if len(blk.literals) != len(todo) || len(todo) != e.o.blockSize {
-				err = blk.encode(e.o.noEntropy)
+				err = blk.encode(e.o.noEntropy, !e.o.allLitEntropy)
 			}
 
 			switch err {

zstd/encoder_options.go

@@ -12,16 +12,18 @@ type EOption func(*encoderOptions) error
 
 // options retains accumulated state of multiple options.
 type encoderOptions struct {
-	concurrent   int
-	level        EncoderLevel
-	single       *bool
-	pad          int
-	blockSize    int
-	windowSize   int
-	crc          bool
-	fullZero     bool
-	noEntropy    bool
-	customWindow bool
+	concurrent      int
+	level           EncoderLevel
+	single          *bool
+	pad             int
+	blockSize       int
+	windowSize      int
+	crc             bool
+	fullZero        bool
+	noEntropy       bool
+	allLitEntropy   bool
+	customWindow    bool
+	customALEntropy bool
 }
 
 func (o *encoderOptions) setDefault() {
@@ -207,6 +209,10 @@ func WithEncoderLevel(l EncoderLevel) EOption {
 				o.windowSize = 16 << 20
 			}
 		}
+		if !o.customALEntropy {
+			o.allLitEntropy = l > SpeedFastest
+		}
+
 		return nil
 	}
 }
@@ -221,6 +227,18 @@ func WithZeroFrames(b bool) EOption {
 	}
 }
 
+// WithAllLitEntropyCompression will apply entropy compression if no matches are found.
+// Disabling this will skip incompressible data faster, but in cases with no matches but
+// skewed character distribution compression is lost.
+// Default value depends on the compression level selected.
+func WithAllLitEntropyCompression(b bool) EOption {
+	return func(o *encoderOptions) error {
+		o.customALEntropy = true
+		o.allLitEntropy = b
+		return nil
+	}
+}
+
 // WithNoEntropyCompression will always skip entropy compression of literals.
 // This can be useful if content has matches, but unlikely to benefit from entropy
 // compression. Usually the slight speed improvement is not worth enabling this.

zstd/encoder_test.go

@@ -927,9 +927,9 @@ func BenchmarkEncoder_EncodeAllPi(b *testing.B) {
 	}
 }
 
-func BenchmarkRandomEncodeAllFastest(b *testing.B) {
+func BenchmarkRandom4KEncodeAllFastest(b *testing.B) {
 	rng := rand.New(rand.NewSource(1))
-	data := make([]byte, 10<<20)
+	data := make([]byte, 4<<10)
 	for i := range data {
 		data[i] = uint8(rng.Intn(256))
 	}
@@ -948,12 +948,29 @@ func BenchmarkRandomEncodeAllFastest(b *testing.B) {
 	}
 }
 
-func BenchmarkRandomEncodeAllDefault(b *testing.B) {
+func BenchmarkRandom10MBEncodeAllFastest(b *testing.B) {
 	rng := rand.New(rand.NewSource(1))
 	data := make([]byte, 10<<20)
-	for i := range data {
-		data[i] = uint8(rng.Intn(256))
+	rng.Read(data)
+	enc, _ := NewWriter(nil, WithEncoderLevel(SpeedFastest), WithEncoderConcurrency(1))
+	defer enc.Close()
+	dst := enc.EncodeAll(data, nil)
+	wantSize := len(dst)
+	b.ResetTimer()
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	for i := 0; i < b.N; i++ {
+		dst := enc.EncodeAll(data, dst[:0])
+		if len(dst) != wantSize {
+			b.Fatal(len(dst), "!=", wantSize)
+		}
 	}
+}
+
+func BenchmarkRandom4KEncodeAllDefault(b *testing.B) {
+	rng := rand.New(rand.NewSource(1))
+	data := make([]byte, 4<<10)
+	rng.Read(data)
 	enc, _ := NewWriter(nil, WithEncoderLevel(SpeedDefault), WithEncoderConcurrency(1))
 	defer enc.Close()
 	dst := enc.EncodeAll(data, nil)
@@ -969,12 +986,29 @@ func BenchmarkRandomEncodeAllDefault(b *testing.B) {
 	}
 }
 
-func BenchmarkRandomEncoderFastest(b *testing.B) {
+func BenchmarkRandomEncodeAllDefault(b *testing.B) {
 	rng := rand.New(rand.NewSource(1))
 	data := make([]byte, 10<<20)
-	for i := range data {
-		data[i] = uint8(rng.Intn(256))
+	rng.Read(data)
+	enc, _ := NewWriter(nil, WithEncoderLevel(SpeedDefault), WithEncoderConcurrency(1))
+	defer enc.Close()
+	dst := enc.EncodeAll(data, nil)
+	wantSize := len(dst)
+	b.ResetTimer()
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	for i := 0; i < b.N; i++ {
+		dst := enc.EncodeAll(data, dst[:0])
+		if len(dst) != wantSize {
+			b.Fatal(len(dst), "!=", wantSize)
+		}
 	}
+}
+
+func BenchmarkRandom10MBEncoderFastest(b *testing.B) {
+	rng := rand.New(rand.NewSource(1))
+	data := make([]byte, 10<<20)
+	rng.Read(data)
 	wantSize := int64(len(data))
 	enc, _ := NewWriter(ioutil.Discard, WithEncoderLevel(SpeedFastest))
 	defer enc.Close()
@@ -1003,9 +1037,7 @@ func BenchmarkRandomEncoderFastest(b *testing.B) {
 func BenchmarkRandomEncoderDefault(b *testing.B) {
 	rng := rand.New(rand.NewSource(1))
 	data := make([]byte, 10<<20)
-	for i := range data {
-		data[i] = uint8(rng.Intn(256))
-	}
+	rng.Read(data)
 	wantSize := int64(len(data))
 	enc, _ := NewWriter(ioutil.Discard, WithEncoderLevel(SpeedDefault))
 	defer enc.Close()

zstd/snappy.go

@@ -178,7 +178,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) {
 				r.err = ErrSnappyCorrupt
 				return written, r.err
 			}
-			err = r.block.encode(false)
+			err = r.block.encode(false, false)
 			switch err {
 			case errIncompressible:
 				r.block.popOffsets()