Tweaks and cleanup

zstd/decoder.go

@@ -313,6 +313,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
 		}
 		frame.rawInput = nil
 		frame.bBuf = nil
+		if frame.history.decoders.br != nil {
+			frame.history.decoders.br.in = nil
+		}
 		d.decoders <- block
 	}()
 	frame.bBuf = input

zstd/decoder_test.go

@@ -477,7 +477,7 @@ func TestNewDecoderBigFile(t *testing.T) {
 	}
 	defer f.Close()
 	start := time.Now()
-	dec, err := NewReader(f, WithDecoderConcurrency(4), WithDecoderLowmem(true))
+	dec, err := NewReader(f)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1159,7 +1159,7 @@ func BenchmarkDecoder_DecodeAllParallel(b *testing.B) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	dec, err := NewReader(nil)
+	dec, err := NewReader(nil, WithDecoderConcurrency(runtime.GOMAXPROCS(0)))
 	if err != nil {
 		b.Fatal(err)
 		return
@@ -1199,150 +1199,6 @@ func BenchmarkDecoder_DecodeAllParallel(b *testing.B) {
 	}
 }
 
-/*
-func BenchmarkDecoder_DecodeAllCgo(b *testing.B) {
-	fn := "testdata/benchdecoder.zip"
-	data, err := ioutil.ReadFile(fn)
-	if err != nil {
-		b.Fatal(err)
-	}
-	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
-	if err != nil {
-		b.Fatal(err)
-	}
-	for _, tt := range zr.File {
-		if !strings.HasSuffix(tt.Name, ".zst") {
-			continue
-		}
-		b.Run(tt.Name, func(b *testing.B) {
-			tt := tt
-			r, err := tt.Open()
-			if err != nil {
-				b.Fatal(err)
-			}
-			defer r.Close()
-			in, err := ioutil.ReadAll(r)
-			if err != nil {
-				b.Fatal(err)
-			}
-			got, err := zstd.Decompress(nil, in)
-			if err != nil {
-				b.Fatal(err)
-			}
-			b.SetBytes(int64(len(got)))
-			b.ReportAllocs()
-			b.ResetTimer()
-			for i := 0; i < b.N; i++ {
-				got, err = zstd.Decompress(got, in)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}
-
-func BenchmarkDecoder_DecodeAllParallelCgo(b *testing.B) {
-	fn := "testdata/benchdecoder.zip"
-	data, err := ioutil.ReadFile(fn)
-	if err != nil {
-		b.Fatal(err)
-	}
-	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
-	if err != nil {
-		b.Fatal(err)
-	}
-	for _, tt := range zr.File {
-		if !strings.HasSuffix(tt.Name, ".zst") {
-			continue
-		}
-		b.Run(tt.Name, func(b *testing.B) {
-			r, err := tt.Open()
-			if err != nil {
-				b.Fatal(err)
-			}
-			defer r.Close()
-			in, err := ioutil.ReadAll(r)
-			if err != nil {
-				b.Fatal(err)
-			}
-			got, err := zstd.Decompress(nil, in)
-			if err != nil {
-				b.Fatal(err)
-			}
-			b.SetBytes(int64(len(got)))
-			b.ReportAllocs()
-			b.ResetTimer()
-			b.RunParallel(func(pb *testing.PB) {
-				got := make([]byte, len(got))
-				for pb.Next() {
-					got, err = zstd.Decompress(got, in)
-					if err != nil {
-						b.Fatal(err)
-					}
-				}
-			})
-		})
-	}
-}
-
-func BenchmarkDecoderSilesiaCgo(b *testing.B) {
-	fn := "testdata/silesia.tar.zst"
-	data, err := ioutil.ReadFile(fn)
-	if err != nil {
-		if os.IsNotExist(err) {
-			b.Skip("Missing testdata/silesia.tar.zst")
-			return
-		}
-		b.Fatal(err)
-	}
-	dec := zstd.NewReader(bytes.NewBuffer(data))
-	n, err := io.Copy(ioutil.Discard, dec)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	b.SetBytes(n)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		dec := zstd.NewReader(bytes.NewBuffer(data))
-		_, err := io.CopyN(ioutil.Discard, dec, n)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-func BenchmarkDecoderEnwik9Cgo(b *testing.B) {
-	fn := "testdata/enwik9-1.zst"
-	data, err := ioutil.ReadFile(fn)
-	if err != nil {
-		if os.IsNotExist(err) {
-			b.Skip("Missing " + fn)
-			return
-		}
-		b.Fatal(err)
-	}
-	dec := zstd.NewReader(bytes.NewBuffer(data))
-	n, err := io.Copy(ioutil.Discard, dec)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	b.SetBytes(n)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		dec := zstd.NewReader(bytes.NewBuffer(data))
-		_, err := io.CopyN(ioutil.Discard, dec, n)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-*/
-
 func BenchmarkDecoderSilesia(b *testing.B) {
 	fn := "testdata/silesia.tar.zst"
 	data, err := ioutil.ReadFile(fn)

zstd/history.go

@@ -41,7 +41,7 @@ func (h *history) reset() {
 	if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined {
 		fseDecoderPool.Put(f)
 	}
-	h.decoders = sequenceDecs{}
+	h.decoders = sequenceDecs{br: h.decoders.br}
 	if h.huffTree != nil {
 		if h.dict == nil || h.dict.litEnc != h.huffTree {
 			huffDecoderPool.Put(h.huffTree)

zstd/seqdec.go

@@ -68,12 +68,11 @@ type sequenceDecs struct {
 	dict         []byte
 	literals     []byte
 	out          []byte
-	//seq          []seqVals
-	nSeqs      int
-	br         *bitReader
-	seqSize    int
-	windowSize int
-	maxBits    uint8
+	nSeqs        int
+	br           *bitReader
+	seqSize      int
+	windowSize   int
+	maxBits      uint8
 }
 
 // initialize all 3 decoders from the stream input.
@@ -256,64 +255,62 @@ func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error {
 	}
 
 	for _, seq := range seqs {
-		ll, ml, mo := seq.ll, seq.ml, seq.mo
-
 		// Add literals
-		s.out = append(s.out, s.literals[:ll]...)
-		s.literals = s.literals[ll:]
+		s.out = append(s.out, s.literals[:seq.ll]...)
+		s.literals = s.literals[seq.ll:]
 		out := s.out
 
 		// Copy form dictionary...
-		if mo > len(s.out)+len(hist) || mo > s.windowSize {
+		if seq.mo > len(s.out)+len(hist) || seq.mo > s.windowSize {
 			if len(s.dict) == 0 {
-				return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist))
+				return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, len(s.out)+len(hist))
 			}
 
 			// we may be in dictionary.
-			dictO := len(s.dict) - (mo - (len(s.out) + len(hist)))
+			dictO := len(s.dict) - (seq.mo - (len(s.out) + len(hist)))
 			if dictO < 0 || dictO >= len(s.dict) {
-				return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", mo, len(s.out)+len(hist)+len(s.dict))
+				return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", seq.mo, len(s.out)+len(hist)+len(s.dict))
 			}
-			end := dictO + ml
+			end := dictO + seq.ml
 			if end > len(s.dict) {
 				out = append(out, s.dict[dictO:]...)
-				mo -= len(s.dict) - dictO
-				ml -= len(s.dict) - dictO
+				seq.mo -= len(s.dict) - dictO
+				seq.ml -= len(s.dict) - dictO
 			} else {
 				s.out = append(out, s.dict[dictO:end]...)
 				continue
 			}
 		}
 
 		// Copy from history.
-		if v := mo - len(s.out); v > 0 {
+		if v := seq.mo - len(s.out); v > 0 {
 			// v is the start position in history from end.
 			start := len(hist) - v
-			if ml > v {
+			if seq.ml > v {
 				// Some goes into current block.
 				// Copy remainder of history
 				out = append(out, hist[start:]...)
-				mo -= v
-				ml -= v
+				seq.mo -= v
+				seq.ml -= v
 			} else {
-				s.out = append(out, hist[start:start+ml]...)
+				s.out = append(out, hist[start:start+seq.ml]...)
 				continue
 			}
 		}
 		// We must be in current buffer now
-		if ml > 0 {
-			start := len(s.out) - mo
-			if ml <= len(s.out)-start {
+		if seq.ml > 0 {
+			start := len(s.out) - seq.mo
+			if seq.ml <= len(s.out)-start {
 				// No overlap
-				s.out = append(out, s.out[start:start+ml]...)
+				s.out = append(out, s.out[start:start+seq.ml]...)
 				continue
 			} else {
 				// Overlapping copy
 				// Extend destination slice and copy one byte at the time.
-				out = out[:len(out)+ml]
-				src := out[start : start+ml]
+				out = out[:len(out)+seq.ml]
+				src := out[start : start+seq.ml]
 				// Destination is the space we just added.
-				dst := out[len(out)-ml:]
+				dst := out[len(out)-seq.ml:]
 				dst = dst[:len(src)]
 				for i := range src {
 					dst[i] = src[i]