decoder slowness when reading a zlib/zstd .NewReader() due to small .Read() buffers #635

Activity

1. folays commented on Mar 16, 2025

   

   folays

   on Mar 16, 2025

   Author

   More actions

   here below is a copy of the test code https://go.dev/play/p/8eiq0ZpZLBG :

   package main
   
   import (
   	"bytes"
   	"compress/zlib"
   	"encoding/json"
   	"fmt"
   	"io"
   	"runtime"
   	"time"
   
   	"github.com/fxamacker/cbor/v2"
   	"github.com/klauspost/compress/zstd"
   )
   
   // go get github.com/fxamacker/cbor/v2
   // go get github.com/klauspost/compress@latest
   
   type State struct {
   	A map[int]*A
   }
   
   type A struct {
   	B map[int]int
   }
   
   var state State
   
   func init() {
   	state.A = make(map[int]*A)
   	for i := 0; i < 128; i++ {
   		state.A[i] = &A{make(map[int]int)}
   		for j := 0; j < 128*1024; j++ {
   			state.A[i].B[j] = j
   		}
   	}
   }
   
   func main() {
   	test_serializer(
   		func(w io.Writer) interface{ Encode(v any) error } { return json.NewEncoder(w) },
   		func(r io.Reader) interface{ Decode(v any) error } { return json.NewDecoder(r) },
   	)
   	test_serializer(
   		func(w io.Writer) interface{ Encode(v any) error } { return cbor.NewEncoder(w) },
   		func(r io.Reader) interface{ Decode(v any) error } { return cbor.NewDecoder(r) },
   	)
   }
   
   type plain struct {
   	io.Writer
   }
   
   func (plain) Close() error { return nil }
   
   type zstdDecoder struct {
   	*zstd.Decoder
   }
   
   func (z zstdDecoder) Close() error { z.Decoder.Close(); return nil }
   
   func test_serializer(
   	encoder func(io.Writer) interface{ Encode(v any) error },
   	decoder func(io.Reader) interface{ Decode(v any) error },
   ) {
   	var data bytes.Buffer
   	encoder(&data).Encode(state)
   
   	test_compressor(data.Bytes(),
   		func(w io.Writer) io.WriteCloser { return plain{w} },
   		func(r io.Reader) io.ReadCloser { return io.NopCloser(r) },
   		decoder,
   	)
   	test_compressor(data.Bytes(),
   		func(w io.Writer) io.WriteCloser { return zlib.NewWriter(w) },
   		func(r io.Reader) io.ReadCloser { r2, _ := zlib.NewReader(r); return r2 },
   		decoder,
   	)
   	test_compressor(data.Bytes(),
   		func(w io.Writer) io.WriteCloser { w2, _ := zstd.NewWriter(w); return w2 },
   		func(r io.Reader) io.ReadCloser { r2, _ := zstd.NewReader(r); return zstdDecoder{r2} },
   		decoder,
   	)
   }
   
   func test_compressor(data []byte,
   	compressor func(w io.Writer) io.WriteCloser,
   	decompressor func(r io.Reader) io.ReadCloser,
   	decoder func(io.Reader) interface{ Decode(v any) error },
   ) {
   	var zData bytes.Buffer
   	compressWriter := compressor(&zData)
   	compressWriter.Write(data)
   	compressWriter.Close()
   
   	for _, greedy := range []bool{false, true} {
   		state = State{}
   		runtime.GC()
   		runtime.GC()
   
   		start := time.Now()
   		compressReader := decompressor(bytes.NewReader(zData.Bytes()))
   		greeder := &greedyReader{
   			r:      compressReader,
   			greedy: greedy,
   		}
   		decoder(greeder).Decode(&state)
   		compressReader.Close()
   		elapsed := time.Since(start).Seconds()
   
   		fmt.Printf("%T/%-20T/greedy=%-5t: %3d/%3d MBs raw/compressed, %.1f seconds (decompressor .Read() calls %d average return %d kB)\n",
   			decoder(greeder), compressReader, greedy, len(data)/1e6, zData.Len()/1e6, elapsed, greeder.nbCalls, greeder.sumBytesRead/greeder.nbCalls/1024)
   	}
   }
   
   type greedyReader struct {
   	r            io.Reader
   	greedy       bool
   	nbCalls      int
   	sumBytesRead int
   }
   
   // Read(): if `greedy` is false, it will behave as if its underlying io.Reader.Read() were called (+ some stats)
   func (g *greedyReader) Read(p []byte) (n int, err error) {
   	for len(p) > 0 && err == nil {
   		var n2 int
   		n2, err = g.r.Read(p)
   		n += n2
   		p = p[n2:]
   		if g.greedy == false {
   			break
   		}
   	}
   
   	g.nbCalls++
   	g.sumBytesRead += n
   
   	return
   }
   
2. fxamacker commented on Mar 17, 2025

   

   fxamacker

   on Mar 17, 2025

   Owner

   More actions

   @folays Thanks for opening this issue and providing detailed information!

   I'll take a closer look (most likely next weekend).
3. fxamacker

   self-assigned this

   on Mar 25, 2025