From 2bd38e07aa59305dcde43817f578447425eefc67 Mon Sep 17 00:00:00 2001 From: Wyndham Blanton Date: Wed, 6 Jan 2021 13:37:54 -0700 Subject: [PATCH] zstd: Make decoder allocations smaller (#306) * keep allocations as small as possible Co-authored-by: bo --- zstd/seqdec.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zstd/seqdec.go b/zstd/seqdec.go index b5c8ef1332..1dd39e63b7 100644 --- a/zstd/seqdec.go +++ b/zstd/seqdec.go @@ -181,11 +181,18 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { return fmt.Errorf("output (%d) bigger than max block size", size) } if size > cap(s.out) { - // Not enough size, will be extremely rarely triggered, + // Not enough size, which can happen under high volume block streaming conditions // but could be if destination slice is too small for sync operations. - // We add maxBlockSize to the capacity. - s.out = append(s.out, make([]byte, maxBlockSize)...) - s.out = s.out[:len(s.out)-maxBlockSize] + // over-allocating here can create a large amount of GC pressure so we try to keep + // it as contained as possible + used := len(s.out) - startSize + addBytes := 256 + ll + ml + used>>2 + // Clamp to max block size. + if used+addBytes > maxBlockSize { + addBytes = maxBlockSize - used + } + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] } if ml > maxMatchLen { return fmt.Errorf("match len (%d) bigger than max allowed length", ml)