gzhttp: Add SHA256 as paranoid option (#769)

```
Benchmark2kJitter-32    	   67309	     17580 ns/op	 116.50 MB/s	    3478 B/op	      17 allocs/op
Benchmark2kJitterParanoid-32    	   54398	     21564 ns/op	  94.97 MB/s	    3438 B/op	      16 allocs
```

### Paranoid?

The padding size is determined by the remainder of a CRC32 of the content.

Since the payload contains elements unknown to the attacker, there is no reason to believe they can derive any information
from this remainder, or predict it.

However, for those that feel uncomfortable with a CRC32 being used for this can enable "paranoid" mode which will use SHA256 for determining the padding.

The hashing itself is about 2 orders of magnitude slower, but in overall terms will maybe only reduce speed by 10%.

Paranoid mode has no effect if buffer is < 0 (non-content aware padding).

Author: klauspost

gzhttp/README.md

@@ -235,21 +235,35 @@ but if you do, or you are in doubt, you can apply mitigations.
 // This should cover the sensitive part of your response.
 // This can be used to obfuscate the exact compressed size.
 // Specifying 0 will use a buffer size of 64KB.
+// 'paranoid' will use a slower hashing function, that MAY provide more safety. 
 // If a negative buffer is given, the amount of jitter will not be content dependent.
 // This provides *less* security than applying content based jitter.
-func RandomJitter(n, buffer int) option {
+func RandomJitter(n, buffer int, paranoid bool) option
 ...	
 ```
 
 The jitter is added as a "Comment" field. This field has a 1 byte overhead, so actual extra size will be 2 -> n+1 (inclusive).
 
-A good option would be to apply 32 random bytes, with default 64KB buffer: `gzhttp.RandomJitter(32, 0)`.
+A good option would be to apply 32 random bytes, with default 64KB buffer: `gzhttp.RandomJitter(32, 0, false)`.
 
 Note that flushing the data forces the padding to be applied, which means that only data before the flush is considered for content aware padding.
 
 The *padding* in the comment is the text `Padding-Padding-Padding-Padding-Pad....`
 
-The *length* is `1 + sha256(payload) MOD n`, or just random from `crypto/rand` if buffer < 0.
+The *length* is `1 + crc32c(payload) MOD n` or `1 + sha256(payload) MOD n` (paranoid), or just random from `crypto/rand` if buffer < 0.
+
+### Paranoid?
+
+The padding size is determined by the remainder of a CRC32 of the content. 
+
+Since the payload contains elements unknown to the attacker, there is no reason to believe they can derive any information
+from this remainder, or predict it.
+
+However, for those that feel uncomfortable with a CRC32 being used for this can enable "paranoid" mode which will use SHA256 for determining the padding.
+
+The hashing itself is about 2 orders of magnitude slower, but in overall terms will maybe only reduce speed by 10%.
+
+Paranoid mode has no effect if buffer is < 0 (non-content aware padding).
 
 ### Examples
 

gzhttp/compress.go

@@ -8,8 +8,10 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"math"
+	"math/bits"
 	"mime"
 	"net"
 	"net/http"
@@ -73,6 +75,7 @@ type GzipResponseWriter struct {
 	setContentType   bool   // Add content type, if missing and detected.
 	suffixETag       string // Suffix to add to ETag header if response is compressed.
 	dropETag         bool   // Drop ETag header if response is compressed (supersedes suffixETag).
+	sha256Jitter     bool   // Use sha256 for jitter.
 	randomJitter     []byte // Add random bytes to output as header field.
 	jitterBuffer     int    // Maximum buffer to accumulate before doing jitter.
 
@@ -167,6 +170,8 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
+var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
+
 // startGzip initializes a GZIP writer and writes the buffer.
 func (w *GzipResponseWriter) startGzip(remain []byte) error {
 	// Set the GZIP header.
@@ -216,18 +221,32 @@ func (w *GzipResponseWriter) startGzip(remain []byte) error {
 		if len(w.randomJitter) > 0 {
 			var jitRNG uint32
 			if w.jitterBuffer > 0 {
-				h := sha256.New()
-				h.Write(w.buf)
-				// Use only up to "w.jitterBuffer", otherwise the output depends on write sizes.
-				if len(remain) > 0 && len(w.buf) < w.jitterBuffer {
-					remain := remain
-					if len(remain)+len(w.buf) > w.jitterBuffer {
-						remain = remain[:w.jitterBuffer-len(w.buf)]
+				if w.sha256Jitter {
+					h := sha256.New()
+					h.Write(w.buf)
+					// Use only up to "w.jitterBuffer", otherwise the output depends on write sizes.
+					if len(remain) > 0 && len(w.buf) < w.jitterBuffer {
+						remain := remain
+						if len(remain)+len(w.buf) > w.jitterBuffer {
+							remain = remain[:w.jitterBuffer-len(w.buf)]
+						}
+						h.Write(remain)
+					}
+					var tmp [sha256.BlockSize]byte
+					jitRNG = binary.LittleEndian.Uint32(h.Sum(tmp[:0]))
+				} else {
+					h := crc32.New(castagnoliTable)
+					h.Write(w.buf)
+					// Use only up to "w.jitterBuffer", otherwise the output depends on write sizes.
+					if len(remain) > 0 && len(w.buf) < w.jitterBuffer {
+						remain := remain
+						if len(remain)+len(w.buf) > w.jitterBuffer {
+							remain = remain[:w.jitterBuffer-len(w.buf)]
+						}
+						h.Write(remain)
 					}
-					h.Write(remain)
+					jitRNG = bits.RotateLeft32(h.Sum32(), 19) ^ 0xab0755de
 				}
-				var tmp [sha256.BlockSize]byte
-				jitRNG = binary.LittleEndian.Uint32(h.Sum(tmp[:0]))
 			} else {
 				// Get from rand.Reader
 				var tmp [4]byte
@@ -441,6 +460,7 @@ func NewWrapper(opts ...option) (func(http.Handler) http.HandlerFunc, error) {
 					setContentType:    c.setContentType,
 					randomJitter:      c.randomJitter,
 					jitterBuffer:      c.jitterBuffer,
+					sha256Jitter:      c.sha256Jitter,
 				}
 				if len(gw.buf) > 0 {
 					gw.buf = gw.buf[:0]
@@ -507,6 +527,7 @@ type config struct {
 	dropETag         bool
 	jitterBuffer     int
 	randomJitter     []byte
+	sha256Jitter     bool
 }
 
 func (c *config) validate() error {
@@ -692,11 +713,14 @@ func DropETag() option {
 // This should cover the sensitive part of your response.
 // This can be used to obfuscate the exact compressed size.
 // Specifying 0 will use a buffer size of 64KB.
+// 'paranoid' will use a slower hashing function, that MAY provide more safety.
+// See README.md for more information.
 // If a negative buffer is given, the amount of jitter will not be content dependent.
 // This provides *less* security than applying content based jitter.
-func RandomJitter(n, buffer int) option {
+func RandomJitter(n, buffer int, paranoid bool) option {
 	return func(c *config) {
 		if n > 0 {
+			c.sha256Jitter = paranoid
 			c.randomJitter = bytes.Repeat([]byte("Padding-"), 1+(n/8))
 			c.randomJitter = c.randomJitter[:(n + 1)]
 			c.jitterBuffer = buffer

gzhttp/compress_test.go

@@ -989,7 +989,7 @@ func TestRandomJitter(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	wrapper, err := NewWrapper(RandomJitter(256, 1024), MinSize(10))
+	wrapper, err := NewWrapper(RandomJitter(256, 1024, false), MinSize(10))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1162,7 +1162,223 @@ func TestRandomJitter(t *testing.T) {
 	}
 
 	// Test non-content aware jitter
-	wrapper, err = NewWrapper(RandomJitter(256, -1), MinSize(10))
+	wrapper, err = NewWrapper(RandomJitter(256, -1, false), MinSize(10))
+	if err != nil {
+		t.Fatal(err)
+	}
+	handler = wrapper(writePayload)
+	changed = false
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i > 0 {
+			changed = changed || len(b2) != len(b)
+		}
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-len(refBody))
+		if len(b2) <= len(refBody) {
+			t.Errorf("no padding applied,")
+		}
+
+		// Do not mutate...
+		// Update last payload.
+		b = b2
+	}
+	if !changed {
+		t.Errorf("no change after 9 attempts")
+	}
+}
+
+func TestRandomJitterParanoid(t *testing.T) {
+	r := httptest.NewRequest("GET", "/", nil)
+	r.Header.Set("Accept-Encoding", "gzip")
+
+	// 4KB input, incompressible to avoid compression variations.
+	rng := rand.New(rand.NewSource(0))
+	payload := make([]byte, 4096)
+	_, err := io.ReadFull(rng, payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wrapper, err := NewWrapper(RandomJitter(256, 1024, true), MinSize(10))
+	if err != nil {
+		t.Fatal(err)
+	}
+	writePayload := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	})
+	referenceHandler := GzipHandler(writePayload)
+	w := httptest.NewRecorder()
+	referenceHandler.ServeHTTP(w, r)
+	result := w.Result()
+	refBody, err := io.ReadAll(result.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Unmodified length: %d", len(refBody))
+
+	handler := wrapper(writePayload)
+	w = httptest.NewRecorder()
+	handler.ServeHTTP(w, r)
+
+	result = w.Result()
+	b, err := io.ReadAll(result.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(refBody) == len(b) {
+		t.Fatal("padding was not applied")
+	}
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	changed := false
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		changed = changed || len(b2) != len(b)
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-len(refBody))
+		if len(b2) <= len(refBody) {
+			t.Errorf("no padding applied,")
+		}
+		if i == 0 && changed {
+			t.Error("length changed without payload change", len(b), "->", len(b2))
+		}
+		// Mutate...
+		payload[0]++
+		b = b2
+	}
+	if !changed {
+		t.Errorf("no change after 9 attempts")
+	}
+
+	// Write one byte at the time to test buffer flushing.
+	handler = wrapper(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for i := range payload {
+			w.Write([]byte{payload[i]})
+		}
+	}))
+
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-len(refBody))
+		if len(b2) <= len(refBody) {
+			t.Errorf("no padding applied,")
+		}
+		if i > 0 && len(b2) != len(b) {
+			t.Error("length changed without payload change", len(b), "->", len(b2))
+		}
+		// Mutate, buf after the buffer...
+		payload[2048]++
+		b = b2
+	}
+
+	// Write less than buffer
+	handler = wrapper(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload[:512])
+	}))
+	changed = false
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i > 0 {
+			changed = changed || len(b2) != len(b)
+		}
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-512)
+		if len(b2) <= 512 {
+			t.Errorf("no padding applied,")
+		}
+		// Mutate...
+		payload[500]++
+		b = b2
+	}
+	if !changed {
+		t.Errorf("no change after 9 attempts")
+	}
+
+	// Write less than buffer, with flush in between.
+	// Checksum should be of all before flush.
+	handler = wrapper(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload[:256])
+		w.(http.Flusher).Flush()
+		w.Write(payload[256:512])
+	}))
+
+	changed = false
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i > 0 {
+			changed = changed || len(b2) != len(b)
+		}
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-512)
+		if len(b2) <= 512 {
+			t.Errorf("no padding applied,")
+		}
+		// Mutate...
+		payload[200]++
+		b = b2
+	}
+	if !changed {
+		t.Errorf("no change after 9 attempts")
+	}
+
+	// Mutate *after* the flush.
+	// Should no longer affect length.
+	for i := 0; i < 10; i++ {
+		w = httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+		result = w.Result()
+		b2, err := io.ReadAll(result.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i > 0 {
+			changed = len(b2) != len(b)
+			if changed {
+				t.Errorf("mutating after flush seems to have affected output")
+			}
+		}
+		t.Logf("attempt %d length: %d. padding: %d.", i, len(b2), len(b2)-512)
+		if len(b2) <= 512 {
+			t.Errorf("no padding applied,")
+		}
+		// Mutate...
+		payload[400]++
+		b = b2
+	}
+
+	// Test non-content aware jitter
+	wrapper, err = NewWrapper(RandomJitter(256, -1, true), MinSize(10))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1477,10 +1693,15 @@ func BenchmarkGzipBestSpeedHandler_P100k(b *testing.B) {
 }
 
 func Benchmark2kJitter(b *testing.B) {
-	benchmark(b, false, 2048, CompressionLevel(gzip.BestSpeed), RandomJitter(32, 0))
+	benchmark(b, false, 2048, CompressionLevel(gzip.BestSpeed), RandomJitter(32, 0, false))
 }
+
+func Benchmark2kJitterParanoid(b *testing.B) {
+	benchmark(b, false, 2048, CompressionLevel(gzip.BestSpeed), RandomJitter(32, 0, true))
+}
+
 func Benchmark2kJitterRNG(b *testing.B) {
-	benchmark(b, false, 2048, CompressionLevel(gzip.BestSpeed), RandomJitter(32, -1))
+	benchmark(b, false, 2048, CompressionLevel(gzip.BestSpeed), RandomJitter(32, -1, false))
 }
 
 // --------------------------------------------------------------------