Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

grpc: fix receiving empty messages when compression is enabled and maxReceiveMessageSize is MaxInt #7753 #7918

Open
wants to merge 34 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
b08845a
Fix test cases and update code to latest branch
vinothkumarr227 Dec 10, 2024
a02e58c
small tweaks
vinothkumarr227 Dec 11, 2024
54da70d
small tweaks
vinothkumarr227 Dec 12, 2024
4efd6fe
update code based on review feedback
vinothkumarr227 Dec 12, 2024
851d13a
fixed the changes based on review feedback
vinothkumarr227 Dec 13, 2024
26bf731
Added test cases for Handles maxReceiveMessageSize as MaxInt64
vinothkumarr227 Dec 17, 2024
3cf9054
Fixed the review changes
vinothkumarr227 Dec 18, 2024
27a68a0
Remove the casting for doesReceiveMessageOverflow
vinothkumarr227 Dec 18, 2024
3daef9b
Resolve all the issues
vinothkumarr227 Dec 18, 2024
8815cbd
Refactored code changes done
vinothkumarr227 Dec 18, 2024
5541486
test: switching to stubserver in tests instead of testservice (#7925)
pvsravani Dec 18, 2024
a21e192
cleanup: replace dial with newclient (#7920)
janardhanvissa Dec 18, 2024
09132bf
deps: update crypto dependency to resolve CVE-2024-45337 (#7956)
TomerJLevy Dec 20, 2024
5181e7b
test: Workaround slow SRV lookups in flaking test (#7957)
arjan-bal Dec 20, 2024
46faf72
envconfig: enable xDS client fallback by default (#7949)
easwars Dec 20, 2024
bd4989a
test: Add a test for decompression exceeding max receive message size…
arjan-bal Dec 23, 2024
985965a
outlierdetection: Support health listener for ejection updates (#7908)
arjan-bal Dec 23, 2024
8a0db4c
Revert "test: switching to stubserver in tests instead of testservice…
vinothkumarr227 Dec 23, 2024
cf12ace
Revert "cleanup: replace dial with newclient (#7920)"
vinothkumarr227 Dec 23, 2024
0e927fb
Revert "deps: update crypto dependency to resolve CVE-2024-45337 (#79…
vinothkumarr227 Dec 23, 2024
ee88fc0
Revert "envconfig: enable xDS client fallback by default (#7949)"
vinothkumarr227 Dec 23, 2024
4ef6aab
Revert "test: Add a test for decompression exceeding max receive mess…
vinothkumarr227 Dec 23, 2024
7ef57fd
Revert "outlierdetection: Support health listener for ejection update…
vinothkumarr227 Dec 23, 2024
20d4dc6
Revert the changes
vinothkumarr227 Dec 23, 2024
0907f9d
Revert the changes for test cases
vinothkumarr227 Dec 23, 2024
84e8a4f
Fix decompression size check and refactor based on review feedback
vinothkumarr227 Dec 23, 2024
77ea230
small tweaks
vinothkumarr227 Dec 24, 2024
6ff9321
Fixed the space formatting issues
vinothkumarr227 Dec 24, 2024
82124a4
Rename the test cases and refactor some of them
vinothkumarr227 Dec 24, 2024
4e9c665
Fix reviewed changes
vinothkumarr227 Jan 10, 2025
da82d30
small tweaks
vinothkumarr227 Jan 10, 2025
33962c2
Fixed test cases for decompress function
vinothkumarr227 Jan 10, 2025
23baa53
Fixed test cases for dock block
vinothkumarr227 Jan 10, 2025
7664e75
small tweaks
vinothkumarr227 Jan 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixed the changes based on review feedback
  • Loading branch information
vinothkumarr227 committed Dec 18, 2024
commit 851d13a0ebc511f77b1820a168bfc286fbb28f8c
42 changes: 22 additions & 20 deletions rpc_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -809,8 +809,8 @@ func (p *payloadInfo) free() {
}
}

// Global error for message size exceeding the limit
var ErrMaxMessageSizeExceeded = errors.New("max message size exceeded")
// errMaxMessageSizeExceeded represents an error due to exceeding the maximum message size limit.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment is unnecessary.

var errMaxMessageSizeExceeded = errors.New("max message size exceeded")

// recvAndDecompress reads a message from the stream, decompressing it if necessary.
//
Expand Down Expand Up @@ -844,13 +844,13 @@ func recvAndDecompress(p *parser, s recvCompressor, dc Decompressor, maxReceiveM
out = mem.BufferSlice{mem.SliceBuffer(uncompressedBuf)}
}
} else {
out, _, err = decompress(compressor, compressed, maxReceiveMessageSize, p.bufferPool)
out, err = decompress(compressor, compressed, maxReceiveMessageSize, p.bufferPool)
}
if err != nil {
return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message: %v", err)
}

if err == ErrMaxMessageSizeExceeded {
if err == errMaxMessageSizeExceeded {
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
out.Free()
// TODO: Revisit the error code. Currently keep it consistent with java
// implementation.
Expand All @@ -870,41 +870,43 @@ func recvAndDecompress(p *parser, s recvCompressor, dc Decompressor, maxReceiveM
}

// Using compressor, decompress d, returning data and size.
// If the decompressed data exceeds maxReceiveMessageSize, it returns nil, 0, and an error.
func decompress(compressor encoding.Compressor, d mem.BufferSlice, maxReceiveMessageSize int, pool mem.BufferPool) (mem.BufferSlice, int, error) {
// If the decompressed data exceeds maxReceiveMessageSize, it returns errMaxMessageSizeExceeded.
func decompress(compressor encoding.Compressor, d mem.BufferSlice, maxReceiveMessageSize int, pool mem.BufferPool) (mem.BufferSlice, error) {
dcReader, err := compressor.Decompress(d.Reader())
if err != nil {
return nil, 0, err
return nil, err
}

out, err := mem.ReadAll(io.LimitReader(dcReader, int64(maxReceiveMessageSize)), pool)
if err != nil {
out.Free()
return nil, 0, ErrMaxMessageSizeExceeded
return nil, err
}
if err = checkReceiveMessageOverflow(int64(out.Len()), int64(maxReceiveMessageSize), dcReader); err != nil {
return nil, out.Len() + 1, err

if doesReceiveMessageOverflow(int64(out.Len()), int64(maxReceiveMessageSize), dcReader) {
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
return nil, errMaxMessageSizeExceeded
dfawley marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this just return the status.Errorf(...) instead of returning a magic error that gets converted into that?

(We'd want to make sure this method always returns an error that is appropriate to just return directly from recvAndDecompress.)

}
return out, out.Len(), nil
return out, nil
}

// checkReceiveMessageOverflow checks if the number of bytes read from the stream exceeds
// the maximum receive message size allowed by the client. If the `readBytes` equals
// `maxReceiveMessageSize`, the function attempts to read one more byte from the `dcReader`
// to detect if there's an overflow.
// doesReceiveMessageOverflow checks if the number of bytes read from the stream
// exceeds the maximum receive message size allowed by the client. If the `readBytes`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: exceed instead of exceeds.

allowed by the client should be allowed by the receiver, the receiver could be the client or the server.

// is greater than or equal to `maxReceiveMessageSize`, the function attempts to read
// one more byte from the `dcReader` to detect if there's an overflow.
//
// If additional data is read, or an error other than `io.EOF` is encountered, the function
// returns an error indicating that the message size has exceeded the permissible limit.
func checkReceiveMessageOverflow(readBytes, maxReceiveMessageSize int64, dcReader io.Reader) error {
// returns `true` to indicate that the message size has exceeded the permissible limit.
// Otherwise, it returns `false` indicating no overflow.
func doesReceiveMessageOverflow(readBytes, maxReceiveMessageSize int64, dcReader io.Reader) bool {
if readBytes < maxReceiveMessageSize {
return nil
return false
}

b := make([]byte, 1)
if n, err := dcReader.Read(b); n > 0 || err != io.EOF {
return fmt.Errorf("overflow: received message size is larger than the allowed maxReceiveMessageSize (%d bytes)", maxReceiveMessageSize)
return true
}
return nil
return false
}

type recvCompressor interface {
Expand Down
44 changes: 23 additions & 21 deletions rpc_util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package grpc
import (
"bytes"
"compress/gzip"
"errors"
"io"
"math"
"reflect"
Expand Down Expand Up @@ -326,52 +327,53 @@ func TestDecompress(t *testing.T) {
input []byte
maxReceiveMessageSize int
want []byte
error error
wantErr error
}{
{
name: "Decompresses successfully with sufficient buffer size",
compressor: c,
input: []byte("decompressed data"),
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
maxReceiveMessageSize: 50,
want: []byte("decompressed data"),
error: nil,
wantErr: nil,
},
{
name: "failure, empty receive message",
name: "Fails due to exceeding maxReceiveMessageSize",
compressor: c,
input: []byte{},
maxReceiveMessageSize: 10,
input: []byte("small message that is too large"),
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
maxReceiveMessageSize: 5,
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
want: nil,
error: nil,
wantErr: errMaxMessageSizeExceeded,
},
{
name: "overflow failure, receive message exceeds maxReceiveMessageSize",
name: "Decompresses to exactly maxReceiveMessageSize",
compressor: c,
input: []byte("small message"),
maxReceiveMessageSize: 5,
want: nil,
error: ErrMaxMessageSizeExceeded,
input: []byte("exact size message"),
maxReceiveMessageSize: len("exact size message"),
want: []byte("exact size message"),
wantErr: nil,
},
}

for _, tt := range tests {
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
t.Run(tt.name, func(t *testing.T) {
compressedMsg := compressInput(tt.input)
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
output, numSliceInBuf, err := decompress(tt.compressor, compressedMsg, tt.maxReceiveMessageSize, mem.DefaultBufferPool())
var wantMsg mem.BufferSlice
if tt.want != nil {
wantMsg = mem.BufferSlice{mem.NewBuffer(&tt.want, nil)}
}
if tt.error != nil && err == nil {
t.Fatalf("decompress() error, got err=%v, want err=%v", err, tt.error)
output, err := decompress(tt.compressor, compressedMsg, tt.maxReceiveMessageSize, mem.DefaultBufferPool())

if tt.wantErr != nil {
if !errors.Is(err, tt.wantErr) {
arjan-bal marked this conversation as resolved.
Show resolved Hide resolved
t.Fatalf("decompress() error = %v, wantErr = %v", err, tt.wantErr)
}
return
}
if tt.error == nil && numSliceInBuf != wantMsg.Len() {
t.Fatalf("decompress() number of slices mismatch, got = %d, want = %d", numSliceInBuf, wantMsg.Len())

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nix new line

if err != nil {
t.Fatalf("decompress() unexpected error = %v", err)
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nix new line

if diff := cmp.Diff(tt.want, output.Materialize()); diff != "" {
t.Fatalf("decompress() mismatch (-want +got):\n%s", diff)
}

})
}
}