-
Notifications
You must be signed in to change notification settings - Fork 189
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Relay rate limits #906
Relay rate limits #906
Changes from 1 commit
02e9069
dc39c21
d99fd39
e39eaf9
876ec69
9500f4f
ffe18c6
79d9614
c1d83e6
b5dc37c
b420cca
517b78e
7982aec
366fdf7
f2c10e4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
Signed-off-by: Cody Littley <cody@eigenlabs.org>
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -143,7 +143,7 @@ func NewServer( | |
// GetBlob retrieves a blob stored by the relay. | ||
func (s *Server) GetBlob(ctx context.Context, request *pb.GetBlobRequest) (*pb.GetBlobReply, error) { | ||
|
||
// Future work: | ||
// TODO(cody-littley): | ||
// - timeouts | ||
|
||
err := s.blobRateLimiter.BeginGetBlobOperation(time.Now()) | ||
ian-shim marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
@@ -188,7 +188,7 @@ func (s *Server) GetBlob(ctx context.Context, request *pb.GetBlobRequest) (*pb.G | |
// GetChunks retrieves chunks from blobs stored by the relay. | ||
func (s *Server) GetChunks(ctx context.Context, request *pb.GetChunksRequest) (*pb.GetChunksReply, error) { | ||
|
||
// Future work: | ||
// TODO(cody-littley): | ||
// - authentication | ||
// - timeouts | ||
|
||
|
@@ -219,7 +219,10 @@ func (s *Server) GetChunks(ctx context.Context, request *pb.GetChunksRequest) (* | |
"error fetching metadata for blob, check if blob exists and is assigned to this relay: %w", err) | ||
} | ||
|
||
requiredBandwidth := computeChunkRequestRequiredBandwidth(request, mMap) | ||
requiredBandwidth, err := computeChunkRequestRequiredBandwidth(request, mMap) | ||
if err != nil { | ||
return nil, fmt.Errorf("error computing required bandwidth: %w", err) | ||
} | ||
err = s.chunkRateLimiter.RequestGetChunkBandwidth(time.Now(), clientID, requiredBandwidth) | ||
if err != nil { | ||
return nil, err | ||
|
@@ -324,26 +327,31 @@ func gatherChunkDataToSend( | |
} | ||
|
||
// computeChunkRequestRequiredBandwidth computes the bandwidth required to fulfill a GetChunks request. | ||
func computeChunkRequestRequiredBandwidth(request *pb.GetChunksRequest, mMap metadataMap) int { | ||
func computeChunkRequestRequiredBandwidth(request *pb.GetChunksRequest, mMap metadataMap) (int, error) { | ||
requiredBandwidth := 0 | ||
for _, req := range request.ChunkRequests { | ||
var metadata *blobMetadata | ||
var key v2.BlobKey | ||
var requestedChunks int | ||
|
||
if req.GetByIndex() != nil { | ||
key := v2.BlobKey(req.GetByIndex().GetBlobKey()) | ||
key = v2.BlobKey(req.GetByIndex().GetBlobKey()) | ||
metadata = mMap[key] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. check if the key exists in the map? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should not be possible, since process will fail before this point if there is missing metadata. But it's a useful safety check, so I don't mind adding it. |
||
requestedChunks = len(req.GetByIndex().ChunkIndices) | ||
} else { | ||
key := v2.BlobKey(req.GetByRange().GetBlobKey()) | ||
key = v2.BlobKey(req.GetByRange().GetBlobKey()) | ||
metadata = mMap[key] | ||
requestedChunks = int(req.GetByRange().EndIndex - req.GetByRange().StartIndex) | ||
} | ||
|
||
if metadata == nil { | ||
return 0, fmt.Errorf("metadata not found for key %s", key.Hex()) | ||
} | ||
|
||
requiredBandwidth += requestedChunks * int(metadata.chunkSizeBytes) | ||
ian-shim marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
return requiredBandwidth | ||
return requiredBandwidth, nil | ||
|
||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cert.BlobHeader.BlobCommitments.Length
gives you the blob length in symbols.I think what we want is here
cert.BlobHeader.BlobSize
(size of raw blob data in bytes) insteadThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There is currently no such field in
BlobHeader
. Is that something that is going to be added?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah my bad. I meant BlobMetadata. Otherwise, we can also multiply
Length
byBYTES_PER_SYMBOL
, but this will overestimate the size