Skip to content

Commit a878b61

Browse files
alanprotalanprot
committed
Changing store gateway error code from ResourceExhausted to PermissionDenied
Signed-off-by: Alan Protasio <alanprot@gmail.com>
1 parent 81fcf88 commit a878b61

File tree

7 files changed

+242
-66
lines changed

7 files changed

+242
-66
lines changed

‎pkg/querier/blocks_store_queryable.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,10 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
648648
if s.Code() == codes.ResourceExhausted {
649649
return validation.LimitError(s.Message())
650650
}
651+
652+
if s.Code() == codes.PermissionDenied {
653+
return validation.AccessDeniedError(s.Message())
654+
}
651655
}
652656
return errors.Wrapf(err, "failed to receive series from %s", c.RemoteAddress())
653657
}
@@ -816,6 +820,10 @@ func (q *blocksStoreQuerier) fetchLabelNamesFromStore(
816820
return validation.LimitError(s.Message())
817821
}
818822
}
823+
824+
if s.Code() == codes.PermissionDenied {
825+
return validation.AccessDeniedError(s.Message())
826+
}
819827
return errors.Wrapf(err, "failed to fetch label names from %s", c.RemoteAddress())
820828
}
821829

@@ -907,6 +915,10 @@ func (q *blocksStoreQuerier) fetchLabelValuesFromStore(
907915
if s.Code() == codes.ResourceExhausted {
908916
return validation.LimitError(s.Message())
909917
}
918+
919+
if s.Code() == codes.PermissionDenied {
920+
return validation.AccessDeniedError(s.Message())
921+
}
910922
}
911923
return errors.Wrapf(err, "failed to fetch label values from %s", c.RemoteAddress())
912924
}

‎pkg/querier/blocks_store_queryable_test.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,44 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
638638
},
639639
},
640640
},
641+
"all store-gateways return PermissionDenied": {
642+
finderResult: bucketindex.Blocks{
643+
{ID: block1},
644+
},
645+
expectedErr: validation.AccessDeniedError("PermissionDenied"),
646+
storeSetResponses: []interface{}{
647+
map[BlocksStoreClient][]ulid.ULID{
648+
&storeGatewayClientMock{
649+
remoteAddr: "1.1.1.1",
650+
mockedSeriesResponses: []*storepb.SeriesResponse{
651+
mockSeriesResponse(labels.Labels{metricNameLabel, series1Label}, minT, 2),
652+
mockHintsResponse(block1),
653+
},
654+
mockedSeriesStreamErr: status.Error(codes.PermissionDenied, "PermissionDenied"),
655+
}: {block1},
656+
},
657+
map[BlocksStoreClient][]ulid.ULID{
658+
&storeGatewayClientMock{
659+
remoteAddr: "2.2.2.2",
660+
mockedSeriesResponses: []*storepb.SeriesResponse{
661+
mockSeriesResponse(labels.Labels{metricNameLabel, series1Label}, minT, 2),
662+
mockHintsResponse(block1),
663+
},
664+
mockedSeriesStreamErr: status.Error(codes.PermissionDenied, "PermissionDenied"),
665+
}: {block1},
666+
},
667+
},
668+
limits: &blocksStoreLimitsMock{},
669+
queryLimiter: noOpQueryLimiter,
670+
expectedSeries: []seriesResult{
671+
{
672+
lbls: labels.New(metricNameLabel, series1Label),
673+
values: []valueResult{
674+
{t: minT, v: 2},
675+
},
676+
},
677+
},
678+
},
641679
"multiple store-gateways has the block, but one of them fails to return on stream": {
642680
finderResult: bucketindex.Blocks{
643681
{ID: block1},

‎pkg/storage/bucket/sse_bucket_client.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ import (
44
"context"
55
"io"
66

7+
cortex_errors "github.com/cortexproject/cortex/pkg/util/errors"
8+
"github.com/gogo/status"
79
"github.com/minio/minio-go/v7/pkg/encrypt"
810
"github.com/pkg/errors"
911
"github.com/thanos-io/objstore"
1012
"github.com/thanos-io/objstore/providers/s3"
13+
"google.golang.org/grpc/codes"
1114

1215
cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
1316
)
@@ -101,12 +104,23 @@ func (b *SSEBucketClient) Iter(ctx context.Context, dir string, f func(string) e
101104

102105
// Get implements objstore.Bucket.
103106
func (b *SSEBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error) {
104-
return b.bucket.Get(ctx, name)
107+
r, err := b.bucket.Get(ctx, name)
108+
109+
if err != nil && b.bucket.IsCustomerManagedKeyError(err) {
110+
return nil, cortex_errors.WithCause(err, status.Error(codes.PermissionDenied, err.Error()))
111+
}
112+
113+
return r, err
105114
}
106115

107116
// GetRange implements objstore.Bucket.
108117
func (b *SSEBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) {
109-
return b.bucket.GetRange(ctx, name, off, length)
118+
r, err := b.bucket.GetRange(ctx, name, off, length)
119+
if err != nil && b.bucket.IsCustomerManagedKeyError(err) {
120+
return nil, cortex_errors.WithCause(err, status.Error(codes.PermissionDenied, err.Error()))
121+
}
122+
123+
return r, err
110124
}
111125

112126
// Exists implements objstore.Bucket.

‎pkg/storage/tsdb/testutil/objstore.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,12 @@ import (
77
"strings"
88
"testing"
99

10+
"github.com/cortexproject/cortex/pkg/util"
1011
"github.com/pkg/errors"
1112
"github.com/stretchr/testify/require"
1213
"github.com/thanos-io/objstore"
1314
"go.uber.org/atomic"
1415

15-
"github.com/cortexproject/cortex/pkg/util"
16-
1716
"github.com/cortexproject/cortex/pkg/storage/bucket/filesystem"
1817
)
1918

@@ -51,6 +50,20 @@ func (m *MockBucketFailure) Delete(ctx context.Context, name string) error {
5150
return m.Bucket.Delete(ctx, name)
5251
}
5352

53+
func (m *MockBucketFailure) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) {
54+
m.GetCalls.Add(1)
55+
for prefix, err := range m.GetFailures {
56+
if strings.HasPrefix(name, prefix) {
57+
return nil, err
58+
}
59+
}
60+
if e, ok := m.GetFailures[name]; ok {
61+
return nil, e
62+
}
63+
64+
return m.Bucket.GetRange(ctx, name, off, length)
65+
}
66+
5467
func (m *MockBucketFailure) Get(ctx context.Context, name string) (io.ReadCloser, error) {
5568
m.GetCalls.Add(1)
5669
for prefix, err := range m.GetFailures {

‎pkg/storegateway/bucket_stores.go

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -306,18 +306,14 @@ func (u *BucketStores) Series(req *storepb.SeriesRequest, srv storepb.Store_Seri
306306
err := u.getStoreError(userID)
307307

308308
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
309-
return httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
309+
return httpgrpc.Errorf(int(codes.PermissionDenied), "store error: %s", err)
310310
}
311311

312312
err = store.Series(req, spanSeriesServer{
313313
Store_SeriesServer: srv,
314314
ctx: spanCtx,
315315
})
316316

317-
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
318-
return httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
319-
}
320-
321317
return err
322318
}
323319

@@ -339,15 +335,11 @@ func (u *BucketStores) LabelNames(ctx context.Context, req *storepb.LabelNamesRe
339335
err := u.getStoreError(userID)
340336

341337
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
342-
return nil, httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
338+
return nil, httpgrpc.Errorf(int(codes.PermissionDenied), "store error: %s", err)
343339
}
344340

345341
resp, err := store.LabelNames(ctx, req)
346342

347-
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
348-
return resp, httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
349-
}
350-
351343
return resp, err
352344
}
353345

@@ -369,15 +361,11 @@ func (u *BucketStores) LabelValues(ctx context.Context, req *storepb.LabelValues
369361
err := u.getStoreError(userID)
370362

371363
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
372-
return nil, httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
364+
return nil, httpgrpc.Errorf(int(codes.PermissionDenied), "store error: %s", err)
373365
}
374366

375367
resp, err := store.LabelValues(ctx, req)
376368

377-
if err != nil && cortex_errors.ErrorIs(err, u.bucket.IsCustomerManagedKeyError) {
378-
return resp, httpgrpc.Errorf(int(codes.ResourceExhausted), "store error: %s", err)
379-
}
380-
381369
return resp, err
382370
}
383371

0 commit comments

Comments
 (0)