Change ingester to return 400 instead of 429 on series/metadata limits.

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [CHANGE] Require explicit flag `-<prefix>.tls-enabled` to enable TLS in GRPC clients. Previously it was enough to specify a TLS flag to enable TLS validation. #3156
 * [CHANGE] Query-frontend: removed `-querier.split-queries-by-day` (deprecated in Cortex 0.4.0). You should use `-querier.split-queries-by-interval` instead. #3813
 * [CHANGE] Store-gateway: the chunks pool controlled by `-blocks-storage.bucket-store.max-chunk-pool-bytes` is now shared across all tenants. #3830
+* [CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. #3833
 * [FEATURE] Adds support to S3 server side encryption using KMS. Deprecated `-<prefix>.s3.sse-encryption`, you should use the following CLI flags that have been added. #3651 #3810
   - `-<prefix>.s3.sse.type`
   - `-<prefix>.s3.sse.kms-key-id`

pkg/ingester/errors.go

@@ -20,7 +20,7 @@ func makeLimitError(errorType string, err error) error {
 	return &validationError{
 		errorType: errorType,
 		err:       err,
-		code:      http.StatusTooManyRequests,
+		code:      http.StatusBadRequest,
 	}
 }
 
@@ -44,7 +44,7 @@ func makeMetricLimitError(errorType string, labels labels.Labels, err error) err
 	return &validationError{
 		errorType: errorType,
 		err:       err,
-		code:      http.StatusTooManyRequests,
+		code:      http.StatusBadRequest,
 		labels:    labels,
 	}
 }

pkg/ingester/ingester_test.go

@@ -577,7 +577,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) {
 			testLimits := func() {
 				// Append to two series, expect series-exceeded error.
 				_, err = ing.Push(ctx, client.ToWriteRequest([]labels.Labels{labels1, labels3}, []client.Sample{sample2, sample3}, nil, client.API))
-				if resp, ok := httpgrpc.HTTPResponseFromError(err); !ok || resp.Code != http.StatusTooManyRequests {
+				if resp, ok := httpgrpc.HTTPResponseFromError(err); !ok || resp.Code != http.StatusBadRequest {
 					t.Fatalf("expected error about exceeding metrics per user, got %v", err)
 				}
 				// Append two metadata, expect no error since metadata is a best effort approach.
@@ -698,7 +698,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) {
 			testLimits := func() {
 				// Append two series, expect series-exceeded error.
 				_, err = ing.Push(ctx, client.ToWriteRequest([]labels.Labels{labels1, labels3}, []client.Sample{sample2, sample3}, nil, client.API))
-				if resp, ok := httpgrpc.HTTPResponseFromError(err); !ok || resp.Code != http.StatusTooManyRequests {
+				if resp, ok := httpgrpc.HTTPResponseFromError(err); !ok || resp.Code != http.StatusBadRequest {
 					t.Fatalf("expected error about exceeding series per metric, got %v", err)
 				}