[release-2.6.x] [k102] Don't interrupt the query path if deletes aren't available

pkg/loki/modules.go

@@ -984,7 +984,7 @@ func (t *Loki) deleteRequestsClient() (deletion.DeleteRequestsClient, error) {
 		return nil, err
 	}
 
-	return deletion.NewDeleteRequestsClient(compactorAddress, &http.Client{Timeout: 5 * time.Second})
+	return deletion.NewDeleteRequestsClient(compactorAddress, &http.Client{Timeout: 5 * time.Second}, prometheus.DefaultRegisterer)
 }
 
 func calculateMaxLookBack(pc config.PeriodConfig, maxLookBackConfig, minDuration time.Duration) (time.Duration, error) {

pkg/querier/querier.go

@@ -177,7 +177,7 @@ func (q *SingleTenantQuerier) SelectSamples(ctx context.Context, params logql.Se
 
 	params.SampleQueryRequest.Deletes, err = q.deletesForUser(ctx, params.Start, params.End)
 	if err != nil {
-		return nil, err
+		level.Error(spanlogger.FromContext(ctx)).Log("msg", "failed loading deletes for user", "err", err)
 	}
 
 	ingesterQueryInterval, storeQueryInterval := q.buildQueryIntervals(params.Start, params.End)

pkg/storage/stores/shipper/compactor/deletion/delete_requests_client.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/grafana/loki/pkg/util/log"
 )
@@ -32,6 +33,8 @@ type deleteRequestsClient struct {
 	cache         map[string][]DeleteRequest
 	cacheDuration time.Duration
 
+	metrics *deleteRequestClientMetrics
+
 	stopChan chan struct{}
 }
 
@@ -47,7 +50,7 @@ func WithRequestClientCacheDuration(d time.Duration) DeleteRequestsStoreOption {
 	}
 }
 
-func NewDeleteRequestsClient(addr string, c httpClient, opts ...DeleteRequestsStoreOption) (DeleteRequestsClient, error) {
+func NewDeleteRequestsClient(addr string, c httpClient, r prometheus.Registerer, opts ...DeleteRequestsStoreOption) (DeleteRequestsClient, error) {
 	u, err := url.Parse(addr)
 	if err != nil {
 		level.Error(log.Logger).Log("msg", "error parsing url", "err", err)
@@ -60,6 +63,7 @@ func NewDeleteRequestsClient(addr string, c httpClient, opts ...DeleteRequestsSt
 		httpClient:    c,
 		cacheDuration: 5 * time.Minute,
 		cache:         make(map[string][]DeleteRequest),
+		metrics:       newDeleteRequestClientMetrics(r),
 		stopChan:      make(chan struct{}),
 	}
 
@@ -76,8 +80,10 @@ func (c *deleteRequestsClient) GetAllDeleteRequestsForUser(ctx context.Context,
 		return cachedRequests, nil
 	}
 
+	c.metrics.deleteRequestsLookupsTotal.Inc()
 	requests, err := c.getRequestsFromServer(ctx, userID)
 	if err != nil {
+		c.metrics.deleteRequestsLookupsFailedTotal.Inc()
 		return nil, err
 	}
 

pkg/storage/stores/shipper/compactor/deletion/delete_requests_client_test.go

@@ -14,7 +14,7 @@ import (
 func TestGetCacheGenNumberForUser(t *testing.T) {
 	t.Run("it requests results from the api", func(t *testing.T) {
 		httpClient := &mockHTTPClient{ret: `[{"request_id":"test-request"}]`}
-		client, err := NewDeleteRequestsClient("http://test-server", httpClient)
+		client, err := NewDeleteRequestsClient("http://test-server", httpClient, nil)
 		require.Nil(t, err)
 
 		deleteRequests, err := client.GetAllDeleteRequestsForUser(context.Background(), "userID")
@@ -30,7 +30,7 @@ func TestGetCacheGenNumberForUser(t *testing.T) {
 
 	t.Run("it caches the results", func(t *testing.T) {
 		httpClient := &mockHTTPClient{ret: `[{"request_id":"test-request"}]`}
-		client, err := NewDeleteRequestsClient("http://test-server", httpClient, WithRequestClientCacheDuration(100*time.Millisecond))
+		client, err := NewDeleteRequestsClient("http://test-server", httpClient, nil, WithRequestClientCacheDuration(100*time.Millisecond))
 		require.Nil(t, err)
 
 		deleteRequests, err := client.GetAllDeleteRequestsForUser(context.Background(), "userID")

pkg/storage/stores/shipper/compactor/deletion/metrics.go

@@ -5,6 +5,29 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+type deleteRequestClientMetrics struct {
+	deleteRequestsLookupsTotal       prometheus.Counter
+	deleteRequestsLookupsFailedTotal prometheus.Counter
+}
+
+func newDeleteRequestClientMetrics(r prometheus.Registerer) *deleteRequestClientMetrics {
+	m := deleteRequestClientMetrics{}
+
+	m.deleteRequestsLookupsTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Namespace: "loki",
+		Name:      "delete_request_lookups_total",
+		Help:      "Number times the client has looked up delete requests",
+	})
+
+	m.deleteRequestsLookupsFailedTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Namespace: "loki",
+		Name:      "delete_request_lookups_failed_total",
+		Help:      "Number times the client has failed to look up delete requests",
+	})
+
+	return &m
+}
+
 type deleteRequestHandlerMetrics struct {
 	deleteRequestsReceivedTotal *prometheus.CounterVec
 }