Deprecate -ruler.evaluation-delay-duration

Signed-off-by: Daniel Deluiggi <ddeluigg@amazon.com>

Author: danielblando

docs/configuration/config-file-reference.md

@@ -3340,11 +3340,6 @@ query_rejection:
   # them.
   [query_attributes: <list of QueryAttribute> | default = []]
 
-# Duration to delay the evaluation of rules to ensure the underlying metrics
-# have been pushed to Cortex.
-# CLI flag: -ruler.evaluation-delay-duration
-[ruler_evaluation_delay_duration: <duration> | default = 0s]
-
 # The default tenant's shard size when the shuffle-sharding strategy is used by
 # ruler. When this setting is specified in the per-tenant overrides, a value of
 # 0 disables shuffle sharding for the tenant.

pkg/ruler/compat.go

@@ -13,7 +13,6 @@ import (
 	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/metadata"
-	"github.com/prometheus/prometheus/model/value"
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -46,27 +45,15 @@ type PusherAppender struct {
 	histogramLabels []labels.Labels
 	histograms      []cortexpb.Histogram
 	userID          string
-	evaluationDelay time.Duration
 }
 
 func (a *PusherAppender) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
 	if h == nil && fh == nil {
 		return 0, errors.New("no histogram")
 	}
-
 	if h != nil {
-		// A histogram sample is considered stale if its sum is set to NaN.
-		// https://github.com/prometheus/prometheus/blob/b6ef745016fa9472fdd0ae20f75a9682e01d1e5c/tsdb/head_append.go#L339-L346
-		if a.evaluationDelay > 0 && (value.IsStaleNaN(h.Sum)) {
-			t -= a.evaluationDelay.Milliseconds()
-		}
 		a.histograms = append(a.histograms, cortexpb.HistogramToHistogramProto(t, h))
 	} else {
-		// A histogram sample is considered stale if its sum is set to NaN.
-		// https://github.com/prometheus/prometheus/blob/b6ef745016fa9472fdd0ae20f75a9682e01d1e5c/tsdb/head_append.go#L339-L346
-		if a.evaluationDelay > 0 && (value.IsStaleNaN(fh.Sum)) {
-			t -= a.evaluationDelay.Milliseconds()
-		}
 		a.histograms = append(a.histograms, cortexpb.FloatHistogramToHistogramProto(t, fh))
 	}
 	a.histogramLabels = append(a.histogramLabels, l)
@@ -75,19 +62,6 @@ func (a *PusherAppender) AppendHistogram(_ storage.SeriesRef, l labels.Labels, t
 
 func (a *PusherAppender) Append(_ storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
 	a.labels = append(a.labels, l)
-
-	// Adapt staleness markers for ruler evaluation delay. As the upstream code
-	// is using the actual time, when there is a no longer available series.
-	// This then causes 'out of order' append failures once the series is
-	// becoming available again.
-	// see https://github.com/prometheus/prometheus/blob/6c56a1faaaad07317ff585bda75b99bdba0517ad/rules/manager.go#L647-L660
-	// Similar to staleness markers, the rule manager also appends actual time to the ALERTS and ALERTS_FOR_STATE series.
-	// See: https://github.com/prometheus/prometheus/blob/ae086c73cb4d6db9e8b67d5038d3704fea6aec4a/rules/alerting.go#L414-L417
-	metricName := l.Get(labels.MetricName)
-	if a.evaluationDelay > 0 && (value.IsStaleNaN(v) || metricName == "ALERTS" || metricName == "ALERTS_FOR_STATE") {
-		t -= a.evaluationDelay.Milliseconds()
-	}
-
 	a.samples = append(a.samples, cortexpb.Sample{
 		TimestampMs: t,
 		Value:       v,
@@ -164,16 +138,14 @@ func (t *PusherAppendable) Appender(ctx context.Context) storage.Appender {
 		failedWrites: t.failedWrites,
 		totalWrites:  t.totalWrites,
 
-		ctx:             ctx,
-		pusher:          t.pusher,
-		userID:          t.userID,
-		evaluationDelay: t.rulesLimits.EvaluationDelay(t.userID),
+		ctx:    ctx,
+		pusher: t.pusher,
+		userID: t.userID,
 	}
 }
 
 // RulesLimits defines limits used by Ruler.
 type RulesLimits interface {
-	EvaluationDelay(userID string) time.Duration
 	MaxQueryLength(userID string) time.Duration
 	RulerTenantShardSize(userID string) int
 	RulerMaxRuleGroupsPerTenant(userID string) int
@@ -182,7 +154,7 @@ type RulesLimits interface {
 	DisabledRuleGroups(userID string) validation.DisabledRuleGroups
 }
 
-// EngineQueryFunc returns a new engine query function by passing an altered timestamp.
+// EngineQueryFunc returns a new engine query function validating max queryLength.
 // Modified from Prometheus rules.EngineQueryFunc
 // https://github.com/prometheus/prometheus/blob/v2.39.1/rules/manager.go#L189.
 func EngineQueryFunc(engine promql.QueryEngine, q storage.Queryable, overrides RulesLimits, userID string, lookbackDelta time.Duration) rules.QueryFunc {
@@ -202,8 +174,7 @@ func EngineQueryFunc(engine promql.QueryEngine, q storage.Queryable, overrides R
 			}
 		}
 
-		evaluationDelay := overrides.EvaluationDelay(userID)
-		q, err := engine.NewInstantQuery(ctx, q, nil, qs, t.Add(-evaluationDelay))
+		q, err := engine.NewInstantQuery(ctx, q, nil, qs, t)
 		if err != nil {
 			return nil, err
 		}

pkg/ruler/compat_test.go

@@ -43,7 +43,6 @@ func TestPusherAppendable(t *testing.T) {
 
 	lbls1 := cortexpb.FromLabelsToLabelAdapters(labels.FromMap(map[string]string{labels.MetricName: "foo_bar"}))
 	lbls2 := cortexpb.FromLabelsToLabelAdapters(labels.FromMap(map[string]string{labels.MetricName: "ALERTS", labels.AlertName: "boop"}))
-	lbls3 := cortexpb.FromLabelsToLabelAdapters(labels.FromMap(map[string]string{labels.MetricName: "ALERTS_FOR_STATE", labels.AlertName: "boop"}))
 
 	testHistogram := tsdbutil.GenerateTestHistogram(1)
 	testFloatHistogram := tsdbutil.GenerateTestFloatHistogram(2)
@@ -55,14 +54,13 @@ func TestPusherAppendable(t *testing.T) {
 	for _, tc := range []struct {
 		name           string
 		series         string
-		evalDelay      time.Duration
 		value          float64
 		histogram      *histogram.Histogram
 		floatHistogram *histogram.FloatHistogram
 		expectedReq    *cortexpb.WriteRequest
 	}{
 		{
-			name:   "tenant without delay, normal value",
+			name:   "tenant, normal value",
 			series: "foo_bar",
 			value:  1.234,
 			expectedReq: &cortexpb.WriteRequest{
@@ -80,7 +78,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:   "tenant without delay, stale nan value",
+			name:   "tenant, stale nan value",
 			series: "foo_bar",
 			value:  math.Float64frombits(value.StaleNaN),
 			expectedReq: &cortexpb.WriteRequest{
@@ -98,45 +96,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:      "tenant with delay, normal value",
-			series:    "foo_bar",
-			value:     1.234,
-			evalDelay: time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Samples: []cortexpb.Sample{
-								{Value: 1.234, TimestampMs: 120_000},
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:      "tenant with delay, stale nan value",
-			series:    "foo_bar",
-			value:     math.Float64frombits(value.StaleNaN),
-			evalDelay: time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Samples: []cortexpb.Sample{
-								{Value: math.Float64frombits(value.StaleNaN), TimestampMs: 60_000},
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:   "ALERTS without delay, normal value",
+			name:   "ALERTS, normal value",
 			series: `ALERTS{alertname="boop"}`,
 			value:  1.234,
 			expectedReq: &cortexpb.WriteRequest{
@@ -154,7 +114,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:   "ALERTS without delay, stale nan value",
+			name:   "ALERTS, stale nan value",
 			series: `ALERTS{alertname="boop"}`,
 			value:  math.Float64frombits(value.StaleNaN),
 			expectedReq: &cortexpb.WriteRequest{
@@ -172,45 +132,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:      "ALERTS with delay, normal value",
-			series:    `ALERTS{alertname="boop"}`,
-			value:     1.234,
-			evalDelay: time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls2,
-							Samples: []cortexpb.Sample{
-								{Value: 1.234, TimestampMs: 60_000},
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:      "ALERTS with delay, stale nan value",
-			series:    `ALERTS_FOR_STATE{alertname="boop"}`,
-			value:     math.Float64frombits(value.StaleNaN),
-			evalDelay: time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls3,
-							Samples: []cortexpb.Sample{
-								{Value: math.Float64frombits(value.StaleNaN), TimestampMs: 60_000},
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:      "tenant without delay, normal histogram",
+			name:      "tenant, normal histogram",
 			series:    "foo_bar",
 			histogram: testHistogram,
 			expectedReq: &cortexpb.WriteRequest{
@@ -228,7 +150,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:           "tenant without delay, float histogram",
+			name:           "tenant, float histogram",
 			series:         "foo_bar",
 			floatHistogram: testFloatHistogram,
 			expectedReq: &cortexpb.WriteRequest{
@@ -246,7 +168,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:      "tenant without delay, both sample and histogram",
+			name:      "tenant, both sample and histogram",
 			series:    "foo_bar",
 			value:     1.234,
 			histogram: testHistogram,
@@ -273,7 +195,7 @@ func TestPusherAppendable(t *testing.T) {
 			},
 		},
 		{
-			name:           "tenant without delay, both sample and float histogram",
+			name:           "tenant, both sample and float histogram",
 			series:         "foo_bar",
 			value:          1.234,
 			floatHistogram: testFloatHistogram,
@@ -299,106 +221,9 @@ func TestPusherAppendable(t *testing.T) {
 				Source: cortexpb.RULE,
 			},
 		},
-		{
-			name:      "tenant with delay and NaN sample, normal histogram",
-			series:    "foo_bar",
-			value:     math.Float64frombits(value.StaleNaN),
-			evalDelay: time.Minute,
-			histogram: testHistogram,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Samples: []cortexpb.Sample{
-								{Value: math.Float64frombits(value.StaleNaN), TimestampMs: 60_000},
-							},
-						},
-					},
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Histograms: []cortexpb.Histogram{
-								cortexpb.HistogramToHistogramProto(120_000, testHistogram),
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:           "tenant with delay and NaN sample, float histogram",
-			series:         "foo_bar",
-			value:          math.Float64frombits(value.StaleNaN),
-			evalDelay:      time.Minute,
-			floatHistogram: testFloatHistogram,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Samples: []cortexpb.Sample{
-								{Value: math.Float64frombits(value.StaleNaN), TimestampMs: 60_000},
-							},
-						},
-					},
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Histograms: []cortexpb.Histogram{
-								cortexpb.FloatHistogramToHistogramProto(120_000, testFloatHistogram),
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:      "tenant with delay, NaN histogram",
-			series:    "foo_bar",
-			histogram: testHistogramWithNaN,
-			evalDelay: time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Histograms: []cortexpb.Histogram{
-								cortexpb.HistogramToHistogramProto(60_000, testHistogramWithNaN),
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
-		{
-			name:           "tenant with delay, NaN float histogram",
-			series:         "foo_bar",
-			floatHistogram: testFloatHistogramWithNaN,
-			evalDelay:      time.Minute,
-			expectedReq: &cortexpb.WriteRequest{
-				Timeseries: []cortexpb.PreallocTimeseries{
-					{
-						TimeSeries: &cortexpb.TimeSeries{
-							Labels: lbls1,
-							Histograms: []cortexpb.Histogram{
-								cortexpb.FloatHistogramToHistogramProto(60_000, testFloatHistogramWithNaN),
-							},
-						},
-					},
-				},
-				Source: cortexpb.RULE,
-			},
-		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx := context.Background()
-			pa.rulesLimits = &ruleLimits{
-				evalDelay: tc.evalDelay,
-			}
 
 			lbls, err := parser.ParseMetric(tc.series)
 			require.NoError(t, err)
@@ -461,7 +286,7 @@ func TestPusherErrors(t *testing.T) {
 			writes := prometheus.NewCounter(prometheus.CounterOpts{})
 			failures := prometheus.NewCounter(prometheus.CounterOpts{})
 
-			pa := NewPusherAppendable(pusher, "user-1", ruleLimits{evalDelay: 10 * time.Second}, writes, failures)
+			pa := NewPusherAppendable(pusher, "user-1", ruleLimits{}, writes, failures)
 
 			lbls, err := parser.ParseMetric("foo_bar")
 			require.NoError(t, err)

pkg/ruler/ruler_ring_test.go

@@ -223,7 +223,7 @@ func TestGetReplicationSetForListRule(t *testing.T) {
 			}
 
 			r, _ := buildRuler(t, cfg, nil, store, nil)
-			r.limits = ruleLimits{evalDelay: 0}
+			r.limits = ruleLimits{}
 
 			rulerRing := r.ring
 			// We start ruler's ring, but nothing else (not even lifecycler).