diff --git a/internal/timer/timingwheel/timingwheel.go b/internal/timer/timingwheel/timingwheel.go index cb0565a88..b9cfa2b71 100644 --- a/internal/timer/timingwheel/timingwheel.go +++ b/internal/timer/timingwheel/timingwheel.go @@ -283,6 +283,9 @@ func (tw *timingWheel) Push(ctx context.Context, e *ce.Event) bool { "expiration": tm.getExpiration().Format(time.RFC3339Nano), }) + metrics.TimerScheduledEventDelayTime.WithLabelValues(metrics.LabelScheduledEventDelayTime). + Observe(time.Until(tm.getExpiration()).Seconds()) + if tm.hasExpired() { // Already expired return tw.getDistributionStation().push(ctx, tm) == nil @@ -400,7 +403,11 @@ func (tw *timingWheel) runReceivingStation(ctx context.Context) { defer wg.Done() waitCtx, cancel := context.WithCancel(ctx) wait.Until(func() { + startTime := time.Now() if tw.Push(ctx, e) { + metrics.TimerPushEventTime.WithLabelValues(metrics.LabelTimerPushScheduledEventTime). + Observe(time.Since(startTime).Seconds()) + metrics.TimerPushEventTPSCounterVec.WithLabelValues(metrics.LabelTimer).Inc() cancel() } else { log.Warning(ctx, "push event to timingwheel failed, retry until it succeed", map[string]interface{}{ @@ -499,7 +506,11 @@ func (tw *timingWheel) runDistributionStation(ctx context.Context) { defer wg.Done() waitCtx, cancel := context.WithCancel(ctx) wait.Until(func() { + startTime := time.Now() if err = tw.deliver(ctx, e); err == nil { + metrics.TimerDeliverEventTime.WithLabelValues(metrics.LabelTimerDeliverScheduledEventTime). + Observe(time.Since(startTime).Seconds()) + metrics.TimerDeliverEventTPSCounterVec.WithLabelValues(metrics.LabelTimer).Inc() cancel() } else { log.Warning(ctx, "deliver event failed, retry until it succeed", map[string]interface{}{ diff --git a/observability/metrics/label.go b/observability/metrics/label.go index 5e4a6b5a8..f92862a21 100644 --- a/observability/metrics/label.go +++ b/observability/metrics/label.go @@ -24,6 +24,8 @@ const ( LabelTrigger = "trigger" LabelResult = "result" LabelBlock = "block" + + LabelTimer = "timer" ) const ( @@ -35,3 +37,9 @@ const ( LabelSegmentDeletedBecauseCreateFailed = "segment_create_failed" LabelSegmentDeletedBecauseDeleted = "segment_deleted" ) + +const ( + LabelScheduledEventDelayTime = "scheduled_event_delay_time" + LabelTimerPushScheduledEventTime = "push_scheduled_event_time" + LabelTimerDeliverScheduledEventTime = "deliver_scheduled_event_time" +) diff --git a/observability/metrics/metrics.go b/observability/metrics/metrics.go index bec264dbb..cc165806d 100644 --- a/observability/metrics/metrics.go +++ b/observability/metrics/metrics.go @@ -59,6 +59,11 @@ func RegisterTimerMetrics() { prometheus.MustRegister(TimingWheelTickGauge) prometheus.MustRegister(TimingWheelSizeGauge) prometheus.MustRegister(TimingWheelLayersGauge) + prometheus.MustRegister(TimerPushEventTPSCounterVec) + prometheus.MustRegister(TimerDeliverEventTPSCounterVec) + prometheus.MustRegister(TimerScheduledEventDelayTime) + prometheus.MustRegister(TimerPushEventTime) + prometheus.MustRegister(TimerDeliverEventTime) } func RegisterSegmentServerMetrics() { diff --git a/observability/metrics/timer.go b/observability/metrics/timer.go index 711f5eeb3..154fb9d87 100644 --- a/observability/metrics/timer.go +++ b/observability/metrics/timer.go @@ -39,4 +39,39 @@ var ( Name: "timingwheel_layers", Help: "The layers of timingwheel.", }) + + TimerPushEventTPSCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: moduleOfTimer, + Name: "push_scheduled_event_count", + Help: "Total scheduled events for push", + }, []string{LabelTimer}) + + TimerDeliverEventTPSCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: moduleOfTimer, + Name: "deliver_scheduled_event_count", + Help: "Total scheduled events for deliver", + }, []string{LabelTimer}) + + TimerScheduledEventDelayTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: moduleOfTriggerWorker, + Name: "scheduled_event_delay_time", + Help: "The time of scheduled event delay deliver", + }, []string{LabelScheduledEventDelayTime}) + + TimerPushEventTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: moduleOfTriggerWorker, + Name: "push_scheduled_event_time", + Help: "The time of timer push scheduled event", + }, []string{LabelTimerPushScheduledEventTime}) + + TimerDeliverEventTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: moduleOfTriggerWorker, + Name: "deliver_scheduled_event_time", + Help: "The time of timer deliver scheduled event", + }, []string{LabelTimerDeliverScheduledEventTime}) )