feat: add timer metrics (#254)

vanus-labs · Sep 15, 2022 · 81995ed · 81995ed
1 parent 18f3cbb
commit 81995ed
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 0 deletions.
diff --git a/internal/timer/timingwheel/timingwheel.go b/internal/timer/timingwheel/timingwheel.go
@@ -283,6 +283,9 @@ func (tw *timingWheel) Push(ctx context.Context, e *ce.Event) bool {
 		"expiration": tm.getExpiration().Format(time.RFC3339Nano),
 	})
 
+	metrics.TimerScheduledEventDelayTime.WithLabelValues(metrics.LabelScheduledEventDelayTime).
+		Observe(time.Until(tm.getExpiration()).Seconds())
+
 	if tm.hasExpired() {
 		// Already expired
 		return tw.getDistributionStation().push(ctx, tm) == nil
@@ -400,7 +403,11 @@ func (tw *timingWheel) runReceivingStation(ctx context.Context) {
 						defer wg.Done()
 						waitCtx, cancel := context.WithCancel(ctx)
 						wait.Until(func() {
+							startTime := time.Now()
 							if tw.Push(ctx, e) {
+								metrics.TimerPushEventTime.WithLabelValues(metrics.LabelTimerPushScheduledEventTime).
+									Observe(time.Since(startTime).Seconds())
+								metrics.TimerPushEventTPSCounterVec.WithLabelValues(metrics.LabelTimer).Inc()
 								cancel()
 							} else {
 								log.Warning(ctx, "push event to timingwheel failed, retry until it succeed", map[string]interface{}{
@@ -499,7 +506,11 @@ func (tw *timingWheel) runDistributionStation(ctx context.Context) {
 						defer wg.Done()
 						waitCtx, cancel := context.WithCancel(ctx)
 						wait.Until(func() {
+							startTime := time.Now()
 							if err = tw.deliver(ctx, e); err == nil {
+								metrics.TimerDeliverEventTime.WithLabelValues(metrics.LabelTimerDeliverScheduledEventTime).
+									Observe(time.Since(startTime).Seconds())
+								metrics.TimerDeliverEventTPSCounterVec.WithLabelValues(metrics.LabelTimer).Inc()
 								cancel()
 							} else {
 								log.Warning(ctx, "deliver event failed, retry until it succeed", map[string]interface{}{

diff --git a/observability/metrics/label.go b/observability/metrics/label.go
@@ -24,6 +24,8 @@ const (
 	LabelTrigger       = "trigger"
 	LabelResult        = "result"
 	LabelBlock         = "block"
+
+	LabelTimer = "timer"
 )
 
 const (
@@ -35,3 +37,9 @@ const (
 	LabelSegmentDeletedBecauseCreateFailed = "segment_create_failed"
 	LabelSegmentDeletedBecauseDeleted      = "segment_deleted"
 )
+
+const (
+	LabelScheduledEventDelayTime        = "scheduled_event_delay_time"
+	LabelTimerPushScheduledEventTime    = "push_scheduled_event_time"
+	LabelTimerDeliverScheduledEventTime = "deliver_scheduled_event_time"
+)
diff --git a/observability/metrics/metrics.go b/observability/metrics/metrics.go
@@ -59,6 +59,11 @@ func RegisterTimerMetrics() {
 	prometheus.MustRegister(TimingWheelTickGauge)
 	prometheus.MustRegister(TimingWheelSizeGauge)
 	prometheus.MustRegister(TimingWheelLayersGauge)
+	prometheus.MustRegister(TimerPushEventTPSCounterVec)
+	prometheus.MustRegister(TimerDeliverEventTPSCounterVec)
+	prometheus.MustRegister(TimerScheduledEventDelayTime)
+	prometheus.MustRegister(TimerPushEventTime)
+	prometheus.MustRegister(TimerDeliverEventTime)
 }
 
 func RegisterSegmentServerMetrics() {

diff --git a/observability/metrics/timer.go b/observability/metrics/timer.go
@@ -39,4 +39,39 @@ var (
 		Name:      "timingwheel_layers",
 		Help:      "The layers of timingwheel.",
 	})
+
+	TimerPushEventTPSCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: moduleOfTimer,
+		Name:      "push_scheduled_event_count",
+		Help:      "Total scheduled events for push",
+	}, []string{LabelTimer})
+
+	TimerDeliverEventTPSCounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: moduleOfTimer,
+		Name:      "deliver_scheduled_event_count",
+		Help:      "Total scheduled events for deliver",
+	}, []string{LabelTimer})
+
+	TimerScheduledEventDelayTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: moduleOfTriggerWorker,
+		Name:      "scheduled_event_delay_time",
+		Help:      "The time of scheduled event delay deliver",
+	}, []string{LabelScheduledEventDelayTime})
+
+	TimerPushEventTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: moduleOfTriggerWorker,
+		Name:      "push_scheduled_event_time",
+		Help:      "The time of timer push scheduled event",
+	}, []string{LabelTimerPushScheduledEventTime})
+
+	TimerDeliverEventTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: moduleOfTriggerWorker,
+		Name:      "deliver_scheduled_event_time",
+		Help:      "The time of timer deliver scheduled event",
+	}, []string{LabelTimerDeliverScheduledEventTime})
 )