[jwt] fix dashboard and alerts

Author: capcom6

deployments/grafana/dashboards/jwt.json

@@ -149,7 +149,7 @@
                         "uid": "Prometheus"
                     },
                     "editorMode": "code",
-                    "expr": "100 * sum(rate(sms_auth_jwt_validations_total{status=\"success\",instance=~\"$instance\",job=~\"$job\"}[5m])) / clamp_min(sum(rate(sms_auth_jwt_validations_total{instance=~\"$instance\",job=~\"$job\"}[5m])), 1)",
+                    "expr": "100 * sum(rate(sms_auth_jwt_tokens_validated_total{status=\"success\",instance=~\"$instance\",job=~\"$job\"}[5m])) / clamp_min(sum(rate(sms_auth_jwt_tokens_validated_total{instance=~\"$instance\",job=~\"$job\"}[5m])), 1)",
                     "legendFormat": "Validation Success Rate",
                     "range": true,
                     "refId": "A"
@@ -386,13 +386,13 @@
                         "uid": "Prometheus"
                     },
                     "editorMode": "code",
-                    "expr": "sum by (reason) (sms_auth_jwt_issuance_errors_total{instance=~\"$instance\",job=~\"$job\"})",
-                    "legendFormat": "{{reason}}",
+                    "expr": "sum by (status) (sms_auth_jwt_tokens_issued_total{status=\"error\",instance=~\"$instance\",job=~\"$job\"})",
+                    "legendFormat": "{{status}}",
                     "range": true,
                     "refId": "A"
                 }
             ],
-            "title": "Issuance Error Breakdown by Reason",
+            "title": "Issuance Error Count",
             "type": "bargauge"
         },
         {
@@ -606,7 +606,7 @@
                         "uid": "Prometheus"
                     },
                     "editorMode": "code",
-                    "expr": "sum(rate(sms_auth_jwt_validations_total{status=\"success\",instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(sms_auth_jwt_validations_total{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])), 1)",
+                    "expr": "sum(rate(sms_auth_jwt_tokens_validated_total{status=\"success\",instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(sms_auth_jwt_tokens_validated_total{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])), 1)",
                     "legendFormat": "Success Rate",
                     "range": true,
                     "refId": "A"
@@ -671,15 +671,15 @@
                     },
                     "editorMode": "code",
                     "exemplar": false,
-                    "expr": "sum by (reason) (sms_auth_jwt_validation_errors_total{instance=~\"$instance\",job=~\"$job\"})",
+                    "expr": "sum by (status) (sms_auth_jwt_tokens_validated_total{instance=~\"$instance\",job=~\"$job\"})",
                     "format": "heatmap",
                     "instant": true,
-                    "legendFormat": "{{reason}}",
+                    "legendFormat": "{{status}}",
                     "range": false,
                     "refId": "A"
                 }
             ],
-            "title": "Validation Error Classification",
+            "title": "Validation Status Classification",
             "type": "piechart"
         },
         {
@@ -993,7 +993,7 @@
                         "uid": "Prometheus"
                     },
                     "editorMode": "code",
-                    "expr": "sum(rate(sms_auth_jwt_revocation_errors_total{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(sms_auth_jwt_tokens_revoked_total{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])), 1)",
+                    "expr": "sum(rate(sms_auth_jwt_tokens_revoked_total{status=\"error\",instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(sms_auth_jwt_tokens_revoked_total{instance=~\"$instance\",job=~\"$job\"}[$__rate_interval])), 1)",
                     "legendFormat": "Error Rate",
                     "range": true,
                     "refId": "A"

deployments/prometheus/alerts/jwt-alerts.yml

@@ -2,7 +2,7 @@ groups:
   - name: jwt-alerts
     rules:
       - alert: JWT_Validation_ErrorRate_High
-        expr: rate(sms_auth_jwt_tokens_validated_total{status="error"}[5m]) / rate(sms_auth_jwt_tokens_validated_total[5m]) > 0.05
+        expr: rate(sms_auth_jwt_tokens_validated_total{status="error"}[5m]) / clamp_min(rate(sms_auth_jwt_tokens_validated_total[5m]), 1e-9) > 0.05
         for: 5m
         labels:
           severity: warning
@@ -32,7 +32,7 @@ groups:
           description: "JWT revocation database errors occurring at rate {{ $value }}/s"
 
       - alert: JWT_Validation_Failures_High
-        expr: rate(sms_auth_jwt_tokens_validated_total{reason=~"expired|revoked"}[5m]) / rate(sms_auth_jwt_tokens_validated_total[5m]) > 0.1
+        expr: rate(sms_auth_jwt_tokens_validated_total{status="error",reason=~"expired|revoked"}[5m]) / max(rate(sms_auth_jwt_tokens_validated_total[5m]), 0.00001) > 0.1
         for: 5m
         labels:
           severity: warning

internal/sms-gateway/jwt/metrics.go

@@ -98,20 +98,20 @@ func (m *Metrics) IncrementTokensRevoked(status string) {
 // ObserveIssuance observes issuance duration
 func (m *Metrics) ObserveIssuance(f func()) {
 	timer := prometheus.NewTimer(m.issuanceDurationHistogram)
+	defer timer.ObserveDuration()
 	f()
-	timer.ObserveDuration()
 }
 
 // ObserveValidation observes validation duration
 func (m *Metrics) ObserveValidation(f func()) {
 	timer := prometheus.NewTimer(m.validationDurationHistogram)
+	defer timer.ObserveDuration()
 	f()
-	timer.ObserveDuration()
 }
 
 // ObserveRevocation observes revocation duration
 func (m *Metrics) ObserveRevocation(f func()) {
 	timer := prometheus.NewTimer(m.revocationDurationHistogram)
+	defer timer.ObserveDuration()
 	f()
-	timer.ObserveDuration()
 }