[release-4.21] OCPBUGS-74641: split grpc requests slow into read/write alerts

jsonnet/custom.libsonnet

@@ -41,22 +41,27 @@
             },
           },
           {
-            alert: 'etcdGRPCRequestsSlow',
-            expr: |||
-              histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method!="Defragment", grpc_type="unary"}[10m])) without(grpc_type))
-              > on () group_left (type)
-              bottomk(1,
-                1.5 * group by (type) (cluster_infrastructure_provider{type="Azure"})
-                or
-                1 * group by (type) (cluster_infrastructure_provider))
-            |||,
-            'for': '30m',
+            alert: 'etcdGRPCReadRequestsSlow',
+            expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Range", grpc_type="unary"}[10m])) without(grpc_type)) > 3',
+            'for': '10m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC read requests is {{ $value }}s on etcd instance {{ $labels.instance }}.',
+              summary: 'etcd grpc read requests are slow',
+            },
+          },
+          {
+            alert: 'etcdGRPCWriteRequestsSlow',
+            expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Txn", grpc_type="unary"}[10m])) without(grpc_type)) > 5',
+            'for': '10m',
             labels: {
               severity: 'critical',
             },
             annotations: {
-              description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.',
-              summary: 'etcd grpc requests are slow',
+              description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC write requests is {{ $value }}s on etcd instance {{ $labels.instance }}.',
+              summary: 'etcd grpc write requests are slow',
             },
           },
           {

jsonnet/jsonnetfile.lock.json

@@ -8,7 +8,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "e4d6a05f8f1ec972384e24a83c420f707a6644f2",
+      "version": "9a2b574b70e69eb1f47dfc896b2f63245f1873f6",
       "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
@@ -18,7 +18,7 @@
           "subdir": "gen/grafonnet-v10.0.0"
         }
       },
-      "version": "42d098fae987f25f08480e203ca6ddc548c6efbf",
+      "version": "7380c9c64fb973f34c3ec46265621a2b0dee0058",
       "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0="
     },
     {
@@ -38,8 +38,8 @@
           "subdir": ""
         }
       },
-      "version": "4eee017d21cb63a303925d1dcd9fc5c496809b46",
-      "sum": "Kh0GbIycNmJPzk6IOMXn1BbtLNyaiiimclYk7+mvsns="
+      "version": "4d7f8cb24d613430799f9d56809cc6964f35cea9",
+      "sum": "hOrwkOx34tOXqoDVnwuI/Uf/dr9HFFSPWpDPOvnEGrk="
     }
   ],
   "legacyImports": false