[Model Monitoring] Support Prometheus as a time series solution for m…

…odel monitoring (mlrun#69)
laurybueno · Jul 25, 2023 · 785e49e · 785e49e
1 parent f5817c7
commit 785e49e
Show file tree

Hide file tree

Showing 5 changed files with 1,092 additions and 77 deletions.
diff --git a/charts/mlrun-ce/Chart.yaml b/charts/mlrun-ce/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
-version: 0.6.1
+version: 0.6.2-rc1
 name: mlrun-ce
 description: MLRUn Open Source Stack
 home: https://iguazio.com

diff --git a/charts/mlrun-ce/templates/config/model-monitoring-details.yml b/charts/mlrun-ce/templates/config/model-monitoring-details.yml
@@ -1,3 +1,4 @@
+{{- if index .Values "kube-prometheus-stack" "enabled" -}}
 apiVersion: v1
 data:
   model-monitoring-details.json: |
@@ -27,24 +28,36 @@ data:
       "editable": true,
       "fiscalYearStartMonth": 0,
       "graphTooltip": 0,
-      "id": 113,
+      "id": 75,
       "links": [
         {
           "asDropdown": true,
-          "icon": "external link",
+          "icon": "dashboard",
           "includeVars": true,
           "keepTime": true,
           "tags": [],
           "title": "Model Monitoring - Overview",
           "type": "link",
           "url": "d/g0M4uh0Mz/model-monitoring-overview"
+        },
+        {
+          "asDropdown": false,
+          "icon": "dashboard",
+          "includeVars": true,
+          "keepTime": true,
+          "tags": [],
+          "targetBlank": false,
+          "title": "Model Monitoring - Performance",
+          "tooltip": "",
+          "type": "link",
+          "url": "d/9CazA-UGz/model-monitoring-performance"
         }
       ],
       "liveNow": false,
       "panels": [
         {
           "datasource": "iguazio",
-          "description": "",
+          "description": "A summery table of the selected model endpoint:\n\n- Endpoint ID — the ID of the deployed model. \n\n- Function — Function to access the model, usually a Nuclio serving function\n\n- Model — user defined name for the model\n\n- Class — the implementation class that is used by the endpoint\n\n- First Request — first request for production data\n\n- Last Request — most recent request for production data\n\n- Average Latency (1 hour) - Average time that it takes to process an event\n\n- Predictions/s (5 minute avg) - Average amount of predictions per second in the last 5 minutes. ",
           "fieldConfig": {
             "defaults": {
               "custom": {
@@ -200,7 +213,7 @@ data:
             "showHeader": true,
             "sortBy": []
           },
-          "pluginVersion": "9.3.6",
+          "pluginVersion": "9.2.2",
           "targets": [
             {
               "datasource": "iguazio",
@@ -228,14 +241,15 @@ data:
               }
             }
           ],
+          "title": "Model Summary",
           "transformations": [
             {
               "id": "organize",
               "options": {
                 "indexByName": {
                   "first_request": 5,
-                  "last_request": 6,
                   "function_uri": 2,
+                  "last_request": 6,
                   "metrics->\"$.generic.latency_avg_1h\"": 7,
                   "metrics->\"$.generic.predictions_per_second\"": 8,
                   "model": 3,
@@ -244,8 +258,8 @@ data:
                 },
                 "renameByName": {
                   "first_request": "First Request",
-                  "last_request": "Last Request",
                   "function_uri": "Function URI",
+                  "last_request": "Last Request",
                   "metrics->\"$.generic.latency_avg_1h\"": "Average Latency (1 hour)",
                   "metrics->\"$.generic.predictions_per_second\"": "Predictions/s (5 minute avg)",
                   "model": "Model",
@@ -260,7 +274,7 @@ data:
         },
         {
           "datasource": "iguazio",
-          "description": "",
+          "description": "Drift statistics of the selected model according to the latest batch",
           "fieldConfig": {
             "defaults": {
               "custom": {
@@ -407,7 +421,7 @@ data:
               }
             ]
           },
-          "pluginVersion": "9.3.6",
+          "pluginVersion": "9.2.2",
           "targets": [
             {
               "datasource": "iguazio",
@@ -461,7 +475,7 @@ data:
         },
         {
           "datasource": "iguazio",
-          "description": "Feature analysis of the latest batch",
+          "description": "Feature analysis for each feature in the selected model according to the latest batch.",
           "fieldConfig": {
             "defaults": {
               "custom": {
@@ -583,7 +597,7 @@ data:
               }
             ]
           },
-          "pluginVersion": "9.3.6",
+          "pluginVersion": "9.2.2",
           "targets": [
             {
               "dataset": "mlrun",
@@ -908,6 +922,100 @@ data:
           ],
           "transparent": true,
           "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "description": "Sampled data points from actual feature production data by time. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "points",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 24,
+            "x": 0,
+            "y": 13
+          },
+          "id": 26,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "displayMode": "table",
+              "placement": "right",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "prometheus"
+              },
+              "editorMode": "code",
+              "expr": "income_features{project=\"$PROJECT\", endpoint_id=\"$MODELENDPOINT\"}",
+              "legendFormat": "{{` {{ metric }} `}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Income Features",
+          "type": "timeseries"
         }
       ],
       "refresh": false,
@@ -917,11 +1025,7 @@ data:
       "templating": {
         "list": [
           {
-            "current": {
-              "selected": false,
-              "text": "default",
-              "value": "default"
-            },
+            "current": {},
             "datasource": "iguazio",
             "definition": "SELECT DISTINCT project FROM mlrun.model_endpoints",
             "hide": 0,
@@ -941,12 +1045,7 @@ data:
             "useTags": false
           },
           {
-            "current": {
-              "isNone": true,
-              "selected": false,
-              "text": "None",
-              "value": ""
-            },
+            "current": {},
             "datasource": "iguazio",
             "definition": "SELECT uid FROM mlrun.model_endpoints where project='$PROJECT';",
             "hide": 0,
@@ -975,11 +1074,12 @@ data:
       "timezone": "",
       "title": "Model Monitoring - Details",
       "uid": "AohIXhAMk",
-      "version": 6,
+      "version": 1,
       "weekStart": ""
     }
 kind: ConfigMap
 metadata:
   labels:
     grafana_dashboard: "1"
-  name: model-monitoring-details
+  name: model-monitoring-details
+{{- end -}}