From bac63e879e751ec66b9843fc99646ab75c6ae062 Mon Sep 17 00:00:00 2001
From: Jakub Skokan <jakub.skokan@havefun.cz>
Date: Sun, 2 Jul 2023 08:51:59 +0200
Subject: [PATCH] clusterconf/monitor: relax vps free space rules except on
 node23

We'll rely on datasets being expanded by vpsAdmin. The alerts should
fire only should that fail. node23 is set aside until it is updated.
---
 modules/clusterconf/monitor/rules/nodes.nix | 22 ++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/modules/clusterconf/monitor/rules/nodes.nix b/modules/clusterconf/monitor/rules/nodes.nix
index 56a879bc..8703ad08 100644
--- a/modules/clusterconf/monitor/rules/nodes.nix
+++ b/modules/clusterconf/monitor/rules/nodes.nix
@@ -686,7 +686,7 @@
       {
         alert = "VpsOsCritFreeSpace";
         expr = ''osctl_container_dataset_avail_bytes{job="nodes"} < 256 * 1024 * 1024'';
-        for = "2m";
+        for = "5m";
         labels = {
           alertclass = "vpsdiskspace";
           severity = "critical";
@@ -706,6 +706,26 @@
       {
         alert = "VpsOsFatalFreeSpace";
         expr = ''osctl_container_dataset_avail_bytes{alias=~"node21.prg|node22.prg|node23.prg|node24.prg"} < 256 * 1024 * 1024'';
+        for = "5m";
+        labels = {
+          alertclass = "vpsdiskspace";
+          severity = "fatal";
+          frequency = "2m";
+        };
+        annotations = {
+          summary = "Not enough free space (instance {{ $labels.instance }})";
+          description = ''
+            VPS has less than 256 MB of diskspace left
+
+            VALUE = {{ $value }}
+            LABELS: {{ $labels }}
+          '';
+        };
+      }
+
+      {
+        alert = "VpsOsFatalFreeSpaceNode23";
+        expr = ''osctl_container_dataset_avail_bytes{alias="node23.prg"} < 256 * 1024 * 1024'';
         for = "2m";
         labels = {
           alertclass = "vpsdiskspace";