Enhance windows observ lib with detailed alerts and add promtool tests.

.github/workflows/lint-mixins.yml

@@ -85,3 +85,6 @@ jobs:
         working-directory: ./${{ matrix.mixin }}
         run: "make && git diff --exit-code || ( echo 'Error: Generated files are not up to date. Run make and commit the local diff'; exit 1; )"
 
+      - name: Run promtool test
+        working-directory: ./${{ matrix.mixin }}
+        run: make test

Makefile

@@ -1,13 +1,40 @@
 JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
 SHELL := /bin/bash
+PROMTOOL_VERSION := 3.2.0
 
-install-ci-deps:
+# Detect OS and architecture
+UNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+UNAME_M := $(shell uname -m)
+
+# Map architecture names
+ifeq ($(UNAME_M),x86_64)
+    ARCH := amd64
+else ifeq ($(UNAME_M),aarch64)
+    ARCH := arm64
+else ifeq ($(UNAME_M),armv7l)
+    ARCH := armv7
+else
+    ARCH := $(UNAME_M)
+endif
+
+install-ci-deps: install-promtool
 	go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
 	go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.20.0
 	go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0
 	go install github.com/monitoring-mixins/mixtool/cmd/mixtool@main
 	go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1
 
+.PHONY: install-promtool
+install-promtool:
+	@echo "Installing promtool $(PROMTOOL_VERSION) for $(UNAME_S)/$(ARCH)..."
+	@wget https://github.com/prometheus/prometheus/releases/download/v$(PROMTOOL_VERSION)/prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH).tar.gz
+	@mkdir -p prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH)
+	@tar xvf prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH).tar.gz
+	@sudo mv prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH)/promtool /usr/local/bin/
+	@rm -rf prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH) prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH).tar.gz
+	@echo "promtool $(PROMTOOL_VERSION) installed successfully"
+
+
 fmt:
 	@find . -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
 			xargs -n 1 -- $(JSONNET_FMT) -i

Makefile_mixin

@@ -1,7 +1,7 @@
 JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 1 --string-style s --comment-style s
 
 .PHONY: all
-all: build dashboards_out prometheus_alerts.yaml prometheus_rules.yaml
+all: build dashboards_out prometheus_alerts.yaml prometheus_rules.yaml test
 
 vendor: $(wildcard jsonnetfile.json)
 ifneq ("$(wildcard jsonnetfile.json)","")
@@ -42,6 +42,14 @@ prometheus_rules.yaml: $(wildcard vendor/**/rules.yaml) $(wildcard vendor/**/rul
 		touch prometheus_rules_out/prometheus_rules.yaml; \
 	fi
 
+.PHONY: test
+test:
+	@if [ -f tests/prometheus_*.yaml ]; then \
+		promtool test rules tests/prometheus_*.yaml; \
+	else \
+		echo "No tests/prometheus_*.yaml files found, skipping promtool test."; \
+	fi
+
 .PHONY: deploy deploy_rules deploy_dashboards
 deploy: deploy_rules deploy_dashboards
 

README.md

@@ -26,8 +26,6 @@ Based on format described [here](https://monitoring.mixins.dev/):
 * [`caddy-mixin`](caddy-mixin/): A set of reusable and extensible dashboards
   for Caddy.
 
-
-
 * [`jira-mixin`](jira-mixin/): A set of reusable and extensible dashboards and alerts for JIRA.
 
 You can find more in directories with `-mixin` suffix.
@@ -54,6 +52,14 @@ Examples:
  - [golang-observ-lib](golang-observ-lib/)
  - [csp-mixin](csp-mixin/)
 
+ ## Prometheus rules testing for monitoring mixins and observability libraries
+
+It is highly recommended to test prometheus alerts with [promtool test rules](https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules) command when complex PromQL queries are used or when additional queries are used in alerts' annotations.
+
+promtool tests files should be placed in tests directory in the root of the library and should be named like `prometheus_*.yaml`. This will enable running tests ing Github Actions and with `make test` command.
+
+A good example of promtool tests can be found in windows-observ-lib: [prometheus_alerts_test.yaml](windows-observ-lib/tests/prometheus_alerts_test.yaml)
+
 ## LICENSE
 
 [Apache-2.0](LICENSE)

windows-observ-lib/alerts.libsonnet

@@ -109,7 +109,11 @@
             annotations: {
               summary: 'High memory usage on Windows host.',
               description: |||
-                Memory usage on host {{ $labels.instance }} is above %(alertMemoryUsageThresholdCritical)s%%. The current value is {{ $value | printf "%%.2f" }}%%.
+                Memory usage on host {{ $labels.instance }} is critically high, with {{ printf "%%.2f" $value }}%% of total memory used.
+                This exceeds the threshold of %(alertMemoryUsageThresholdCritical)s%%.
+                Current memory free: {{ with printf `windows_os_physical_memory_free_bytes{%(filteringSelector)s}` | query | first | value | humanize }}{{ . }}{{ end }}.
+                Total memory: {{ with printf `windows_cs_physical_memory_bytes{%(filteringSelector)s}` | query | first | value | humanize }}{{ . }}{{ end }}.
+                Consider investigating processes consuming high memory or increasing available memory.
               ||| % this.config,
             },
           },
@@ -126,7 +130,11 @@
             annotations: {
               summary: 'Disk is almost full on Windows host.',
               description: |||
-                Volume {{ $labels.volume }} is almost full on host {{ $labels.instance }}, more than %(alertDiskUsageThresholdCritical)s%% of space is used. The current volume utilization is {{ $value | printf "%%.2f" }}%%.
+                Disk space on volume {{ $labels.volume }} of host {{ $labels.instance }} is critically low, with {{ printf "%%.2f" $value }}%% of total space used.
+                This exceeds the threshold of %(alertDiskUsageThresholdCritical)s%%.
+                Current disk free: {{ with printf `windows_logical_disk_free_bytes{volume="%%s", %(filteringSelector)s}` $labels.volume | query | first | value | humanize }}{{ . }}{{ end }}.
+                Total disk size: {{ with printf `windows_logical_disk_size_bytes{volume="%%s", %(filteringSelector)s}` $labels.volume | query | first | value | humanize }}{{ . }}{{ end }}.
+                Consider cleaning up unnecessary files or increasing disk capacity.
               ||| % this.config,
             },
           },

windows-observ-lib/tests/prometheus_alerts_test.yaml

@@ -0,0 +1,49 @@
+rule_files:
+  - ../prometheus_rules_out/prometheus_alerts.yaml
+
+evaluation_interval: 15m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'windows_os_physical_memory_free_bytes{instance="host1"}'
+        values: '10000000x15'
+      - series: 'windows_cs_physical_memory_bytes{instance="host1"}'
+        values: '1000000000x15'
+      - series: 'windows_logical_disk_free_bytes{volume="C:", instance="host1"}'
+        values: '10000000x15'
+      - series: 'windows_logical_disk_size_bytes{volume="C:", instance="host1"}'
+        values: '1000000000x15'
+
+
+    alert_rule_test:
+      - eval_time: 15m
+        alertname: WindowsMemoryHighUtilization
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              instance: host1
+            exp_annotations:
+              description: |
+                Memory usage on host host1 is critically high, with 99.00% of total memory used.
+                This exceeds the threshold of 90%.
+                Current memory free: 10M.
+                Total memory: 1G.
+                Consider investigating processes consuming high memory or increasing available memory.
+              summary: 'High memory usage on Windows host.'
+
+      - eval_time: 15m
+        alertname: WindowsDiskAlmostOutOfSpace
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              instance: host1
+              volume: "C:"
+            exp_annotations:
+              description: |
+                Disk space on volume C: of host host1 is critically low, with 99.00% of total space used.
+                This exceeds the threshold of 90%.
+                Current disk free: 10M.
+                Total disk size: 1G.
+                Consider cleaning up unnecessary files or increasing disk capacity.
+              summary: 'Disk is almost full on Windows host.'