Metrics Follow-Ups (#2105)

Adding comment for traceability to prometheus alert rules. Move prometheus installation to new script to clean up makefile.

Signed-off-by: Daniel Franz <dfranz@redhat.com>

Author: dtfranz

Makefile

@@ -282,22 +282,13 @@ test-experimental-e2e: run image-registry prometheus experimental-e2e e2e e2e-me
 .PHONY: prometheus
 prometheus: PROMETHEUS_NAMESPACE := olmv1-system
 prometheus: PROMETHEUS_VERSION := v0.83.0
-prometheus: TMPDIR := $(shell mktemp -d)
 prometheus: #EXHELP Deploy Prometheus into specified namespace
-	trap 'echo "Cleaning up $(TMPDIR)"; rm -rf "$(TMPDIR)"' EXIT; \
-	curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$(PROMETHEUS_VERSION)/kustomization.yaml" > "$(TMPDIR)/kustomization.yaml"; \
-	curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$(PROMETHEUS_VERSION)/bundle.yaml" > "$(TMPDIR)/bundle.yaml"; \
-	(cd $(TMPDIR) && $(KUSTOMIZE) edit set namespace $(PROMETHEUS_NAMESPACE)) && kubectl create -k "$(TMPDIR)"
-	kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) -l app.kubernetes.io/name=prometheus-operator
-	$(KUSTOMIZE) build config/overlays/prometheus | sed "s/cert-git-version/cert-$(VERSION)/g" | kubectl apply -f -
-	kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) -l app.kubernetes.io/name=prometheus-operator --timeout=60s
-	kubectl wait --for=create pods -n $(PROMETHEUS_NAMESPACE) prometheus-prometheus-0 --timeout=60s
-	kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) prometheus-prometheus-0 --timeout=120s
+	./hack/test/install-prometheus.sh $(PROMETHEUS_NAMESPACE) $(PROMETHEUS_VERSION) $(KUSTOMIZE) $(VERSION)
 
 # The output alerts.out file contains any alerts, pending or firing, collected during a test run in json format.
 .PHONY: e2e-metrics
 e2e-metrics: ALERTS_FILE_PATH := $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/alerts.out
-e2e-metrics: #EXHELP Request metrics from prometheus; select only actively firing alerts; place in ARTIFACT_PATH if set
+e2e-metrics: #EXHELP Request metrics from prometheus; place in ARTIFACT_PATH if set
 	curl -X GET http://localhost:30900/api/v1/alerts | jq 'if (.data.alerts | length) > 0 then .data.alerts.[] else empty end' > $(ALERTS_FILE_PATH)
 
 .PHONY: extension-developer-e2e

config/README.md

@@ -33,6 +33,8 @@ Overlay containing manifest files which enable prometheus scraping of the catalo
 
 These manifests will not end up in the `manifests/` folder, as they must be applied in two distinct steps to avoid issues with applying prometheus CRDs and CRs simultaneously.
 
+Performance alert settings can be found in: `config/overlays/prometheus/prometheus_rule.yaml`
+
 ## config/overlays/experimental
 
 This provides additional configuration used to support experimental features, including CRDs. This configuration requires cert-manager.

hack/test/install-prometheus.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+help="install-prometheus.sh is used to set up prometheus monitoring for e2e testing.
+Usage:
+  install-prometheus.sh [PROMETHEUS_NAMESPACE] [PROMETHEUS_VERSION] [KUSTOMIZE] [GIT_VERSION]
+"
+
+if [[ "$#" -ne 4 ]]; then
+  echo "Illegal number of arguments passed"
+  echo "${help}"
+  exit 1
+fi
+
+PROMETHEUS_NAMESPACE="$1"
+PROMETHEUS_VERSION="$2"
+KUSTOMIZE="$3"
+GIT_VERSION="$4"
+
+TMPDIR="$(mktemp -d)"
+trap 'echo "Cleaning up $TMPDIR"; rm -rf "$TMPDIR"' EXIT
+
+echo "Downloading Prometheus resources..."
+curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/kustomization.yaml" > "${TMPDIR}/kustomization.yaml"
+curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/bundle.yaml" > "${TMPDIR}/bundle.yaml"
+
+echo "Patching namespace to ${PROMETHEUS_NAMESPACE}..."
+(cd "$TMPDIR" && $KUSTOMIZE edit set namespace "$PROMETHEUS_NAMESPACE")
+
+echo "Applying Prometheus base..."
+kubectl apply -k "$TMPDIR" --server-side
+
+echo "Waiting for Prometheus Operator pod to become ready..."
+kubectl wait --for=condition=Ready pod -n "$PROMETHEUS_NAMESPACE" -l app.kubernetes.io/name=prometheus-operator
+
+echo "Applying overlay config..."
+$KUSTOMIZE build config/overlays/prometheus | sed "s/cert-git-version/cert-${VERSION}/g" | kubectl apply -f -
+
+echo "Waiting for metrics scraper to become ready..."
+kubectl wait --for=create pods -n "$PROMETHEUS_NAMESPACE" prometheus-prometheus-0 --timeout=60s
+kubectl wait --for=condition=Ready pods -n "$PROMETHEUS_NAMESPACE" prometheus-prometheus-0 --timeout=120s
+
+echo "Prometheus deployment completed successfully."