Skip to content

disk_alert update: add sending metrics to VictoriaMetrics #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

disk_alert update: add sending metrics to VictoriaMetrics #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions disk_alert/disk_alert.conf.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,8 @@
#declare -A DISK_ALERT_INODE_WARNING
#DISK_ALERT_INODE_CRITICAL["/mount/point"]="95"
#DISK_ALERT_INODE_WARNING["/mount/point"]="90"

# Victoria Metrics. If the VMAGENT_URL variable is set, the script will attempt to send metrics
#VMAGENT_URL="https://victoriametrics.example.org"
#VMUSER="Basic Auth Username"
#VMUSER_PWD="Basic Auth Password"
94 changes: 94 additions & 0 deletions disk_alert/disk_alert.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,40 @@ declare -A DISK_ALERT_INODE_WARNING
# Seconds since unix epoch
TIMESTAMP=$(date '+%s')

# Severity codes for victoria metrics
severity_ok='0'
severity_indeterminate='1'
severity_informational='2'
severity_warning='3'
severity_minor='4'
severity_major='5'
severity_critical='6'
severity_fatal='7'
severity_security='8'

# Function to send disk metric to victoria metrics
function send_disk_metric_to_victoria {
if [[ -z "${VMAGENT_URL}" ]] || [[ -z "${1}" ]] || [[ -z "${2}" ]] ; then
echo "Error: 'send_disk_metric_to_victoria' missing arguments or 'VMAGENT_URL' is not set in 'disk_alert.conf'"
echo "Usage: send_disk_metric_to_victoria <metric_name> <metric_value>"
return 1
fi
local timestamp=$(date +%s000)
local METRIC_NAME=$1
local METRIC_VALUE=$2
curl ${VMUSER:+${VMUSER_PWD:+-u "$VMUSER:$VMUSER_PWD"}} -d @- -X POST "${VMAGENT_URL}/api/v1/import" <<EOF
{
"metric": {
"__name__": "${METRIC_NAME}",
"host": "${HOSTNAME}",
"partition": "${PARTITION}"
},
"values": [${METRIC_VALUE}],
"timestamps": [${timestamp}]
}
EOF
}

# Include config
if [ -f /opt/sysadmws/disk_alert/disk_alert.conf ]; then
. /opt/sysadmws/disk_alert/disk_alert.conf
Expand Down Expand Up @@ -124,6 +158,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage_severity ${severity_fatal}
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage $USEP
fi
# Critical percent message
elif [[ $USEP -ge $CRITICAL ]]; then
echo '{
Expand All @@ -141,6 +179,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage_severity ${severity_critical}
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage $USEP
fi
elif [[ $USEP -ge $WARNING ]]; then
echo '{
"severity": "major",
Expand All @@ -157,6 +199,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage_severity ${severity_major}
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage $USEP
fi
else
echo '{
"severity": "ok",
Expand All @@ -173,6 +219,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage_severity ${severity_ok}
send_disk_metric_to_victoria microdevops_agent_disk_alert_space_usage_percentage $USEP
fi
fi
elif [[ $USAGE_CHECK == "FREE_SPACE" ]]; then
# 0 is always fatal
Expand All @@ -192,6 +242,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$FREE_SPACE_CRITICAL'MB"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb_severity ${severity_fatal}
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb $FREESP
fi
# Critical free space message
elif [[ $FREESP -le $FREE_SPACE_CRITICAL ]]; then
echo '{
Expand All @@ -209,6 +263,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$FREE_SPACE_CRITICAL'MB"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb_severity ${severity_critical}
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb $FREESP
fi
elif [[ $FREESP -le $FREE_SPACE_WARNING ]]; then
echo '{
"severity": "major",
Expand All @@ -225,6 +283,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$FREE_SPACE_CRITICAL'MB"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb_severity ${severity_major}
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb $FREESP
fi
else
echo '{
"severity": "ok",
Expand All @@ -241,6 +303,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"critical threshold": "'$FREE_SPACE_CRITICAL'MB"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb_severity ${severity_ok}
send_disk_metric_to_victoria microdevops_agent_disk_alert_free_space_mb $FREESP
fi
fi
fi
# Add partition usage history by seconds from unix epoch
Expand Down Expand Up @@ -289,6 +355,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"predict critical threshold": "'$PREDICT_CRITICAL'"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec_severity ${severity_minor}
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec $PREDICT_SECONDS
fi
elif [[ $PREDICT_SECONDS -lt $PREDICT_WARNING && $PREDICT_SECONDS -gt 0 ]]; then
echo '{
"severity": "warning",
Expand All @@ -310,6 +380,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"predict critical threshold": "'$PREDICT_CRITICAL'"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec_severity ${severity_warning}
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec $PREDICT_SECONDS
fi
else
echo '{
"severity": "ok",
Expand All @@ -331,6 +405,10 @@ df -P -BM | grep -vE $FILTER | awk '{ print $5 " " $6 " " $4 }' | while read out
"predict critical threshold": "'$PREDICT_CRITICAL'"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec_severity ${severity_ok}
send_disk_metric_to_victoria microdevops_agent_disk_alert_predicted_full_sec $PREDICT_SECONDS
fi
fi
fi
done
Expand Down Expand Up @@ -375,6 +453,10 @@ df -P -i | grep -vE $FILTER | awk '{ print $5 " " $6 }' | while read output; do
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage_severity ${severity_fatal}
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage $USEP
fi
# Critical percent message
elif [[ $USEP -ge $CRITICAL ]]; then
echo '{
Expand All @@ -391,6 +473,10 @@ df -P -i | grep -vE $FILTER | awk '{ print $5 " " $6 }' | while read output; do
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage_severity ${severity_critical}
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage $USEP
fi
elif [[ $USEP -ge $WARNING ]]; then
echo '{
"severity": "major",
Expand All @@ -406,6 +492,10 @@ df -P -i | grep -vE $FILTER | awk '{ print $5 " " $6 }' | while read output; do
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage_severity ${severity_major}
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage $USEP
fi
else
echo '{
"severity": "ok",
Expand All @@ -421,5 +511,9 @@ df -P -i | grep -vE $FILTER | awk '{ print $5 " " $6 }' | while read output; do
"critical threshold": "'$CRITICAL'%"
}
}' | /opt/sysadmws/notify_devilry/notify_devilry.py
if [[ ! -z "${VMAGENT_URL}" ]]; then
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage_severity ${severity_ok}
send_disk_metric_to_victoria microdevops_agent_disk_alert_inode_usage_percentage $USEP
fi
fi
done