Update

throwoutofcoffeeexception · Jun 16, 2021 · 8f5a427 · 8f5a427
1 parent dc0e17d
commit 8f5a427
Show file tree

Hide file tree

Showing 4 changed files with 130 additions and 10 deletions.
diff --git a/templates/cronjob.yml b/templates/cronjob.yml
@@ -10,6 +10,7 @@
 {{- end -}}
 {{- $general := $.Values.cronJobsGeneral -}}
 {{- range $CJ.jobs }}
+{{- $job := . -}}
 {{- $cronJobName := include "helpers.app.fullname" (dict "name" .name "context" $) }}
 ---
 apiVersion: batch/v1beta1
@@ -149,12 +150,17 @@ spec:
             {{- with .containerSecurityContext }}
             securityContext: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
             {{- end }}
-            {{- with .args }}
+            {{- if and .args (not $job.max_duration) }}
             args: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
             {{- end }}
             {{- with .command }}
             {{- if typeIs "string" . }}
-            command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " .) "" )) }}
+            {{- if $job.max_duration }}
+            command: ["/bin/bash"]
+            args: ["-c", "/usr/bin/timeout --preserve-status {{ $job.max_duration }} {{ . }}; exit_status=$?; if [[ $exit_status -eq 143 ]]; then echo 'Max execution exceeded after {{ $job.max_duration }} seconds (Added by task #91828)!'; fi; exit $exit_status"]
+            {{- else }}
+            command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " . ) "" )) }}
+            {{- end }}
             {{- else -}}
             command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
             {{- end }}
@@ -217,9 +223,17 @@ spec:
             {{- with .envFrom }}
             envFrom: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
             {{- end }}
+            {{- if and .args (not $job.max_duration) }}
+            args: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
+            {{- end }}
             {{- with .command }}
             {{- if typeIs "string" . }}
-            command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " .) "" )) }}
+            {{- if $job.max_duration }}
+            command: ["/bin/bash"]
+            args: ["-c", "/usr/bin/timeout --preserve-status {{ $job.max_duration }} {{ . }}; exit_status=$?; if [[ $exit_status -eq 143 ]]; then echo 'Max execution exceeded after {{ $job.max_duration }} seconds (Added by task #91828)!'; fi; exit $exit_status"]
+            {{- else }}
+            command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " . ) "" )) }}
+            {{- end }}
             {{- else -}}
             command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
             {{- end }}

diff --git a/templates/job.yml b/templates/job.yml
@@ -10,6 +10,7 @@
 {{- end -}}
 {{- $general := $.Values.jobsGeneral -}}
 {{- range $j.jobs }}
+{{- $job := . -}}
 {{- $jobName := include "helpers.app.fullname" (dict "name" .name "context" $) }}
 ---
 apiVersion: batch/v1
@@ -128,14 +129,19 @@ spec:
         {{- with .containerSecurityContext }}
         securityContext: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 8 }}
         {{- end }}
-        {{- with .args }}
-        args: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 8 }}
+        {{- if and .args (not $job.max_duration) }}
+        args: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
         {{- end }}
         {{- with .command }}
         {{- if typeIs "string" . }}
-        command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " .) "" )) }}
+        {{- if $job.max_duration }}
+        command: ["/bin/bash"]
+        args: ["-c", "/usr/bin/timeout --preserve-status {{ $job.max_duration }} {{ . }}; exit_status=$?; if [[ $exit_status -eq 143 ]]; then echo 'Max execution exceeded after {{ $job.max_duration }} seconds (Added by task #91828)!'; fi; exit $exit_status"]
+        {{- else }}
+        command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " . ) "" )) }}
+        {{- end }}
         {{- else -}}
-        command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 8 }}
+        command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
         {{- end }}
         {{- end }}
         env:
@@ -196,11 +202,19 @@ spec:
         {{- with .envFrom }}
         envFrom: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 8 }}
         {{- end }}
+        {{- if and .args (not $job.max_duration) }}
+        args: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
+        {{- end }}
         {{- with .command }}
         {{- if typeIs "string" . }}
-        command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " .) "" )) }}
+        {{- if $job.max_duration }}
+        command: ["/bin/bash"]
+        args: ["-c", "/usr/bin/timeout --preserve-status {{ $job.max_duration }} {{ . }}; exit_status=$?; if [[ $exit_status -eq 143 ]]; then echo 'Max execution exceeded after {{ $job.max_duration }} seconds (Added by task #91828)!'; fi; exit $exit_status"]
+        {{- else }}
+        command: {{ printf "[\"%s\"]" (join ("\", \"") (without (splitList " " . ) "" )) }}
+        {{- end }}
         {{- else -}}
-        command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 8 }}
+        command: {{- include "helpers.tplvalues.render" ( dict "value" . "context" $) | nindent 14 }}
         {{- end }}
         {{- end }}
         {{- with .lifecycle }}

diff --git a/templates/prometheusrules.yml b/templates/prometheusrules.yml
@@ -0,0 +1,73 @@
+{{- $CJ := dict -}}
+{{- if .Values.cronJobs -}}
+{{- if kindIs "string" .Values.cronJobs -}}
+{{- $CJ = fromYaml .Values.cronJobs -}}
+{{- else if kindIs "map" .Values.cronJobs -}}
+{{- $CJ = .Values.cronJobs -}}
+{{- end -}}
+{{- else if .Values.cronJobsFileName -}}
+{{- $CJ = fromYaml (.Files.Get .Values.cronJobsFileName) -}}
+{{- end -}}
+{{- range $CJ.jobs }}
+{{- $cronJobName := include "helpers.app.fullname" (dict "name" .name "context" $) }}
+{{- if .duration_alert }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cronjob-{{ $cronJobName }}
+  labels:
+    prometheus: k8s
+    role: alert-rules
+  namespace: nxs-monitoring
+spec:
+  groups:
+  - name: "cronjobs_rules"
+    interval: 1m # period check for alerts
+    rules:
+    - alert: "cronjob-{{ $cronJobName }}-too-long-execution"
+      expr: '((time() - kube_job_status_start_time{namespace="{{ $.Release.Namespace }}", job_name=~"cronjob-{{ $cronJobName }}-[0-9]{10}"}) and kube_job_status_active{namespace="{{ $.Release.Namespace }}", job_name=~"cronjob-{{ $cronJobName }}-[0-9]{10}"} == 1) > {{ .duration_alert }}'
+      for: 3m
+      labels:
+        severity: warning
+      annotations:
+        message: "CronJob {{ $.Release.Namespace }}/cronjob-{{ $cronJobName }} is taking more than {{ .duration_alert }} to complete"
+{{- end }}
+{{- end }}
+---
+{{- $j := dict -}}
+{{- if .Values.jobs -}}
+{{- if kindIs "string" .Values.jobs -}}
+{{- $j = fromYaml .Values.jobs -}}
+{{- else if kindIs "map" .Values.jobs -}}
+{{- $j = .Values.jobs -}}
+{{- end -}}
+{{- else if .Values.jobsFileName -}}
+{{- $j = fromYaml (.Files.Get .Values.jobsFileName) -}}
+{{- end -}}
+{{- range $j.jobs }}
+{{- $jobName := include "helpers.app.fullname" (dict "name" .name "context" $) }}
+{{- if .duration_alert }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: job-{{ $jobName }}
+  labels:
+    prometheus: k8s
+    role: alert-rules
+  namespace: nxs-monitoring
+spec:
+  groups:
+  - name: "jobs_rules"
+    interval: 1m # period check for alerts
+    rules:
+    - alert: "job-{{ $jobName }}-too-long-execution"
+      expr: '((time() - kube_job_status_start_time{namespace="{{ $.Release.Namespace }}", job_name=~"job-{{ $jobName }}-[0-9]{10}"}) and kube_job_status_active{namespace="{{ $.Release.Namespace }}", job_name=~"job-{{ $jobName }}-[0-9]{10}"} == 1) > {{ .duration_alert }}'
+      for: 3m
+      labels:
+        severity: warning
+      annotations:
+        message: "CronJob {{ $.Release.Namespace }}/job-{{ $jobName }} is taking more than {{ .duration_alert }} to complete"
+{{- end }}
+{{- end }}
diff --git a/values.yaml b/values.yaml
@@ -151,4 +151,23 @@ nodeAffinityPreset:
   ##   - e2e-az1
   ##   - e2e-az2
   ##
-  values: []
+  values: []
+
+cronJobs:
+  jobs:
+  - name: generate-data
+    schedule: "*/5 * * * *"
+    command: "php yii crontab/generate-data"
+    single_only: true
+    duration_alert: 600
+    max_duration: 800
+  - name: auto-sender
+    schedule: "*/5 * * * *"
+    command: "php yii crontab/auto-sender"
+
+jobs:
+  jobs:
+  - name: refresh-cache
+    command: "php yii crontab/refresh-cache"
+    duration_alert: 30
+    max_duration: 60