From 835dfebe7f02ff0b7e454aae6521adb2a9d10a1e Mon Sep 17 00:00:00 2001 From: Paul Gier Date: Wed, 29 Jan 2020 14:57:11 -0600 Subject: [PATCH] prevent prometheus sts update loop Changes in kubernetes v1.17 cause an endless update loop due to validation errors cause by setting the 'apiVersion' and 'kind' fields in the StatefulSet spec. These two fields are set to empty strings by the Kube API server, and v1.17 added validation that does not allow these fields to be modified. See #2950 --- pkg/prometheus/operator.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/prometheus/operator.go b/pkg/prometheus/operator.go index 534caba98ff..18787835d68 100644 --- a/pkg/prometheus/operator.go +++ b/pkg/prometheus/operator.go @@ -1145,6 +1145,7 @@ func (c *Operator) sync(key string) error { if err != nil { return errors.Wrap(err, "making statefulset failed") } + sanitizeSTS(sset) if !exists { level.Debug(c.logger).Log("msg", "no current Prometheus statefulset found") @@ -1183,6 +1184,15 @@ func (c *Operator) sync(key string) error { return nil } +// sanitizeSTS removes values for APIVersion and Kind from the VolumeClaimTemplates. +// This prevents update failures due to these fields changing when applied. +func sanitizeSTS(sts *appsv1.StatefulSet) { + for i := range sts.Spec.VolumeClaimTemplates { + sts.Spec.VolumeClaimTemplates[i].APIVersion = "" + sts.Spec.VolumeClaimTemplates[i].Kind = "" + } +} + //checkPrometheusSpecDeprecation checks for deprecated fields in the prometheus spec and logs a warning if applicable func checkPrometheusSpecDeprecation(key string, p *monitoringv1.Prometheus, logger log.Logger) { deprecationWarningf := "prometheus key=%v, field %v is deprecated, '%v' field should be used instead"