Merge pull request #2047 from kube-logging/chore/fine-tune-fluentdpredictedbuffergrowth

chore: fine-tune FluentdPredictedBufferGrowth

Author: csatib02

e2e/logging_metrics_monitoring/logging_metrics_monitoring_test.go

@@ -319,13 +319,18 @@ func installPrometheusOperator(c common.Cluster) error {
 		return fmt.Errorf("failed to add prometheus-community repo: %v", err)
 	}
 
+	if err := manager.RunRepo(helm.WithArgs("update")); err != nil {
+		return fmt.Errorf("failed to update helm repos: %v", err)
+	}
+
 	if err := manager.RunInstall(
 		helm.WithName("prometheus"),
 		helm.WithChart("prometheus-community/kube-prometheus-stack"),
 		helm.WithArgs("--create-namespace"),
 		helm.WithNamespace("monitoring"),
 		helm.WithArgs("--set", "prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false"),
 		helm.WithArgs("--set", "prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false"),
+		helm.WithArgs("--set", "prometheus.prometheusSpec.maximumStartupDurationSeconds=900"),
 		helm.WithWait(),
 	); err != nil {
 		return fmt.Errorf("failed to install prometheus: %v", err)

pkg/resources/fluentd/prometheusrules.go

@@ -121,7 +121,7 @@ func (r *Reconciler) prometheusRules() (runtime.Object, reconciler.DesiredState,
 			},
 			{
 				Alert: "FluentdPredictedBufferGrowth",
-				Expr:  intstr.FromString(fmt.Sprintf("sum(predict_linear(fluentd_output_status_buffer_total_bytes{%[1]s}[10m], 600)) > sum(fluentd_output_status_buffer_total_bytes{%[1]s}) * 1.5", nsJobLabel)),
+				Expr:  intstr.FromString(fmt.Sprintf("sum(predict_linear(fluentd_output_status_buffer_total_bytes{%[1]s}[10m], 600)) > sum(fluentd_output_status_buffer_total_bytes{%[1]s}) * 1.5 and sum(fluentd_output_status_buffer_total_bytes{%[1]s}) > 10 * 1024 * 1024", nsJobLabel)),
 				For:   prometheus_operator.Duration("10m"),
 				Labels: map[string]string{
 					"rulegroup": ruleGroupName,