feat: Kafka Add metadata full to config (#9833)

influxdata · Oct 21, 2021 · 8265969 · 8265969
1 parent 06f2a37
commit 8265969
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 0 deletions.
diff --git a/etc/telegraf.conf b/etc/telegraf.conf
@@ -1081,6 +1081,9 @@
 #   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
 #   # sasl_version = 1
 #
+#   # Disable Kafka metadata full fetch
+#   # metadata_full = false
+#
 #   ## Data format to output.
 #   ## Each data format has its own unique set of configuration options, read
 #   ## more about them here:
@@ -7618,6 +7621,9 @@
 #   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
 #   # sasl_version = 1
 #
+#   # Disable Kafka metadata full fetch
+#   # metadata_full = false
+#
 #   ## Name of the consumer group.
 #   # consumer_group = "telegraf_metrics_consumers"
 #

diff --git a/plugins/common/kafka/config.go b/plugins/common/kafka/config.go
@@ -55,6 +55,9 @@ type Config struct {
 
 	// EnableTLS deprecated
 	EnableTLS *bool `toml:"enable_tls"`
+
+	// Disable full metadata fetching
+	MetadataFull *bool `toml:"metadata_full"`
 }
 
 // SetConfig on the sarama.Config object from the Config struct.
@@ -89,5 +92,10 @@ func (k *Config) SetConfig(config *sarama.Config) error {
 		config.Net.TLS.Enable = true
 	}
 
+	if k.MetadataFull != nil {
+		// Defaults to true in Sarama
+		config.Metadata.Full = *k.MetadataFull
+	}
+
 	return k.SetSASLConfig(config)
 }
diff --git a/plugins/inputs/kafka_consumer/README.md b/plugins/inputs/kafka_consumer/README.md
@@ -59,6 +59,9 @@ and use the old zookeeper connection method.
   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
   # sasl_version = 1
 
+  # Disable Kafka metadata full fetch
+  # metadata_full = false
+
   ## Name of the consumer group.
   # consumer_group = "telegraf_metrics_consumers"
 

diff --git a/plugins/inputs/kafka_consumer/kafka_consumer.go b/plugins/inputs/kafka_consumer/kafka_consumer.go
@@ -66,6 +66,9 @@ const sampleConfig = `
   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
   # sasl_version = 1
 
+  # Disable Kafka metadata full fetch
+  # metadata_full = false
+
   ## Name of the consumer group.
   # consumer_group = "telegraf_metrics_consumers"
 

diff --git a/plugins/outputs/kafka/README.md b/plugins/outputs/kafka/README.md
@@ -136,6 +136,9 @@ This plugin writes to a [Kafka Broker](http://kafka.apache.org/07/quickstart.htm
   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
   # sasl_version = 1
 
+  # Disable Kafka metadata full fetch
+  # metadata_full = false
+
   ## Data format to output.
   ## Each data format has its own unique set of configuration options, read
   ## more about them here:

diff --git a/plugins/outputs/kafka/kafka.go b/plugins/outputs/kafka/kafka.go
@@ -212,6 +212,9 @@ var sampleConfig = `
   ## SASL protocol version.  When connecting to Azure EventHub set to 0.
   # sasl_version = 1
 
+  # Disable Kafka metadata full fetch
+  # metadata_full = false
+
   ## Data format to output.
   ## Each data format has its own unique set of configuration options, read
   ## more about them here: