From 0316e7085dd4c0c16f1c1c94fe080816c3d00e68 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Tue, 2 Jul 2019 15:26:35 +0200 Subject: [PATCH] Update kafka output metadata fetch (#12738) Change default metadata request configuration in the kafka output to only collect metadata of topics we publish to. --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 ++-- filebeat/filebeat.reference.yml | 4 ++-- heartbeat/heartbeat.reference.yml | 4 ++-- journalbeat/journalbeat.reference.yml | 4 ++-- libbeat/_meta/config.reference.yml.tmpl | 4 ++-- libbeat/docs/outputconfig.asciidoc | 2 +- libbeat/outputs/kafka/config.go | 2 +- metricbeat/metricbeat.reference.yml | 4 ++-- packetbeat/packetbeat.reference.yml | 4 ++-- winlogbeat/winlogbeat.reference.yml | 4 ++-- x-pack/auditbeat/auditbeat.reference.yml | 4 ++-- x-pack/filebeat/filebeat.reference.yml | 4 ++-- x-pack/metricbeat/metricbeat.reference.yml | 4 ++-- x-pack/winlogbeat/winlogbeat.reference.yml | 4 ++-- 15 files changed, 27 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a6630e2b494f..30ef8b0474c8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update to Golang 1.12.1. {pull}11330[11330] - Update to Golang 1.12.4. {pull}11782[11782] - Update to ECS 1.0.1. {pull}12284[12284] {pull}12317[12317] +- Default of output.kafka.metadata.full is set to false by now. This reduced the amount of metadata to be queried from a kafka cluster. {pull}12738[12738] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 95c31e50b7eb..a0f2799142b7 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -668,8 +668,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 823b172c4a05..38d123b40a6d 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1372,8 +1372,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 008eb35636d6..9525e30cff8b 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -812,8 +812,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 42989dfd6196..d29f6d5752f2 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -608,8 +608,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index e48ff516b806..5a297e398818 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -556,8 +556,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 4edb606ed2cb..b039351bdc21 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -1142,7 +1142,7 @@ brokers, topics, partition, and active leaders to use for publishing. *`full`*:: Strategy to use when fetching metadata, when this option is `true`, the client will maintain a full set of metadata for all the available topics, if the this option is set to `false` it will only refresh the -metadata for the configured topics. The default is true. +metadata for the configured topics. The default is false. *`retry.max`*:: Total number of metadata update retries when cluster is in middle of leader election. The default is 3. diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 7a3d7208b087..e96f96ef7a5e 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -93,7 +93,7 @@ func defaultConfig() kafkaConfig { Backoff: 250 * time.Millisecond, }, RefreshFreq: 10 * time.Minute, - Full: true, + Full: false, }, KeepAlive: 0, MaxMessageBytes: nil, // use library default diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4b1c9e50e472..92e69b9d08a2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1326,8 +1326,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index e95ef3fd4351..a35116ec6546 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1045,8 +1045,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cba009eed0fd..67700af19ffe 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -589,8 +589,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 4c1961fb2f4a..253ab0f1d8eb 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -719,8 +719,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index ac938746b1da..f2436f5d630c 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1520,8 +1520,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ffac94653ad0..a9222ac900d8 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1421,8 +1421,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1 diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index ae1594907c26..3ba4221449a9 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -601,8 +601,8 @@ output.elasticsearch: # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m - # Strategy for fetching the topics metadata from the broker. Default is true. - #full: true + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false # The number of concurrent load-balanced Kafka output workers. #worker: 1