Filebeat: Add option to convert kafka module timezones to UTC

CHANGELOG.asciidoc

@@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
 - Add Slow log fileset to the Elasticsearch module. {pull}7473[7473]
 - Add deprecation fileset to the Elasticsearch module. {pull}7474[7474]
+- Add `config_timezone` option to Kafka module to convert dates to UTC. {issue}7546[7546] {pull}7578[7578]
 
 *Heartbeat*
 

filebeat/docs/modules/kafka.asciidoc

@@ -63,6 +63,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/var-convert-timezone.asciidoc[]
+
 
 [float]
 === Fields

filebeat/filebeat.reference.yml

@@ -194,6 +194,8 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
 
 #------------------------------- kibana Module -------------------------------
 - module: kibana

filebeat/module/kafka/_meta/config.yml

@@ -11,3 +11,5 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false

filebeat/module/kafka/_meta/docs.asciidoc

@@ -57,3 +57,5 @@ include::../include/config-option-intro.asciidoc[]
 ==== `log` fileset settings
 
 include::../include/var-paths.asciidoc[]
+
+include::../include/var-convert-timezone.asciidoc[]

filebeat/module/kafka/log/config/log.yml

@@ -8,3 +8,7 @@ multiline:
   pattern: '^\['
   negate: true
   match: after
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}

filebeat/module/kafka/log/ingest/pipeline.json

@@ -52,7 +52,9 @@
       "date": {
         "field": "kafka.log.timestamp",
         "target_field": "@timestamp",
-        "formats": ["yyyy-MM-dd HH:mm:ss,SSS"]
+        "formats": ["yyyy-MM-dd HH:mm:ss,SSS"],
+        {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >}
+        "ignore_failure": true
       }
     },
     {"remove": {"field": "kafka.log.timestamp" }}

filebeat/module/kafka/log/manifest.yml

@@ -9,6 +9,13 @@ var:
       - "{{.kafka_home}}/logs/server.log*"
       - "{{.kafka_home}}/logs/state-change.log*"
       - "{{.kafka_home}}/logs/kafka-*.log*"
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/log.yml

filebeat/modules.d/kafka.yml.disabled

@@ -11,3 +11,5 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false