Filebeat Kibana module log fileset (elastic#7052)

* Filebeat Kibana module with log fileset

This adds the Kibana module to Filebeat reading the Kibana log. The Kibana log is in JSON format which simplifies reading it but at the same time it provides many fields which are not necessarily know in advance. To solve this issue The data is put under `kibana.log.meta` and the most common known fields are picked into `kibana.log.*` or directly put into the global field like `message`. The fields under `meta` are stored as keyword. This make sure all the meta information around the event is still in the index but prevents potential type conflicts like long vs double.

Additional change:

* Module tests always overwrite the pipeline. This should make development easier.

* add beta label

* Fix review comment for pipeline

CHANGELOG.asciidoc

@@ -133,6 +133,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Ship fields.yml as part of the binary {pull}4834[4834]
 - Added options to dev-tools/cmd/dashboards/export_dashboard.go: -indexPattern to include index-pattern in output, -quiet to be quiet. {pull}7101[7101]
 - Add Indexer indexing by pod uid. Enable pod uid metadata gathering in add_kubernetes_metadata. Extended Matcher log_path matching to support volume mounts {pull}7072[7072]
+- Add Kibana module with log fileset. {pull}7052[7052]
 
 *Auditbeat*
 

filebeat/_meta/fields.common.yml

@@ -96,3 +96,8 @@
       required: false
       description: >
         The severity of the event.
+
+    - name: service.name
+      type: keyword
+      description: >
+        Service name.

filebeat/docs/fields.asciidoc

@@ -21,6 +21,7 @@ grouped in the following categories:
 * <<exported-fields-icinga>>
 * <<exported-fields-iis>>
 * <<exported-fields-kafka>>
+* <<exported-fields-kibana>>
 * <<exported-fields-kubernetes-processor>>
 * <<exported-fields-log>>
 * <<exported-fields-logstash>>
@@ -1649,6 +1650,53 @@ type: text
 The full trace in the log line.
 
 
+--
+
+[[exported-fields-kibana]]
+== kibana fields
+
+kibana Module
+
+
+
+[float]
+== kibana fields
+
+
+
+
+[float]
+== log fields
+
+Kafka log lines.
+
+
+
+*`kibana.log.tags`*::
++
+--
+type: keyword
+
+Kibana logging tags.
+
+
+--
+
+*`kibana.log.state`*::
++
+--
+type: keyword
+
+Current state of Kibana.
+
+
+--
+
+*`kibana.log.meta`*::
++
+--
+type: object
+
 --
 
 [[exported-fields-kubernetes-processor]]
@@ -1920,6 +1968,16 @@ required: False
 The severity of the event.
 
 
+--
+
+*`service.name`*::
++
+--
+type: keyword
+
+Service name.
+
+
 --
 
 [[exported-fields-logstash]]

filebeat/docs/modules/kibana.asciidoc

@@ -0,0 +1,42 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-kibana]]
+:modulename: kibana
+
+== Kibana module
+
+beta[]
+
+This is the Kibana module.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The Kibana modules is compatible with Kibana 6.3 and newer.
+
+include::../include/running-modules.asciidoc[]
+
+
+include::../include/configuring-intro.asciidoc[]
+
+//set the fileset name used in the included example
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `{fileset}` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-kibana,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -8,6 +8,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-icinga>>
   * <<filebeat-module-iis>>
   * <<filebeat-module-kafka>>
+  * <<filebeat-module-kibana>>
   * <<filebeat-module-logstash>>
   * <<filebeat-module-mongodb>>
   * <<filebeat-module-mysql>>
@@ -27,6 +28,7 @@ include::modules/auditd.asciidoc[]
 include::modules/icinga.asciidoc[]
 include::modules/iis.asciidoc[]
 include::modules/kafka.asciidoc[]
+include::modules/kibana.asciidoc[]
 include::modules/logstash.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]

filebeat/filebeat.reference.yml

@@ -161,6 +161,16 @@ filebeat.modules:
     #var.paths:
 
 
+#------------------------------- kibana Module -------------------------------
+- module: kibana
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------ logstash Module ------------------------------
 #- module: logstash
   # logs

filebeat/module/kibana/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: kibana
+  # All logs
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/kibana/_meta/docs.asciidoc

@@ -0,0 +1,29 @@
+:modulename: kibana
+
+== Kibana module
+
+beta[]
+
+This is the Kibana module.
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+The Kibana modules is compatible with Kibana 6.3 and newer.
+
+include::../include/running-modules.asciidoc[]
+
+
+include::../include/configuring-intro.asciidoc[]
+
+//set the fileset name used in the included example
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `{fileset}` log fileset settings
+
+include::../include/var-paths.asciidoc[]

filebeat/module/kibana/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: kibana
+  title: "kibana"
+  description: >
+    kibana Module
+  fields:
+    - name: kibana
+      type: group
+      description: >
+      fields:

filebeat/module/kibana/log/_meta/config.reference.yml

@@ -0,0 +1,7 @@
+- module: kibana
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/kibana/log/_meta/config.yml

@@ -0,0 +1,7 @@
+- module: kibana
+  log:
+    enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/kibana/log/_meta/fields.yml

@@ -0,0 +1,16 @@
+- name: log
+  type: group
+  description: >
+    Kafka log lines.
+  fields:
+    - name: tags
+      type: keyword
+      description: >
+        Kibana logging tags.
+    - name: state
+      type: keyword
+      description: >
+        Current state of Kibana.
+    - name: meta
+      type: object
+      object_type: keyword

filebeat/module/kibana/log/config/log.yml

@@ -0,0 +1,9 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+json.keys_under_root: false
+json.add_error_key: true

filebeat/module/kibana/log/ingest/pipeline.json

@@ -0,0 +1,63 @@
+{
+    "description": "Pipeline for parsing Kibana logs",
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
+        }
+    ],
+    "processors": [
+        {
+            "rename": {
+                "field": "json",
+                "target_field": "kibana.log.meta"
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.@timestamp",
+                "target_field": "read_timestamp"
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.message",
+                "target_field": "message"
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.state",
+                "target_field": "kibana.log.state",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.pid",
+                "target_field": "process.pid"
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.tags",
+                "target_field": "kibana.log.tags"
+            }
+        },
+        {
+            "date": {
+                "field": "read_timestamp",
+                "formats" : ["ISO8601"],
+                "target_field": "@timestamp"
+            }
+        },
+        {
+            "append": {
+                "field": "service.name",
+                "value": "kibana"
+            }
+        }
+    ]
+}

filebeat/module/kibana/log/manifest.yml

@@ -0,0 +1,9 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/kibana/kibana.stdout
+
+ingest_pipeline: ingest/pipeline.json
+prospector: config/log.yml