[Filebeat] Improve ECS categorization field mappings in ibmmq (elasti…

…c#16532) - event.kind - convert pipeline to yaml Closes elastic#16163
leehinman · Mar 11, 2020 · a91cc9e · a91cc9e
1 parent 96e4506
commit a91cc9e
Show file tree

Hide file tree

Showing 7 changed files with 298 additions and 109 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -172,6 +172,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in logstash module. {issue}16169[16169] {pull}16668[16668]
 - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354]
 - Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533]
+- Improve ECS categorization field mappings in ibmmq module. {issue}16163[16163] {pull}16532[16532]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.json b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.json
diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml
@@ -0,0 +1,76 @@
+description: Pipeline for parsing MQ error logs.
+processors:
+- gsub:
+    field: message
+    pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,}
+    replacement: ""
+- gsub:
+    field: message
+    pattern: |2+
+
+    replacement: ' '
+- gsub:
+    field: message
+    pattern: '[ ]{2,}'
+    replacement: ' '
+- trim:
+    field: message
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- grok:
+    field: message
+    patterns:
+    - ^%{DATA:log_timestamp} -
+- grok:
+    field: message
+    patterns:
+    - 'Process\(%{DATA:process.pid}\) User\(%{WORD:user.name}\) Program\(%{DATA:process.title}\)
+      Host\(%{DATA:host.hostname}\) Installation\(%{WORD:ibmmq.errorlog.installation}\)
+      VRMF\(%{DATA:service.version}\)( QMgr\(%{DATA:ibmmq.errorlog.qmgr}\))?( Time\(%{TIMESTAMP_ISO8601:@timestamp}\))?(
+      RemoteHost\(%{DATA:destination.address}\))?( ArithInsert1\(%{DATA:ibmmq.errorlog.arithinsert1}\))?(
+      ArithInsert2\(%{DATA:ibmmq.errorlog.arithinsert2}\))?( CommentInsert1\(%{DATA:ibmmq.errorlog.commentinsert1}\))?(
+      CommentInsert2\(%{DATA:ibmmq.errorlog.commentinsert2}\))?( CommentInsert3\(%{DATA:ibmmq.errorlog.commentinsert3}\))?
+      (?=AMQ[0-9]{4})%{DATA:ibmmq.errorlog.code}((?<=AMQ[0-9]{4}[A-Z])%{DATA:log.level})?:
+      %{DATA:ibmmq.errorlog.errordescription} [^\ ]+:( %{DATA:ibmmq.errorlog.explanation})?
+      [^\ ]+:( %{DATA:ibmmq.errorlog.action})?$'
+- date:
+    field: log_timestamp
+    target_field: '@timestamp'
+    formats:
+    - MM/dd/yyyy hh:mm:ss aa
+    - dd/MM/yyyy HH:mm:ss
+    ignore_failure: true
+- append:
+    field: ibmmq.errorlog.commentinsert
+    value:
+    - '{{ibmmq.errorlog.commentinsert1}}'
+    - '{{ibmmq.errorlog.commentinsert2}}'
+    - '{{ibmmq.errorlog.commentinsert3}}'
+    ignore_failure: true
+- append:
+    field: ibmmq.errorlog.arithinsert
+    value:
+    - '{{ibmmq.errorlog.arithinsert1}}'
+    - '{{ibmmq.errorlog.arithinsert2}}'
+    ignore_failure: true
+- remove:
+    field:
+    - log_timestamp
+    - message
+    - ibmmq.errorlog.arithinsert1
+    - ibmmq.errorlog.arithinsert2
+    - ibmmq.errorlog.commentinsert1
+    - ibmmq.errorlog.commentinsert2
+    - ibmmq.errorlog.commentinsert3
+    ignore_missing: true
+- rename:
+    field: ibmmq.errorlog.errordescription
+    target_field: message
+- set:
+    field: event.kind
+    value: event
+on_failure:
+- set:
+    field: error.message
+    value: 'pipeline-entry: {{ _ingest.on_failure_message }}'
diff --git a/x-pack/filebeat/module/ibmmq/errorlog/manifest.yml b/x-pack/filebeat/module/ibmmq/errorlog/manifest.yml
@@ -9,5 +9,5 @@ var:
       - C:\ProgramData\IBM\MQ\errors\*.LOG*
       - C:\ProgramData\IBM\MQ\qmgrs\*\errors\*.LOG*
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/errorlog.yml
diff --git a/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01.log-expected.json b/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01.log-expected.json
@@ -2,6 +2,7 @@
     {
         "@timestamp": "2018-10-11T08:39:30.731Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -34,6 +35,7 @@
     {
         "@timestamp": "2018-10-11T08:39:30.729Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -66,6 +68,7 @@
     {
         "@timestamp": "2018-10-11T10:46:25.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -98,6 +101,7 @@
     {
         "@timestamp": "2018-10-11T10:46:26.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -130,6 +134,7 @@
     {
         "@timestamp": "2018-10-11T10:46:26.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -162,6 +167,7 @@
     {
         "@timestamp": "2018-10-17T11:50:15.982Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -194,6 +200,7 @@
     {
         "@timestamp": "2018-10-17T11:50:18.439Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -226,6 +233,7 @@
     {
         "@timestamp": "2018-10-18T14:13:58.401Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -258,6 +266,7 @@
     {
         "@timestamp": "2018-10-28T15:12:07.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -290,6 +299,7 @@
     {
         "@timestamp": "2018-10-28T15:12:07.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -322,6 +332,7 @@
     {
         "@timestamp": "2018-10-28T15:12:08.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -354,6 +365,7 @@
     {
         "@timestamp": "2018-10-28T15:12:08.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -386,6 +398,7 @@
     {
         "@timestamp": "2018-10-29T16:48:52.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -418,6 +431,7 @@
     {
         "@timestamp": "2018-10-29T16:48:52.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -450,6 +464,7 @@
     {
         "@timestamp": "2018-10-29T16:48:53.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -482,6 +497,7 @@
     {
         "@timestamp": "2018-10-29T16:48:53.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -514,6 +530,7 @@
     {
         "@timestamp": "2018-10-29T16:49:35.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -546,6 +563,7 @@
     {
         "@timestamp": "2018-10-29T16:49:35.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -578,6 +596,7 @@
     {
         "@timestamp": "2018-10-29T16:49:36.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",
@@ -610,6 +629,7 @@
     {
         "@timestamp": "2018-10-29T16:49:36.000Z",
         "event.dataset": "ibmmq.errorlog",
+        "event.kind": "event",
         "event.module": "ibmmq",
         "fileset.name": "errorlog",
         "host.hostname": "FELIX-ELASTIC",