[cisco_meraki] Fix handling of security events without dhost and with action. (#8384)

* Fix handling of security events without dhost and with action.

* Add changelog PR

* Trim values

Author: marc-gr

packages/cisco_meraki/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.18.1"
+  changes:
+    - description: Fix handling of security events without dhost and with action.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8384
 - version: "1.18.0"
   changes:
     - description: Simplify IPflows pipeline to cover ICMP events.

packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log

@@ -1,4 +1,5 @@
 <134>1 1637691198.348361125 MX84 security_event ids_alerted signature=1:29708:4 priority=1 timestamp=1637691198.330873 dhost=D0:AB:D5:7B:43:73 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=allowed message: BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected
+<134>1 1698065891.323413683 MX84 security_event ids_alerted signature=1:45749:2 priority=1 timestamp=1698065891.322786 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=blocked action=allow message: SERVER-WEBAPP PHPUnit PHP remote code execution attempt
 <134>1 1637691298.984398273 MX84 security_event security_filtering_file_scanned url=http://www.eicar.org/download/eicar.com.txt src=192.168.128.2:53150 dst=67.43.156.15:80 mac=98:5A:EB:E1:81:2F name='EICAR:EICAR_Test_file_not_a_virus-tpd' sha256=275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f disposition=malicious action=block
 <134>1 1637783435.239819833 MX84 security_event security_filtering_disposition_change name=EICAR:EICAR_Test_file_not_a_virus-tpd sha256=275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f disposition=malicious action=allow
 <134>1 1637783891.345984502 MX84 ids-alerts signature=129:4:1 priority=3 timestamp=1637783891.512569 direction=ingress protocol=tcp/ip src=67.43.156.15:80

packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log-expected.json

@@ -60,11 +60,76 @@
             ],
             "threat": {
                 "indicator": {
-                    "description": " BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
+                    "description": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
                     "last_seen": "2021-11-23T18:13:18.330Z"
                 }
             }
         },
+        {
+            "@timestamp": "2023-10-23T12:58:11.323Z",
+            "cisco_meraki": {
+                "event_subtype": "ids_alerted",
+                "event_type": "security_event",
+                "security": {
+                    "action": "allow",
+                    "decision": "blocked",
+                    "priority": "1",
+                    "signature": "1:45749:2"
+                }
+            },
+            "destination": {
+                "ip": "10.0.3.162",
+                "port": 56391
+            },
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "action": "ids-signature-matched",
+                "category": [
+                    "network",
+                    "threat"
+                ],
+                "original": "<134>1 1698065891.323413683 MX84 security_event ids_alerted signature=1:45749:2 priority=1 timestamp=1698065891.322786 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=blocked action=allow message: SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
+                "type": [
+                    "info",
+                    "indicator"
+                ]
+            },
+            "network": {
+                "direction": "ingress",
+                "protocol": "tcp/ip"
+            },
+            "observer": {
+                "hostname": "MX84"
+            },
+            "source": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.12",
+                "port": 80
+            },
+            "tags": [
+                "forwarded",
+                "preserve_original_event"
+            ],
+            "threat": {
+                "indicator": {
+                    "description": "SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
+                    "last_seen": "2023-10-23T12:58:11.322Z"
+                }
+            }
+        },
         {
             "@timestamp": "2021-11-23T18:14:58.984Z",
             "cisco_meraki": {

packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/security.yml

@@ -10,57 +10,44 @@ processors:
     target_field: cisco_meraki.event_subtype
 
 # scan event based on event type
-- dissect:
-    field: event.original
-    pattern: "%{} ids_alerted %{*sig}=%{&sig} %{*pri}=%{&pri} %{*ts}=%{&ts} %{*dhost}=%{&dhost} %{*dir}=%{&dir} %{*prot}=%{&prot} %{*src}=%{&src} %{*dst}=%{&dst} %{}"
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
-- dissect:
-    field: event.original
-    pattern: "%{} security_filtering_file_scanned %{*url}=%{&url} %{*src}=%{&src} %{*dst}=%{&dst} %{*mac}=%{&mac} %{*name}='%{&name}' %{*sha256}=%{&sha256} %{*disp}=%{&disp} %{*action}=%{&action}"
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned'
-- dissect:
+- grok:
     field: event.original
-    pattern: "%{} security_filtering_disposition_change %{*name}=%{&name} %{*sha256}=%{&sha256} %{*disp}=%{&disp} %{*action}=%{&action}"
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_disposition_change'
+    patterns:
+      - ^%{DATA} (security_event|ids-alerts) (%{WORD}\s)?%{DATA:_temp.kvs}(\smessage:\s?%{DATA:message})?$
+- kv:
+    field: _temp.kvs
+    field_split: " "
+    value_split: "="
+    trim_value: " '\""
 
 # handle fields of ids_alerted type
 - rename:
     field: priority
     target_field: cisco_meraki.security.priority
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    ignore_missing: true
 - rename:
     field: signature
     target_field: cisco_meraki.security.signature
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    ignore_missing: true
 - date:
     field: timestamp
     target_field: threat.indicator.last_seen
     formats: ['UNIX']
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    if: ctx.timestamp != null
 - gsub:
     field: dhost
     target_field: cisco_meraki.security.dhost
     pattern: '[-:.]'
     replacement: '-'
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    ignore_missing: true
 - rename:
     field: direction
     target_field: network.direction
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    ignore_missing: true
 - lowercase:
     field: protocol
     target_field: network.protocol
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
-# Process the remaining after dst=. It can have "decision=<word> message: *" or just "message: *"
-- dissect:
-    field: event.original
-    pattern: "%{} dst=%{?ignore} %{*decision}=%{&decision} %{*message}:%{&message}"
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
-    ignore_failure: true
-- dissect:
-    field: event.original
-    pattern: "%{} dst=%{?ignore} %{*message}:%{&message}"
-    if: ctx?.decision == null && ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
+    ignore_missing: true
 - rename:
     field: message
     target_field: threat.indicator.description
@@ -70,35 +57,34 @@ processors:
     field: decision
     target_field: cisco_meraki.security.decision
     ignore_missing: true
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
 
 # handle fields of security_filtering_file_scanned or security_filtering_disposition_change type
 - rename:
     field: url
     target_field: threat.indicator.reference
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned'
+    ignore_missing: true
 - gsub:
     field: mac
     target_field: cisco_meraki.security.mac
     pattern: '[-:.]'
     replacement: '-'
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned'
+    ignore_missing: true
 - rename:
     field: name
     target_field: threat.indicator.file.name
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned' || ctx?.cisco_meraki?.event_subtype == 'security_filtering_disposition_change'
+    ignore_missing: true
 - rename:
     field: sha256
     target_field: threat.indicator.file.hash.sha256
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned' || ctx?.cisco_meraki?.event_subtype == 'security_filtering_disposition_change'
+    ignore_missing: true
 - rename:
     field: disposition
     target_field: cisco_meraki.disposition
     ignore_missing: true
 - rename:
     field: action
     target_field: cisco_meraki.security.action
-    if: ctx?.cisco_meraki?.event_subtype == 'security_filtering_file_scanned' || ctx?.cisco_meraki?.event_subtype == 'security_filtering_disposition_change'
+    ignore_missing: true
 # fields common to more than one event type
 # src processing
 - grok:
@@ -117,13 +103,14 @@ processors:
     type: ip
     field: _temp.src_ip
     target_field: source.ip
-    if: ctx?.cisco_meraki?.event_subtype != 'security_filtering_disposition_change'
+    ignore_missing: true
 - convert:
     field: sport
     target_field: source.port
     type: long
-    if: ctx?.sport != "0" && ctx?.cisco_meraki?.event_subtype != 'security_filtering_disposition_change'
+    ignore_missing: true
     ignore_failure: true
+    if: ctx?.sport != "0"
 # dst processing
 - grok:
     field: dst
@@ -142,13 +129,14 @@ processors:
     field: _temp.dst_ip
     target_field: destination.ip
     ignore_failure: true
-    if: ctx?.cisco_meraki?.event_subtype != 'security_filtering_disposition_change'
+    ignore_missing: true
 - convert:
     field: dport
     target_field: destination.port
     type: long
     if: ctx?.dport != "0" && ctx?.cisco_meraki?.event_subtype != 'security_filtering_disposition_change'
     ignore_failure: true
+    ignore_missing: true
 on_failure:
 - set:
     field: event.kind

packages/cisco_meraki/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: cisco_meraki
 title: Cisco Meraki
-version: "1.18.0"
+version: "1.18.1"
 description: Collect logs from Cisco Meraki with Elastic Agent.
 type: integration
 categories: