[stormshield] Fix timestamp fields, move stormshield.msg to message (#…

…11177)

- Use stormshield.time for @timestamp
- Use stormshield.starttime for event.start
- Move stormshield.msg to message
- Release integration as GA

packages/stormshield/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.0.0
+  changes:
+    - description: Fix timestamp handling and move stormshield.msg to message. Release integration as GA.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11177
 - version: 0.1.0
   changes:
     - description: Initial creation of the integration.

packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -57,38 +57,53 @@ processors:
           }
         }
 
+  - remove:
+      field: message
+      ignore_missing: true
+  - rename:
+      tag: set_message
+      target_field: message
+      field: stormshield.msg
+      ignore_missing: true
+
   - grok:
       field: stormshield.tz
       patterns:
         - "(?:%{OFFSET:_temp_.tz_offset})(?:%{HOUR:_temp_.tz_hour}):?(?:%{MINUTE:_temp_.tz_minute})"
       pattern_definitions:
         OFFSET: "[+-]?"
-
+  - remove:
+      field: stormshield.tz
+      ignore_missing: true
+
   # rename some fields
 
   - set:
       field: "event.timezone"
       value: "{{{_temp_.tz_offset}}}{{{_temp_.tz_hour}}}:{{{_temp_.tz_minute}}}"
       if: ctx._temp_?.tz_hour != null
 
+  - date:
+      tag: process_time
+      field: stormshield.time
+      formats:
+        - "yyyy-MM-dd HH:mm:ss"
+      timezone: "{{{event.timezone}}}"
+      if: ctx.stormshield?.time != null
+
   - date:
       tag: format_startime
       field: stormshield.startime
-      target_field: "event.created"
+      target_field: "event.start"
       formats:
         - "yyyy-MM-dd HH:mm:ss"
       timezone: "{{{event.timezone}}}"
+      if: ctx.stormshield?.startime != null
       on_failure:
         - remove:
-            field: event.created
+            field: event.start
             ignore_missing: true
 
-  # Set @timestamp to the time when the log indicates.
-  - set:
-      copy_from: event.created
-      field: '@timestamp'
-      if: ctx.event?.created != null
-
   - pipeline:
       name: '{{ IngestPipeline "filterstat" }}'
       if: ctx.stormshield?.logtype == 'filterstat'
@@ -488,11 +503,6 @@ processors:
       if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: message
-      ignore_failure: true
-      ignore_missing: true
-
 
 on_failure:
 - append:

packages/stormshield/data_stream/log/fields/ecs.yml

@@ -44,12 +44,16 @@
   name: destination.as.organization.name
 - external: ecs
   name: destination.ip
+- external: ecs
+  name: destination.mac
 - external: ecs
   name: destination.nat.ip
 - external: ecs
   name: destination.port
 - external: ecs
   name: destination.nat.port
+- external: ecs
+  name: message
 - external: ecs
   name: source.geo.city_name
 - external: ecs

packages/stormshield/data_stream/log/sample_event.json

@@ -1,118 +1,83 @@
 {
+    "@timestamp": "2024-03-08T10:14:08.000Z",
     "agent": {
-        "name": "ubuntu",
-        "id": "4ef1feb0-bb75-4728-9415-c84920a76292",
+        "ephemeral_id": "fbc0ca2c-7300-45ce-84b1-6ece2bc46905",
+        "id": "e0b60804-99ad-435e-865c-35384901d186",
+        "name": "elastic-agent-45518",
         "type": "filebeat",
-        "ephemeral_id": "713a4ebd-2cc4-47dd-9940-b836b1612470",
-        "version": "8.11.4"
+        "version": "8.14.1"
     },
-    "process": {
-        "name": "asqd"
+    "data_stream": {
+        "dataset": "stormshield.log",
+        "namespace": "65295",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.11.0"
+    },
+    "elastic_agent": {
+        "id": "e0b60804-99ad-435e-865c-35384901d186",
+        "snapshot": false,
+        "version": "8.14.1"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "stormshield.log",
+        "ingested": "2024-09-18T14:01:36Z",
+        "original": "id=firewall time=\"2024-03-08 10:14:08\" fw=\"stormy-1\" tz=+0000 startime=\"2024-03-08 10:14:08\" error=0 user=\"admin\" address=192.168.197.1 sessionid=1 msg=\"PKI SEARCH scope=local type=ca\" logtype=\"server\"",
+        "start": "2024-03-08T10:14:08.000Z",
+        "timezone": "+00:00"
+    },
+    "input": {
+        "type": "tcp"
     },
     "log": {
         "source": {
-            "address": "192.168.197.134:17502"
+            "address": "172.27.0.3:40024"
         },
         "syslog": {
-            "severity": {
-                "code": 6,
-                "name": "Informational"
-            },
-            "hostname": "stormy-1",
-            "appname": "asqd",
-            "priority": 14,
-            "version": "1",
+            "appname": "serverd",
             "facility": {
                 "code": 1,
                 "name": "user-level"
-            }
+            },
+            "hostname": "stormy-1",
+            "priority": 13,
+            "severity": {
+                "code": 5,
+                "name": "Notice"
+            },
+            "version": "1"
         }
     },
-    "elastic_agent": {
-        "id": "4ef1feb0-bb75-4728-9415-c84920a76292",
-        "version": "8.11.4",
-        "snapshot": false
-    },
-    "destination": {
-        "geo": {
-            "country_iso_code": "gb"
-        },
-        "port": 80,
-        "ip": "185.125.190.48",
-        "domain": "connectivity-check.ubuntu.com"
-    },
-    "rule": {
-        "id": "7"
-    },
-    "source": {
-        "geo": {
-            "country_iso_code": "au"
-        },
-        "as": {
-            "number": 1221,
-            "organization": {
-                "name": "Telstra Pty Ltd"
-            }
-        },
-        "port": 33828,
-        "bytes": 0,
-        "ip": "1.128.7.128",
-        "mac": "00-0C-29-8D-6C-55"
-    },
-    "tags": [
-        "forwarded"
-    ],
-    "network": {
-        "protocol": "http",
-        "transport": "tcp",
-        "type": "ipv4"
-    },
-    "input": {
-        "type": "udp"
-    },
+    "message": "PKI SEARCH scope=local type=ca",
     "observer": {
+        "name": "stormy-1",
+        "type": "firewall",
         "vendor": "Stormshield"
     },
-    "@timestamp": "2024-06-06T04:32:19.000Z",
-    "ecs": {
-        "version": "8.11.0"
-    },
-    "data_stream": {
-        "namespace": "default",
-        "type": "logs",
-        "dataset": "stormshield.log"
+    "related": {
+        "user": [
+            "admin"
+        ]
     },
     "stormshield": {
+        "fw": "stormy-1",
+        "logtype": "server",
         "metadata": {
-            "confid": "01",
-            "slotlevel": "2",
-            "dstcontinent": "eu",
-            "srcname": "vm-internal",
-            "pri": "5",
-            "dstportname": "http",
-            "rcvd": "0",
+            "address": "192.168.197.1",
+            "error": "0",
             "id": "firewall",
-            "srccontinent": "oc",
-            "srcportname": "ephemeral_fw_tcp",
-            "sent": "0"
+            "sessionid": "1"
         },
-        "tz": "+0000",
-        "srcifname": "segment0",
-        "srcif": "Ethernet1",
-        "fw": "stormy-1",
-        "logtype": "filter",
-        "dstif": "Ethernet0",
-        "dstifname": "out",
-        "startime": "2024-06-06 04:32:19",
-        "time": "2024-06-06 04:32:19"
+        "startime": "2024-03-08 10:14:08",
+        "time": "2024-03-08 10:14:08"
     },
-    "event": {
-        "duration": 0,
-        "agent_id_status": "verified",
-        "ingested": "2024-06-06T04:33:16Z",
-        "timezone": "+00:00",
-        "created": "2024-06-06T04:32:19.000Z",
-        "action": "pass",
-        "dataset": "stormshield.log"
+    "tags": [
+        "preserve_original_event",
+        "forwarded"
+    ],
+    "user": {
+        "name": "admin"
     }
-}
+}