Merge branch 'main' into main

Author: trisch-me

packages/filestream/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add logs stream support
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/14846
 - version: "1.2.0"
   changes:
     - description: Add the native file identity

packages/filestream/data_stream/generic/agent/stream/filestream.yml.hbs

@@ -1,5 +1,10 @@
+{{#if use_logs_stream}}
+index: logs
+{{else}}
 data_stream:
   dataset: {{data_stream.dataset}}
+{{/if}}
+
 paths:
 {{#each paths as |path i|}}
   - {{path}}

packages/filestream/data_stream/generic/manifest.yml

@@ -14,6 +14,14 @@ streams:
         show_user: true
         default:
           - /var/log/*.log
+      - name: use_logs_stream
+        type: bool
+        title: Use the "logs" data stream
+        description: |
+          Enabling this will send all the ingested data to the "logs" data stream. This feature is disabled by default. If enabled the Dataset name option is ignored. "Write to logs streams" option must be enabled in the output settings for this to work.
+        required: false
+        show_user: true
+        default: false
       - name: data_stream.dataset
         type: text
         title: Dataset name

packages/filestream/manifest.yml

@@ -3,10 +3,10 @@ name: filestream
 title: Custom Logs (Filestream)
 description: Collect log data using filestream with Elastic Agent.
 type: integration
-version: 1.2.0
+version: 1.3.0
 conditions:
   kibana:
-    version: "^8.15.0 || ^9.0.0"
+    version: "^9.2.0"
 categories:
   - custom
   - custom_logs

packages/sophos/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.15.2"
+  changes:
+    - description: Add conditions to GeoIP processors to prevent failure on empty strings
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15130
 - version: "3.15.1"
   changes:
     - description: Changed owners.

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml

@@ -121,6 +121,7 @@ processors:
       target_field: client.geo
       ignore_missing: true
       tag: geo_client_ip
+      if: ctx.client?.ip != ''
 
   # IP Autonomous System (AS) Lookup
   - geoip:
@@ -132,6 +133,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_client_as
+      if: ctx.client?.ip != ''
   - rename:
       field: client.as.asn
       target_field: client.as.number

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml

@@ -55,6 +55,7 @@ processors:
       target_field: server.geo
       ignore_missing: true
       tag: geo_server_ip
+      if: ctx.server?.ip != ''
 
   # IP Autonomous System (AS) Lookup
   - geoip:
@@ -66,6 +67,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_server_as
+      if: ctx.server?.ip != ''
   - rename:
       field: server.as.asn
       target_field: server.as.number

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml

@@ -209,11 +209,13 @@ processors:
       target_field: source.geo
       ignore_missing: true
       tag: geo_source_ip
+      if: ctx.source?.ip != ''
   - geoip:
       field: destination.ip
       target_field: destination.geo
       ignore_missing: true
       tag: geo_destination_ip
+      if: ctx.destination?.ip != ''
 
   # IP Autonomous System (AS) Lookup
   - geoip:
@@ -225,6 +227,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_source_as
+      if: ctx.source?.ip != ''
   - geoip:
       database_file: GeoLite2-ASN.mmdb
       field: destination.ip
@@ -234,6 +237,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_destination_as
+      if: ctx.destination?.ip != ''
   - rename:
       field: source.as.asn
       target_field: source.as.number

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml

@@ -190,11 +190,13 @@ processors:
       target_field: source.geo
       ignore_missing: true
       tag: geo_source_ip
+      if: ctx.source?.ip != ''
   - geoip:
       field: destination.ip
       target_field: destination.geo
       ignore_missing: true
       tag: geo_destination_ip
+      if: ctx.destination?.ip != ''
 
   # IP Autonomous System (AS) Lookup
   - geoip:
@@ -206,6 +208,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_source_as
+      if: ctx.source?.ip != ''
   - geoip:
       database_file: GeoLite2-ASN.mmdb
       field: destination.ip
@@ -215,6 +218,7 @@ processors:
           - organization_name
       ignore_missing: true
       tag: geo_destination_as
+      if: ctx.destination?.ip != ''
   - rename:
       field: source.as.asn
       target_field: source.as.number

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml

@@ -525,12 +525,12 @@ processors:
     field: source.ip
     target_field: source.geo
     ignore_missing: true
-    if: "ctx.source?.geo == null"
+    if: "ctx.source?.geo == null && ctx.source?.ip != ''"
 - geoip:
     field: destination.ip
     target_field: destination.geo
     ignore_missing: true
-    if: "ctx.destination?.geo == null"
+    if: "ctx.destination?.geo == null && ctx.destination?.ip != ''"
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: source.ip
@@ -539,6 +539,7 @@ processors:
       - asn
       - organization_name
     ignore_missing: true
+    if: "ctx.source?.ip != ''"
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: destination.ip
@@ -547,16 +548,17 @@ processors:
       - asn
       - organization_name
     ignore_missing: true
+    if: "ctx.destination?.ip != ''"
 - geoip:
     field: source.nat.ip
     target_field: source.geo
     ignore_missing: true
-    if: "ctx.source?.geo == null"
+    if: "ctx.source?.geo == null && ctx.source?.nat?.ip != ''"
 - geoip:
     field: destination.nat.ip
     target_field: destination.geo
     ignore_missing: true
-    if: "ctx.destination?.geo == null"
+    if: "ctx.destination?.geo == null && ctx.destination?.nat?.ip != ''"
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: source.nat.ip
@@ -565,7 +567,7 @@ processors:
       - asn
       - organization_name
     ignore_missing: true
-    if: "ctx.source?.as == null"
+    if: "ctx.source?.as == null && ctx.source?.nat?.ip != ''"
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: destination.nat.ip
@@ -574,7 +576,7 @@ processors:
       - asn
       - organization_name
     ignore_missing: true
-    if: "ctx.destination?.as == null"
+    if: "ctx.destination?.as == null && ctx.destination?.nat?.ip != ''"
 - rename:
     field: source.as.asn
     target_field: source.as.number