feat: Preserve event.original when pipeline errors in integration-experience integrations (part 4)

packages/sophos/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.16.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15806
 - version: "3.15.4"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml

@@ -226,6 +226,12 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - append:
       field: error.message
@@ -237,3 +243,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml

@@ -169,3 +169,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml

@@ -97,3 +97,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml

@@ -324,3 +324,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml

@@ -276,3 +276,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/antispam.yml

@@ -163,3 +163,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/antivirus.yml

@@ -263,3 +263,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/atp.yml

@@ -144,3 +144,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/cfilter.yml

@@ -200,3 +200,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml

@@ -869,6 +869,12 @@ processors:
       - sophos.xg.severity
       - sophos.xg.src_country
     ignore_missing: true
+- append:
+    tag: append_preserve_original_event_on_error
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false
+    if: ctx.error?.message != null
 on_failure:
 - set:
     field: event.kind
@@ -880,3 +886,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/event.yml

@@ -151,3 +151,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/firewall.yml

@@ -276,3 +276,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/idp.yml

@@ -138,3 +138,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/sandstorm.yml

@@ -159,3 +159,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/systemhealth.yml

@@ -230,3 +230,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/waf.yml

@@ -207,3 +207,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/wifi.yml

@@ -36,3 +36,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/sophos/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: sophos
 title: Sophos
-version: "3.15.4"
+version: "3.16.0"
 description: Collect logs from Sophos with Elastic Agent.
 categories:
   - security

packages/squid/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15806
 - version: "1.3.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/squid/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -248,6 +248,12 @@ processors:
       field:
         - _tmp
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 
 on_failure:
   - remove:
@@ -265,3 +271,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/squid/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: squid
 title: Squid Proxy
-version: "1.3.2"
+version: "1.4.0"
 description: Collect and parse logs from Squid devices with Elastic Agent.
 categories:
   # Observability is a parent category for web

packages/stormshield/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15806
 - version: "1.3.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/count.yml

@@ -56,3 +56,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -556,6 +556,12 @@ processors:
       field: _temp_
       ignore_missing: true
       ignore_failure: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 
 on_failure:
 - append:
@@ -568,3 +574,7 @@ on_failure:
 - set:
     field: event.kind
     value: pipeline_error
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/filterstat.yml

@@ -62,3 +62,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/monitor.yml

@@ -158,3 +158,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/stormshield/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.1.1
 name: stormshield
 title: "StormShield SNS"
-version: "1.3.2"
+version: "1.4.0"
 source:
   license: "Elastic-2.0"
 description: "Stormshield SNS integration."

packages/suricata/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.26.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15806
 - version: "2.25.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml

@@ -788,6 +788,12 @@ processors:
         - dns.question.domain
         - _tmp_
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -799,3 +805,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v1.yml

@@ -44,3 +44,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns-answer-v2.yml

@@ -48,3 +48,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/dns.yml

@@ -111,3 +111,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/tls.yml

@@ -294,3 +294,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/suricata/manifest.yml

@@ -1,6 +1,6 @@
 name: suricata
 title: Suricata
-version: "2.25.2"
+version: "2.26.0"
 description: Collect logs from Suricata with Elastic Agent.
 type: integration
 icons:

packages/syslog_router/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15806
 - version: "0.3.1"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/syslog_router/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -5,6 +5,12 @@ processors:
       tag: set_ecs_version_135371bc
       field: ecs.version
       value: 8.16.0
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 
 on_failure:
   - append:
@@ -17,3 +23,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/syslog_router/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: syslog_router
 title: "Syslog Router"
-version: 0.3.1
+version: 0.4.0
 description: "Route syslog events to integrations with Elastic Agent."
 type: integration
 categories: