feat: Preserve event.original when pipeline errors in integration-experience integrations (part 2)

packages/citrix_waf/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.19.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "1.18.3"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/cef.yml

@@ -135,3 +135,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -166,6 +166,12 @@ processors:
         - _tmp
         - _conf
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -182,3 +188,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/native.yml

@@ -39,3 +39,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_waf/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: citrix_waf
 title: "Citrix Web App Firewall"
-version: "1.18.3"
+version: "1.19.0"
 description: Ingest events from Citrix Systems Web App Firewall.
 type: integration
 categories:

packages/endace/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "0.1.2"
   changes:
     - description: Normalize error handler.

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/compatibility.yml

@@ -34,3 +34,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/default.yml

@@ -109,6 +109,12 @@ processors:
     field: _conf
     ignore_missing: true
     tag: remove_conf
+- append:
+    tag: append_preserve_original_event_on_error
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false
+    if: ctx.error?.message != null
 
 on_failure:
   - append:
@@ -118,3 +124,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/endace.yml

@@ -7,15 +7,15 @@ processors:
       value: "ip_conv={{ source.ip }}%26{{ destination.ip }}"
       if: (ctx.destination?.ip != null && ctx.destination?.ip != '') && (ctx.source?.ip != null && ctx.source?.ip != '')
       tag: endace conversation set
-  
-  - set: 
+
+  - set:
       description: "Set IP Conversation if only destination.ip is present"
       field: _conf.ip_conv
       value: "ip={{ destination.ip }}"
       if: (ctx.destination?.ip != null && ctx.destination.ip != '') && (ctx.source?.ip == null || ctx.source?.ip == '')
       tag: endace destination ip set
 
-  - set: 
+  - set:
       description: "Set IP Conversation if only source.ip is present"
       field: _conf.ip_conv
       value: "ip={{ source.ip }}"
@@ -90,3 +90,7 @@ on_failure:
       field: event.kind
       description: "Event Kind"
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/flow/elasticsearch/ingest_pipeline/geoip.yml

@@ -109,3 +109,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -134,6 +134,12 @@ processors:
   - remove:
       tag: remove_conf_fields
       field: "_conf"
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -145,3 +151,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/data_stream/log/elasticsearch/ingest_pipeline/endace-netflow.yml

@@ -92,3 +92,7 @@ on_failure:
       field: event.kind
       description: "Event Kind"
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/endace/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: endace
 title: "Endace"
-version: 0.1.2
+version: 0.2.0
 source:
   license: "Elastic-2.0"
 description: "This Endace integration configures Network Packet Capture for flow generation and adds a pivot field to your Endace platform."

packages/fortinet_fortiedr/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.20.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "1.19.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/fortinet_fortiedr/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -278,6 +278,12 @@ processors:
       field:
         - _temp_
       ignore_failure: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -289,3 +295,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortiedr/manifest.yml

@@ -1,6 +1,6 @@
 name: fortinet_fortiedr
 title: Fortinet FortiEDR Logs
-version: "1.19.2"
+version: "1.20.0"
 description: Collect logs from Fortinet FortiEDR instances with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/fortinet_fortigate/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.36.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "1.35.0"
   changes:
     - description: Add timezone mapping for Fortinet FortiGate logs.

packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -34,7 +34,7 @@ processors:
   - gsub:
       tag: gsub_syslog5424_msg_b526c35d
       field: syslog5424_msg
-      pattern: "[\u0000-\u001F\u007F]"
+      pattern: "[\x00-\x1f\x7f]"
       replacement: ""
   - script:
       tag: script_2073fe5f
@@ -805,6 +805,12 @@ processors:
         - fortinet.firewall.time
         - fortinet.firewall.duration
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -816,3 +822,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml

@@ -345,3 +345,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/login.yml

@@ -166,3 +166,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml

@@ -320,3 +320,7 @@ on_failure:
       {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
       {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
       failed with message '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml

@@ -594,3 +594,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortigate/manifest.yml

@@ -1,6 +1,6 @@
 name: fortinet_fortigate
 title: Fortinet FortiGate Firewall Logs
-version: "1.35.0"
+version: "1.36.0"
 description: Collect logs from Fortinet FortiGate firewalls with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/fortinet_fortimail/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "2.16.1"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -295,6 +295,12 @@ processors:
       field: event.kind
       value: pipeline_error
       if: ctx.error?.message != null
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - append:
       field: error.message
@@ -306,3 +312,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_antispam.yml

@@ -133,3 +133,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_antivirus.yml

@@ -79,3 +79,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_encryption.yml

@@ -74,3 +74,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_history.yml

@@ -289,3 +289,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_mail.yml

@@ -171,3 +171,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/data_stream/log/elasticsearch/ingest_pipeline/pipeline_system.yml

@@ -240,3 +240,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/fortinet_fortimail/manifest.yml

@@ -1,6 +1,6 @@
 name: fortinet_fortimail
 title: Fortinet FortiMail
-version: "2.16.1"
+version: "2.17.0"
 description: Collect logs from Fortinet FortiMail instances with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/fortinet_fortimanager/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15803
 - version: "2.16.2"
   changes:
     - description: Generate processor tags and normalize error handler.