feat: Preserve event.original when errors occur in pipelines

- Added append processor to global on_failure to preserve event original
- Added append processor to default piplines to preserve event original if error.message is set

Affects the following integrations:

- imperva
- iptables
- juniper_srx
- modsecurity
- netflow
- panw
- pfsense
- proxysg
- qnap_nas
- snort
- sonicwall_firewall

Author: taylor-swanson

packages/imperva/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.8.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml

@@ -336,6 +336,12 @@ processors:
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
       if: ctx.error?.message != null
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - append:
       field: error.message
@@ -347,3 +353,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/imperva/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.3
 name: imperva
 title: Imperva
-version: "1.8.2"
+version: "1.9.0"
 description: Collect logs from Imperva devices with Elastic Agent.
 categories: ["network", "security"]
 type: integration

packages/iptables/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.21.4"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -415,6 +415,12 @@ processors:
       field:
         - _tmp
       ignore_failure: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -430,3 +436,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/iptables/manifest.yml

@@ -1,6 +1,6 @@
 name: iptables
 title: Iptables
-version: "1.21.4"
+version: "1.22.0"
 description: Collect logs from Iptables with Elastic Agent.
 type: integration
 icons:

packages/juniper_srx/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.26.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.25.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml

@@ -427,3 +427,7 @@ on_failure:
 - set:
     field: event.kind
     value: pipeline_error
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false

packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -434,6 +434,12 @@ processors:
         - juniper.srx.dstzone
         - syslog_pri
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - append:
       field: error.message
@@ -445,3 +451,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml

@@ -427,3 +427,7 @@ on_failure:
 - set:
     field: event.kind
     value: pipeline_error
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false