feat: Preserve event.original when pipeline errors in integration-experience integrations (part 1)

packages/arista_ngfw/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.4.2"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/admin_login.yml

@@ -77,3 +77,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -987,6 +987,12 @@ processors:
         - _conf
         - _ingest
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -998,3 +1004,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/http_event.yml

@@ -130,3 +130,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/interface_stats.yml

@@ -50,3 +50,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/intrusion_prevention.yml

@@ -69,3 +69,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/session_stats.yml

@@ -37,3 +37,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/system_stats.yml

@@ -148,3 +148,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/web_filter.yml

@@ -40,3 +40,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/arista_ngfw/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: arista_ngfw
 title: "Arista NG Firewall"
-version: "1.4.2"
+version: "1.5.0"
 source:
   license: "Elastic-2.0"
 description: "Collect logs and metrics from Arista NG Firewall."

packages/cef/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.22.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "2.21.3"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml

@@ -403,3 +403,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -213,6 +213,12 @@ processors:
         - cef.extensions._cefVer
         - _tmp
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -228,3 +234,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml

@@ -36,3 +36,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cef/manifest.yml

@@ -1,6 +1,6 @@
 name: cef
 title: Common Event Format (CEF)
-version: "2.21.3"
+version: "2.22.0"
 description: Collect logs from CEF Logs with Elastic Agent.
 categories:
   - security

packages/checkpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.42.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.41.3"
   changes:
     - description: Generate processor tags.

packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml

@@ -17,10 +17,8 @@ processors:
       field: event.original
       tag: "grok_syslog_line"
       patterns:
-        - '%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP}|-)
-          +(?:%{SYSLOG5424PRINTASCII:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app})
-          +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(?::-|%{SYSLOG5424PRINTASCII:syslog5424_msgid})
-          +\[%{GREEDYDATA:syslog5424_sd}\]'
+        - '%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP}|-) +(?:%{SYSLOG5424PRINTASCII:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(?::-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +\[%{GREEDYDATA:syslog5424_sd}\]'
+
       pattern_definitions:
         TIMESTAMP: "%{TIMESTAMP_ISO8601:syslog5424_ts}(?:-?%{ISO8601_TIMEZONE:_temp_.tz})?"
         TIMESTAMP_ISO8601: "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?"
@@ -1686,6 +1684,12 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -1697,3 +1701,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/checkpoint/manifest.yml

@@ -1,6 +1,6 @@
 name: checkpoint
 title: Check Point
-version: "1.41.3"
+version: "1.42.0"
 description: Collect logs from Check Point with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/cisco_aironet/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.18.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.17.1"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/cisco_aironet/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -381,6 +381,12 @@ processors:
       field: _conf
       ignore_failure: true
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 
 on_failure:
   - set:
@@ -393,3 +399,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cisco_aironet/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_aironet
 title: "Cisco Aironet"
-version: "1.17.1"
+version: "1.18.0"
 description: "Integration for Cisco Aironet WLC Logs"
 type: integration
 categories:

packages/cisco_asa/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.44.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "2.43.11"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -3301,6 +3301,12 @@ processors:
         - _temp_
         - _conf
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 
 on_failure:
   # Copy any fields under _temp_.cisco to its final destination. Those can help
@@ -3325,3 +3331,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cisco_asa/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_asa
 title: Cisco ASA
-version: "2.43.11"
+version: "2.44.0"
 description: Collect logs from Cisco ASA with Elastic Agent.
 type: integration
 categories:

packages/cisco_ftd/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.12.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "3.11.0"
   changes:
     - description: Add support for Security Group Tag (SGT) and Endpoint Group (EPG) fields in connection events.

packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -3173,6 +3173,12 @@ processors:
       tag: community_id_99f56bc8
       ignore_missing: true
       ignore_failure: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   # Copy any fields under _temp_.cisco to its final destination. Those can help
   # with diagnosing the failure.
@@ -3196,3 +3202,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cisco_ftd/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ftd
 title: Cisco FTD
-version: "3.11.0"
+version: "3.12.0"
 description: Collect logs from Cisco FTD with Elastic Agent.
 type: integration
 categories:

packages/cisco_ios/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.33.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.32.1"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -567,6 +567,12 @@ processors:
        - _temp_
        - _conf
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -583,3 +589,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/cisco_ios/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ios
 title: Cisco IOS
-version: "1.32.1"
+version: "1.33.0"
 description: Collect logs from Cisco IOS with Elastic Agent.
 type: integration
 categories:

packages/cisco_ise/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.0"
+  changes:
+    - description: Preserve event.original on pipeline error.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15798
 - version: "1.29.1"
   changes:
     - description: Generate processor tags and normalize error handler.

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -271,6 +271,12 @@ processors:
           return false;
         }
         dropEmptyFields(ctx);
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - set:
       field: event.kind
@@ -282,3 +288,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false