Merge pull request #26 from criblpacks/release/1.1.0

Release 1.1.0
criblpacks · Dec 14, 2022 · 228480c · 228480c
2 parents 843ae92 + 2d8ea7e
commit 228480c
Show file tree

Hide file tree

Showing 21 changed files with 1,714 additions and 346 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -16,7 +16,7 @@ jobs:
         uses: rlespinasse/github-slug-action@3.5.1
 
       - name: Pack it up!
-        run: tar -czf /tmp/${{ github.event.repository.name }}-${{ env.GITHUB_SHA_SHORT }}-${{ env.GITHUB_REF_SLUG_CS }}.crbl .
+        run: tar -czf /tmp/${{ github.event.repository.name }}-${{ env.GITHUB_SHA_SHORT }}-${{ env.GITHUB_REF_SLUG_CS }}.crbl data default LICENSE package.json README.md
 
       - name: Release
         if: ${{ success() }}

diff --git a/README.md b/README.md
@@ -10,13 +10,13 @@ You should expect to see 15-30% reduction in the size of your Palo Alto Firewall
 
 ## Installation
 ---
-1. Download the most recent .crbl file from the repo [releases page](https://github.com/criblpacks/cribl-palo-alto-networks/releases).
-2. Create a Route with with a filter for your Palo Alto Firewall events. A sample filter to match all events:
+1. Install this pack from the [Cribl Pack Dispensary](https://packs.cribl.io), use the Git clone feature inside Cribl Stream, or download the most recent .crbl file from the repo [releases page](https://github.com/criblpacks/cribl-palo-alto-networks/releases).
+2. Create a Route with a filter for your Palo Alto Firewall events. A sample filter to match all events:
 ```
 (sourcetype=='pan:log' || sourcetype=='pan_log' || /^[^,]+,[^,]+,[^,]+,(THREAT|TRAFFIC|SYSTEM|CONFIG|HIPMATCH|CORRELATION|USERID|GLOBALPROTECT),/.test(_raw))
 ```
-3. Select the `PAN` pack as the pipeline.
-4. Configure the pack pipelines with the appropriate index for your Palo Alto logs. By default the index field will be set to `pan_logs`.
+3. Select the `cribl-palo-alto-networks` pack as the pipeline.
+4. Configure the Global Variable (`pan_default_index`) inside the Pack with the appropriate Splunk index for your Palo Alto logs. By default, the index field will be set to `pan_logs`.
 
 ### Configure Device Information
 This pack assumes all of your firewalls use UTC/GMT for their time zone configuration. If you use local time zones, please configure the `device_info.csv` lookup file (located in the pack's Knowledge content).
@@ -33,6 +33,13 @@ FW-.*,Etc/GMT+1
 
 ## Release Notes
 ---
+### Version 1.1.0 - 2022-04-12
+* Fixes incorrect sourcetype set in Decryption pipeline
+* Add explanations why fields are dropped
+* New feature: use Global Variables to define default `index` and `source` field values. Change in one location instead of every pipeline!
+* Rewrites pipeline logic to separate parser reserialize function into separate parser extract and serialize functions
+* New feature: set the global variable `pan_device_name_as_host` to use set the `host` field value from the `dvc_host` field value instead of the syslog header.
+
 ### Version 1.0.0 - 2022-03-22
 * Update to version 1.0.0 - major release for new Pack Dispensary 🎉
 * Changes Pack ID from `PAN` to `cribl-palo-alto-networks` to match naming convention of Cribl built Packs.

diff --git a/default/pipelines/pan_config/conf.yml b/default/pipelines/pan_config/conf.yml
@@ -1,5 +1,8 @@
 output: default
-groups: {}
+groups:
+  yicqwn:
+    name: Reserialization of Events
+    index: 6
 asyncFuncTimeout: 1000
 functions:
   - id: comment
@@ -14,24 +17,29 @@ functions:
 
         #2. Simple eval to set the host, sourcetype, source, index, and cleanup the _raw message to remove the syslog header
 
-        #3-4. Use the Auto Timestamp function to set the event timestamp to the "generated time"
+        #3. The parser function extracts all field values to the top level for event processing
 
-        #5. Reshape the events using the parser to remove unnecessary fields
+        #4. If the pan_device_name_as_host Global Variable is set to true, use the dvc_name field as the host value
+
+        #5-6. Use the Auto Timestamp function to set the event timestamp to the "generated time"
+
+        #7-10. Reserialization of Events back into CSV dropping fields that are not relevant
   - id: eval
     filter: "true"
     disabled: null
     conf:
       add:
         - name: host
-          value: _raw.match(/[A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\s([^\s]+)\s/)[1] || host
+          value: _raw.match(/[A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\s([^\s]+)\s/)[1]
+            || host
+        - value: (message || _raw).substring((message || _raw).indexOf(','))
+          name: _raw
         - value: "'pan:config'"
           name: sourcetype
         - name: source
-          value: source || 'pan:syslog'
-        - value: index || 'pan_logs'
+          value: source || C.vars.pan_default_source
+        - value: index || C.vars.pan_default_index
           name: index
-        - value: (message || _raw).substring((message || _raw).indexOf(','))
-          name: _raw
       keep:
         - _raw
         - _time
@@ -41,6 +49,49 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: serde
+    filter: "true"
+    disabled: null
+    conf:
+      mode: extract
+      type: csv
+      srcField: _raw
+      fields:
+        - future_use1
+        - receive_time
+        - serial_number
+        - type
+        - log_subtype
+        - version
+        - generated_time
+        - host_name
+        - vsys
+        - command
+        - admin
+        - client
+        - result
+        - configuration_path
+        - sequence_number
+        - action_flags
+        - before_change_detail
+        - after_change_detail
+        - devicegroup_level1
+        - devicegroup_level2
+        - devicegroup_level3
+        - devicegroup_level4
+        - vsys_name
+        - dvc_name
+        - dvc_group
+        - audit_comment
+  - id: eval
+    filter: C.vars.pan_device_name_as_host
+    disabled: null
+    conf:
+      add:
+        - name: host
+          value: dvc_name || host
+    description: If Global Variable is true, set the host field value to the value
+      from the dvc_name field
   - id: lookup
     filter: "true"
     disabled: null
@@ -57,32 +108,54 @@ functions:
       outFields:
         - lookupField: tz
           eventField: __tz
-    description: Add time zone offset as an internal field to the event. Uses the host
-      field to look up the value from device_info.csv file.
+    description: Add time zone offset as an internal field to the event. Uses the
+      host field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
-      srcField: _raw
+      srcField: generated_time
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 :
+        time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now
       latestDateAllowed: +1week
       earliestDateAllowed: -420weeks
       timestamps:
-        - regex: /^(?:[^,]*,){6}([^,]+)/
+        - regex: /(.*)/
           strptime: "%Y/%m/%d %H:%M:%S"
     description: Use the correct "generated time" as the timestamp for the event.
-  - id: serde
+  - id: comment
     filter: "true"
     disabled: null
     conf:
-      mode: reserialize
+      comment: >-
+        The following fields are dropped from the original message:
+
+        * future_use_* - there is no defined usage of these fields
+
+        * *_time - redundant fields. The "generated_time" field is true timestamp of the event and is used for the Auto Timestamp function.
+    groupId: yicqwn
+  - id: eval
+    filter: "true"
+    disabled: false
+    conf:
+      remove:
+        - future_use*
+        - "*_time"
+      keep:
+        - _time
+    groupId: yicqwn
+    description: Remove fields from event where the values should not be serialized
+  - id: serialize
+    filter: "true"
+    disabled: false
+    conf:
       type: csv
-      srcField: _raw
+      dstField: _raw
       fields:
         - future_use1
         - receive_time
@@ -110,6 +183,21 @@ functions:
         - dvc_name
         - dvc_group
         - audit_comment
+    description: Serialize PAN OS events to the fields used in the Splunk TA
+    groupId: yicqwn
+  - id: eval
+    filter: "true"
+    disabled: false
+    conf:
+      keep:
+        - _raw
+        - _time
+        - index
+        - host
+        - source
+        - sourcetype
       remove:
-        - future_use*
-        - "*_time"
+        - "*"
+    description: Final field cleanup
+    groupId: yicqwn
+description: CONFIG log type