Add correlation route, pipeline, and sample

README.md

@@ -33,6 +33,9 @@ FW-.*,Etc/GMT+1
 
 ## Release Notes
 ---
+### Version 0.6.3 - 2022-03-03
+* Adds Correlation event log pipeline.
+
 ### Version 0.6.2 - 2021-11-17
 * `device_info.csv` now uses Olson formatted timezones (e.g. `America/Chicago`) instead of static offsets and the [`C.Time.adjustTZ`](https://docs.cribl.io/logstream/cribl-reference/#time) function for better time zone support. A listing of time zones can be found [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).
 
@@ -41,16 +44,16 @@ FW-.*,Etc/GMT+1
 * Routes use `indexOf` filter instead of `test` for higher performance. 
 
 ### Version 0.6.0 - 2021-09-14
-Adds `device_info.csv` lookup file and lookup function in pipelines to adjust time zones per firewall.
+* Adds `device_info.csv` lookup file and lookup function in pipelines to adjust time zones per firewall.
 
 ### Version 0.5.2 - 2021-08-12
-Adds pack display name for LogStream v3.1
+* Adds pack display name for LogStream v3.1
 
 ### Version 0.5.1 - 2021-07-21
-Fixes README
+* Fixes README
 
 ### Version 0.5.0 - 2021-07-20
-Initial release
+* Initial release
 
 
 ## Contributing to the Pack
@@ -63,4 +66,4 @@ The author of this pack is Brendan Dalpe and can be contacted at <bdalpe@cribl.i
 
 ## License
 ---
-This Pack uses the following license: [`Apache 2.0`](https://github.com/criblio/appscope/blob/master/LICENSE).
+This Pack uses the following license: [`Apache 2.0`](https://github.com/criblpacks/cribl-palo-alto-networks/blob/master/LICENSE).

data/samples/BvsW2V.json

@@ -0,0 +1 @@
+[{"_time":1626825542,"cribl_breaker":"Break on newlines","message":"1,2021/07/20 23:59:02,012345678902,CORRELATION,,,2021/07/20 23:59:02,1.2.3.4,username,,compromised-host,medium,0,0,0,0,,us2,,beacon-heuristics,6005,\"Host visited known malware URL (100 times).\"","severity":6,"facility":1,"host":"192.168.1.198","severityName":"info","facilityName":"user","_raw":"<14>Jul 20 23:59:02 1,2021/07/20 23:59:02,012345678902,CORRELATION,,,2021/07/20 23:59:02,1.2.3.4,username,,compromised-host,medium,0,0,0,0,,us2,,beacon-heuristics,6005,\"Host visited known malware URL (100 times).\""}]

default/pipelines/pan_correlation/conf.yml

@@ -0,0 +1,114 @@
+output: default
+groups: {}
+asyncFuncTimeout: 1000
+functions:
+  - id: comment
+    filter: "true"
+    disabled: null
+    conf:
+      comment: >-
+        This pipeline will process PAN Correlation event logs.
+
+
+        There are multiple functions in this pipeline:
+
+        #2. Simple eval to set the host, sourcetype, source, index, and cleanup the _raw message to remove the syslog header
+
+        #3. Use the Auto Timestamp function to set the event timestamp to the "generated time"
+
+        #4. Reshape the events using the parser to remove unnecessary fields
+  - id: eval
+    filter: "true"
+    disabled: null
+    conf:
+      add:
+        - name: host
+          value: _raw.match(/[A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\s([^\s]+)\s/)[1]
+            || host
+        - name: sourcetype
+          value: "'pan:correlation'"
+        - name: source
+          value: source || 'pan:syslog'
+        - name: index
+          value: index || 'pan_logs'
+        - name: _raw
+          value: (message || _raw).substring((message || _raw).indexOf(','))
+      keep:
+        - _raw
+        - _time
+        - index
+        - host
+        - source
+        - sourcetype
+      remove:
+        - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
+  - id: auto_timestamp
+    filter: "true"
+    disabled: null
+    conf:
+      srcField: _raw
+      dstField: _time
+      defaultTimezone: utc
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
+      offset: 0
+      maxLen: 150
+      defaultTime: now
+      latestDateAllowed: +1week
+      earliestDateAllowed: -420weeks
+      timestamps:
+        - regex: /^(?:[^,]*,){6}([^,]+)/
+          strptime: "%Y/%m/%d %H:%M:%S"
+    description: Use the correct "generated time" as the timestamp for the event.
+  - id: serde
+    filter: "true"
+    disabled: null
+    conf:
+      mode: reserialize
+      type: csv
+      srcField: _raw
+      fields:
+        - future_use1
+        - receive_time
+        - serial_number
+        - type
+        - content_threat_type
+        - future_use2
+        - generated_time
+        - source_address_source_user
+        - virtual_system
+        - category
+        - severity
+        - device_group_hierarchy_level_1
+        - device_group_hierarchy_level_2
+        - device_group_hierarchy_level_3
+        - device_group_hierarchy_level_4
+        - virtual_system_name
+        - device_name
+        - virtual_system_id
+        - object_name
+        - object_id
+        - evidence
+      remove:
+        - future_use*
+        - "*_time"
+        - timestamp
+        - device_group_*

default/pipelines/route.yml

@@ -73,6 +73,17 @@ routes:
     clones: []
     filter: _raw.indexOf(",DECRYPTION,") > -1
     output: default
+  - id: Uwyn3m
+    name: Correlation
+    final: true
+    disabled: false
+    pipeline: pan_correlation
+    description: ""
+    clones: [ ]
+    enableOutputExpression: false
+    outputExpression: null
+    filter: _raw.indexOf(",CORRELATION,") > -1
+    output: default
   - id: default
     name: default
     final: true

default/samples.yml

@@ -29,3 +29,8 @@ zU8yWr:
   created: 1626806657410
   size: 1032
   numEvents: 1
+BvsW2V:
+  sampleName: pan_correlation.log
+  created: 1646346512671
+  size: 584
+  numEvents: 1

package.json

@@ -1 +1 @@
-{"name":"PAN","version":"0.6.2","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
+{"name":"PAN","version":"0.6.3","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}