Release 0.6.0: Add support for time zones per device

criblpacks · Sep 14, 2021 · 66fc7b3 · 66fc7b3
2 parents cb6fecb + 60feef1
commit 66fc7b3
Show file tree

Hide file tree

Showing 12 changed files with 193 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -18,8 +18,27 @@ You should expect to see 15-30% reduction in the size of your Palo Alto Firewall
 3. Select the `PAN` pack as the pipeline.
 4. Configure the pack pipelines with the appropriate index for your Palo Alto logs. By default the index field will be set to `pan_logs`.
 
+### Configure Device Information
+This pack assumes all of your firewalls use UTC/GMT for their time zone configuration. If you use local time zones, please configure the `device_info.csv` lookup file (located in the pack's Knowledge content).
+
+The `device_info.csv` file uses a regular expression lookup function in each pipeline. You can use wildcards (e.g. `.*`, `KCMO-FW-\d+`, `FW-.*`) in the hostname field. The time zone (`tz`) field must be formatted as an integer (e.g. -05, +11, etc.). The regex lookup will return the most specific regex as the time zone value.
+
+Here is an example lookup file:
+```
+host,tz
+KCMO-FW-\d+,-05
+FW-.*,+01
+.*,-04
+```
+
 ## Release Notes
 ---
+### Version 0.6.0 - 2021-09-14
+Adds `device_info.csv` lookup file and lookup function in pipelines to adjust time zones per firewall.
+
+### Version 0.5.2 - 2021-08-12
+Adds pack display name for LogStream v3.1
+
 ### Version 0.5.1 - 2021-07-21
 Fixes README
 

diff --git a/data/lookups/device_info.csv b/data/lookups/device_info.csv
@@ -0,0 +1 @@
+host,tz
diff --git a/data/lookups/device_info.yml b/data/lookups/device_info.yml
@@ -0,0 +1,4 @@
+size: 7
+description: "Provides time zone information based on host configuration. Time zones should
+  be specified in this format: -05, +11, etc."
+rows: 0
diff --git a/default/pipelines/pan_config/conf.yml b/default/pipelines/pan_config/conf.yml
@@ -41,14 +41,33 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_decryption/conf.yml b/default/pipelines/pan_decryption/conf.yml
@@ -41,14 +41,33 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now
@@ -57,6 +76,7 @@ functions:
       timestamps:
         - regex: /^(?:[^,]*,){6}([^,]+)/
           strptime: "%Y/%m/%d %H:%M:%S"
+    description: Use the correct "generated time" as the timestamp for the event.
   - id: serde
     filter: "true"
     disabled: null

diff --git a/default/pipelines/pan_globalprotect/conf.yml b/default/pipelines/pan_globalprotect/conf.yml
@@ -41,14 +41,33 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now
@@ -57,6 +76,7 @@ functions:
       timestamps:
         - regex: /^(?:[^,]*,){6}([^,]+)/
           strptime: "%Y/%m/%d %H:%M:%S"
+    description: Use the correct "generated time" as the timestamp for the event.
   - id: serde
     filter: "true"
     disabled: null

diff --git a/default/pipelines/pan_hipmatch/conf.yml b/default/pipelines/pan_hipmatch/conf.yml
@@ -42,14 +42,33 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_system/conf.yml b/default/pipelines/pan_system/conf.yml
@@ -3,7 +3,7 @@ groups:
   RARnbB:
     name: Suppress Duplicate Events
     disabled: true
-    index: 4
+    index: 5
 asyncFuncTimeout: 1000
 functions:
   - id: comment
@@ -50,14 +50,33 @@ functions:
         - "*"
     description: Set fields to correct values and remove the remainder. Cleanup the _raw
       field by removing the syslog header.
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_threat/conf.yml b/default/pipelines/pan_threat/conf.yml
@@ -42,14 +42,33 @@ functions:
         - source
         - sourcetype
         - _time
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now
@@ -58,6 +77,7 @@ functions:
       timestamps:
         - regex: /^(?:[^,]*,){6}([^,]+)/
           strptime: "%Y/%m/%d %H:%M:%S"
+    description: Use the correct "generated time" as the timestamp for the event.
   - id: serde
     filter: "true"
     disabled: false

diff --git a/default/pipelines/pan_traffic/conf.yml b/default/pipelines/pan_traffic/conf.yml
@@ -4,11 +4,11 @@ groups:
     name: Sampling of Events
     description: Enable these functions if you want to sample events
     disabled: true
-    index: 4
+    index: 5
   yjDKQN:
     name: Drop irrelevant logs
     disabled: true
-    index: 5
+    index: 6
 asyncFuncTimeout: 1000
 functions:
   - id: comment
@@ -56,14 +56,33 @@ functions:
         - "*"
     description: Set fields to correct values and remove the remainder. Cleanup the _raw
       field by removing the syslog header.
-  - id: auto_timestamp
+  - id: lookup
     filter: "true"
     disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
+  - id: auto_timestamp
+    filter: "true"
+    disabled: false
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_userid/conf.yml b/default/pipelines/pan_userid/conf.yml
@@ -41,14 +41,33 @@ functions:
         - sourcetype
       remove:
         - "*"
+  - id: lookup
+    filter: "true"
+    disabled: null
+    conf:
+      matchMode: regex
+      matchType: specific
+      reloadPeriodSec: 60
+      addToEvent: false
+      inFields:
+        - eventField: host
+          lookupField: host
+      ignoreCase: false
+      file: device_info.csv
+      outFields:
+        - lookupField: tz
+          eventField: __tz
+    description: Add time zone offset as an internal field to the event. Uses the host
+      field to look up the value from device_info.csv file.
   - id: auto_timestamp
     filter: "true"
     disabled: null
     conf:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: time.getTime() / 1000
+      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
+        1000)"
       offset: 0
       maxLen: 150
       defaultTime: now
@@ -57,6 +76,7 @@ functions:
       timestamps:
         - regex: /^(?:[^,]*,){6}([^,]+)/
           strptime: "%Y/%m/%d %H:%M:%S"
+    description: Use the correct "generated time" as the timestamp for the event.
   - id: serde
     filter: "true"
     disabled: null

diff --git a/package.json b/package.json
@@ -1 +1 @@
-{"name":"PAN","version":"0.5.2","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
+{"name":"PAN","version":"0.6.0","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"name":"PAN","version":"0.5.2","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
		{"name":"PAN","version":"0.6.0","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}