Merge pull request #9 from criblpacks/release/0.6.2

Time zone support++
criblpacks · Nov 17, 2021 · 53322de · 53322de
2 parents 1acd969 + bb03a0b
commit 53322de
Show file tree

Hide file tree

Showing 11 changed files with 16 additions and 21 deletions.
diff --git a/README.md b/README.md
@@ -26,13 +26,16 @@ The `device_info.csv` file uses a regular expression lookup function in each pip
 Here is an example lookup file:
 ```
 host,tz
-KCMO-FW-\d+,-05
-FW-.*,+01
-.*,-04
+KCMO-FW-\d+,America/Chicago
+FW-.*,Etc/GMT+1
+.*,US/Eastern
 ```
 
 ## Release Notes
 ---
+### Version 0.6.2 - 2021-11-17
+* `device_info.csv` now uses Olson formatted timezones (e.g. `America/Chicago`) instead of static offsets and the [`C.Time.adjustTZ`](https://docs.cribl.io/logstream/cribl-reference/#time) function for better time zone support. A listing of time zones can be found [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).
+
 ### Version 0.6.1 - 2021-11-02
 * Bug fix - Corrects an issue in pipelines where the hostname is not correctly extracted if the date is a single digit. Unifies the hostname extraction across all pipelines.
 * Routes use `indexOf` filter instead of `test` for higher performance. 

diff --git a/data/lookups/device_info.yml b/data/lookups/device_info.yml
@@ -1,4 +1,4 @@
 size: 7
 description: "Provides time zone information based on host configuration. Time zones should
-  be specified in this format: -05, +11, etc."
+  be specified in Olson format. E.g. America/Chicago, Etc/GMT-5, US/Central, etc."
 rows: 0
diff --git a/default/pipelines/pan_config/conf.yml b/default/pipelines/pan_config/conf.yml
@@ -66,8 +66,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_decryption/conf.yml b/default/pipelines/pan_decryption/conf.yml
@@ -66,8 +66,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_globalprotect/conf.yml b/default/pipelines/pan_globalprotect/conf.yml
@@ -66,8 +66,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_hipmatch/conf.yml b/default/pipelines/pan_hipmatch/conf.yml
@@ -67,8 +67,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_system/conf.yml b/default/pipelines/pan_system/conf.yml
@@ -74,8 +74,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_threat/conf.yml b/default/pipelines/pan_threat/conf.yml
@@ -66,8 +66,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_traffic/conf.yml b/default/pipelines/pan_traffic/conf.yml
@@ -81,8 +81,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/default/pipelines/pan_userid/conf.yml b/default/pipelines/pan_userid/conf.yml
@@ -66,8 +66,7 @@ functions:
       srcField: _raw
       dstField: _time
       defaultTimezone: utc
-      timeExpression: "__tz ? (time.getTime() / 1000) + (__tz * 3600) : (time.getTime() /
-        1000)"
+      timeExpression: "__tz ? C.Time.adjustTZ(time.getTime(), 'UTC', __tz) / 1000 : time.getTime() / 1000"
       offset: 0
       maxLen: 150
       defaultTime: now

diff --git a/package.json b/package.json
@@ -1 +1 @@
-{"name":"PAN","version":"0.6.1","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
+{"name":"PAN","version":"0.6.2","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"name":"PAN","version":"0.6.1","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}
		{"name":"PAN","version":"0.6.2","author":"Brendan Dalpe - Cribl","description":"Process, reduce, and transform Palo Alto Networks Firewall logs.","displayName":"Palo Alto Networks"}