Teach elasticsearch/audit fileset to handle timestamps correctly (#…

…15942) (#15963) * Add samples from ES 7.1.1 audit log * When JSON logs contain @timestamp and TZ is detected, use detected TZ * Add CHANGELOG entry * Remove redundant guard * Revert unrelated change * s/teach/improve/ in changelog entry
elastic · Feb 5, 2020 · 6a23e8f · 6a23e8f
1 parent d71beed
commit 6a23e8f
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 10 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -55,6 +55,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
 - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
 - Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
+- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
 
 *Heartbeat*
 

diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -6,5 +6,4 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-# Locale for timezone is only needed in non-json logs
-- add_locale.when.not.regexp.message: "^{"
+- add_locale: ~
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
@@ -16,7 +16,23 @@
             "remove": {
                 "field": "elasticsearch.audit.type",
                 "ignore_missing": true
-
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null",
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "elasticsearch.audit.@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}"
+            }
+        },
+        {
+            "remove": {
+                "if": "ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null",
+                "field": "event.timezone"
             }
         },
         {

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log b/filebeat/module/elasticsearch/audit/test/test-audit-711.log
@@ -0,0 +1,3 @@
+{"@timestamp":"2019-09-05T14:02:37,921", "node.id":"UwRu4mReRtyJO1-FWAPvIQ", "event.type":"transport", "event.action":"authentication_success", "user.name":"_system", "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "realm":"__fallback", "request.id":"474ZciqtQteOhjLO3OdZIw", "action":"indices:monitor/stats", "request.name":"IndicesStatsRequest"}
+{"@timestamp":"2020-01-29T09:41:10,856", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear", "request.name":"ClearRealmCacheRequest"}
+{"@timestamp":"2020-01-29T09:41:10,859", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear[n]", "request.name":"Node"}
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json
@@ -0,0 +1,77 @@
+[
+    {
+        "@timestamp": "2019-09-05T16:02:37.921Z",
+        "elasticsearch.audit.action": "indices:monitor/stats",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "local_node",
+        "elasticsearch.audit.realm": "__fallback",
+        "elasticsearch.audit.request.id": "474ZciqtQteOhjLO3OdZIw",
+        "elasticsearch.audit.request.name": "IndicesStatsRequest",
+        "elasticsearch.node.id": "UwRu4mReRtyJO1-FWAPvIQ",
+        "event.action": "authentication_success",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "{\"@timestamp\":\"2019-09-05T14:02:37,921\", \"node.id\":\"UwRu4mReRtyJO1-FWAPvIQ\", \"event.type\":\"transport\", \"event.action\":\"authentication_success\", \"user.name\":\"_system\", \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"realm\":\"__fallback\", \"request.id\":\"474ZciqtQteOhjLO3OdZIw\", \"action\":\"indices:monitor/stats\", \"request.name\":\"IndicesStatsRequest\"}",
+        "service.type": "elasticsearch",
+        "source.address": "127.0.0.1:9300",
+        "source.ip": "127.0.0.1",
+        "source.port": 9300,
+        "user.name": "_system"
+    },
+    {
+        "@timestamp": "2020-01-29T11:41:10.856Z",
+        "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "local_node",
+        "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg",
+        "elasticsearch.audit.request.name": "ClearRealmCacheRequest",
+        "elasticsearch.audit.user.realm": "__attach",
+        "elasticsearch.audit.user.roles": [
+            "superuser"
+        ],
+        "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q",
+        "event.action": "access_granted",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "log.offset": 363,
+        "message": "{\"@timestamp\":\"2020-01-29T09:41:10,856\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}",
+        "service.type": "elasticsearch",
+        "source.address": "127.0.0.1:9300",
+        "source.ip": "127.0.0.1",
+        "source.port": 9300,
+        "user.name": "_xpack_security"
+    },
+    {
+        "@timestamp": "2020-01-29T11:41:10.859Z",
+        "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "local_node",
+        "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg",
+        "elasticsearch.audit.request.name": "Node",
+        "elasticsearch.audit.user.realm": "__attach",
+        "elasticsearch.audit.user.roles": [
+            "superuser"
+        ],
+        "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q",
+        "event.action": "access_granted",
+        "event.dataset": "elasticsearch.audit",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "log.offset": 785,
+        "message": "{\"@timestamp\":\"2020-01-29T09:41:10,859\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}",
+        "service.type": "elasticsearch",
+        "source.address": "127.0.0.1:9300",
+        "source.ip": "127.0.0.1",
+        "source.port": 9300,
+        "user.name": "_xpack_security"
+    }
+]
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json
@@ -1,12 +1,13 @@
 [
     {
-        "@timestamp": "2018-10-31T09:34:25.109Z",
+        "@timestamp": "2018-10-31T11:34:25.109Z",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin.type": "rest",
         "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
         "event.action": "authentication_failed",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 0,
@@ -19,13 +20,14 @@
         "user.name": "elastic"
     },
     {
-        "@timestamp": "2018-10-31T09:34:25.207Z",
+        "@timestamp": "2018-10-31T11:34:25.207Z",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin.type": "rest",
         "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
         "event.action": "authentication_failed",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 274,
@@ -38,7 +40,7 @@
         "user.name": "elastic"
     },
     {
-        "@timestamp": "2018-10-31T09:35:11.428Z",
+        "@timestamp": "2018-10-31T11:35:11.428Z",
         "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear",
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin.type": "local_node",
@@ -51,6 +53,7 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 558,
@@ -62,7 +65,7 @@
         "user.name": "_xpack_security"
     },
     {
-        "@timestamp": "2018-10-31T09:35:11.430Z",
+        "@timestamp": "2018-10-31T11:35:11.430Z",
         "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]",
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin.type": "local_node",
@@ -75,6 +78,7 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 941,
@@ -86,7 +90,7 @@
         "user.name": "_xpack_security"
     },
     {
-        "@timestamp": "2018-10-31T09:35:12.303Z",
+        "@timestamp": "2018-10-31T11:35:12.303Z",
         "elasticsearch.audit.action": "cluster:admin/xpack/security/user/change_password",
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin.type": "rest",
@@ -99,6 +103,7 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1309,
@@ -110,7 +115,7 @@
         "user.name": "elastic"
     },
     {
-        "@timestamp": "2018-10-31T09:35:12.314Z",
+        "@timestamp": "2018-10-31T11:35:12.314Z",
         "elasticsearch.audit.action": "indices:admin/create",
         "elasticsearch.audit.indices": [
             ".security-6"
@@ -126,6 +131,7 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1676,
@@ -137,7 +143,7 @@
         "user.name": "_xpack_security"
     },
     {
-        "@timestamp": "2019-01-27T20:15:10.380Z",
+        "@timestamp": "2019-01-27T22:15:10.380Z",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin.type": "rest",
         "elasticsearch.audit.realm": "default_file",
@@ -147,6 +153,7 @@
         "event.action": "authentication_success",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "http.request.body.content": "\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n",
         "http.request.method": "GET",