Cherry-pick #18948 to 7.8: Filebeat: Fix o365 module issues (#19062)

- Fix scary data-loss warning on startup - Avoid API errors being processed by the JS pipeline - Fix dissect error about overiding client.port - Fix module passing API settings to the input - Document max_period using the right units (cherry picked from commit 83bbd57)
elastic · Jun 9, 2020 · bf707dd · bf707dd
1 parent 7eddfae
commit bf707dd
Show file tree

Hide file tree

Showing 8 changed files with 13 additions and 15 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -109,6 +109,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525]
 - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
 - Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591]
+- Fix `o365` module ignoring `var.api` settings. {pull}18948[18948]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -852,7 +852,7 @@ filebeat.modules:
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m
 

diff --git a/x-pack/filebeat/input/o365audit/state.go b/x-pack/filebeat/input/o365audit/state.go
@@ -114,10 +114,7 @@ func (s *stateStorage) Load(key stream) (cursor, error) {
 	}
 	cur, err := s.persister.Load(key)
 	if err != nil {
-		if err != errStateNotFound {
-			return cur, err
-		}
-		cur = newCursor(key, time.Time{})
+		return newCursor(key, time.Time{}), err
 	}
 	return cur, s.saveUnsafe(cur)
 }

diff --git a/x-pack/filebeat/input/o365audit/state_test.go b/x-pack/filebeat/input/o365audit/state_test.go
@@ -21,18 +21,14 @@ func TestNoopState(t *testing.T) {
 	t.Run("new state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		empty := newCursor(myStream, time.Time{})
 		assert.Equal(t, empty, cur)
 	})
 	t.Run("update state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		advanced := cur.TryAdvance(content{
 			Type:       tn,
 			ID:         "1234",

diff --git a/x-pack/filebeat/module/o365/_meta/config.yml b/x-pack/filebeat/module/o365/_meta/config.yml
@@ -40,6 +40,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m
diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml
@@ -21,7 +21,7 @@ content_type:
 {{ if .api }}
 api:
 {{ range $k, $v := .api }}
- - {{ $k }}: {{ $v -}}
+  {{ $k }}: {{ $v -}}
 {{ end }}
 {{ end }}
 
@@ -49,6 +49,7 @@ processors:
         - 2006-01-02T15:04:05
 {{ end }}
   - script:
+      when.has_fields: ['o365audit']
       lang: javascript
       id: o365audit_script
       file: ${path.home}/module/o365/audit/config/pipeline.js

diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js
@@ -729,7 +729,10 @@ function AuditProcessor(tenant_names, debug) {
         tokenizer: '[%{_ip}]:%{port}',
         field: 'client.address',
         target_prefix: 'client',
-        'when.contains.client.address': ']:',
+        'when.and': [
+            {'not.has_fields': ['client._ip', 'client.port']},
+            {'contains.client.address': ']:'},
+        ],
     }));
     builder.Add("extractClientIPv4Port", new processor.Dissect({
         tokenizer: '%{_ip}:%{port}',

diff --git a/x-pack/filebeat/modules.d/o365.yml.disabled b/x-pack/filebeat/modules.d/o365.yml.disabled
@@ -43,6 +43,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m