o365: add fallback for missing startTime (#8374)

Improve clarity of CEL code:
* make evaluation step clearly defined
* remove redundant as macro use
* use string for RFC 3339 timestamp formatting
* use more vertical whitespace

Reduce repeated work:
* hoist request query parsing out of struct

Add fallback for case where request URL in response has no startTime.

Add mappings for missing fields:
* o365.audit.AdditionalInfo
* o365.audit.AppAccessContext

Author: efd6

packages/o365/_dev/deploy/docker/config.yml

@@ -100,7 +100,7 @@ rules:
         body: |-
           [
             {"OriginatingServer": "HE1PR0102MB3228 (15.20.207.17)", "ClientAppId": "", "OrganizationName": "testsiem.onmicrosoft.com", "RecordType": 1, "ObjectId": "testsiem.onmicrosoft.com\\Transport Settings", "Workload": "Exchange", "OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", "UserId": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "CreationTime": "2020-02-10T07:37:13", "Parameters": [{"Name": "DomainController", "Value": ""}, {"Name": "Identity", "Value": "testsiem.onmicrosoft.com"}, {"Name": "OrganizationFederatedMailbox", "Value": "FederatedEmail.4c1f4d8b-8179-4148-93bf-00a95fa1e042@testsiem.onmicrosoft.com"}], "UserType": 3, "Version": 1, "ResultStatus": "True", "AppId": "", "ExternalAccess": true, "UserKey": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "Operation": "Set-TransportConfig", "Id": "ea769bfc-fa67-465c-767a-08d7adfc0b7b"},
-            {"OrganizationName":"testsiem.onmicrosoft.com","UserKey":"S-1-5-18","MailboxGuid":"26286ffa-073d-45ff-9fe9-539891984d69","Operation":"ModifyFolderPermissions","OrganizationId":"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd","ClientIPAddress":"::1","Item":{"Id":"LgAAAACklF6sEsJgSK/ulVd531/WAQCzgXIUnq3lQqXFeCmxHwmHAAAAAAENAAAC","ParentFolder":{"Path":"\\Calendar","MemberRights":"ReadAny, Create, EditOwned, DeleteOwned, EditAny, DeleteAny, Visible, FreeBusySimple, FreeBusyDetailed","MemberSid":"S-1-8-640184314-1174341437-2555636127-1766693009-0","Id":"LgAAAACklF6sEsJgSK/ulVd531/WAQCzgXIUnq3lQqXFeCmxHwmHAAAAAAENAAAC","MemberUpn":"Owner@local","Name":"Calendar"}},"LogonUserSid":"S-1-5-18","OriginatingServer":"AM6PR01MB4535 (15.20.229.32)\n","RecordType":2,"Version":1,"ClientInfoString":"Client=WebServices;Action=ConfigureGroupMailbox","MailboxOwnerUPN":"SIEMTest@testsiem.onmicrosoft.com","MailboxOwnerMasterAccountSid":"S-1-5-10","MailboxOwnerSid":"S-1-5-21-3422892061-1135328251-2670905592-26680073","ResultStatus":"Succeeded","ExternalAccess":true,"LogonType":1,"ClientIP":"::1","Workload":"Exchange","InternalLogonType":1,"UserId":"S-1-5-18","CreationTime":"2020-02-17T17:12:03","Id":"284dfe85-ab53-48ad-0863-08d7b3cc81f7","UserType":2}
+            {"OrganizationName":"testsiem.onmicrosoft.com","UserKey":"S-1-5-18","MailboxGuid":"26286ffa-073d-45ff-9fe9-539891984d69","Operation":"ModifyFolderPermissions","OrganizationId":"b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd","ClientIPAddress":"::1","Item":{"Id":"LgAAAACklF6sEsJgSK/ulVd531/WAQCzgXIUnq3lQqXFeCmxHwmHAAAAAAENAAAC","ParentFolder":{"Path":"\\Calendar","MemberRights":"ReadAny, Create, EditOwned, DeleteOwned, EditAny, DeleteAny, Visible, FreeBusySimple, FreeBusyDetailed","MemberSid":"S-1-8-640184314-1174341437-2555636127-1766693009-0","Id":"LgAAAACklF6sEsJgSK/ulVd531/WAQCzgXIUnq3lQqXFeCmxHwmHAAAAAAENAAAC","MemberUpn":"Owner@local","Name":"Calendar"}},"AdditionalInfo":{"EnvironmentName":"Default-ae81357b-84e6-4e54-b02c-db7dddd1e869"},"AppAccessContext":{"ClientAppId":"89118fb4-83bf-46d9-bb84-78cca4c122d5"},"LogonUserSid":"S-1-5-18","OriginatingServer":"AM6PR01MB4535 (15.20.229.32)\n","RecordType":2,"Version":1,"ClientInfoString":"Client=WebServices;Action=ConfigureGroupMailbox","MailboxOwnerUPN":"SIEMTest@testsiem.onmicrosoft.com","MailboxOwnerMasterAccountSid":"S-1-5-10","MailboxOwnerSid":"S-1-5-21-3422892061-1135328251-2670905592-26680073","ResultStatus":"Succeeded","ExternalAccess":true,"LogonType":1,"ClientIP":"::1","Workload":"Exchange","InternalLogonType":1,"UserId":"S-1-5-18","CreationTime":"2020-02-17T17:12:03","Id":"284dfe85-ab53-48ad-0863-08d7b3cc81f7","UserType":2}
           ]
   - path: /api/v1.0/tenant-id/activity/feed/subscriptions/content
     methods: [GET]

packages/o365/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.25.1"
+  changes:
+    - description: Add start time fallback for responses that do not include the `startTime` query.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8374
+    - description: Add mappings for `o365.audit.AdditionalInfo.*` and `o365.audit.AppAccessContext.*`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8374
 - version: "1.25.0"
   changes:
     - description: Improve 'event.original' check to avoid errors if set.

packages/o365/data_stream/audit/_dev/test/system/test-cel-config.yml

@@ -12,3 +12,5 @@ data_stream:
     azure_tenant_id: test-cel-tenant-id
     content_types: "Audit.SharePoint, Audit.General"
     initial_interval: 12h
+assert:
+  hit_count: 6

packages/o365/data_stream/audit/_dev/test/system/test-o365audit-config.yml

@@ -14,3 +14,5 @@ data_stream:
     api: |
       authentication_endpoint: http://{{Hostname}}:{{Port}}/
       resource: http://{{Hostname}}:{{Port}}
+assert:
+  hit_count: 27