Added attributes types and new attribute status_reasons

dftimewolf/lib/processors/gcp_logging_timesketch.py

@@ -1,5 +1,42 @@
 # -*- coding: utf-8 -*-
-"""Processes Google Cloud Platform (GCP) logs for loading into Timesketch."""
+"""Processes Google Cloud Platform (GCP) logs for loading into Timesketch.
+
+The following attributes are extracted by the processor:
+  data_type: Timesketch data type i.e. gcp:log:json.
+  datetime: event date time.
+  dcsa_emails: default compute service account, a service account attached when
+      a Compute Engine instance is created.
+  dcsa_scopes: OAuth scopes granted to the default service account attached to a
+      Compute Engine instance.
+  delegation_chain: service account impersonation/delegation chain.
+  event_subtype: event subtype.
+  gcloud_command_id: unique gcloud command execution ID.
+  gcloud_command_partial: partial gcloud command related to the operation.
+  message: summary message of the operation.
+  method_name: operation performed.
+  permissions: IAM permissions used for the operation.
+  policy_delta: IAM policy delta.
+  principal_email: email address of the requester.
+  principal_subject: subject of the requester.
+  query: Google Cloud log filtering query.
+  resource_label_instance_id: Compute Engine instance ID.
+  resource_name: resource name.
+  service_account_delegation: service accounts delegation in
+      authentication.
+  service_account_display_name: display name of the service account.
+  service_account_key_name: service account key name used in
+      authentication.
+  service_name: name of the service.
+  severity: log entry severity.
+  source_images: source images of disks attached to a Compute Engine instance.
+  source_ranges: firewall source ranges.
+  status_code: operation success or failure code.
+  status_message: operation success or failure message.
+  status_reason: operation failure reasons.
+  textPayload: text payload for logs not using a JSON or proto payload.
+  timestamp_desc: description of timestamp.
+  user: user or requestor.
+"""
 
 import json
 import re
@@ -225,6 +262,20 @@ def _ParseProtoPayloadStatus(
     timesketch_record['status_code'] = status_code
     timesketch_record['status_message'] = status_message
 
+    # `protoPayload.status` struction may contain `details` attribute when
+    # opertion fails. The reason attribute contains the reason the operation
+    # failed.
+    status_reasons = []
+
+    status_details = status.get('details', [])
+    for status_detail in status_details:
+      reason = status_detail.get('reason')
+      if reason:
+        status_reasons.append(reason)
+
+    if status_reasons:
+      timesketch_record['status_reason'] = ', '.join(status_reasons)
+
   def _ParseServiceData(
       self,
       proto_payload: Dict[str, Any],
@@ -332,7 +383,7 @@ def _ParseComputeInstancesInsert(
         dcsa_scopes.extend(scopes)
 
     if dcsa_emails:
-      timesketch_record['dcsa_email'] = ','.join(dcsa_emails)
+      timesketch_record['dcsa_emails'] = dcsa_emails
 
     if dcsa_scopes:
       timesketch_record['dcsa_scopes'] = dcsa_scopes

tests/lib/processors/gcp_logging_timesketch.py

@@ -347,7 +347,7 @@ def testGCECreateLog(self):
             '2019-06-06T09:29:04.499000Z',
         'timestamp_desc':
             'Event Recorded',
-        'dcsa_email': 'my-sa2@ketchup-research.iam.gserviceaccount.com',
+        'dcsa_emails': ['my-sa2@ketchup-research.iam.gserviceaccount.com'],
         'dcsa_scopes': [
           'https://www.googleapis.com/auth/devstorage.full_control',
           'https://www.googleapis.com/auth/logging.read',
@@ -703,6 +703,137 @@ def testComputeInstancesInsert(self):
     actual_timesketch_record = json.loads(actual_timesketch_record)
     self.assertDictEqual(expected_timesketch_record, actual_timesketch_record)
 
+  def testServiceAccountCreateFailed(self):
+    """Test the failed service account create logs with reasons."""
+    test_state = state.DFTimewolfState(config.Config)
+    processor = gcp_logging_timesketch.GCPLoggingTimesketch(test_state)
+
+    service_create_log = {
+      'protoPayload': {
+        '@type': 'type.googleapis.com/google.cloud.audit.AuditLog',
+        'status': {
+          'code': 7,
+          'message': ('Permission \'iam.serviceAccounts.create\' denied on '
+              'resource (or it may not exist).'),
+          'details': [
+            {
+              '@type': 'type.googleapis.com/google.rpc.ErrorInfo',
+              'reason': 'IAM_PERMISSION_DENIED',
+              'domain': 'iam.googleapis.com',
+              'metadata': {
+                'permission': 'iam.serviceAccounts.create'
+              }
+            }
+          ]
+        },
+        'authenticationInfo': {
+          'principalEmail': ('dvwa-service-account@ketchup'
+              '.iam.gserviceaccount.com'),
+          'serviceAccountDelegationInfo': [
+            {
+              'firstPartyPrincipal': {
+                'principalEmail': ('service-1234567890@compute-system.iam.'
+                    'gserviceaccount.com')
+              }
+            }
+          ],
+          'principalSubject': ('serviceAccount:dvwa-service-account@'
+              'ketchup.iam.gserviceaccount.com')
+        },
+        'requestMetadata': {
+          'callerIp': '34.72.217.225',
+          'callerSuppliedUserAgent': '(gzip),gzip(gfe)',
+          'requestAttributes': {
+            'time': '2024-12-03T17:58:45.019694350Z',
+            'auth': {}
+          },
+          'destinationAttributes': {}
+        },
+        'serviceName': 'iam.googleapis.com',
+        'methodName': 'google.iam.admin.v1.CreateServiceAccount',
+        'authorizationInfo': [
+          {
+            'resource': 'projects/ketchup',
+            'permission': 'iam.serviceAccounts.create',
+            'resourceAttributes': {
+              'type': 'iam.googleapis.com/ServiceAccount'
+            },
+            'permissionType': 'ADMIN_WRITE'
+          }
+        ],
+        'resourceName': 'projects/ketchup',
+        'request': {
+          'service_account': {
+            'display_name': 'This is the attacker account'
+          },
+          'account_id': 'theattacker',
+          'name': 'projects/ketchup',
+          '@type': ('type.googleapis.com/google.iam.admin.v1.'
+              'CreateServiceAccountRequest')
+        },
+        'response': {
+          '@type': 'type.googleapis.com/google.iam.admin.v1.ServiceAccount'
+        }
+      },
+      'insertId': '1awjxggeaxqgz',
+      'resource': {
+        'type': 'service_account',
+        'labels': {
+          'unique_id': '',
+          'project_id': 'ketchup',
+          'email_id': ''
+        }
+      },
+      'timestamp': '2024-12-03T17:58:44.882119699Z',
+      'severity': 'ERROR',
+      'logName': ('projects/ketchup/logs/cloudaudit.'
+          'googleapis.com%2Factivity'),
+      'receiveTimestamp': '2024-12-03T17:58:45.716564605Z'
+    }
+
+    expected_timesketch_record = {
+      'query': 'test_query',
+      'data_type': 'gcp:log:json',
+      'datetime': '2024-12-03T17:58:44.882119699Z',
+      'timestamp_desc': 'Event Recorded',
+      'caller_ip': '34.72.217.225',
+      'delegation_chain': ('service-1234567890@compute-system.iam.'
+          'gserviceaccount.com'),
+      'email_id': '',
+      'message': ('User dvwa-service-account@ketchup.iam.'
+          'gserviceaccount.com performed google.iam.admin.v1.'
+          'CreateServiceAccount on projects/ketchup'),
+      'method_name': 'google.iam.admin.v1.CreateServiceAccount',
+     'permissions': ['iam.serviceAccounts.create'],
+      'principal_email': ('dvwa-service-account@ketchup.'
+          'iam.gserviceaccount.com'),
+      'principal_subject': ('serviceAccount:dvwa-service-account@'
+          'ketchup.iam.gserviceaccount.com'),
+      'project_id': 'ketchup',
+      'request_account_id': 'theattacker',
+      'request_name': 'projects/ketchup',
+      'resource_name': 'projects/ketchup',
+      'service_account_delegation': [
+          'service-1234567890@compute-system.iam.gserviceaccount.com'],
+      'service_account_display_name': 'This is the attacker account',
+      'service_name': 'iam.googleapis.com',
+      'severity': 'ERROR',
+      'status_code': '7',
+      'status_message': ('Permission \'iam.serviceAccounts.create\' denied on'
+          ' resource (or it may not exist).'),
+      'status_reason': 'IAM_PERMISSION_DENIED',
+      'unique_id': '',
+      'user_agent': '(gzip),gzip(gfe)'
+    }
+
+    failed_service_account_create_log = json.dumps(service_create_log)
+
+    # pylint: disable=protected-access
+    actual_timesketch_record = processor._ProcessLogLine(
+        failed_service_account_create_log, 'test_query')
+
+    actual_timesketch_record = json.loads(actual_timesketch_record)
+    self.assertDictEqual(expected_timesketch_record, actual_timesketch_record)
 
 if __name__ == '__main__':
   unittest.main()