[gcp] Add policy_violation_info metadata and related fields to GCP audit logs (#10886)

haetamoudi · web-flow · commit 813097607458 · 2024-09-16T10:39:40.000+02:00
* add missing fields gcp audit logs

* update changelog entry
diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.38.0"
+  changes:
+    - description: Add `policy_violation_info`, `metadata` and `related` fields to audit logs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10886
+    - description: Update GCP audit log dashboard to use correct `email` field.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10886
 - version: "2.37.2"
   changes:
     - description: Fix definition of subfields of nested objects
diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log
diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json
diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json
@@ -39,6 +39,11 @@
                 "level": "INFO",
                 "logger": "organizations/123456789098/logs/cloudaudit.googleapis.com%2Fdata_access"
             },
+            "related": {
+                "user": [
+                    "joel.miller@contoso.com"
+                ]
+            },
             "service": {
                 "name": "cloudresourcemanager.googleapis.com"
             },
diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -178,6 +178,7 @@ processors:
       field: gcp.audit.labels
       copy_from: json.labels
       if: ctx.json?.labels != null
+
 ##
 # RequestMetadata
 # .protoPayload.requestMetadata
@@ -196,6 +197,48 @@ processors:
   - user_agent:
       field: user_agent.original
       ignore_missing: true
+
+##
+# Metadata
+# .protoPayload.metadata
+# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog
+##
+  - set:
+      field: gcp.audit.metadata
+      copy_from: json.protoPayload.metadata
+      if: ctx.json?.protoPayload?.metadata != null
+
+##
+# PolicyViolationInfo
+# .protoPayload.orgPolicyViolationInfo
+# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#policyviolationinfo
+##
+  - set:
+      field: gcp.audit.policy_violation_info.violations
+      copy_from: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.violationInfo
+      ignore_failure: true
+  - foreach:
+      field: gcp.audit.policy_violation_info.violations
+      ignore_missing: true
+      ignore_failure: true
+      processor:
+        rename:
+          field: _ingest._value.resourceAttributes
+          target_field: _ingest._value.resource_attributes
+      if: ctx.gcp?.audit?.policy_violation_info instanceof List
+  - rename:
+      field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.payload
+      target_field: gcp.audit.policy_violation_info.payload
+      ignore_failure: true
+  - rename:
+      field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.resourceType
+      target_field: gcp.audit.policy_violation_info.resource_type
+      ignore_failure: true
+  - rename:
+      field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.resourceTags
+      target_field: gcp.audit.policy_violation_info.resource_tags
+      ignore_failure: true
+
 ##
 # LogEntryOperation
 # .operation
@@ -300,6 +343,10 @@ processors:
       field: gcp.audit.status.message
       copy_from: json.protoPayload.status.message
       ignore_failure: true
+  - set:
+      field: gcp.audit.status.details
+      copy_from: json.protoPayload.status.details
+      ignore_empty_value: true
   - set:
       field: event.outcome
       value: success
@@ -365,6 +412,16 @@ processors:
       target_field: source.as.organization.name
       ignore_missing: true
 
+# Related fields
+  - append:
+      field: related.ip
+      value: "{{{source.ip}}}"
+      allow_duplicates: false
+  - append:
+      field: related.user
+      value: "{{{client.user.email}}}"
+      if: ctx.client?.user?.email != null
+      allow_duplicates: false
 ##
 # clean-up
 ##
diff --git a/packages/gcp/data_stream/audit/fields/fields.yml b/packages/gcp/data_stream/audit/fields/fields.yml
@@ -72,6 +72,10 @@
         - name: last
           type: boolean
           description: "Optional. Set this to True if this is the last log entry in the operation."
+    - name: metadata
+      type: flattened
+      description: |
+        Service-specific data about the request, response, and other information associated with the current audited event.
     - name: method_name
       type: keyword
       description: |
@@ -80,6 +84,35 @@
       type: long
       description: |
         The number of items returned from a List or Query API method, if applicable.
+    - name: policy_violation_info
+      type: group
+      fields:
+        - name: payload
+          type: flattened
+          description: Resource payload that is currently in scope and is subjected to orgpolicy conditions.
+        - name: resource_tags
+          type: flattened
+          description: Tags referenced on the resource at the time of evaluation.
+        - name: resource_type
+          type: keyword
+          description: Resource type that the orgpolicy is checked against.
+        - name: violations
+          type: nested
+          description: |
+            Provides information about the Policy violation info for the request.
+          fields:
+            - name: constraint
+              type: keyword
+              description: "Constraint name."
+            - name: errorMessage
+              type: keyword
+              description: "Error message that policy is indicating."
+            - name: checkedValue
+              type: keyword
+              description: "Value that is being checked for the policy."
+            - name: policyType
+              type: keyword
+              description: "Indicates the type of the policy."
     - name: request
       type: flattened
     - name: request_metadata
@@ -121,6 +154,10 @@
         - name: message
           type: keyword
           description: "A developer-facing error message, which should be in English. Any user-facing  error message should be localized and sent in the google.rpc.Status.details  field, or localized by the client."
+        - name: details
+          type: flattened
+          description: |
+            A list of messages that carry the error details.
     - name: flattened
       type: flattened
       description: Contains the full audit document as sent by GCP.
diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md
@@ -249,8 +249,16 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.logentry_operation.id | Optional. An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation. | keyword |
 | gcp.audit.logentry_operation.last | Optional. Set this to True if this is the last log entry in the operation. | boolean |
 | gcp.audit.logentry_operation.producer | Optional. An arbitrary producer identifier. The combination of id and producer must be globally unique. | keyword |
+| gcp.audit.metadata | Service-specific data about the request, response, and other information associated with the current audited event. | flattened |
 | gcp.audit.method_name | The name of the service method or operation. For API calls, this  should be the name of the API method.  For example, 'google.datastore.v1.Datastore.RunQuery'. | keyword |
 | gcp.audit.num_response_items | The number of items returned from a List or Query API method, if applicable. | long |
+| gcp.audit.policy_violation_info.payload | Resource payload that is currently in scope and is subjected to orgpolicy conditions. | flattened |
+| gcp.audit.policy_violation_info.resource_tags | Tags referenced on the resource at the time of evaluation. | flattened |
+| gcp.audit.policy_violation_info.resource_type | Resource type that the orgpolicy is checked against. | keyword |
+| gcp.audit.policy_violation_info.violations.checkedValue | Value that is being checked for the policy. | keyword |
+| gcp.audit.policy_violation_info.violations.constraint | Constraint name. | keyword |
+| gcp.audit.policy_violation_info.violations.errorMessage | Error message that policy is indicating. | keyword |
+| gcp.audit.policy_violation_info.violations.policyType | Indicates the type of the policy. | keyword |
 | gcp.audit.request |  | flattened |
 | gcp.audit.request_metadata.caller_ip | The IP address of the caller. | ip |
 | gcp.audit.request_metadata.caller_supplied_user_agent | The user agent of the caller. This information is not authenticated and  should be treated accordingly. | keyword |
@@ -260,6 +268,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.response |  | flattened |
 | gcp.audit.service_name | The name of the API service performing the operation.  For example, datastore.googleapis.com. | keyword |
 | gcp.audit.status.code | The status code, which should be an enum value of google.rpc.Code. | integer |
+| gcp.audit.status.details | A list of messages that carry the error details. | flattened |
 | gcp.audit.status.message | A developer-facing error message, which should be in English. Any user-facing  error message should be localized and sent in the google.rpc.Status.details  field, or localized by the client. | keyword |
 | gcp.audit.type | Type property. | keyword |
 | gcp.destination.instance.project_id | ID of the project containing the VM. | keyword |
diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md
@@ -38,8 +38,16 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.logentry_operation.id | Optional. An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation. | keyword |
 | gcp.audit.logentry_operation.last | Optional. Set this to True if this is the last log entry in the operation. | boolean |
 | gcp.audit.logentry_operation.producer | Optional. An arbitrary producer identifier. The combination of id and producer must be globally unique. | keyword |
+| gcp.audit.metadata | Service-specific data about the request, response, and other information associated with the current audited event. | flattened |
 | gcp.audit.method_name | The name of the service method or operation. For API calls, this  should be the name of the API method.  For example, 'google.datastore.v1.Datastore.RunQuery'. | keyword |
 | gcp.audit.num_response_items | The number of items returned from a List or Query API method, if applicable. | long |
+| gcp.audit.policy_violation_info.payload | Resource payload that is currently in scope and is subjected to orgpolicy conditions. | flattened |
+| gcp.audit.policy_violation_info.resource_tags | Tags referenced on the resource at the time of evaluation. | flattened |
+| gcp.audit.policy_violation_info.resource_type | Resource type that the orgpolicy is checked against. | keyword |
+| gcp.audit.policy_violation_info.violations.checkedValue | Value that is being checked for the policy. | keyword |
+| gcp.audit.policy_violation_info.violations.constraint | Constraint name. | keyword |
+| gcp.audit.policy_violation_info.violations.errorMessage | Error message that policy is indicating. | keyword |
+| gcp.audit.policy_violation_info.violations.policyType | Indicates the type of the policy. | keyword |
 | gcp.audit.request |  | flattened |
 | gcp.audit.request_metadata.caller_ip | The IP address of the caller. | ip |
 | gcp.audit.request_metadata.caller_supplied_user_agent | The user agent of the caller. This information is not authenticated and  should be treated accordingly. | keyword |
@@ -49,6 +57,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.response |  | flattened |
 | gcp.audit.service_name | The name of the API service performing the operation.  For example, datastore.googleapis.com. | keyword |
 | gcp.audit.status.code | The status code, which should be an enum value of google.rpc.Code. | integer |
+| gcp.audit.status.details | A list of messages that carry the error details. | flattened |
 | gcp.audit.status.message | A developer-facing error message, which should be in English. Any user-facing  error message should be localized and sent in the google.rpc.Status.details  field, or localized by the client. | keyword |
 | gcp.audit.type | Type property. | keyword |
 | gcp.destination.instance.project_id | ID of the project containing the VM. | keyword |
diff --git a/packages/gcp/kibana/search/gcp-d88364c0-73a1-11ea-a345-f985c61fe654.json b/packages/gcp/kibana/search/gcp-d88364c0-73a1-11ea-a345-f985c61fe654.json
@@ -2,7 +2,7 @@
     "attributes": {
         "columns": [
             "event.action",
-            "user.email",
+            "client.user.email",
             "service.name",
             "gcp.audit.type",
             "event.outcome",
diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml
@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.37.2"
+version: "2.38.0"
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration
 icons:
