[GCP] Enhance billing data stream

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 2.45.0
+  changes:
+    - description: Enhance GCP billing data stream with additional fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15828
 - version: 2.44.0
   changes:
     - description: Add support for parsing sensitive action notifications event in the audit dataset.

packages/gcp/data_stream/billing/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,2 @@
+dynamic_fields:
+  "event.ingested": ".*"

packages/gcp/data_stream/billing/_dev/test/pipeline/test-gcp-compute.json

@@ -0,0 +1,63 @@
+{
+    "events": [
+        {
+            "@timestamp": "2017-10-12T08:05:34.853Z",
+            "cloud": {
+                "account": {
+                    "id": "01475F-5B1080-1137E7"
+                },
+                "project": {
+                    "id": "elastic-bi",
+                    "name": "elastic-containerlib-prod"
+                },
+                "provider": "gcp"
+            },
+            "event": {
+                "dataset": "gcp.billing",
+                "duration": 115000,
+                "module": "gcp"
+            },
+            "gcp": {
+                "billing": {
+                    "billing_account_id": "01475F-5B1080-1137E7",
+                    "cost_type": "regular",
+                    "invoice_month": "202106",
+                    "project_id": "containerlib-prod-12763",
+                    "project_name": "elastic-containerlib-prod",
+                    "total": 4717.170681,
+                    "sku_id": "0D56-2F80-52A5",
+                    "service_id": "6F81-5844-456A",
+                    "sku_description": "Network Inter Region Ingress from Jakarta to Americas",
+                    "service_description": "Compute Engine",
+                    "effective_price": 0.00292353,
+                    "tags": [
+                        {
+                            "key": "stage",
+                            "value": "prod"
+                        },
+                        {
+                            "key": "size",
+                            "value": "standard"
+                        }
+                    ],
+                    "labels": {
+                        "test_label": "value"
+                    },
+                    "location": {
+                        "region": "us-central1",
+                        "country": "US"
+                    },
+                    "usage_start_time": "2023-10-22T22:00:00.000Z",
+                    "usage_end_time": "2023-10-22T23:00:00.000Z"
+                }
+            },
+            "metricset": {
+                "name": "billing",
+                "period": 10000
+            },
+            "service": {
+                "type": "gcp"
+            }
+        }
+    ]
+}

packages/gcp/data_stream/billing/_dev/test/pipeline/test-gcp-compute.json-expected.json

@@ -0,0 +1,65 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-10-22T23:00:00.000Z",
+            "cloud": {
+                "account": {
+                    "id": "01475F-5B1080-1137E7"
+                },
+                "project": {
+                    "id": "elastic-bi",
+                    "name": "elastic-containerlib-prod"
+                },
+                "provider": "gcp"
+            },
+            "event": {
+                "dataset": "gcp.billing",
+                "duration": 115000,
+                "end": "2023-10-22T23:00:00.000Z",
+                "module": "gcp",
+                "start": "2023-10-22T22:00:00.000Z"
+            },
+            "gcp": {
+                "billing": {
+                    "billing_account_id": "01475F-5B1080-1137E7",
+                    "cost_type": "regular",
+                    "effective_price": 0.00292353,
+                    "invoice_month": "202106",
+                    "labels": {
+                        "test_label": "value"
+                    },
+                    "location": {
+                        "country": "US",
+                        "region": "us-central1"
+                    },
+                    "project_id": "containerlib-prod-12763",
+                    "project_name": "elastic-containerlib-prod",
+                    "service_description": "Compute Engine",
+                    "service_id": "6F81-5844-456A",
+                    "sku_description": "Network Inter Region Ingress from Jakarta to Americas",
+                    "sku_id": "0D56-2F80-52A5",
+                    "tags": [
+                        {
+                            "key": "stage",
+                            "value": "prod"
+                        },
+                        {
+                            "key": "size",
+                            "value": "standard"
+                        }
+                    ],
+                    "total": 4717.170681,
+                    "usage_end_time": "2023-10-22T23:00:00.000Z",
+                    "usage_start_time": "2023-10-22T22:00:00.000Z"
+                }
+            },
+            "metricset": {
+                "name": "billing",
+                "period": 10000
+            },
+            "service": {
+                "type": "gcp"
+            }
+        }
+    ]
+}

packages/gcp/data_stream/billing/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,28 @@
+---
+description: Pipeline for parsing GCP Billing metrics.
+processors:
+  - date:
+      field: gcp.billing.usage_start_time
+      formats:
+        - ISO8601
+  - date:
+      field: gcp.billing.usage_end_time
+      formats:
+        - ISO8601
+  - set:
+      field: event.start
+      copy_from: "gcp.billing.usage_start_time"
+  - set:
+      field: event.end
+      copy_from: "gcp.billing.usage_end_time"
+on_failure:
+- set:
+    field: event.kind
+    value: pipeline_error
+- append:
+    field: tags
+    value: preserve_original_event
+    allow_duplicates: false
+- set:
+    field: error.message
+    value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'

packages/gcp/data_stream/billing/fields/fields.yml

@@ -42,3 +42,25 @@
       type: keyword
     - name: tags.value
       type: keyword
+    - name: usage_start_time
+      type: date
+      description: The start time of the usage period for this billing record.
+    - name: usage_end_time
+      type: date
+      description: The end time of the usage period for this billing record.
+    - name: location
+      type: group
+      fields:
+        - name: region
+          type: keyword
+          description: The geographic region where the resource was used (e.g., us-central1, europe-west1).
+        - name: zone
+          type: keyword
+          description: The specific zone within the region (e.g., us-central1-a).
+        - name: country
+          type: keyword
+          description: The country code for the resource location (e.g., US, GB).
+    - name: labels
+      type: object
+      object_type: keyword
+      description: Resource labels as key-value pairs. Labels are user-defined metadata that can be attached to GCP resources.

packages/gcp/data_stream/billing/sample_event.json

@@ -37,7 +37,16 @@
                     "key": "size",
                     "value": "standard"
                 }
-            ]
+            ],
+            "labels": {
+                "test_label": "value"
+            },
+            "location": {
+                "region": "us-central1",
+                "country": "US"
+            },
+            "usage_start_time": "2023-10-22T22:00:00.000Z",
+            "usage_end_time": "2023-10-22T23:00:00.000Z"
         }
     },
     "metricset": {

packages/gcp/docs/README.md

@@ -1187,6 +1187,10 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword |
 | gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float |
 | gcp.billing.invoice_month | Billing report month. | keyword |
+| gcp.billing.labels | Resource labels as key-value pairs. Labels are user-defined metadata that can be attached to GCP resources. | object |
+| gcp.billing.location.country | The country code for the resource location (e.g., US, GB). | keyword |
+| gcp.billing.location.region | The geographic region where the resource was used (e.g., us-central1, europe-west1). | keyword |
+| gcp.billing.location.zone | The specific zone within the region (e.g., us-central1-a). | keyword |
 | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword |
 | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword |
 | gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword |
@@ -1197,6 +1201,8 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.billing.tags.key |  | keyword |
 | gcp.billing.tags.value |  | keyword |
 | gcp.billing.total | Total billing amount. | float |
+| gcp.billing.usage_end_time | The end time of the usage period for this billing record. | date |
+| gcp.billing.usage_start_time | The start time of the usage period for this billing record. | date |
 | host.containerized | If the host is a container. | boolean |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
@@ -1244,7 +1250,16 @@ An example event for `billing` looks as following:
                     "key": "size",
                     "value": "standard"
                 }
-            ]
+            ],
+            "labels": {
+                "test_label": "value"
+            },
+            "location": {
+                "region": "us-central1",
+                "country": "US"
+            },
+            "usage_start_time": "2023-10-22T22:00:00.000Z",
+            "usage_end_time": "2023-10-22T23:00:00.000Z"
         }
     },
     "metricset": {

packages/gcp/docs/billing.md

@@ -107,7 +107,16 @@ An example event for `billing` looks as following:
                     "key": "size",
                     "value": "standard"
                 }
-            ]
+            ],
+            "labels": {
+                "test_label": "value"
+            },
+            "location": {
+                "region": "us-central1",
+                "country": "US"
+            },
+            "usage_start_time": "2023-10-22T22:00:00.000Z",
+            "usage_end_time": "2023-10-22T23:00:00.000Z"
         }
     },
     "metricset": {
@@ -141,6 +150,10 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword |
 | gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float |
 | gcp.billing.invoice_month | Billing report month. | keyword |
+| gcp.billing.labels | Resource labels as key-value pairs. Labels are user-defined metadata that can be attached to GCP resources. | object |
+| gcp.billing.location.country | The country code for the resource location (e.g., US, GB). | keyword |
+| gcp.billing.location.region | The geographic region where the resource was used (e.g., us-central1, europe-west1). | keyword |
+| gcp.billing.location.zone | The specific zone within the region (e.g., us-central1-a). | keyword |
 | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword |
 | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword |
 | gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword |
@@ -151,6 +164,8 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.billing.tags.key |  | keyword |
 | gcp.billing.tags.value |  | keyword |
 | gcp.billing.total | Total billing amount. | float |
+| gcp.billing.usage_end_time | The end time of the usage period for this billing record. | date |
+| gcp.billing.usage_start_time | The start time of the usage period for this billing record. | date |
 | host.containerized | If the host is a container. | boolean |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |

packages/gcp/manifest.yml

@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.44.0"
+version: "2.45.0"
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration
 icons:
@@ -15,7 +15,7 @@ categories:
   - observability
 conditions:
   kibana:
-    version: "^8.13.0 || ^9.0.0"
+    version: "^9.1.0"
 screenshots:
   - src: /img/filebeat-gcp-audit.png
     title: filebeat gcp audit