x-pack/metricbeat/module/gcp: Add omitted labels #40062

Linu-Elias · 2024-07-01T06:48:26Z

Proposed commit message

When parsing time series objects, we're omitting key label instance_id which was correct for services that has its own metadata logic, but we were missing crucial labels for services without additional metadata logic. It is now fixed with an explicit condition for instance_id.

Checklist

My code follows the style guidelines of this project
I have commented my code, particularly in hard-to-understand areas
I have made corresponding changes to the documentation
I have made corresponding change to the default configuration files
I have added tests that prove my fix is effective or that my feature works
I have added an entry in CHANGELOG.next.asciidoc or CHANGELOG-developer.next.asciidoc.

Closes GCP labels are being omitted #40033

Screenshots

Compute Service Document:

Spanner Service Before Change:

Spanner Service After Change:

Before Change

{
  "_index": ".ds-metricbeat-8.15.0-2024.07.03-000001",
  "_id": "ymisd5AB56NXeJqdMPru",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2024-07-03T08:13:00.000Z",
    "agent": {
      "name": "EPINPUNW06A8",
      "type": "metricbeat",
      "version": "8.15.0",
      "ephemeral_id": "2e83ec6e-b29b-407b-99fa-e8383f908e74",
      "id": "f49fbc8d-cc6a-4629-89b1-454f33f48d82"
    },
    "metricset": {
      "name": "metrics",
      "period": 60000
    },
    "gcp": {
      "labels": {},
      "metrics": {
        "instance": {
          "uptime": {
            "sec": 60
          },
          "cpu": {
            "usage": {
              "pct": 0.006929705028543509
            },
            "reserved_cores": {
              "value": 0.2
            },
            "usage_time": {
              "sec": 0.08315646034252211
            }
          }
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "cloud": {
      "availability_zone": "us-central1-a",
      "region": "us-central1",
      "instance": {
        "name": "demo-crest-group-j76r",
        "id": "2314488029850962513"
      },
      "machine": {
        "type": "f1-micro"
      },
      "account": {
        "id": "elastic-obs-integrations-dev",
        "name": "elastic-obs-integrations-dev"
      },
      "provider": "gcp"
    },
    "host": {
      "hostname": "epinpunw06a8",
      "architecture": "x86_64",
      "os": {
        "platform": "ubuntu",
        "version": "22.04.3 LTS (Jammy Jellyfish)",
        "family": "debian",
        "name": "Ubuntu",
        "kernel": "5.15.146.1-microsoft-standard-WSL2",
        "codename": "jammy",
        "type": "linux"
      },
      "containerized": false,
      "ip": [
        "172.29.134.71",
        "fe80::215:5dff:fe11:8275"
      ],
      "name": "epinpunw06a8",
      "id": "2fef4836d29e4bf6b57df5049e8f58e4",
      "mac": [
        "00-15-5D-11-82-75"
      ]
    },
    "event": {
      "dataset": "gcp.metrics",
      "module": "gcp",
      "duration": 2541915617
    },
    "ecs": {
      "version": "8.0.0"
    }
  },
  "fields": {
    "beats_state.state.host.hostname": [
      "epinpunw06a8"
    ],
    "host.os.name.text": [
      "Ubuntu"
    ],
    "host.hostname": [
      "epinpunw06a8"
    ],
    "host.mac": [
      "00-15-5D-11-82-75"
    ],
    "cloud.availability_zone": [
      "us-central1-a"
    ],
    "service.type": [
      "gcp"
    ],
    "cloud.instance.id": [
      "2314488029850962513"
    ],
    "host.ip": [
      "172.29.134.71",
      "fe80::215:5dff:fe11:8275"
    ],
    "agent.type": [
      "metricbeat"
    ],
    "gcp.metrics.instance.cpu.reserved_cores.value": [
      0.2
    ],
    "logstash_stats.timestamp": [
      "2024-07-03T08:13:00.000Z"
    ],
    "event.module": [
      "gcp"
    ],
    "host.os.version": [
      "22.04.3 LTS (Jammy Jellyfish)"
    ],
    "host.os.kernel": [
      "5.15.146.1-microsoft-standard-WSL2"
    ],
    "host.os.name": [
      "Ubuntu"
    ],
    "agent.name": [
      "EPINPUNW06A8"
    ],
    "host.name": [
      "epinpunw06a8"
    ],
    "beats_state.timestamp": [
      "2024-07-03T08:13:00.000Z"
    ],
    "host.id": [
      "2fef4836d29e4bf6b57df5049e8f58e4"
    ],
    "beats_state.state.host.name": [
      "epinpunw06a8"
    ],
    "timestamp": [
      "2024-07-03T08:13:00.000Z"
    ],
    "cloud.account.name": [
      "elastic-obs-integrations-dev"
    ],
    "cloud.region": [
      "us-central1"
    ],
    "host.os.type": [
      "linux"
    ],
    "kibana_stats.timestamp": [
      "2024-07-03T08:13:00.000Z"
    ],
    "metricset.period": [
      60000
    ],
    "host.os.codename": [
      "jammy"
    ],
    "gcp.metrics.instance.cpu.usage.pct": [
      0.006929705028543509
    ],
    "agent.hostname": [
      "EPINPUNW06A8"
    ],
    "gcp.metrics.instance.cpu.usage_time.sec": [
      0.08315646034252211
    ],
    "event.duration": [
      2541915617
    ],
    "host.architecture": [
      "x86_64"
    ],
    "metricset.name": [
      "metrics"
    ],
    "cloud.machine.type": [
      "f1-micro"
    ],
    "cloud.provider": [
      "gcp"
    ],
    "@timestamp": [
      "2024-07-03T08:13:00.000Z"
    ],
    "agent.id": [
      "f49fbc8d-cc6a-4629-89b1-454f33f48d82"
    ],
    "cloud.account.id": [
      "elastic-obs-integrations-dev"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.containerized": [
      false
    ],
    "host.os.platform": [
      "ubuntu"
    ],
    "agent.ephemeral_id": [
      "2e83ec6e-b29b-407b-99fa-e8383f908e74"
    ],
    "agent.version": [
      "8.15.0"
    ],
    "beats_state.state.host.architecture": [
      "x86_64"
    ],
    "host.os.family": [
      "debian"
    ],
    "gcp.metrics.instance.uptime.sec": [
      60
    ],
    "event.dataset": [
      "gcp.metrics"
    ],
    "cloud.instance.name": [
      "demo-crest-group-j76r"
    ]
  }
}

After Change:

{
  "_index": ".ds-metricbeat-8.15.0-2024.07.03-000001",
  "_id": "0c4gd5ABrcpmtJO-0-eX",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2024-07-03T05:41:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "metricset": {
      "name": "metrics",
      "period": 60000
    },
    "gcp": {
      "metrics": {
        "instance": {
          "cpu": {
            "usage": {
              "pct": 0.006597337449646782
            },
            "reserved_cores": {
              "value": 0.2
            },
            "usage_time": {
              "sec": 0.07916804939576139
            }
          },
          "uptime": {
            "sec": 60
          }
        }
      },
      "labels": {}
    },
    "service": {
      "type": "gcp"
    },
    "cloud": {
      "region": "us-central1",
      "instance": {
        "id": "2314488029850962513",
        "name": "demo-crest-group-j76r"
      },
      "machine": {
        "type": "f1-micro"
      },
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      },
      "provider": "gcp",
      "availability_zone": "us-central1-a"
    },
    "host": {
      "name": "epinpunw06a8",
      "architecture": "x86_64",
      "os": {
        "codename": "jammy",
        "type": "linux",
        "platform": "ubuntu",
        "version": "22.04.3 LTS (Jammy Jellyfish)",
        "family": "debian",
        "name": "Ubuntu",
        "kernel": "5.15.146.1-microsoft-standard-WSL2"
      },
      "containerized": false,
      "ip": [
        "172.29.134.71",
        "fe80::215:5dff:fe10:5cff"
      ],
      "mac": [
        "00-15-5D-10-5C-FF"
      ],
      "hostname": "epinpunw06a8",
      "id": "2fef4836d29e4bf6b57df5049e8f58e4"
    },
    "event": {
      "dataset": "gcp.metrics",
      "module": "gcp",
      "duration": 2606976189
    },
    "agent": {
      "ephemeral_id": "145cb68a-7481-4a5e-887b-41fe3f0af245",
      "id": "f49fbc8d-cc6a-4629-89b1-454f33f48d82",
      "name": "EPINPUNW06A8",
      "type": "metricbeat",
      "version": "8.15.0"
    }
  },
  "fields": {
    "beats_state.state.host.hostname": [
      "epinpunw06a8"
    ],
    "host.os.name.text": [
      "Ubuntu"
    ],
    "host.hostname": [
      "epinpunw06a8"
    ],
    "host.mac": [
      "00-15-5D-10-5C-FF"
    ],
    "cloud.availability_zone": [
      "us-central1-a"
    ],
    "service.type": [
      "gcp"
    ],
    "cloud.instance.id": [
      "2314488029850962513"
    ],
    "host.ip": [
      "172.29.134.71",
      "fe80::215:5dff:fe10:5cff"
    ],
    "agent.type": [
      "metricbeat"
    ],
    "gcp.metrics.instance.cpu.reserved_cores.value": [
      0.2
    ],
    "logstash_stats.timestamp": [
      "2024-07-03T05:41:00.000Z"
    ],
    "event.module": [
      "gcp"
    ],
    "host.os.version": [
      "22.04.3 LTS (Jammy Jellyfish)"
    ],
    "host.os.kernel": [
      "5.15.146.1-microsoft-standard-WSL2"
    ],
    "host.os.name": [
      "Ubuntu"
    ],
    "agent.name": [
      "EPINPUNW06A8"
    ],
    "host.name": [
      "epinpunw06a8"
    ],
    "beats_state.timestamp": [
      "2024-07-03T05:41:00.000Z"
    ],
    "host.id": [
      "2fef4836d29e4bf6b57df5049e8f58e4"
    ],
    "beats_state.state.host.name": [
      "epinpunw06a8"
    ],
    "timestamp": [
      "2024-07-03T05:41:00.000Z"
    ],
    "cloud.account.name": [
      "elastic-obs-integrations-dev"
    ],
    "cloud.region": [
      "us-central1"
    ],
    "host.os.type": [
      "linux"
    ],
    "kibana_stats.timestamp": [
      "2024-07-03T05:41:00.000Z"
    ],
    "metricset.period": [
      60000
    ],
    "host.os.codename": [
      "jammy"
    ],
    "gcp.metrics.instance.cpu.usage.pct": [
      0.006597337449646782
    ],
    "agent.hostname": [
      "EPINPUNW06A8"
    ],
    "gcp.metrics.instance.cpu.usage_time.sec": [
      0.07916804939576139
    ],
    "event.duration": [
      2606976189
    ],
    "host.architecture": [
      "x86_64"
    ],
    "metricset.name": [
      "metrics"
    ],
    "cloud.machine.type": [
      "f1-micro"
    ],
    "cloud.provider": [
      "gcp"
    ],
    "@timestamp": [
      "2024-07-03T05:41:00.000Z"
    ],
    "agent.id": [
      "f49fbc8d-cc6a-4629-89b1-454f33f48d82"
    ],
    "cloud.account.id": [
      "elastic-obs-integrations-dev"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.containerized": [
      false
    ],
    "host.os.platform": [
      "ubuntu"
    ],
    "agent.ephemeral_id": [
      "145cb68a-7481-4a5e-887b-41fe3f0af245"
    ],
    "agent.version": [
      "8.15.0"
    ],
    "beats_state.state.host.architecture": [
      "x86_64"
    ],
    "host.os.family": [
      "debian"
    ],
    "gcp.metrics.instance.uptime.sec": [
      60
    ],
    "event.dataset": [
      "gcp.metrics"
    ],
    "cloud.instance.name": [
      "demo-crest-group-j76r"
    ]
  }
}

botelastic · 2024-07-01T06:48:31Z

This pull request doesn't have a Team:<team> label.

mergify · 2024-07-01T06:49:05Z

This pull request does not have a backport label.
If this is a bug or security fix, could you label this PR @Linu-Elias? 🙏.
For such, you'll need to label your PR with:

The upcoming major version of the Elastic Stack
The upcoming minor version of the Elastic Stack (if you're not pushing a breaking change)

To fixup this pull request, you need to add the backport labels for the needed
branches, such as:

backport-v8./d.0 is the label to automatically backport to the 8./d branch. /d is the digit

CHANGELOG.next.asciidoc

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go

gpop63 · 2024-07-02T12:14:57Z

@Linu-Elias do we have a before & after document for compute for example? We have to check that events are not being affected by this change. I see that in the PR description you added a document but not sure if that's before or after the change.

mergify · 2024-07-03T05:22:12Z

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b gcp-label-fix upstream/gcp-label-fix
git merge upstream/main
git push upstream gcp-label-fix

Linu-Elias · 2024-07-03T08:32:41Z

@Linu-Elias do we have a before & after document for compute for example? We have to check that events are not being affected by this change. I see that in the PR description you added a document but not sure if that's before or after the change.

@gpop63 I have added an example of compute before and after the change. I don't any difference in both the documents hence it will not break the existing flow.

CHANGELOG.next.asciidoc

efd6

Approving purely as requested codeowner. Looks fine, but leaving real approval by domain owners.

mergify · 2024-07-08T17:28:36Z

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b gcp-label-fix upstream/gcp-label-fix
git merge upstream/main
git push upstream gcp-label-fix

gpop63 · 2024-07-09T12:57:49Z

I did some tests and it seems fine to me. Services that have no metadata will include cloud.instance.id and services that already have metadata will not be affected.

spanner (service without metadata) diff

{
-  "@timestamp": "2024-07-09T12:28:46.200Z",
+  "@timestamp": "2024-07-09T12:29:46.200Z",
   "cloud": {
+    "instance": {
+      "id": "spanner-1"
+    },
     "account": {
       "id": "elastic-obs-integrations-dev",
       "name": "elastic-obs-integrations-dev"
     },
     "provider": "gcp"
   },
   "event": {
-    "duration": 269604388,
+    "duration": 262185450,
     "dataset": "gcp.metrics",
     "module": "gcp"
   },
   "metricset": {
     "name": "metrics",
     "period": 60000
   },
   "agent": {
     "id": "c3a8e245-a074-4133-8a29-30808077cace",
     "name": "gab-MS-7C95",
     "type": "metricbeat",
     "version": "8.16.0",
-    "ephemeral_id": "e6d85758-d871-4681-a7cd-0fe247ebf801"
+    "ephemeral_id": "eb9d711f-a1be-4dc6-8e94-9802fb4de460"
   },
   "ecs": {
     "version": "8.0.0"
   },
   "host": {
     // ... (host details unchanged)
   },
   "gcp": {
     "labels": {
       "resource": {
         "location": "europe-west1",
         "instance_config": "regional-europe-west1"
       },
       "metrics": {
         "database": "db-1",
         "region": "europe-west1"
       }
     },
     "metrics": {
       "instance": {
         "leader_percentage_by_region": {
           "value": 1
         }
       }
     }
   },
   "service": {
     "type": "gcp"
   }
}

compute (service with metadata) diff

{
-  "@timestamp": "2024-07-09T12:19:00.000Z",
+  "@timestamp": "2024-07-09T12:22:00.000Z",
   "cloud": {
     "account": {
       "id": "elastic-obs-integrations-dev",
       "name": "elastic-obs-integrations-dev"
     },
     "provider": "gcp",
     "availability_zone": "asia-south1-c",
     "region": "asia-south1",
     "instance": {
       "id": "6751872764960737742",
       "name": "service-integration-dev-idc-1"
     },
     "machine": {
       "type": "n1-standard-8"
     }
   },
   "host": {
     // ... (host details unchanged)
   },
   "event": {
     "module": "gcp",
-    "duration": 2486983199,
+    "duration": 2092800246,
     "dataset": "gcp.metrics"
   },
   "metricset": {
     "name": "metrics",
     "period": 60000
   },
   "agent": {
     "name": "gab-MS-7C95",
     "type": "metricbeat",
     "version": "8.16.0",
-    "ephemeral_id": "9ee881eb-973f-4851-b12c-995ee83202f8",
+    "ephemeral_id": "23a53a7b-96dc-431c-97af-9afa30a372cd",
     "id": "c3a8e245-a074-4133-8a29-30808077cace"
   },
   "ecs": {
     "version": "8.0.0"
   },
   "gcp": {
     "labels": {
       "user": {
         "division": "engineering",
         "org": "obs",
         "project": "agithomas",
         "team": "service_integration"
       }
     },
     "metrics": {
       "instance": {
         "cpu": {
           "usage_time": {
-            "sec": 27.490263244137168
+            "sec": 26.798008978366852
           },
           "usage": {
-            "pct": 0.0572713817586191
+            "pct": 0.05582918537159761
           },
           "reserved_cores": {
             "value": 8
           }
         },
         "uptime": {
           "sec": 60
         }
       }
     }
   },
   "service": {
     "type": "gcp"
   }
}

gpop63 · 2024-07-12T09:13:30Z

@elastic/obs-ds-hosted-services hey guys, could someone take a look at this PR please?

gcp ommitted label fix

6699280

Linu-Elias requested review from a team as code owners July 1, 2024 06:48

botelastic bot added the needs_team Indicates that the issue/PR needs a Team:* label label Jul 1, 2024

mergify bot assigned Linu-Elias Jul 1, 2024

Merge branch 'elastic:main' into gcp-label-fix

c7afb5f

shmsr reviewed Jul 1, 2024

View reviewed changes

CHANGELOG.next.asciidoc Outdated Show resolved Hide resolved

shmsr reviewed Jul 1, 2024

View reviewed changes

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go Outdated Show resolved Hide resolved

ishleenk17 reviewed Jul 1, 2024

View reviewed changes

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go Show resolved Hide resolved

Linu-Elias added 2 commits July 1, 2024 16:31

added comments

8fa9e49

Merge branch 'elastic:main' into gcp-label-fix

40c1f0d

ishleenk17 reviewed Jul 2, 2024

View reviewed changes

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go Outdated Show resolved Hide resolved

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go Show resolved Hide resolved

Linu-Elias added 2 commits July 2, 2024 15:11

Merge branch 'elastic:main' into gcp-label-fix

0b40deb

resolved comments

1f9e36a

Merge branch 'main' into gcp-label-fix

8269244

Merge branch 'main' into gcp-label-fix

eee0637

gpop63 reviewed Jul 3, 2024

View reviewed changes

CHANGELOG.next.asciidoc Outdated Show resolved Hide resolved

shmsr changed the title ~~GCP: fix omitted labels~~ x-pack/metricbeat/module/gcp: Add omitted labels Jul 3, 2024

efd6 approved these changes Jul 4, 2024

View reviewed changes

Linu-Elias added 2 commits July 4, 2024 10:35

Merge branch 'elastic:main' into gcp-label-fix

d9e21c2

CHANGELOG.next.asciidoc

c3fab79

Merge branch 'main' into gcp-label-fix

7487e15

gpop63 approved these changes Jul 9, 2024

View reviewed changes

Linu-Elias added 2 commits July 11, 2024 14:03

Merge branch 'main' into gcp-label-fix

893eedd

Merge branch 'main' into gcp-label-fix

5197e8a

ishleenk17 approved these changes Jul 12, 2024

View reviewed changes

shmsr approved these changes Jul 12, 2024

View reviewed changes

Linu-Elias added 2 commits July 12, 2024 12:00

Merge branch 'elastic:main' into gcp-label-fix

1391cfb

Merge branch 'elastic:main' into gcp-label-fix

b0beeba

kaiyan-sheng approved these changes Jul 16, 2024

View reviewed changes

gpop63 merged commit 4301c0a into elastic:main Jul 17, 2024
19 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

x-pack/metricbeat/module/gcp: Add omitted labels #40062

x-pack/metricbeat/module/gcp: Add omitted labels #40062

Linu-Elias commented Jul 1, 2024 •

edited

Loading

botelastic bot commented Jul 1, 2024

mergify bot commented Jul 1, 2024

gpop63 commented Jul 2, 2024 •

edited

Loading

mergify bot commented Jul 3, 2024

Linu-Elias commented Jul 3, 2024

efd6 left a comment

mergify bot commented Jul 8, 2024

gpop63 commented Jul 9, 2024 •

edited

Loading

gpop63 commented Jul 12, 2024

x-pack/metricbeat/module/gcp: Add omitted labels #40062

x-pack/metricbeat/module/gcp: Add omitted labels #40062

Conversation

Linu-Elias commented Jul 1, 2024 • edited Loading

Proposed commit message

Checklist

Screenshots

botelastic bot commented Jul 1, 2024

mergify bot commented Jul 1, 2024

gpop63 commented Jul 2, 2024 • edited Loading

mergify bot commented Jul 3, 2024

Linu-Elias commented Jul 3, 2024

efd6 left a comment

Choose a reason for hiding this comment

mergify bot commented Jul 8, 2024

gpop63 commented Jul 9, 2024 • edited Loading

gpop63 commented Jul 12, 2024

Linu-Elias commented Jul 1, 2024 •

edited

Loading

gpop63 commented Jul 2, 2024 •

edited

Loading

gpop63 commented Jul 9, 2024 •

edited

Loading