From 71e8ca1505a8292089de069ac7d8874cef784471 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Wed, 4 Mar 2020 18:24:02 +0100 Subject: [PATCH] [Metricbeat] Add PubSub metricset to Google Cloud Platform module (#15536) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/googlecloud.asciidoc | 9 ++- .../docs/modules/googlecloud/pubsub.asciidoc | 18 +++++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/metricbeat.reference.yml | 3 + .../module/googlecloud/_meta/config.yml | 3 + .../module/googlecloud/_meta/docs.asciidoc | 2 +- .../metricbeat/module/googlecloud/fields.go | 2 +- .../metricbeat/module/googlecloud/module.yml | 1 + .../pubsub/_meta/data_subscription.json | 53 ++++++++++++++ .../googlecloud/pubsub/_meta/data_topic.json | 61 ++++++++++++++++ .../googlecloud/pubsub/_meta/docs.asciidoc | 25 +++++++ .../googlecloud/pubsub/_meta/fields.yml | 70 +++++++++++++++++++ .../module/googlecloud/pubsub/manifest.yml | 27 +++++++ .../modules.d/googlecloud.yml.disabled | 3 + 15 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 metricbeat/docs/modules/googlecloud/pubsub.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2283b9779f41..b704cce68309 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -209,6 +209,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] - Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754] +- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 07eedf702409..7928316598b6 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -44,7 +44,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` @@ -83,7 +83,10 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false @@ -99,7 +102,11 @@ The following metricsets are available: * <> +* <> + include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] +include::googlecloud/pubsub.asciidoc[] + diff --git a/metricbeat/docs/modules/googlecloud/pubsub.asciidoc b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc new file mode 100644 index 000000000000..66798829a79c --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-pubsub]] +=== Google Cloud Platform pubsub metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 3ac1af27bce4..30efcd5e63ab 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -107,8 +107,9 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.2+| .2+| |<> beta[] +.3+| .3+| |<> beta[] |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index bca4bab82afd..bf144e2e36f9 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -473,7 +473,10 @@ metricbeat.modules: - module: googlecloud metricsets: - compute + - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 6ff566f37b3d..b33dfaa03d68 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -1,7 +1,10 @@ - module: googlecloud metricsets: - compute + - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index c52f1ce67725..a25e364e1f01 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -34,7 +34,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index e726b128b299..61e4e93847a8 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsW0tz2zgSvvtX9M3JVuIc9ubDViWqrSS1ycY1cXJVQWBLwhhEc4CmFc2vnwL4ECmJJGiLSmZKN5uP7q+b6O9rPPQaHnB7CyuilUapKU+uAFixxlu4fh+uwsxfhjsteEk2vb4CsKhROLyFBbK4AkjQSasyVmRu4T9XAADvZ3eQUpJrvAJYKtSJuw03XoMRKbZc3mixQO3CbQDeZngLtPgdJZeXmu83beQO7c2/6stH3909nSKLRLAY9YZV0sW/4LaOMY1/XlKa5YyNx/dT2zazspRnjautxLc+16ywXIXQeGc/mU08yjgWRmLrZpfzLmNNg0tlcSO0Pnigz2if4abxxFKWYTJfbBndXFJu+OjzlS9NZtXxQCuRH42kVJkVBMOVG1hsgdfYF9IhtEzIB+RJwJWmo+HVYy7LJ/kaFh3aR0zmkiy63lgTyhd6f5Adjfb/ebpAC7SEYLV2AmRCtGty7O/6vzsGbxtlzkqrP4W3fiKI9z7pVkj/XwVFaE1SMCYwu/sGvBYMyoHMrUXDegvKeO6qgogD7sQK56zSLlBjcX/zBmFJ1qMt06sMOJRkksPvV49s5R4mGj/i9KU885b8VylK2fuApaW0K4w9OJRNAsY7L7B8/AKUoQ3j8XjNVHg2VjFOnR/vhNEA03CCCkBTZih4GEhRXR5ZR2n0Amg5/0Cb8Fyoye+fYS0cLBAN2NwYZVavYsrDIG/ITlUhEtXjBIJ3UCWFn6JSfDa6ozqC7vSaV+OrNO9pCB0anjp33gfQI9pxqM6Qs35kFRpNIlkILYxULT+n6ks/kUjgXeVgZHu6Zs72S6+7rJ4BogtIE8zC59Ukcy0YjVRHW54WhLee1diqRd5sFYrXtyCFlrkOPUMY1Js1Ft2BxT9ydAwbUX7DstHLLP3Yeqb2/5RgIDesdOP2YaFUTxa+HYdx68FYdBkZhzeTcNczBvZuNib6e7ZBC4vcl0JRZP0daqwpyrqVeyhtMJC6prsvvX7ifMX4a/r874+MDBpWQr8L0fa5j4cQC6MJZWVpw+v5UkgmO/hWVC986MRJEfF0BA8fmjZ5Ol8qEzq4qFQO+NnnoJIjBsQtvhXycxpTT70ayiZcxUeuoJQP9/d3b74G2YBCN7zAUIXMHbJJF/ZTo65xljPGxbYG5W8fAx4DtiDJM2W6cFamukb/gixIIdf40md6XCBLS4Z9JCyzueVj4CNE67f7e0/HLrdeWsgCCrkGScZgMQ9eIG986yy1CsGYpBCki7hcxOUiLoPGfwlxUYbRGnHKtdsxvfM/p3seHrz9QzaG5iJGTgTVjbASQ3fjzfVTXhwLDNd/PPHF804c6zyZ/sYQYDwFPp0ER9LgeCIcRYXPJMMBX7sVrpguexD4EzvtopVy8S3fIfApIB9ps8fDi2qqT5nYVmPdNYcpM94HnYmFPrOM1arUkrJ97VoRw9vZ/5pjB8gUolXFH9JyUaxnmbso1rOBxIOBi2JF+BqnV1OsCo3WqlilOuFi0FhgP2/hZ7Q+xavT31CbLstFHaYuy0WjIcTCgMty0VE/9a7xv286VovOvD1bAcKVRed6ybondXFEHdjr46d3u0WhipLhxZIs3M/uYKlp40DxtStYZ3eqkAyILNNKhoMt4NiiSIGM3r7cZ729oPrPCjwtrNYhgZ7ASk3wYXWhVOZcuS8xMbXATpP7KqrzJP9oZF3QLHOPxg8ofGsnJ5wO8anb7eS4sL3jUYR0HgpyH9EOLwQ/ca4/KMNRbw9L8BgzffJ7mtn0kPCedqo5UnKnmXWOE9sRUjtOaKNl9hkiOyCxLLO5c3oeOuifK7KN8tXkwonvmi6eNSdqnvbeEVA4Pr1Bi8BoU2XCFKSiqjdfv37q2loe0Qw8DeW+IH3/3JDK3IVT+v3QTrMd71IiXmMS2PyFMpC6lztWb07Frl3gd8dCPrwqdu1TZXLGliBqsUVbRpcJV05D6y2zIsDL3OwyN7vMzQaN/yJb+eeiwF3n+v1zHAUa3JxNQKTF0epBGZqTAZwVPwFqNP+Us2NhEp+pNmhL+WodqLsD6evOU9nhR52B5BOrHrFZCqf/IeFfAQAA///d9YAm" + return "eJzsXE1z47gRvftX9M3jlEdzyM2V2qpZJ9mdykzWFXvmqgLBloQYBBigYa3216cAfoiURBKURO3Olm6WRKBfN9DvNT7o9/CKmwdYar2UyKV26Q0ACZL4ALc/hW/h0X8NT5LRQpvs9gbAoERm8QESJHYDkKLlRuQktHqAH24AAH56fIJMp07iDcBCoEztQ/jhPSiWYcvkTLIEpQ0/A9AmxwfQyX+RU/lVs32zD2fRzP5Sf32w7fbpDImljNioFkZwG9/AbixhFv8811nuCBuP74a23c3SaJc3vm0FvjVcj0XPlQuNNrvBbOIRyhJTHFs/dhnv6qzZ4UIYXDMp9x7o67Sv42bnqdF5juk82RDaOddO0cHnK1tSq2XHA61AflJcZ0ItIXRcmYFkA7TCPpf2oeWMvyJNAq7sOhpePedyN8loGLRo3jCdc23Q9vqaapfI3Ul20Nt/uyxBA3oBodfaCGgVvF1pS/5X/3fH5G2jdCSk+I353s8E8cUH3TDuP1VQmJSaM8IUHp++Aq0YgbDAnTGoSG5AKM9dlRNxwC1b4pxE1gVqLO6vvkNYaOPRluEVCixyrdL98atntrCvE80fdv5UfvQ9+VEpUtnbgIXRWZcbO3B0PgkYb7zA8ukX0DmaMB8P50yFZ20E4dTx8UYIFZAeDlABaMoIBQsDIarTI+9IjV4ALeM/63V4LuTkty+wYhYSRAXGKSXU8j4mPRTSWpupMoSjeJtA8PaypLBTZIqPRrdXB9CdX/NqfJXmHYfQoqKpY+dtgH5DMw7VBWLWj6xCIzVLEyaZ4qJl51x16WfNUvixMjCyPF0R5bup151WJ4DoAtIEk/i4qnQuGaHi4mDJ04Lw0bMaGZG4ZqlQNN8AZ5I7GWqGMKnXKyyqA4P/c2gJ1qwcw7LQy43+deOZ2n8owYBTJGTj5/1EqZ4sbFsK89aDMWhzrSzOJuGuEyb2djXG+mu2wR4S51OhSLL+CjW2K513K/dQ2GAgdE1zv/TaibMVY69p8x+/5lqhIsHkj8HbPvPxEGJhNKEsjV7Tar5gnLQZbBVVC+8bsZxFPB3Bw/tdK5fNF0KFCi4qlAN2djmo5IgBcYsvhfyaRtVLr4ayMVvxkS0o5eeXl6cPz0E2oNANLzC6Qmb32aQL+7lR1zjLFWOyqUH5nw8BjwFbkOSFIl0YK0Ndo3+nDXDGV3jnIz3OkYXRirwnxPO5oUPgI0TrPy8vno6tM15atAFkfAVcK4XFOjhBWvvSmUsRnFFpIUhXcbmKy1VcBjv/Q4iLUIRGsXPu3Y6pnf881fPw5O2fsjE0FzFzIqhuRC8xdDe+u37Ki2OB4fyPJ7543oljnaPpbwwBxlPg8SQ4kgbHE+EoKjyRDAdsbXe4YqrsQeBHVtpFKWXjS7594FNAPlBmj4cXVVSfM7CtwrprDVNGvA86aWLywjJWq1JLyna1a6kJPj7+qzl3QKtCtCr/Q1iuinVSd1fFOhlIPBi4KlaErXF6NcWu0GitilWqM24GjQX2+238jNaneHX6DrXpul3U0dV1u2g0hFgYcN0uOminPjX+66xjt+jCx7MVIFwatLaXrHtCF0fUgb0+ff5xuylUUTK8W2gDL49PsJB6bUHQrS1YZ3urUCtgeS4FDxdbwJJBloFWcnO3y3o7TvXfFTjOrdYlgR7HSk3wbnWhFOpSsS8xkW6BnSb2lVeXCf5Bz7qgGaIejR9Q+NZJTrgd4kO3Pcmx4XjHowjh3BfkPqId3gg+cq0/KMNRrYcleEw3ffJ7ntX0kPCed6k5UnKnWXWOE9sRUjtOaKNl9gSRHZBY4vncWjkPFfTvK7KN9JXahhvfNV2ctCZq3vbeElC4Pr1Gg0BoMqHCEqSiqg/Pz5+7jpZHFAPHodwVpG9fGlLpbLil3w/tPMfxNtOaVpgGNn8nFGT2bsvqzaXYrQ38bonx1/vi1D4TyhG2BFGyDZrSu5zZchlaH5kVDl7XZte12XVtNtj5H+Qo/1IUuK1cv32Jo0CF64sJCDc4Wj10jupsAB+LV4Aaxb92ZImp1EeqDdpot1wF6u5AWiHMXWJd0jB9rsvaTy55dsnIS9rWJXV/x9Upz67+4F0JI3ZcfcL46zxDG16dOn3WP7rMSUbiDQuZ8sNX9m69KaXXEtNlIbkft5/r3ef7wvfigRSleEOzCQD6bx5KvSxS97QdZ01MFhdWrPgNq+LBqRb02qF3bPY6Y7MKQP3DHQgFrDXMPcntsnljhleDca4sr8GWwcSwV9EGd2shd3YFqNJcC0X3kDgCpQk2SK1h63fDqdrIud2YdAi0TNHSPNjYQp+z5WlvS31cYig1yxei7qrpVJjrcKnLo1EO5U7KeTOz69fDJsrxhie1I9tX0lpJbdA6STP4pzbAIMUg5mWlfqipxQbXftiSbSsUHzKdBodTZKkUCrs8j4jZ+Q7YhiJ1kPTK+PTjvNS4emPf9YBOO5ghPEeOol1NDs6ugBFhlh8GB1+VFK8YvLD3xQrYtwkHwAZElkvMUFGx4k012qAJCSO+Cv9YombbGTzrYqlcnXRqJTe+WCMmlAWtsNVgFrZ3m8aMnwLFvjMao42fJ16mluINVastcBb2ppEZyJwkkUsEEhn2nPK2ot132Bsf8b/vbDIEZyrvaxPFVoPgRlciMGqWhBcfL1SdVVd/26lJukpLL8TlcfoxRVpxguALnIuL03Nl+smn659Aqg7EcloqqQ22GQ/WglagtHrvOWbTCq1IxxHijk+Xmhs7nl1qQvyN6xR/OGpaxEawvo5zoSlR3scZHPTtzZtc8ONWvy++6Ynr3u8kWtBWg/SClV8ihV19txR53vcm+yIUVf+979xxCv9dLJw2pMYL6QmbUcP/0er/AQAA//8JtEe4" } diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index 6b6704ca08d1..10e35ce5a9fa 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -1,4 +1,5 @@ name: googlecloud metricsets: - compute +- pubsub - loadbalancing diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json new file mode 100644 index 000000000000..1503d6aa80fa --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2020-01-14T11:46:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "subscription": { + "num_undelivered_messages": 0, + "oldest_unacked_message_age": 0, + "backlog_bytes": 0 + } + } + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228" + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "dataset": "googlecloud.pubsub", + "module": "googlecloud", + "duration": 933264720 + }, + "metricset": { + "name": "pubsub", + "period": 300000 + } +} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json new file mode 100644 index 000000000000..f85d81487820 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json @@ -0,0 +1,61 @@ +{ + "@timestamp": "2020-01-14T11:42:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "ecs": { + "version": "1.4.0" + }, + "cloud": { + "provider": "googlecloud", + "account": { + "id": "elastic-metricbeat" + } + }, + "metricset": { + "name": "pubsub", + "period": 300000 + }, + "event": { + "duration": 932060852, + "dataset": "googlecloud.pubsub", + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "topic": { + "message_sizes": { + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 4, + "scale": 1, + "num_finite_buckets": 16 + } + } + } + } + } + } + }, + "service": { + "type": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0", + "type": "metricbeat" + } +} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc new file mode 100644 index 000000000000..9fa3ea4f50c7 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -0,0 +1,25 @@ +PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. + +The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +No special permissions are needed apart from the ones detailed in the module section of the docs. + +- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. +- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. +- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. +- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. +- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. +- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. +- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. +- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml new file mode 100644 index 000000000000..64063c444973 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -0,0 +1,70 @@ +- name: pubsub + release: beta + type: group + description: Google Cloud PubSub metrics + fields: + - name: subscription + type: group + description: Suscription related metrics + fields: + - name: ack_message_count + type: long + description: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. + - name: backlog_bytes + type: long + description: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: num_outstanding_messages + type: long + description: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. + - name: num_undelivered_messages + type: long + description: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: oldest_unacked_message_age + type: long + description: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. + - name: pull_ack_message_operation_count + type: long + description: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_ack_request_count + type: long + description: Cumulative count of acknowledge requests, grouped by result. + - name: pull_message_operation_count + type: long + description: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_request_count + type: long + description: Cumulative count of pull requests, grouped by result. + - name: push_request_count + type: long + description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. + - name: push_request_latencies + type: long + description: Distribution of push request latencies (in microseconds), grouped by result. + - name: sent_message_count + type: long + description: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. + - name: streaming_pull_ack_message_operation_count + type: long + description: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: streaming_pull_ack_request_count + type: long + description: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. + - name: streaming_pull_message_operation_count + type: long + description: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: topic + type: group + description: Topic related metrics + fields: + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: send_message_operation_count + type: long + description: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: send_request_count + type: long + description: Cumulative count of publish requests, grouped by result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml new file mode 100644 index 000000000000..e7e3e032b714 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -0,0 +1,27 @@ +default: true +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: pubsub + metrics: + - "pubsub.googleapis.com/subscription/ack_message_count" + - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/num_outstanding_messages" + - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_ack_request_count" + - "pubsub.googleapis.com/subscription/pull_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_request_count" + - "pubsub.googleapis.com/subscription/push_request_count" + - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/sent_message_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/topic/message_sizes" + - "pubsub.googleapis.com/topic/send_message_operation_count" + - "pubsub.googleapis.com/topic/send_request_count" diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 9f5408ca3529..15304fdba006 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -4,7 +4,10 @@ - module: googlecloud metricsets: - compute + - pubsub + - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false