From 44e4375dfdcf57754979502b9b1bd35db77f1f87 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 2 Mar 2020 17:40:48 -0500 Subject: [PATCH] Add cloudfoundry module to metricbeat (#16671) * Split up parts into different metricsets. * Add value and container metricsets. * Add documentation. * Revert metricbeat.yml. * Run make update. * Fix documentation add changelog. * Fix bad change in azure module. * Run make update. * Improve reference config. * Document Kibana Space ID setting (#16622) * Document Kibana Space ID setting * Update reference YML files * Update libbeat/docs/shared-kibana-config.asciidoc Co-Authored-By: DeDe Morton * Updating reference YML files * Adding link to doc Co-authored-by: DeDe Morton * Remove the special pins type for `ca_sha256` (#16615) We have changed the type from a special custom type to a slice of string. This reduce the number of exposed types and make it easier in general to work with in the tests. * Remove feature.Describer and enhance Details (#16658) * Make memqueue.Broker internal (now memqueue.broker) (#16667) * [docs] Add skeleton files for 7.7 release highlights and breaking changes (#16584) * [Metricbeat] Add vpc metricset for aws module (#16111) * Add vpn into vpc metricset * Add NATGateway and TrasitGateway * Update cisco.asciidoc (#16434) * Update cisco.asciidoc Making Elasticsearch explicit in the instructions * Update docs source in the module dir Co-authored-by: Loek van Gool * Fix wording in dev guide (#16497) * Added NetBeat (#12612) * Close files earlier when generating fields files (#16681) Fields generator keeps all open files while generating the global fields.yml file. This is not needed and may lead to "too many open files" errors in some environments. * [Metricbeat] Add new module for Redis Enterprise (#16482) Add metricsets: node, proxy * Add doppler_address, uaa_address, rlp_address to config.reference.yml. Co-authored-by: Shaunak Kashyap Co-authored-by: DeDe Morton Co-authored-by: Pier-Hugues Pellerin Co-authored-by: Steffen Siering Co-authored-by: Fae Charlton Co-authored-by: kaiyan-sheng Co-authored-by: Loek van Gool Co-authored-by: Harold Schreckengost Co-authored-by: Jaime Soriano Pastor Co-authored-by: Marcin Tojek (cherry picked from commit 07b03a7f58e0a8a00942003abb9316bf084cc34e) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 172 ++++++++++++++++++ metricbeat/docs/modules/cloudfoundry.asciidoc | 151 +++++++++++++++ .../modules/cloudfoundry/container.asciidoc | 24 +++ .../modules/cloudfoundry/counter.asciidoc | 24 +++ .../docs/modules/cloudfoundry/value.asciidoc | 24 +++ metricbeat/docs/modules_list.asciidoc | 5 + x-pack/metricbeat/include/list.go | 4 + x-pack/metricbeat/metricbeat.reference.yml | 11 ++ .../cloudfoundry/_meta/config.reference.yml | 12 ++ .../module/cloudfoundry/_meta/config.yml | 9 + .../module/cloudfoundry/_meta/docs.asciidoc | 99 ++++++++++ .../module/cloudfoundry/_meta/fields.yml | 19 ++ .../module/cloudfoundry/cloudfoundry.go | 145 +++++++++++++++ .../cloudfoundry/container/_meta/data.json | 35 ++++ .../container/_meta/docs.asciidoc | 2 + .../cloudfoundry/container/_meta/fields.yml | 30 +++ .../cloudfoundry/container/container.go | 45 +++++ .../cloudfoundry/counter/_meta/data.json | 29 +++ .../cloudfoundry/counter/_meta/docs.asciidoc | 2 + .../cloudfoundry/counter/_meta/fields.yml | 18 ++ .../module/cloudfoundry/counter/counter.go | 45 +++++ x-pack/metricbeat/module/cloudfoundry/doc.go | 8 + .../metricbeat/module/cloudfoundry/fields.go | 23 +++ .../metricbeat/module/cloudfoundry/module.yml | 5 + .../module/cloudfoundry/value/_meta/data.json | 29 +++ .../cloudfoundry/value/_meta/docs.asciidoc | 2 + .../cloudfoundry/value/_meta/fields.yml | 18 ++ .../module/cloudfoundry/value/value.go | 45 +++++ .../modules.d/cloudfoundry.yml.disabled | 12 ++ 30 files changed, 1048 insertions(+) create mode 100644 metricbeat/docs/modules/cloudfoundry.asciidoc create mode 100644 metricbeat/docs/modules/cloudfoundry/container.asciidoc create mode 100644 metricbeat/docs/modules/cloudfoundry/counter.asciidoc create mode 100644 metricbeat/docs/modules/cloudfoundry/value.asciidoc create mode 100644 x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/_meta/config.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/cloudfoundry/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go create mode 100644 x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json create mode 100644 x-pack/metricbeat/module/cloudfoundry/container/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/container/container.go create mode 100644 x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json create mode 100644 x-pack/metricbeat/module/cloudfoundry/counter/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/cloudfoundry/counter/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/counter/counter.go create mode 100644 x-pack/metricbeat/module/cloudfoundry/doc.go create mode 100644 x-pack/metricbeat/module/cloudfoundry/fields.go create mode 100644 x-pack/metricbeat/module/cloudfoundry/module.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json create mode 100644 x-pack/metricbeat/module/cloudfoundry/value/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/cloudfoundry/value/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/cloudfoundry/value/value.go create mode 100644 x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b493a4501b0..05b296f6b6a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -273,6 +273,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Add Load Balancing metricset to GCP {pull}15559[15559] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] +- Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index dd99335dc4c..e4cabfcb168 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -22,6 +22,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -4628,6 +4629,177 @@ alias to: cloud.region -- +[[exported-fields-cloudfoundry]] +== cloudfoundry fields + +Cloud Foundry module + + + +[float] +=== cloudfoundry + + + + +[float] +=== app + +The application the metric is associated with. + + + +*`cloudfoundry.app.id`*:: ++ +-- +The ID of the application. + + +type: keyword + +-- + +[float] +=== container + +`container` contains container metrics from Cloud Foundry. + + + +*`cloudfoundry.container.instance_index`*:: ++ +-- +Index of the instance the metric belongs to. + + +type: long + +-- + +*`cloudfoundry.container.cpu.pct`*:: ++ +-- +CPU usage percentage. + + +type: float + +-- + +*`cloudfoundry.container.memory.bytes`*:: ++ +-- +Bytes of used memory. + + +type: long + +-- + +*`cloudfoundry.container.memory.quota.bytes`*:: ++ +-- +Bytes of available memory. + + +type: long + +-- + +*`cloudfoundry.container.disk.bytes`*:: ++ +-- +Bytes of used storage. + + +type: long + +-- + +*`cloudfoundry.container.disk.quota.bytes`*:: ++ +-- +Bytes of available storage. + + +type: long + +-- + +[float] +=== counter + +`counter` contains counter metrics from Cloud Foundry. + + + +*`cloudfoundry.counter.name`*:: ++ +-- +The name of the counter. + + +type: keyword + +-- + +*`cloudfoundry.counter.delta`*:: ++ +-- +The difference between the last time the counter event occurred. + + +type: long + +-- + +*`cloudfoundry.counter.total`*:: ++ +-- +The total value for the counter. + + +type: long + +-- + +[float] +=== value + +`value` contains counter metrics from Cloud Foundry. + + + +*`cloudfoundry.value.name`*:: ++ +-- +The name of the value. + + +type: keyword + +-- + +*`cloudfoundry.value.unit`*:: ++ +-- +The unit of the value. + + +type: keyword + +-- + +*`cloudfoundry.value.value`*:: ++ +-- +The value of the value. + + +type: float + +-- + [[exported-fields-cockroachdb]] == CockroachDB fields diff --git a/metricbeat/docs/modules/cloudfoundry.asciidoc b/metricbeat/docs/modules/cloudfoundry.asciidoc new file mode 100644 index 00000000000..ce079c0f452 --- /dev/null +++ b/metricbeat/docs/modules/cloudfoundry.asciidoc @@ -0,0 +1,151 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-cloudfoundry]] +[role="xpack"] +== cloudfoundry module + +experimental[] + +This is the cloudfoundry module. + +The Cloud Foundry module connects to Cloud Foundry loggregator to gather container, counter, and value metrics into a common data platform where it can be used for analysis, visualization, and alerting. + + +The cloudfoundry module metrics are numerical values that describe some aspect of a system at a particular point in time. They are collected when pushed from the loggregator and are identified with a timestamp, a name, a value, and one or more defining labels. + +The cloudfoundry module mericsets are `container`, `counter` and `value`. + +[float] +=== Module-specific configuration notes + +All metrics come from the Cloud Foundry loggregator API. The loggregator API authenticates through the Cloud Foundry UAA API. +This requires that a new client be added to UAA with the correct permissions. This can be done using the `uaac` client. + +[source,bash] +---- +$ export CLOUDFOUNDRY_CLIENT_ID=metricbeat +$ export CLOUDFOUNDRY_CLIENT_SECRET=yoursecret +$ uaac client add $CLOUDFOUNDRY_CLIENT_ID --name $CLOUDFOUNDRY_CLIENT_ID --secret $CLOUDFOUNDRY_CLIENT_SECRET --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +---- + +Then configuration of the module needs to contain the created `client_id` and `client_secret`. + +[source,yaml] +---- +- module: cloudfoundry + api_address: https://api.dev.cfdev.sh + client_id: "${CLOUDFOUNDRY_CLIENT_ID}" + client_secret: "${CLOUDFOUNDRY_CLIENT_SECRET}" + ssl: + verification_mode: none +---- + + +[float] +== Metricsets + +[float] +=== `container` +The container metricset of Cloud Foundry module allows you to collect container metrics that the +loggregator sends to metricbeat. + +[float] +=== `counter` +The counter metricset of Cloud Foundry module allows you to collect counter metrics that the +loggregator sends to metricbeat. + +[float] +=== `value` +The value metricset of Cloud Foundry module allows you to collect value metrics that the +loggregator sends to metricbeat. + + +[float] +== Configuration options + +The `cloudfoundry` input supports the following configuration options. + +[float] +=== `api_address` + +The URL of the Cloud Foundry API. Optional. Default: "http://api.bosh-lite.com". + +[float] +=== `doppler_address` + +The URL of the Cloud Foundry Doppler Websocket. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `uaa_address` + +The URL of the Cloud Foundry UAA API. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `rlp_address` + +The URL of the Cloud Foundry RLP Gateway. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `client_id` + +Client ID to authenticate with Cloud Foundry. Default: "". + +[float] +=== `client_secret` + +Client Secret to authenticate with Cloud Foundry. Default: "". + +[float] +=== `shard_id` + +Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events +from the RLP Gateway. Default: "(generated UUID)". + +[float] +=== `ssl` + +This specifies SSL/TLS common config. Default: not used. + + +[float] +=== Example configuration + +The cloudfoundry module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: cloudfoundry + metricsets: + - container + - counter + - value + enabled: true + api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + doppler_address: '${CLOUDFOUNDRY_DOPPLER_ADDRESS:""}' + uaa_address: '${CLOUDFOUNDRY_UAA_ADDRESS:""}' + rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' + client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' + client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::cloudfoundry/container.asciidoc[] + +include::cloudfoundry/counter.asciidoc[] + +include::cloudfoundry/value.asciidoc[] + diff --git a/metricbeat/docs/modules/cloudfoundry/container.asciidoc b/metricbeat/docs/modules/cloudfoundry/container.asciidoc new file mode 100644 index 00000000000..99e55464986 --- /dev/null +++ b/metricbeat/docs/modules/cloudfoundry/container.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-cloudfoundry-container]] +=== cloudfoundry container metricset + +experimental[] + +include::../../../../x-pack/metricbeat/module/cloudfoundry/container/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/cloudfoundry/counter.asciidoc b/metricbeat/docs/modules/cloudfoundry/counter.asciidoc new file mode 100644 index 00000000000..254fbb4c950 --- /dev/null +++ b/metricbeat/docs/modules/cloudfoundry/counter.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-cloudfoundry-counter]] +=== cloudfoundry counter metricset + +experimental[] + +include::../../../../x-pack/metricbeat/module/cloudfoundry/counter/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/cloudfoundry/value.asciidoc b/metricbeat/docs/modules/cloudfoundry/value.asciidoc new file mode 100644 index 00000000000..4c25ef3df34 --- /dev/null +++ b/metricbeat/docs/modules/cloudfoundry/value.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-cloudfoundry-value]] +=== cloudfoundry value metricset + +experimental[] + +include::../../../../x-pack/metricbeat/module/cloudfoundry/value/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index f51ea8b9709..47a88fd71fd 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -56,6 +56,10 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> +|<> experimental[] |image:./images/icon-no.png[No prebuilt dashboards] | +.3+| .3+| |<> experimental[] +|<> experimental[] +|<> experimental[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | @@ -269,6 +273,7 @@ include::modules/aws.asciidoc[] include::modules/azure.asciidoc[] include::modules/beat.asciidoc[] include::modules/ceph.asciidoc[] +include::modules/cloudfoundry.asciidoc[] include::modules/cockroachdb.asciidoc[] include::modules/consul.asciidoc[] include::modules/coredns.asciidoc[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 876e25874a3..d08c50b6189 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -23,6 +23,10 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm_scaleset" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/storage" + _ "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry" + _ "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry/container" + _ "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry/counter" + _ "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry/value" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/cockroachdb" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/coredns" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/coredns/stats" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ec81f50e309..1cf1201d967 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -333,6 +333,17 @@ metricbeat.modules: hosts: ["localhost:5000"] enabled: true +#----------------------------- Cloudfoundry Module ----------------------------- +- module: cloudfoundry + metricsets: + - container + - counter + - value + enabled: true + api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' + client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + #----------------------------- CockroachDB Module ----------------------------- - module: cockroachdb metricsets: ['status'] diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml new file mode 100644 index 00000000000..c157d5deeff --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml @@ -0,0 +1,12 @@ +- module: cloudfoundry + metricsets: + - container + - counter + - value + enabled: true + api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + doppler_address: '${CLOUDFOUNDRY_DOPPLER_ADDRESS:""}' + uaa_address: '${CLOUDFOUNDRY_UAA_ADDRESS:""}' + rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' + client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' + client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml new file mode 100644 index 00000000000..a2803b06d40 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml @@ -0,0 +1,9 @@ +- module: cloudfoundry + metricsets: + - container + - counter + - value + enabled: true + api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' + client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc new file mode 100644 index 00000000000..762d3e4f34f --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc @@ -0,0 +1,99 @@ +This is the cloudfoundry module. + +The Cloud Foundry module connects to Cloud Foundry loggregator to gather container, counter, and value metrics into a common data platform where it can be used for analysis, visualization, and alerting. + + +The cloudfoundry module metrics are numerical values that describe some aspect of a system at a particular point in time. They are collected when pushed from the loggregator and are identified with a timestamp, a name, a value, and one or more defining labels. + +The cloudfoundry module mericsets are `container`, `counter` and `value`. + +[float] +=== Module-specific configuration notes + +All metrics come from the Cloud Foundry loggregator API. The loggregator API authenticates through the Cloud Foundry UAA API. +This requires that a new client be added to UAA with the correct permissions. This can be done using the `uaac` client. + +[source,bash] +---- +$ export CLOUDFOUNDRY_CLIENT_ID=metricbeat +$ export CLOUDFOUNDRY_CLIENT_SECRET=yoursecret +$ uaac client add $CLOUDFOUNDRY_CLIENT_ID --name $CLOUDFOUNDRY_CLIENT_ID --secret $CLOUDFOUNDRY_CLIENT_SECRET --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +---- + +Then configuration of the module needs to contain the created `client_id` and `client_secret`. + +[source,yaml] +---- +- module: cloudfoundry + api_address: https://api.dev.cfdev.sh + client_id: "${CLOUDFOUNDRY_CLIENT_ID}" + client_secret: "${CLOUDFOUNDRY_CLIENT_SECRET}" + ssl: + verification_mode: none +---- + + +[float] +== Metricsets + +[float] +=== `container` +The container metricset of Cloud Foundry module allows you to collect container metrics that the +loggregator sends to metricbeat. + +[float] +=== `counter` +The counter metricset of Cloud Foundry module allows you to collect counter metrics that the +loggregator sends to metricbeat. + +[float] +=== `value` +The value metricset of Cloud Foundry module allows you to collect value metrics that the +loggregator sends to metricbeat. + + +[float] +== Configuration options + +The `cloudfoundry` input supports the following configuration options. + +[float] +=== `api_address` + +The URL of the Cloud Foundry API. Optional. Default: "http://api.bosh-lite.com". + +[float] +=== `doppler_address` + +The URL of the Cloud Foundry Doppler Websocket. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `uaa_address` + +The URL of the Cloud Foundry UAA API. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `rlp_address` + +The URL of the Cloud Foundry RLP Gateway. Optional. Default: "(value from ${api_address}/v2/info)". + +[float] +=== `client_id` + +Client ID to authenticate with Cloud Foundry. Default: "". + +[float] +=== `client_secret` + +Client Secret to authenticate with Cloud Foundry. Default: "". + +[float] +=== `shard_id` + +Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events +from the RLP Gateway. Default: "(generated UUID)". + +[float] +=== `ssl` + +This specifies SSL/TLS common config. Default: not used. diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/fields.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/fields.yml new file mode 100644 index 00000000000..0190cac342e --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/fields.yml @@ -0,0 +1,19 @@ +- key: cloudfoundry + title: "cloudfoundry" + description: > + Cloud Foundry module + release: experimental + fields: + - name: cloudfoundry + type: group + description: > + fields: + - name: app + type: group + description: > + The application the metric is associated with. + fields: + - name: id + type: keyword + description: > + The ID of the application. diff --git a/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go b/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go new file mode 100644 index 00000000000..bc4fecde76d --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go @@ -0,0 +1,145 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudfoundry + +import ( + "context" + "sync" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + cfcommon "github.com/elastic/beats/x-pack/libbeat/common/cloudfoundry" +) + +// ModuleName is the name of this module. +const ModuleName = "cloudfoundry" + +type Module struct { + mb.BaseModule + + log *logp.Logger + + hub *cfcommon.Hub + listener *cfcommon.RlpListener + listenerLock sync.Mutex + + counterReporter mb.PushReporterV2 + valueReporter mb.PushReporterV2 + containerReporter mb.PushReporterV2 +} + +func init() { + if err := mb.Registry.AddModule(ModuleName, newModule); err != nil { + panic(err) + } +} + +func newModule(base mb.BaseModule) (mb.Module, error) { + var cfg cfcommon.Config + if err := base.UnpackConfig(&cfg); err != nil { + return nil, err + } + + log := logp.NewLogger("cloudfoundry") + hub := cfcommon.NewHub(&cfg, "metricbeat", log) + + // early check that listener can be created + _, err := hub.RlpListener(cfcommon.RlpListenerCallbacks{}) + if err != nil { + return nil, err + + } + + return &Module{ + BaseModule: base, + log: log, + hub: hub, + }, nil +} + +func (m *Module) RunCounterReporter(reporter mb.PushReporterV2) { + m.listenerLock.Lock() + m.runReporters(reporter, m.valueReporter, m.containerReporter) + m.listenerLock.Unlock() + + <-reporter.Done() + + m.listenerLock.Lock() + m.runReporters(nil, m.valueReporter, m.containerReporter) + m.listenerLock.Unlock() +} + +func (m *Module) RunValueReporter(reporter mb.PushReporterV2) { + m.listenerLock.Lock() + m.runReporters(m.counterReporter, reporter, m.containerReporter) + m.listenerLock.Unlock() + + <-reporter.Done() + + m.listenerLock.Lock() + m.runReporters(m.counterReporter, nil, m.containerReporter) + m.listenerLock.Unlock() +} + +func (m *Module) RunContainerReporter(reporter mb.PushReporterV2) { + m.listenerLock.Lock() + m.runReporters(m.counterReporter, m.valueReporter, reporter) + m.listenerLock.Unlock() + + <-reporter.Done() + + m.listenerLock.Lock() + m.runReporters(m.counterReporter, m.valueReporter, nil) + m.listenerLock.Unlock() +} + +func (m *Module) runReporters(counterReporter, valueReporter, containerReporter mb.PushReporterV2) { + if m.listener != nil { + m.listener.Stop() + m.listener = nil + } + m.counterReporter = counterReporter + m.valueReporter = valueReporter + m.containerReporter = containerReporter + + start := false + callbacks := cfcommon.RlpListenerCallbacks{} + if m.counterReporter != nil { + start = true + callbacks.Counter = func(evt *cfcommon.EventCounter) { + m.counterReporter.Event(mb.Event{ + Timestamp: evt.Timestamp(), + RootFields: evt.ToFields(), + }) + } + } + if m.valueReporter != nil { + start = true + callbacks.ValueMetric = func(evt *cfcommon.EventValueMetric) { + m.valueReporter.Event(mb.Event{ + Timestamp: evt.Timestamp(), + RootFields: evt.ToFields(), + }) + } + } + if m.containerReporter != nil { + start = true + callbacks.ContainerMetric = func(evt *cfcommon.EventContainerMetric) { + m.containerReporter.Event(mb.Event{ + Timestamp: evt.Timestamp(), + RootFields: evt.ToFields(), + }) + } + } + if start { + l, err := m.hub.RlpListener(callbacks) + if err != nil { + m.log.Errorf("failed to create RlpListener: %v", err) + return + } + l.Start(context.Background()) + m.listener = l + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json new file mode 100644 index 00000000000..3d1f51694e4 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2020-02-27T18:09:35.583Z", + "cloudfoundry": { + "container": { + "instance_index": 0, + "cpu.pct": 0.18815706013687805, + "memory.bytes": 23320985, + "memory.quota.bytes": 134217728, + "disk.bytes": 186187776, + "timestamp": "2020-02-27T18:09:35.583Z", + "type": "container", + "disk.quota.bytes": 2122136037 + }, + "envelope": { + "origin": "rep", + "deployment": "cf", + "ip": "10.144.0.13", + "job": "diego-cell", + "index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0" + }, + "app": { + "id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a" + } + }, + "event": { + "dataset": "cloudfoundry.container", + "module": "cloudfoundry" + }, + "metricset": { + "name": "container" + }, + "service": { + "type": "cloudfoundry" + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/container/_meta/docs.asciidoc new file mode 100644 index 00000000000..defeb16cae6 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +The container metricset of Cloud Foundry module allows you to collect container metrics that the +loggregator sends to metricbeat. diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml b/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml new file mode 100644 index 00000000000..7711469f160 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/fields.yml @@ -0,0 +1,30 @@ +- name: container + type: group + description: > + `container` contains container metrics from Cloud Foundry. + release: experimental + fields: + - name: instance_index + type: long + description: > + Index of the instance the metric belongs to. + - name: cpu.pct + type: float + description: > + CPU usage percentage. + - name: memory.bytes + type: long + description: > + Bytes of used memory. + - name: memory.quota.bytes + type: long + description: > + Bytes of available memory. + - name: disk.bytes + type: long + description: > + Bytes of used storage. + - name: disk.quota.bytes + type: long + description: > + Bytes of available storage. diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container.go b/x-pack/metricbeat/module/cloudfoundry/container/container.go new file mode 100644 index 00000000000..1d92e9340cf --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/container.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container + +import ( + "fmt" + + "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry" + + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("cloudfoundry", "container", New, mb.DefaultMetricSet()) +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + + mod *cloudfoundry.Module +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(*cloudfoundry.Module) + if !ok { + return nil, fmt.Errorf("must be child of cloudfoundry module") + } + return &MetricSet{base, mod}, nil +} + +// Run method provides the module with a reporter with which events can be reported. +func (m *MetricSet) Run(reporter mb.PushReporterV2) { + m.mod.RunContainerReporter(reporter) +} diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json new file mode 100644 index 00000000000..d6f155213af --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -0,0 +1,29 @@ +{ + "@timestamp": "2020-02-24T18:49:02.674Z", + "cloudfoundry": { + "counter": { + "total": 0, + "name": "promhttp_metric_handler_errors_total", + "delta": 0, + "timestamp": "2020-02-24T18:49:02.674Z", + "type": "counter" + }, + "envelope": { + "origin": "loggregator_forwarder_agent", + "deployment": "cf", + "ip": "10.144.0.13", + "job": "diego-cell", + "index": "1e58e943-a498-4339-9d5f-2885bf223db9" + } + }, + "event": { + "dataset": "cloudfoundry.counter", + "module": "cloudfoundry" + }, + "metricset": { + "name": "counter" + }, + "service": { + "type": "cloudfoundry" + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/docs.asciidoc new file mode 100644 index 00000000000..e3bcdcb5253 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +The counter metricset of Cloud Foundry module allows you to collect counter metrics that the +loggregator sends to metricbeat. diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/fields.yml b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/fields.yml new file mode 100644 index 00000000000..1a6b5566d41 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/fields.yml @@ -0,0 +1,18 @@ +- name: counter + type: group + description: > + `counter` contains counter metrics from Cloud Foundry. + release: experimental + fields: + - name: name + type: keyword + description: > + The name of the counter. + - name: delta + type: long + description: > + The difference between the last time the counter event occurred. + - name: total + type: long + description: > + The total value for the counter. diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter.go new file mode 100644 index 00000000000..470e128f3f1 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package counter + +import ( + "fmt" + + "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry" + + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("cloudfoundry", "counter", New, mb.DefaultMetricSet()) +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + + mod *cloudfoundry.Module +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(*cloudfoundry.Module) + if !ok { + return nil, fmt.Errorf("must be child of cloudfoundry module") + } + return &MetricSet{base, mod}, nil +} + +// Run method provides the module with a reporter with which events can be reported. +func (m *MetricSet) Run(reporter mb.PushReporterV2) { + m.mod.RunCounterReporter(reporter) +} diff --git a/x-pack/metricbeat/module/cloudfoundry/doc.go b/x-pack/metricbeat/module/cloudfoundry/doc.go new file mode 100644 index 00000000000..a91b647f36a --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/doc.go @@ -0,0 +1,8 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +/* +Package cloudfoundry is a Metricbeat module that contains MetricSets. +*/ +package cloudfoundry diff --git a/x-pack/metricbeat/module/cloudfoundry/fields.go b/x-pack/metricbeat/module/cloudfoundry/fields.go new file mode 100644 index 00000000000..a40c3445172 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package cloudfoundry + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "cloudfoundry", asset.ModuleFieldsPri, AssetCloudfoundry); err != nil { + panic(err) + } +} + +// AssetCloudfoundry returns asset data. +// This is the base64 encoded gzipped contents of module/cloudfoundry. +func AssetCloudfoundry() string { + return "eJzclbFy2zAQRHt+xY178wNYpIgzmXGXwqljCFhKGIEAAxwk8+8zoESbpCDFCWUVVqHiAL5d7uGIe9qiq0gaF1XtolW+K4hYs0FFd+PyXUGkEKTXLWtnK/pSEBE9pC30/bCHGqeiQUHkYSACKsJLC68bWBamIKo1jApV/+g9WdHgRDz9uGtR0dq72B4rGekpbUwUbftay8HOAg+/pw0Swmgp0jLxBtSAvZakA4kQnNSCoWiveVOOHp37GXvSalIebG3R7Z2fr10wNxh8/Eau7q2NrJYnUUhnWWgLvySQ51fK8wAMb+RjNoFq75rpgRiHc+5IvCs8G1hYiV/aKrxkgzTOrv8pxceEGjIcBMa9XiExA7Ers6ZkG8tWctZNbZyYr/ylqQ8/flIMYg1q4WVKZ428cIPG+a5cdYxwnSzoa2KlMGKAGgQuif+OjsXHWBA7oY1YGVz0oXTYfmAEgZ0/24Be+yYJnNh4m+toefFU94zJTPeVG0x0+r/uBzERh3E+vseZ7sGwuFLLkrDSdQ2P9PVYgffA4cYwIjCxbjB2RNjBMjkpo/dQeYPspjkuNdjzaCdMBNXO5xMaxPtti45VT/iEh6p/r3zHotX5e+C/hRPxHcLzbtGSG+hp0JoJ/wkAAP//aoqyCQ==" +} diff --git a/x-pack/metricbeat/module/cloudfoundry/module.yml b/x-pack/metricbeat/module/cloudfoundry/module.yml new file mode 100644 index 00000000000..cf62d2c84c0 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/module.yml @@ -0,0 +1,5 @@ +name: cloudfoundry +metricsets: + - counter + - value + - container diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json new file mode 100644 index 00000000000..bf70762fd38 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -0,0 +1,29 @@ +{ + "@timestamp": "2020-02-27T18:20:48.379Z", + "cloudfoundry": { + "value": { + "unit": "", + "value": 5.125735912782789e-05, + "timestamp": "2020-02-27T18:20:48.379Z", + "type": "value", + "name": "go_memstats_gc_cpu_fraction" + }, + "envelope": { + "origin": "loggregator_forwarder_agent", + "deployment": "cf", + "ip": "10.144.0.11", + "job": "scheduler", + "index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f" + } + }, + "event": { + "dataset": "cloudfoundry.value", + "module": "cloudfoundry" + }, + "metricset": { + "name": "value" + }, + "service": { + "type": "cloudfoundry" + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/value/_meta/docs.asciidoc new file mode 100644 index 00000000000..0332945c2c0 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +The value metricset of Cloud Foundry module allows you to collect value metrics that the +loggregator sends to metricbeat. diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/fields.yml b/x-pack/metricbeat/module/cloudfoundry/value/_meta/fields.yml new file mode 100644 index 00000000000..0ba9f29890a --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/fields.yml @@ -0,0 +1,18 @@ +- name: value + type: group + description: > + `value` contains counter metrics from Cloud Foundry. + release: experimental + fields: + - name: name + type: keyword + description: > + The name of the value. + - name: unit + type: keyword + description: > + The unit of the value. + - name: value + type: float + description: > + The value of the value. diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value.go b/x-pack/metricbeat/module/cloudfoundry/value/value.go new file mode 100644 index 00000000000..4df1e3bc304 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/value.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package value + +import ( + "fmt" + + "github.com/elastic/beats/x-pack/metricbeat/module/cloudfoundry" + + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("cloudfoundry", "value", New, mb.DefaultMetricSet()) +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + + mod *cloudfoundry.Module +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(*cloudfoundry.Module) + if !ok { + return nil, fmt.Errorf("must be child of cloudfoundry module") + } + return &MetricSet{base, mod}, nil +} + +// Run method provides the module with a reporter with which events can be reported. +func (m *MetricSet) Run(reporter mb.PushReporterV2) { + m.mod.RunValueReporter(reporter) +} diff --git a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled new file mode 100644 index 00000000000..ae540f20cfc --- /dev/null +++ b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled @@ -0,0 +1,12 @@ +# Module: cloudfoundry +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-cloudfoundry.html + +- module: cloudfoundry + metricsets: + - container + - counter + - value + enabled: true + api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' + client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' + client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}'