From 16abaa57bf39c3f488e020dd44387fe38e87da6c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 17:56:50 +0200 Subject: [PATCH] [8.4](backport #33032) Add enricher to storageclass metricset (#33054) * Add enricher to storageclass metricset (#33032) * add enricher for storageclass Signed-off-by: Tetiana Kravchenko * add pr number Signed-off-by: Tetiana Kravchenko * fix check-no-changes] Error 1 Signed-off-by: Tetiana Kravchenko * generate notice file Signed-off-by: Tetiana Kravchenko * readd removed metric from test file Signed-off-by: Tetiana Kravchenko Signed-off-by: Tetiana Kravchenko (cherry picked from commit d20fff40fa2c0fc8a49431098572ba3a737680ce) # Conflicts: # go.mod # go.sum * Update go.mod * Update go.sum * update elastic-agent-libs to pass tests Signed-off-by: Tetiana Kravchenko * run make check-default Signed-off-by: Tetiana Kravchenko Signed-off-by: Tetiana Kravchenko Co-authored-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 8 ++--- deploy/kubernetes/metricbeat-kubernetes.yaml | 7 ++++ .../metricbeat-daemonset-configmap.yaml | 3 ++ .../metricbeat/metricbeat-role.yaml | 4 +++ go.mod | 4 +-- go.sum | 9 +++-- .../test/docs/01_playground/metricbeat.yaml | 7 ++++ .../_meta/test/ksm.unit.v1.8.0 | 2 -- .../_meta/test/ksm.unit.v1.8.0.expected | 2 +- .../_meta/test/ksm.v1.8.0.expected | 2 +- .../_meta/test/ksm.v2.0.0.expected | 6 ++-- .../state_storageclass/state_storageclass.go | 34 +++++++++++++++---- .../module/kubernetes/util/kubernetes.go | 2 ++ 14 files changed, 67 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e4120e0c0d2..9775e4a72fd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] - Add missing cluster metadata to k8s module metricsets {pull}32979[32979] +- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 1c5942ca796..62ffded5514 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9414,11 +9414,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.2.1 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.3.0/LICENSE: Apache License Version 2.0, January 2004 @@ -9858,11 +9858,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.2.9 +Version: v0.2.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.11/LICENSE: Apache License Version 2.0, January 2004 diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 17b6c3c4d59..f278fe403e7 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -303,6 +306,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index 4dc934ccb26..9ddab21360c 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml index a9157a7e41d..73ce911439b 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -36,6 +36,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index 89080d5829e..5b8ce85fe6f 100644 --- a/go.mod +++ b/go.mod @@ -201,8 +201,8 @@ require ( github.com/aws/smithy-go v1.12.0 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.2.1 - github.com/elastic/elastic-agent-libs v0.2.9 + github.com/elastic/elastic-agent-autodiscover v0.3.0 + github.com/elastic/elastic-agent-libs v0.2.11 github.com/elastic/elastic-agent-shipper-client v0.2.0 github.com/elastic/elastic-agent-system-metrics v0.4.4 github.com/elastic/go-elasticsearch/v8 v8.2.0 diff --git a/go.sum b/go.sum index 32f215ce130..624020bf325 100644 --- a/go.sum +++ b/go.sum @@ -593,15 +593,14 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= -github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= +github.com/elastic/elastic-agent-autodiscover v0.3.0 h1:kdpNnIDnVk7gvQxxR6PzZY7aM8LyMTRkwI/p+FNS17s= +github.com/elastic/elastic-agent-autodiscover v0.3.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= -github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.7/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-libs v0.2.9 h1:7jOCqNqEWG0kJb3fa8/SC6beSiys1TmAylH9+hWTnrM= -github.com/elastic/elastic-agent-libs v0.2.9/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= +github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= +github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-shipper-client v0.2.0 h1:p+5ep48YCOe+3nICeWmiLwQV11yDLad2n4NunI66Shg= github.com/elastic/elastic-agent-shipper-client v0.2.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q= github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0= diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml index e17b1c3a687..204b4313e54 100644 --- a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml +++ b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml @@ -37,6 +37,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass - module: kubernetes metricsets: - apiserver @@ -293,6 +296,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 index 0a7712e74b7..0efb02acbee 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 @@ -7,5 +7,3 @@ kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09 # HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels. # TYPE kube_storageclass_labels gauge kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1 - - diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected index cd0595a14f4..6fcfce94976 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "foo": "bar" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected index 346cc2cf104..deedb1e2fea 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "addonmanager_kubernetes_io_mode": "EnsureExists" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected index ace72a2ccd9..7e3ff74968e 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected @@ -1,11 +1,13 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": null, "MetricSetFields": { "created": "2021-08-12T14:24:52.000Z", "name": "standard", - "provisioner": "rancher.io/local-path" + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer" }, "Index": "", "ID": "", diff --git a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go index 5a0ce3d0109..927468a676f 100644 --- a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go +++ b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go @@ -23,6 +23,8 @@ import ( p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" + "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) func init() { @@ -38,6 +40,7 @@ type StorageClassMetricSet struct { prometheus p.Prometheus mapping *p.MetricsMapping mod k8smod.Module + enricher util.Enricher } // NewStorageClassMetricSet returns a prometheus based metricset for Storage classes @@ -54,6 +57,7 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, prometheus: prometheus, mod: mod, + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.StorageClass{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ "kube_storageclass_info": p.InfoMetric(), @@ -66,10 +70,12 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { "kube_storageclass_created": p.Metric("created", p.OpUnixTimestampValue()), }, Labels: map[string]p.LabelMap{ - "storageclass": p.KeyLabel("name"), - "provisioner": p.Label("provisioner"), - "reclaimPolicy": p.Label("reclaim_policy"), - "volumeBindingMode": p.Label("volume_binding_mode"), + "storageclass": p.KeyLabel("name"), + "provisioner": p.Label("provisioner"), + "reclaimPolicy": p.Label("reclaim_policy"), + "reclaim_policy": p.Label("reclaim_policy"), + "volumeBindingMode": p.Label("volume_binding_mode"), + "volume_binding_mode": p.Label("volume_binding_mode"), }, }, }, nil @@ -78,6 +84,8 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as // module rooted fields at the event that gets reported func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { + m.enricher.Start() + families, err := m.mod.GetStateMetricsFamilies(m.prometheus) if err != nil { m.Logger().Error(err) @@ -91,12 +99,24 @@ func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { return } + m.enricher.Enrich(events) + for _, event := range events { - event[mb.NamespaceKey] = "storageclass" - reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil)) - if !reported { + + e, err := util.CreateEvent(event, "kubernetes.storageclass") + if err != nil { + m.Logger().Error(err) + } + + if reported := reporter.Event(e); !reported { m.Logger().Debug("error trying to emit event") return } } } + +// Close stops this metricset +func (m *StorageClassMetricSet) Close() error { + m.enricher.Stop() + return nil +} diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index c52f4bf8571..e6addbba791 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -158,6 +158,8 @@ func NewResourceMetadataEnricher( m[id] = metaGen.Generate("persistentvolume", r) case *kubernetes.PersistentVolumeClaim: m[id] = metaGen.Generate("persistentvolumeclaim", r) + case *kubernetes.StorageClass: + m[id] = metaGen.Generate("storageclass", r) default: m[id] = metaGen.Generate(r.GetObjectKind().GroupVersionKind().Kind, r) }