From 122f2075c7f078f96067004cc06edf45b61064b2 Mon Sep 17 00:00:00 2001 From: Jimmy K Seto Date: Mon, 2 Nov 2020 13:39:40 -0500 Subject: [PATCH] fix container resource limits metrics, which got dropped btwn 2.0.0-alpha.1 and 2.0.0-alpha.2 --- docs/pod-metrics.md | 2 +- internal/store/pod.go | 44 +++++++++++++++++++++++++++----------- internal/store/pod_test.go | 8 ++++++- main_test.go | 8 ++++++- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/docs/pod-metrics.md b/docs/pod-metrics.md index e9460ee4b3..44750d8332 100644 --- a/docs/pod-metrics.md +++ b/docs/pod-metrics.md @@ -21,7 +21,7 @@ | kube_pod_container_status_ready | Gauge | `container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | STABLE | | kube_pod_container_status_restarts_total | Counter | `container`=<container-name>
`namespace`=<pod-namespace>
`pod`=<pod-name> | STABLE | | kube_pod_container_resource_requests | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | EXPERIMENTAL | -| kube_pod_container_resource_limits | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace> | EXPERIMENTAL | +| kube_pod_container_resource_limits | Gauge | `resource`=<resource-name>
`unit`=<resource-unit>
`container`=<container-name>
`pod`=<pod-name>
`namespace`=<pod-namespace>
`node`=< node-name> | EXPERIMENTAL | | kube_pod_overhead_cpu_cores | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | EXPERIMENTAL | | kube_pod_overhead_memory_bytes | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | EXPERIMENTAL | | kube_pod_runtimeclass_name_info | Gauge | `pod`=<pod-name>
`namespace`=<pod-namespace> | EXPERIMENTAL | diff --git a/internal/store/pod.go b/internal/store/pod.go index 4dd352a078..b57554925b 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -895,29 +895,47 @@ var ( lim := c.Resources.Limits for resourceName, val := range lim { - if isHugePageResourceName(resourceName) { - ms = append(ms, &metric.Metric{ - LabelValues: []string{c.Name, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, - Value: float64(val.Value()), - }) - } - if isAttachableVolumeResourceName(resourceName) { + switch resourceName { + case v1.ResourceCPU: ms = append(ms, &metric.Metric{ - Value: float64(val.Value()), - LabelValues: []string{c.Name, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitCore)}, + Value: float64(val.MilliValue()) / 1000, }) - } - if isExtendedResourceName(resourceName) { + case v1.ResourceStorage: + fallthrough + case v1.ResourceEphemeralStorage: + fallthrough + case v1.ResourceMemory: ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, Value: float64(val.Value()), - LabelValues: []string{c.Name, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, }) + default: + if isHugePageResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + Value: float64(val.Value()), + }) + } + if isAttachableVolumeResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitByte)}, + }) + } + if isExtendedResourceName(resourceName) { + ms = append(ms, &metric.Metric{ + Value: float64(val.Value()), + LabelValues: []string{c.Name, p.Spec.NodeName, sanitizeLabelName(string(resourceName)), string(constant.UnitInteger)}, + }) + + } } } } for _, metric := range ms { - metric.LabelKeys = []string{"container", "resource", "unit"} + metric.LabelKeys = []string{"container", "node", "resource", "unit"} } return &metric.Family{ diff --git a/internal/store/pod_test.go b/internal/store/pod_test.go index 4d66286bab..73265969e7 100644 --- a/internal/store/pod_test.go +++ b/internal/store/pod_test.go @@ -1403,7 +1403,13 @@ kube_pod_container_status_last_terminated_reason{container="container7",namespac # TYPE kube_pod_init_container_resource_requests_memory_bytes gauge # TYPE kube_pod_init_container_resource_requests_storage_bytes gauge # TYPE kube_pod_init_container_status_last_terminated_reason gauge - kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",pod="pod1",resource="nvidia_com_gpu",unit="integer"} 1 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core"} 0.2 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte"} 3e+08 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte"} 1e+08 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="nvidia_com_gpu",unit="integer"} 1 + kube_pod_container_resource_limits{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="storage",unit="byte"} 4e+08 + kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core"} 0.3 + kube_pod_container_resource_limits{container="pod1_con2",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte"} 2e+08 kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="cpu",unit="core"} 0.2 kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="ephemeral_storage",unit="byte"} 3e+08 kube_pod_container_resource_requests{container="pod1_con1",namespace="ns1",node="",pod="pod1",resource="memory",unit="byte"} 1e+08 diff --git a/main_test.go b/main_test.go index 17739a7e1a..3d74e9666d 100644 --- a/main_test.go +++ b/main_test.go @@ -266,7 +266,13 @@ func TestFullScrapeCycle(t *testing.T) { # TYPE kube_pod_status_unschedulable gauge kube_pod_container_info{namespace="default",pod="pod0",container="container2",image="k8s.gcr.io/hyperkube2",image_id="docker://sha256:bbb",container_id="docker://cd456"} 1 kube_pod_container_info{namespace="default",pod="pod0",container="container3",image="k8s.gcr.io/hyperkube3",image_id="docker://sha256:ccc",container_id="docker://ef789"} 1 -kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",resource="nvidia_com_gpu",unit="integer"} 1 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="nvidia_com_gpu",unit="integer"} 1 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3 +kube_pod_container_resource_limits{namespace="default",pod="pod0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08 kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="cpu",unit="core"} 0.2 kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="ephemeral_storage",unit="byte"} 3e+08 kube_pod_container_resource_requests{namespace="default",pod="pod0",container="pod1_con1",node="node1",resource="memory",unit="byte"} 1e+08