Provide option to control Prometheus labels

This change generalizes the existing ContainerNameToLabelsFunc to allow the user to fully control all labels attached to exported Prometheus metrics. The existing behavior is available as DefaultContainerLabelsFunc and is used if no custom function is provided.

This will allow Kubernetes to filter out its internal Docker labels.

Author: grobie

http/handlers.go

@@ -89,8 +89,11 @@ func RegisterHandlers(mux httpmux.Mux, containerManager manager.Manager, httpAut
 	return nil
 }
 
-func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, containerNameToLabelsFunc metrics.ContainerNameToLabelsFunc) {
-	collector := metrics.NewPrometheusCollector(containerManager, containerNameToLabelsFunc)
+// RegisterPrometheusHandler creates a new PrometheusCollector, registers it
+// on the global registry and configures the provided HTTP mux to handle the
+// given Prometheus endpoint.
+func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, f metrics.ContainerLabelsFunc) {
+	collector := metrics.NewPrometheusCollector(containerManager, f)
 	prometheus.MustRegister(collector)
 	mux.Handle(prometheusEndpoint, prometheus.Handler())
 }

metrics/prometheus.go

@@ -25,13 +25,14 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-// This will usually be manager.Manager, but can be swapped out for testing.
+// infoProvider will usually be manager.Manager, but can be swapped out for testing.
 type infoProvider interface {
-	// Get information about all subcontainers of the specified container (includes self).
+	// SubcontainersInfo provides information about all subcontainers of the
+	// specified container including itself.
 	SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error)
-	// Get information about the version.
+	// GetVersionInfo provides information about the version.
 	GetVersionInfo() (*info.VersionInfo, error)
-	// Get information about the machine.
+	// GetMachineInfo provides information about the machine.
 	GetMachineInfo() (*info.MachineInfo, error)
 }
 
@@ -56,8 +57,8 @@ func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64) metri
 	return values
 }
 
-// A containerMetric describes a multi-dimensional metric used for exposing
-// a certain type of container statistic.
+// containerMetric describes a multi-dimensional metric used for exposing a
+// certain type of container statistic.
 type containerMetric struct {
 	name        string
 	help        string
@@ -71,21 +72,29 @@ func (cm *containerMetric) desc(baseLabels []string) *prometheus.Desc {
 	return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil)
 }
 
-type ContainerNameToLabelsFunc func(containerName string) map[string]string
+// ContainerLabelsFunc defines all base labels and their values attached to
+// each metric exported by cAdvisor.
+type ContainerLabelsFunc func(*info.ContainerInfo) map[string]string
 
 // PrometheusCollector implements prometheus.Collector.
 type PrometheusCollector struct {
-	infoProvider          infoProvider
-	errors                prometheus.Gauge
-	containerMetrics      []containerMetric
-	containerNameToLabels ContainerNameToLabelsFunc
+	infoProvider        infoProvider
+	errors              prometheus.Gauge
+	containerMetrics    []containerMetric
+	containerLabelsFunc ContainerLabelsFunc
 }
 
-// NewPrometheusCollector returns a new PrometheusCollector.
-func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFunc) *PrometheusCollector {
+// NewPrometheusCollector returns a new PrometheusCollector. The passed
+// ContainerLabelsFunc specifies which base labels will be attached to all
+// exported metrics. If left to nil, the DefaultContainerLabels function
+// will be used instead.
+func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc) *PrometheusCollector {
+	if f == nil {
+		f = DefaultContainerLabels
+	}
 	c := &PrometheusCollector{
-		infoProvider:          infoProvider,
-		containerNameToLabels: f,
+		infoProvider:        i,
+		containerLabelsFunc: f,
 		errors: prometheus.NewGauge(prometheus.GaugeOpts{
 			Namespace: "container",
 			Name:      "scrape_error",
@@ -533,10 +542,38 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 const (
-	containerLabelPrefix = "container_label_"
-	containerEnvPrefix   = "container_env_"
+	// ContainerLabelPrefix is the prefix added to all container labels.
+	ContainerLabelPrefix = "container_label_"
+	// ContainerEnvPrefix is the prefix added to all env variable labels.
+	ContainerEnvPrefix = "container_env_"
+	// LabelID is the name of the id label.
+	LabelID = "id"
+	// LabelName is the name of the name label.
+	LabelName = "name"
+	// LabelImage is the name of the image label.
+	LabelImage = "image"
 )
 
+// DefaultContainerLabels implements ContainerLabelsFunc. It exports the
+// container name, first alias, image name as well as all its env and label
+// values.
+func DefaultContainerLabels(container *info.ContainerInfo) map[string]string {
+	set := map[string]string{LabelID: container.Name}
+	if len(container.Aliases) > 0 {
+		set[LabelName] = container.Aliases[0]
+	}
+	if image := container.Spec.Image; len(image) > 0 {
+		set[LabelImage] = image
+	}
+	for k, v := range container.Spec.Labels {
+		set[ContainerLabelPrefix+k] = v
+	}
+	for k, v := range container.Spec.Envs {
+		set[ContainerEnvPrefix+k] = v
+	}
+	return set
+}
+
 func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) {
 	containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1})
 	if err != nil {
@@ -545,56 +582,32 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
 		return
 	}
 	for _, container := range containers {
-		baseLabels := []string{"id"}
-		id := container.Name
-		name := id
-		if len(container.Aliases) > 0 {
-			name = container.Aliases[0]
-			baseLabels = append(baseLabels, "name")
-		}
-		image := container.Spec.Image
-		if len(image) > 0 {
-			baseLabels = append(baseLabels, "image")
-		}
-		baseLabelValues := []string{id, name, image}[:len(baseLabels)]
-
-		if c.containerNameToLabels != nil {
-			newLabels := c.containerNameToLabels(name)
-			for k, v := range newLabels {
-				baseLabels = append(baseLabels, sanitizeLabelName(k))
-				baseLabelValues = append(baseLabelValues, v)
-			}
-		}
-
-		for k, v := range container.Spec.Labels {
-			baseLabels = append(baseLabels, sanitizeLabelName(containerLabelPrefix+k))
-			baseLabelValues = append(baseLabelValues, v)
-		}
-		for k, v := range container.Spec.Envs {
-			baseLabels = append(baseLabels, sanitizeLabelName(containerEnvPrefix+k))
-			baseLabelValues = append(baseLabelValues, v)
+		labels, values := []string{}, []string{}
+		for l, v := range c.containerLabelsFunc(container) {
+			labels = append(labels, sanitizeLabelName(l))
+			values = append(values, v)
 		}
 
 		// Container spec
-		desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil)
-		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...)
+		desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", labels, nil)
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), values...)
 
 		if container.Spec.HasCpu {
-			desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", baseLabels, nil)
-			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), baseLabelValues...)
+			desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", labels, nil)
+			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), values...)
 			if container.Spec.Cpu.Quota != 0 {
-				desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", baseLabels, nil)
-				ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), baseLabelValues...)
+				desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", labels, nil)
+				ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), values...)
 			}
-			desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", baseLabels, nil)
-			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), baseLabelValues...)
+			desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", labels, nil)
+			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), values...)
 
 		}
 		if container.Spec.HasMemory {
-			desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", baseLabels, nil)
-			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), baseLabelValues...)
-			desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", baseLabels, nil)
-			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), baseLabelValues...)
+			desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", labels, nil)
+			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), values...)
+			desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", labels, nil)
+			ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), values...)
 		}
 
 		// Now for the actual metrics
@@ -603,9 +616,9 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
 			if cm.condition != nil && !cm.condition(container.Spec) {
 				continue
 			}
-			desc := cm.desc(baseLabels)
+			desc := cm.desc(labels)
 			for _, metricValue := range cm.getValues(stats) {
-				ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...)
+				ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(values, metricValue.labels...)...)
 			}
 		}
 	}

metrics/prometheus_test.go

@@ -180,10 +180,10 @@ var (
 )
 
 func TestPrometheusCollector(t *testing.T) {
-	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string {
-		return map[string]string{
-			"zone.name": "hello",
-		}
+	c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(container *info.ContainerInfo) map[string]string {
+		s := DefaultContainerLabels(container)
+		s["zone.name"] = "hello"
+		return s
 	})
 	prometheus.MustRegister(c)
 	defer prometheus.Unregister(c)
@@ -212,7 +212,7 @@ func testPrometheusCollector(t *testing.T, c *PrometheusCollector, metricsFile s
 			continue
 		}
 		if want != gotLines[i] {
-			t.Fatalf("want %s, got %s", want, gotLines[i])
+			t.Fatalf("unexpected metric line\nwant: %s\nhave: %s", want, gotLines[i])
 		}
 	}
 }
@@ -250,10 +250,10 @@ func TestPrometheusCollector_scrapeFailure(t *testing.T) {
 		shouldFail:         true,
 	}
 
-	c := NewPrometheusCollector(provider, func(name string) map[string]string {
-		return map[string]string{
-			"zone.name": "hello",
-		}
+	c := NewPrometheusCollector(provider, func(container *info.ContainerInfo) map[string]string {
+		s := DefaultContainerLabels(container)
+		s["zone.name"] = "hello"
+		return s
 	})
 	prometheus.MustRegister(c)
 	defer prometheus.Unregister(c)