Add wildcard support to label to tag mappings for containers. (#2525)

* Add wildcard support to pod label to tag mappings for k8s.

* Add reno

* gofmt

* fix tests

Dockerfiles/agent/README.md

@@ -69,13 +69,27 @@ We automatically collect common tags from [Docker](https://github.com/DataDog/da
 - `DD_KUBERNETES_POD_LABELS_AS_TAGS` : extract pod labels
 - `DD_KUBERNETES_POD_ANNOTATIONS_AS_TAGS` : extract pod annotations
 
-You can either define them in your custom `datadog.yaml`, or set them as JSON maps in these envvars. The map key is the source (label/envvar) name, and the map value the datadog tag name.
+You can either define them in your custom `datadog.yaml`, or set them as JSON maps in these envvars. The map key is the source (label/envvar) name, and the map value the Datadog tag name.
 
 ```shell
 DD_KUBERNETES_POD_LABELS_AS_TAGS='{"app":"kube_app","release":"helm_release"}'
 DD_DOCKER_LABELS_AS_TAGS='{"com.docker.compose.service":"service_name"}'
 ```
 
+You can use shell patterns in label names to define simple rules for mapping labels to Datadog tag names using the same simple template system used by Autodiscovery. This is only supported by `DD_KUBERNETES_POD_LABELS_AS_TAGS`.
+
+To add all pod labels as tags to your metrics where tags names are prefixed by `kube_`, you can use the following:
+
+```shell
+DD_KUBERNETES_POD_LABELS_AS_TAGS='{"*":"kube_%%label%%"}'
+```
+
+To add only pod labels as tags to your metrics that start with `app`, you can use the following:
+
+```shell
+DD_KUBERNETES_POD_LABELS_AS_TAGS='{"app*":"kube_%%label%%"}'
+```
+
 #### Using secret files (BETA)
 
 Integration credentials can be stored in Docker / Kubernetes secrets and used in Autodiscovery templates. See the [setup instructions for the helper script](secrets-helper/README.md) and the [agent documentation](https://github.com/DataDog/datadog-agent/blob/6.4.x/docs/agent/secrets.md) for more information.

pkg/autodiscovery/configresolver/configresolver.go

@@ -11,7 +11,6 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-	"unicode"
 
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 
@@ -55,15 +54,14 @@ func Resolve(tpl integration.Config, svc listeners.Service) (integration.Config,
 		// Copy original content from template
 		vars := tpl.GetTemplateVariablesForInstance(i)
 		for _, v := range vars {
-			name, key := parseTemplateVar(v)
-			if f, found := templateVariables[string(name)]; found {
-				resolvedVar, err := f(key, svc)
+			if f, found := templateVariables[string(v.Name)]; found {
+				resolvedVar, err := f(v.Key, svc)
 				if err != nil {
 					return integration.Config{}, err
 				}
 				// init config vars are replaced by the first found
-				resolvedConfig.InitConfig = bytes.Replace(resolvedConfig.InitConfig, v, resolvedVar, -1)
-				resolvedConfig.Instances[i] = bytes.Replace(resolvedConfig.Instances[i], v, resolvedVar, -1)
+				resolvedConfig.InitConfig = bytes.Replace(resolvedConfig.InitConfig, v.Raw, resolvedVar, -1)
+				resolvedConfig.Instances[i] = bytes.Replace(resolvedConfig.Instances[i], v.Raw, resolvedVar, -1)
 			}
 		}
 		err = resolvedConfig.Instances[i].MergeAdditionalTags(tags)
@@ -178,20 +176,3 @@ func getEnvvar(tplVar []byte, svc listeners.Service) ([]byte, error) {
 	}
 	return []byte(value), nil
 }
-
-// parseTemplateVar extracts the name of the var
-// and the key (or index if it can be cast to an int)
-func parseTemplateVar(v []byte) (name, key []byte) {
-	stripped := bytes.Map(func(r rune) rune {
-		if unicode.IsSpace(r) || r == '%' {
-			return -1
-		}
-		return r
-	}, v)
-	parts := bytes.SplitN(stripped, []byte("_"), 2)
-	name = parts[0]
-	if len(parts) == 2 {
-		return name, parts[1]
-	}
-	return name, []byte("")
-}

pkg/autodiscovery/configresolver/configresolver_test.go

@@ -70,28 +70,6 @@ func (s *dummyService) GetCreationTime() integration.CreationTime {
 	return s.CreationTime
 }
 
-func TestParseTemplateVar(t *testing.T) {
-	name, key := parseTemplateVar([]byte("%%host%%"))
-	assert.Equal(t, "host", string(name))
-	assert.Equal(t, "", string(key))
-
-	name, key = parseTemplateVar([]byte("%%host_0%%"))
-	assert.Equal(t, "host", string(name))
-	assert.Equal(t, "0", string(key))
-
-	name, key = parseTemplateVar([]byte("%%host 0%%"))
-	assert.Equal(t, "host0", string(name))
-	assert.Equal(t, "", string(key))
-
-	name, key = parseTemplateVar([]byte("%%host_0_1%%"))
-	assert.Equal(t, "host", string(name))
-	assert.Equal(t, "0_1", string(key))
-
-	name, key = parseTemplateVar([]byte("%%host_network_name%%"))
-	assert.Equal(t, "host", string(name))
-	assert.Equal(t, "network_name", string(key))
-}
-
 func TestGetFallbackHost(t *testing.T) {
 	ip, err := getFallbackHost(map[string]string{"bridge": "172.17.0.1"})
 	assert.Equal(t, "172.17.0.1", ip)

pkg/autodiscovery/integration/config.go

@@ -8,17 +8,15 @@ package integration
 import (
 	"fmt"
 	"hash/fnv"
-	"regexp"
 	"sort"
 	"strconv"
 
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/DataDog/datadog-agent/pkg/util/log"
+	"github.com/DataDog/datadog-agent/pkg/util/tmplvar"
 )
 
-var tplVarRegex = regexp.MustCompile(`%%.+?%%`)
-
 // Data contains YAML code
 type Data []byte
 
@@ -147,11 +145,11 @@ func (c *Config) AddMetrics(metrics Data) error {
 
 // GetTemplateVariablesForInstance returns a slice of raw template variables
 // it found in a config instance template.
-func (c *Config) GetTemplateVariablesForInstance(i int) (vars [][]byte) {
+func (c *Config) GetTemplateVariablesForInstance(i int) []tmplvar.TemplateVar {
 	if len(c.Instances) < i {
-		return vars
+		return nil
 	}
-	return tplVarRegex.FindAll(c.Instances[i], -1)
+	return tmplvar.Parse(c.Instances[i])
 }
 
 // MergeAdditionalTags merges additional tags to possible existing config tags

pkg/tagger/collectors/common.go

@@ -9,8 +9,13 @@ import (
 	"strings"
 
 	"github.com/DataDog/datadog-agent/pkg/config"
+	"github.com/DataDog/datadog-agent/pkg/util/tmplvar"
 )
 
+var templateVariables = map[string]struct{}{
+	"label": {},
+}
+
 // retrieveMappingFromConfig gets a stringmapstring config key and
 // lowercases all map keys to make envvar and yaml sources consistent
 func retrieveMappingFromConfig(configKey string) map[string]string {
@@ -22,3 +27,16 @@ func retrieveMappingFromConfig(configKey string) map[string]string {
 
 	return labelsList
 }
+
+func resolveTag(tmpl, label string) string {
+	vars := tmplvar.ParseString(tmpl)
+	tagName := tmpl
+	for _, v := range vars {
+		if _, ok := templateVariables[string(v.Name)]; ok {
+			tagName = strings.Replace(tagName, string(v.Raw), label, -1)
+			continue
+		}
+		tagName = strings.Replace(tagName, string(v.Raw), "", -1)
+	}
+	return tagName
+}

pkg/tagger/collectors/common_test.go

@@ -1,6 +1,12 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2018 Datadog, Inc.
+
 package collectors
 
 import (
+	"fmt"
 	"sort"
 	"testing"
 
@@ -49,3 +55,32 @@ func assertTagInfoListEqual(t *testing.T, expectedUpdates []*TagInfo, updates []
 		assertTagInfoEqual(t, expectedUpdates[i], updates[i])
 	}
 }
+
+func TestResolveTag(t *testing.T) {
+	testCases := []struct {
+		tmpl, label, expected string
+	}{
+		{
+			"kube_%%label%%", "app", "kube_app",
+		},
+		{
+			"foo_%%label%%_bar", "app", "foo_app_bar",
+		},
+		{
+			"%%label%%%%label%%", "app", "appapp",
+		},
+		{
+			"kube_", "app", "kube_", // no template variable
+		},
+		{
+			"kube_%%foo%%", "app", "kube_", // unsupported template variable
+		},
+	}
+
+	for i, testCase := range testCases {
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			tagName := resolveTag(testCase.tmpl, testCase.label)
+			assert.Equal(t, testCase.expected, tagName)
+		})
+	}
+}

pkg/tagger/collectors/kubelet_extract.go

@@ -9,6 +9,7 @@ package collectors
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/DataDog/datadog-agent/pkg/util/log"
@@ -38,8 +39,10 @@ func (c *KubeletCollector) parsePods(pods []*kubelet.Pod) ([]*TagInfo, error) {
 
 		// Pod labels
 		for name, value := range pod.Metadata.Labels {
-			if tagName, found := c.labelsAsTags[strings.ToLower(name)]; found {
-				tags.AddAuto(tagName, value)
+			for pattern, tmpl := range c.labelsAsTags {
+				if ok, _ := filepath.Match(pattern, strings.ToLower(name)); ok {
+					tags.AddAuto(resolveTag(tmpl, name), value)
+				}
 			}
 		}
 

pkg/tagger/collectors/kubelet_extract_test.go

@@ -433,6 +433,42 @@ func TestParsePods(t *testing.T) {
 				},
 			}},
 		},
+		{
+			desc: "pod labels as tags with wildcards",
+			pod: &kubelet.Pod{
+				Metadata: kubelet.PodMetadata{
+					Labels: map[string]string{
+						"component":         "kube-proxy",
+						"tier":              "node",
+						"k8s-app":           "kubernetes-dashboard",
+						"pod-template-hash": "490794276",
+					},
+				},
+				Status: dockerContainerStatus,
+				Spec:   dockerContainerSpec,
+			},
+			labelsAsTags: map[string]string{
+				"*":         "foo_%%label%%",
+				"component": "component",
+			},
+			annotationsAsTags: map[string]string{},
+			expectedInfo: []*TagInfo{{
+				Source: "kubelet",
+				Entity: dockerEntityID,
+				LowCardTags: []string{
+					"foo_component:kube-proxy",
+					"component:kube-proxy",
+					"foo_tier:node",
+					"foo_k8s-app:kubernetes-dashboard",
+					"foo_pod-template-hash:490794276",
+					"image_name:datadog/docker-dd-agent",
+					"image_tag:latest5",
+					"kube_container_name:dd-agent",
+					"short_image:docker-dd-agent",
+				},
+				HighCardTags: []string{"container_id:d0242fc32d53137526dc365e7c86ef43b5f50b6f72dfd53dcb948eff4560376f"},
+			}},
+		},
 	} {
 		t.Run(fmt.Sprintf("case %d: %s", nb, tc.desc), func(t *testing.T) {
 			collector := &KubeletCollector{

pkg/util/tmplvar/parse.go

@@ -0,0 +1,54 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2018 Datadog, Inc.
+
+package tmplvar
+
+import (
+	"bytes"
+	"regexp"
+	"unicode"
+)
+
+var tmplVarRegex = regexp.MustCompile(`%%.+?%%`)
+
+// TemplateVar is the info for a parsed template variable.
+type TemplateVar struct {
+	Raw, Name, Key []byte
+}
+
+// ParseString returns parsed template variables found in the input string.
+func ParseString(s string) []TemplateVar {
+	return Parse([]byte(s))
+}
+
+// Parse returns parsed template variables found in the input data.
+func Parse(b []byte) []TemplateVar {
+	var parsed []TemplateVar
+	vars := tmplVarRegex.FindAll(b, -1)
+	for _, v := range vars {
+		name, key := parseTemplateVar(v)
+		parsed = append(parsed, TemplateVar{v, name, key})
+	}
+	return parsed
+}
+
+// parseTemplateVar extracts the name of the var and the key (or index if it can be
+// cast to an int)
+func parseTemplateVar(v []byte) (name, key []byte) {
+	stripped := bytes.Map(func(r rune) rune {
+		if unicode.IsSpace(r) || r == '%' {
+			return -1
+		}
+		return r
+	}, v)
+	split := bytes.SplitN(stripped, []byte("_"), 2)
+	name = split[0]
+	if len(split) == 2 {
+		key = split[1]
+	} else {
+		key = []byte("")
+	}
+	return name, key
+}

pkg/util/tmplvar/parse_test.go

@@ -0,0 +1,53 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2018 Datadog, Inc.
+
+package tmplvar
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseTemplateVar(t *testing.T) {
+	testCases := []struct {
+		tmpl, name, key string
+	}{
+		{
+			"%%host%%",
+			"host",
+			"",
+		},
+		{
+			"%%host_0%%",
+			"host",
+			"0",
+		},
+		{
+			"%%host 0%%",
+			"host0",
+			"",
+		},
+		{
+			"%%host_0_1%%",
+			"host",
+			"0_1",
+		},
+		{
+			"%%host_network_name%%",
+			"host",
+			"network_name",
+		},
+	}
+
+	for i, testCase := range testCases {
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			name, key := parseTemplateVar([]byte(testCase.tmpl))
+			assert.Equal(t, testCase.name, string(name))
+			assert.Equal(t, testCase.key, string(key))
+		})
+	}
+}

releasenotes/notes/support-wildcards-pod-labels-as-tags-45b873f7a94be498.yaml

@@ -0,0 +1,5 @@
+enhancements:
+  - |
+    Added support for wildcards to `DD_KUBERNETES_POD_LABELS_AS_TAGS`. For example,
+    `DD_KUBERNETES_POD_LABELS_AS_TAGS='{"*":"kube_%%label%%"}'` will all pod labels as
+    tags to your metrics with tags names prefixed by `kube_`.