Skip to content

Commit d5bda76

Browse files
kaiyan-shengkaiyan-sheng
authored
Add Dedot for Kubernetes labels and annotations (#9939)
* Never default to a qualifier when none of them are set. (#9148) Remove default version qualifier and rename the environment variable to set it from `BEAT_VERSION_QUALIFIER` to `VERSION_QUALIFIER` this will align with other parts of the stack. **Tested with filebeat.** ``` ❯ ./filebeat version [08:39:01] filebeat version 7.0.0 (amd64), libbeat 7.0.0 [0a0c267 built 2018-11-19 13:38:15 +0000 UTC] ``` **Without the patch** ``` ❯ ./filebeat version [08:40:07] filebeat version 7.0.0-alpha1 (amd64), libbeat 7.0.0-alpha1 [b007837 built 2018-11-19 13:39:59 +0000 UTC] ``` Fixes: #8384 * Add Dedot for Kubernetes labels and annotations * Add dedot options in libbeat kubernetes metadata * Update changelog * Refactor TestGenerateMapStrFromEvent * Update shared-autodiscover.asciidoc with dedot params * Add names for each unit test case in event_test.go * Fix rebase errors
1 parent c55226e commit d5bda76

File tree

9 files changed

+322
-20
lines changed

9 files changed

+322
-20
lines changed

‎CHANGELOG.next.asciidoc

+1
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
126126
- Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709]
127127
- Add docker `event` metricset. {pull}9856[9856]
128128
- Add 'performance' metricset to x-pack mssql module {pull}9826[9826]
129+
- Add DeDot for kubernetes labels and annotations. {issue}9860[9860] {pull}9939[9939]
129130

130131
*Packetbeat*
131132

‎libbeat/common/kubernetes/metadata.go

+19-5
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ type MetaGeneratorConfig struct {
4444

4545
// Undocumented settings, to be deprecated in favor of `drop_fields` processor:
4646
IncludeCreatorMetadata bool `config:"include_creator_metadata"`
47+
LabelsDedot bool `config:"labels.dedot"`
48+
AnnotationsDedot bool `config:"annotations.dedot"`
4749
}
4850

4951
type metaGenerator = MetaGeneratorConfig
@@ -53,6 +55,8 @@ func NewMetaGenerator(cfg *common.Config) (MetaGenerator, error) {
5355
// default settings:
5456
generator := metaGenerator{
5557
IncludeCreatorMetadata: true,
58+
LabelsDedot: false,
59+
AnnotationsDedot: false,
5660
}
5761

5862
err := cfg.Unpack(&generator)
@@ -70,18 +74,23 @@ func (g *metaGenerator) ResourceMetadata(obj Resource) common.MapStr {
7074
labelMap := common.MapStr{}
7175
if len(g.IncludeLabels) == 0 {
7276
for k, v := range obj.GetMetadata().Labels {
73-
safemapstr.Put(labelMap, k, v)
77+
if g.LabelsDedot {
78+
label := common.DeDot(k)
79+
labelMap.Put(label, v)
80+
} else {
81+
safemapstr.Put(labelMap, k, v)
82+
}
7483
}
7584
} else {
76-
labelMap = generateMapSubset(objMeta.Labels, g.IncludeLabels)
85+
labelMap = generateMapSubset(objMeta.Labels, g.IncludeLabels, g.LabelsDedot)
7786
}
7887

7988
// Exclude any labels that are present in the exclude_labels config
8089
for _, label := range g.ExcludeLabels {
8190
delete(labelMap, label)
8291
}
8392

84-
annotationsMap := generateMapSubset(objMeta.Annotations, g.IncludeAnnotations)
93+
annotationsMap := generateMapSubset(objMeta.Annotations, g.IncludeAnnotations, g.AnnotationsDedot)
8594
meta := common.MapStr{}
8695
if objMeta.GetNamespace() != "" {
8796
meta["namespace"] = objMeta.GetNamespace()
@@ -136,7 +145,7 @@ func (g *metaGenerator) ContainerMetadata(pod *Pod, container string) common.Map
136145
return podMeta
137146
}
138147

139-
func generateMapSubset(input map[string]string, keys []string) common.MapStr {
148+
func generateMapSubset(input map[string]string, keys []string, dedot bool) common.MapStr {
140149
output := common.MapStr{}
141150
if input == nil {
142151
return output
@@ -145,7 +154,12 @@ func generateMapSubset(input map[string]string, keys []string) common.MapStr {
145154
for _, key := range keys {
146155
value, ok := input[key]
147156
if ok {
148-
safemapstr.Put(output, key, value)
157+
if dedot {
158+
dedotKey := common.DeDot(key)
159+
output.Put(dedotKey, value)
160+
} else {
161+
safemapstr.Put(output, key, value)
162+
}
149163
}
150164
}
151165

‎libbeat/common/kubernetes/metadata_test.go

+86-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import (
2727
"github.com/elastic/beats/libbeat/common"
2828
)
2929

30-
func TestPodMetadataDeDot(t *testing.T) {
30+
func TestPodMetadata(t *testing.T) {
3131
UID := "005f3b90-4b9d-12f8-acf0-31020a840133"
3232
Deployment := "Deployment"
3333
test := "test"
@@ -104,3 +104,88 @@ func TestPodMetadataDeDot(t *testing.T) {
104104
assert.Equal(t, metaGen.PodMetadata(test.pod), test.meta)
105105
}
106106
}
107+
108+
func TestPodMetadataDeDot(t *testing.T) {
109+
UID := "005f3b90-4b9d-12f8-acf0-31020a840133"
110+
Deployment := "Deployment"
111+
test := "test"
112+
ReplicaSet := "ReplicaSet"
113+
True := true
114+
False := false
115+
tests := []struct {
116+
pod *Pod
117+
meta common.MapStr
118+
config *common.Config
119+
}{
120+
{
121+
pod: &Pod{
122+
Metadata: &metav1.ObjectMeta{
123+
Labels: map[string]string{"a.key": "foo", "a": "bar"},
124+
Uid: &UID,
125+
Namespace: &test,
126+
Annotations: map[string]string{"b.key": "foo", "b": "bar"},
127+
},
128+
Spec: &v1.PodSpec{
129+
NodeName: &test,
130+
},
131+
},
132+
meta: common.MapStr{
133+
"pod": common.MapStr{
134+
"name": "",
135+
"uid": "005f3b90-4b9d-12f8-acf0-31020a840133",
136+
},
137+
"node": common.MapStr{"name": "test"},
138+
"namespace": "test",
139+
"labels": common.MapStr{"a": "bar", "a_key": "foo"},
140+
"annotations": common.MapStr{"b": "bar", "b_key": "foo"},
141+
},
142+
config: common.NewConfig(),
143+
},
144+
{
145+
pod: &Pod{
146+
Metadata: &metav1.ObjectMeta{
147+
Labels: map[string]string{"a.key": "foo", "a": "bar"},
148+
Uid: &UID,
149+
OwnerReferences: []*metav1.OwnerReference{
150+
{
151+
Kind: &Deployment,
152+
Name: &test,
153+
Controller: &True,
154+
},
155+
{
156+
Kind: &ReplicaSet,
157+
Name: &ReplicaSet,
158+
Controller: &False,
159+
},
160+
},
161+
},
162+
Spec: &v1.PodSpec{
163+
NodeName: &test,
164+
},
165+
},
166+
meta: common.MapStr{
167+
"pod": common.MapStr{
168+
"name": "",
169+
"uid": "005f3b90-4b9d-12f8-acf0-31020a840133",
170+
},
171+
"node": common.MapStr{"name": "test"},
172+
"labels": common.MapStr{"a": "bar", "a_key": "foo"},
173+
"deployment": common.MapStr{"name": "test"},
174+
},
175+
config: common.NewConfig(),
176+
},
177+
}
178+
179+
for _, test := range tests {
180+
config, err := common.NewConfigFrom(map[string]interface{}{
181+
"labels.dedot": true,
182+
"annotations.dedot": true,
183+
"include_annotations": []string{"b", "b.key"},
184+
})
185+
metaGen, err := NewMetaGenerator(config)
186+
if err != nil {
187+
t.Fatal(err)
188+
}
189+
assert.Equal(t, metaGen.PodMetadata(test.pod), test.meta)
190+
}
191+
}

‎libbeat/docs/shared-autodiscover.asciidoc

+12
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,18 @@ event:
130130
If the `include_annotations` config is added to the provider config, then the list of annotations present in the config
131131
are added to the event.
132132

133+
If the `include_labels` config is added to the provider config, then the list of labels present in the config
134+
will be added to the event.
135+
136+
If the `exclude_labels` config is added to the provider config, then the list of labels present in the config
137+
will be excluded from the event.
138+
139+
if the `labels.dedot` config is set to be `true` in the provider config, then `.` in labels will be replaced with `_`.
140+
141+
if the `annotations.dedot` config is set to be `true` in the provider config, then `.` in annotations will be replaced
142+
with `_`.
143+
144+
133145
For example:
134146

135147
[source,yaml]

‎metricbeat/module/kubernetes/_meta/config.yml

+2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
# Enriching parameters:
1818
#add_metadata: true
1919
#in_cluster: true
20+
#labels.dedot: false
21+
#annotations.dedot: false
2022
# When used outside the cluster:
2123
#in_cluster: false
2224
#host: node_name

‎metricbeat/module/kubernetes/event/config.go

+10-6
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ import (
2323
)
2424

2525
type kubeEventsConfig struct {
26-
InCluster bool `config:"in_cluster"`
27-
KubeConfig string `config:"kube_config"`
28-
Namespace string `config:"namespace"`
29-
SyncPeriod time.Duration `config:"sync_period"`
26+
InCluster bool `config:"in_cluster"`
27+
KubeConfig string `config:"kube_config"`
28+
Namespace string `config:"namespace"`
29+
SyncPeriod time.Duration `config:"sync_period"`
30+
LabelsDedot bool `config:"labels.dedot"`
31+
AnnotationsDedot bool `config:"annotations.dedot"`
3032
}
3133

3234
type Enabled struct {
@@ -35,8 +37,10 @@ type Enabled struct {
3537

3638
func defaultKubernetesEventsConfig() kubeEventsConfig {
3739
return kubeEventsConfig{
38-
InCluster: true,
39-
SyncPeriod: 1 * time.Second,
40+
InCluster: true,
41+
SyncPeriod: 1 * time.Second,
42+
LabelsDedot: false,
43+
AnnotationsDedot: false,
4044
}
4145
}
4246

‎metricbeat/module/kubernetes/event/event.go

+36-8
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,16 @@ func init() {
4141
// MetricSet implements the mb.PushMetricSet interface, and therefore does not rely on polling.
4242
type MetricSet struct {
4343
mb.BaseMetricSet
44-
watcher kubernetes.Watcher
44+
watcher kubernetes.Watcher
45+
watchOptions kubernetes.WatchOptions
46+
dedotConfig dedotConfig
47+
}
48+
49+
// dedotConfig defines LabelsDedot and AnnotationsDedot.
50+
// Default to be false. If set to true, replace dots in labels with `_`.
51+
type dedotConfig struct {
52+
LabelsDedot bool `config:"labels.dedot"`
53+
AnnotationsDedot bool `config:"annotations.dedot"`
4554
}
4655

4756
// New create a new instance of the MetricSet
@@ -62,17 +71,26 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
6271
return nil, fmt.Errorf("fail to get kubernetes client: %s", err.Error())
6372
}
6473

65-
watcher, err := kubernetes.NewWatcher(client, &kubernetes.Event{}, kubernetes.WatchOptions{
74+
watchOptions := kubernetes.WatchOptions{
6675
SyncTimeout: config.SyncPeriod,
6776
Namespace: config.Namespace,
68-
})
77+
}
78+
79+
watcher, err := kubernetes.NewWatcher(client, &kubernetes.Event{}, watchOptions)
6980
if err != nil {
7081
return nil, fmt.Errorf("fail to init kubernetes watcher: %s", err.Error())
7182
}
7283

84+
dedotConfig := dedotConfig{
85+
LabelsDedot: config.LabelsDedot,
86+
AnnotationsDedot: config.AnnotationsDedot,
87+
}
88+
7389
return &MetricSet{
7490
BaseMetricSet: base,
91+
dedotConfig: dedotConfig,
7592
watcher: watcher,
93+
watchOptions: watchOptions,
7694
}, nil
7795
}
7896

@@ -81,10 +99,10 @@ func (m *MetricSet) Run(reporter mb.PushReporter) {
8199
now := time.Now()
82100
handler := kubernetes.ResourceEventHandlerFuncs{
83101
AddFunc: func(obj kubernetes.Resource) {
84-
reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event)))
102+
reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig))
85103
},
86104
UpdateFunc: func(obj kubernetes.Resource) {
87-
reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event)))
105+
reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig))
88106
},
89107
// ignore events that are deleted
90108
DeleteFunc: nil,
@@ -107,7 +125,7 @@ func (m *MetricSet) Run(reporter mb.PushReporter) {
107125
return
108126
}
109127

110-
func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr {
128+
func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig) common.MapStr {
111129
eventMeta := common.MapStr{
112130
"timestamp": common.MapStr{
113131
"created": kubernetes.Time(eve.Metadata.CreationTimestamp).UTC(),
@@ -123,7 +141,12 @@ func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr {
123141
if len(eve.Metadata.Labels) != 0 {
124142
labels := make(common.MapStr, len(eve.Metadata.Labels))
125143
for k, v := range eve.Metadata.Labels {
126-
safemapstr.Put(labels, k, v)
144+
if dedotConfig.LabelsDedot {
145+
label := common.DeDot(k)
146+
labels.Put(label, v)
147+
} else {
148+
safemapstr.Put(labels, k, v)
149+
}
127150
}
128151

129152
eventMeta["labels"] = labels
@@ -132,7 +155,12 @@ func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr {
132155
if len(eve.Metadata.Annotations) != 0 {
133156
annotations := make(common.MapStr, len(eve.Metadata.Annotations))
134157
for k, v := range eve.Metadata.Annotations {
135-
safemapstr.Put(annotations, k, v)
158+
if dedotConfig.AnnotationsDedot {
159+
annotation := common.DeDot(k)
160+
annotations.Put(annotation, v)
161+
} else {
162+
safemapstr.Put(annotations, k, v)
163+
}
136164
}
137165

138166
eventMeta["annotations"] = annotations

0 commit comments

Comments
 (0)