Cherry-pick to 6.x: Add DeDot in add_docker_metadata processor (#9505) (#9602)

* Cherry-pick to 6.x: Add DeDot in add_docker_metadata processor (#9505)

* Add DeDot in add_docker_metadata processor

* Add dedot into config and default to be false

* Update changelog and documentation

* Add documentation into processors-using.asciidoc

* Run mage fmt update under x-pack filebeat

* Run mage fmt update on x-pack metricbeat

* Run update again

* Run make update fmt from top level dir

* Remove mistakes from rebase

* Remove repeated doc

(cherry picked from commit 58573a9)

* remove extra line in data.json

Author: kaiyan-sheng

CHANGELOG.asciidoc

@@ -259,6 +259,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits]
 - Report configured queue type. {pull}8091[8091]
 - Enable `host` and `cloud` metadata processors by default. {pull}8596[8596]
 - Autodiscovery no longer requires that the `condition` field be set. If left unset all configs will be matched. {pull}9029[9029]
+- Add DeDot method in add_docker_metadata processor in libbeat. {issue}9350[9350] {pull}9505[9505]
 
 *Filebeat*
 

auditbeat/auditbeat.reference.yml

@@ -286,6 +286,7 @@ auditbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

auditbeat/docs/getting-started.asciidoc

@@ -97,6 +97,22 @@ endif::[]
 [[docker]]
 *docker:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {stack-version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
+["source","sh",subs="attributes"]
+------------------------------------------------
+curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz
+tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz
+------------------------------------------------
+
+endif::[]
+
 See <<running-on-docker, Running on Docker>> for deploying Docker containers.
 
 [[win]]

filebeat/filebeat.reference.yml

@@ -965,6 +965,7 @@ filebeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

heartbeat/heartbeat.reference.yml

@@ -424,6 +424,7 @@ heartbeat.scheduler:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

journalbeat/journalbeat.reference.yml

@@ -237,6 +237,7 @@ journalbeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

libbeat/_meta/config.reference.yml

@@ -179,6 +179,7 @@
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

libbeat/docs/processors-using.asciidoc

@@ -848,6 +848,7 @@ processors:
     #match_source_index: 4
     #match_short_id: true
     #cleanup_timeout: 60
+    #labels.dedot: false
     # To connect to Docker over TLS you must specify a client and CA certificate.
     #ssl:
     #  certificate_authority: "/etc/pki/root/ca.pem"
@@ -885,6 +886,9 @@ for container ID. It defaults to 4 to match
 `cleanup_timeout`:: (Optional) Time of inactivity to consider we can clean and
 forget metadata for a container, 60s by default.
 
+`labels.dedot`:: (Optional) Default to be false. If set to true, replace dots in
+ labels with `_`.
+
 [[add-host-metadata]]
 === Add Host metadata
 

libbeat/processors/add_docker_metadata/add_docker_metadata.go

@@ -60,6 +60,7 @@ type addDockerMetadata struct {
 	pidFields []string      // Field names that contain PIDs.
 	cgroups   *common.Cache // Cache of PID (int) to cgropus (map[string]string).
 	hostFS    string        // Directory where /proc is found
+	dedot     bool          // If set to true, replace dots in labels with `_`.
 }
 
 func newDockerMetadataProcessor(cfg *common.Config) (processors.Processor, error) {
@@ -103,6 +104,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
 		sourceProcessor: sourceProcessor,
 		pidFields:       config.MatchPIDs,
 		hostFS:          config.HostFS,
+		dedot:           config.DeDot,
 	}, nil
 }
 
@@ -174,7 +176,12 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
 		if len(container.Labels) > 0 {
 			labels := common.MapStr{}
 			for k, v := range container.Labels {
-				safemapstr.Put(labels, k, v)
+				if d.dedot {
+					label := common.DeDot(k)
+					labels.Put(label, v)
+				} else {
+					safemapstr.Put(labels, k, v)
+				}
 			}
 			meta.Put("container.labels", labels)
 		}

libbeat/processors/add_docker_metadata/add_docker_metadata_test.go

@@ -148,6 +148,51 @@ func TestMatchContainer(t *testing.T) {
 	}, result.Fields)
 }
 
+func TestMatchContainerWithDedot(t *testing.T) {
+	testConfig, err := common.NewConfigFrom(map[string]interface{}{
+		"match_fields": []string{"foo"},
+		"labels.dedot": true,
+	})
+	assert.NoError(t, err)
+
+	p, err := buildDockerMetadataProcessor(testConfig, MockWatcherFactory(
+		map[string]*docker.Container{
+			"container_id": &docker.Container{
+				ID:    "container_id",
+				Image: "image",
+				Name:  "name",
+				Labels: map[string]string{
+					"a.x":   "1",
+					"b":     "2",
+					"b.foo": "3",
+				},
+			},
+		}))
+	assert.NoError(t, err, "initializing add_docker_metadata processor")
+
+	input := common.MapStr{
+		"foo": "container_id",
+	}
+	result, err := p.Run(&beat.Event{Fields: input})
+	assert.NoError(t, err, "processing an event")
+
+	assert.EqualValues(t, common.MapStr{
+		"docker": common.MapStr{
+			"container": common.MapStr{
+				"id":    "container_id",
+				"image": "image",
+				"labels": common.MapStr{
+					"a_x":   "1",
+					"b":     "2",
+					"b_foo": "3",
+				},
+				"name": "name",
+			},
+		},
+		"foo": "container_id",
+	}, result.Fields)
+}
+
 func TestMatchSource(t *testing.T) {
 	// Use defaults
 	testConfig, err := common.NewConfigFrom(map[string]interface{}{})

libbeat/processors/add_docker_metadata/config.go

@@ -33,6 +33,7 @@ type Config struct {
 	SourceIndex  int               `config:"match_source_index"` // Index in the source path split by / to look for container ID.
 	MatchPIDs    []string          `config:"match_pids"`         // A list of fields containing process IDs (PIDs).
 	HostFS       string            `config:"system.hostfs"`      // Specifies the mount point of the host’s filesystem for use in monitoring a host from within a container.
+	DeDot        bool              `config:"labels.dedot"`       // If set to true, replace dots in labels with `_`.
 
 	// Annotations are kept after container is killed, until they haven't been
 	// accessed for a full `cleanup_timeout`:
@@ -45,5 +46,6 @@ func defaultConfig() Config {
 		MatchSource: true,
 		SourceIndex: 4, // Use 4 to match the CID in /var/lib/docker/containers/<container_id>/*.log.
 		MatchPIDs:   []string{"process.pid", "process.ppid"},
+		DeDot:       false,
 	}
 }

metricbeat/metricbeat.reference.yml

@@ -864,6 +864,7 @@ metricbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

metricbeat/module/elasticsearch/node/_meta/data.json

@@ -47,4 +47,4 @@
     "service": {
         "name": "elasticsearch"
     }
-}
+}

packetbeat/packetbeat.reference.yml

@@ -660,6 +660,7 @@ packetbeat.protocols:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

winlogbeat/winlogbeat.reference.yml

@@ -208,6 +208,7 @@ winlogbeat.event_logs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

x-pack/filebeat/filebeat.reference.yml

@@ -993,6 +993,7 @@ filebeat.inputs:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

x-pack/functionbeat/functionbeat.reference.yml

@@ -273,6 +273,7 @@ functionbeat.provider.aws.functions:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"

x-pack/metricbeat/metricbeat.reference.yml

@@ -872,6 +872,7 @@ metricbeat.modules:
 #    match_source_index: 4
 #    match_short_id: false
 #    cleanup_timeout: 60
+#    labels.dedot: false
 #    # To connect to Docker over TLS you must specify a client and CA certificate.
 #    #ssl:
 #    #  certificate_authority: "/etc/pki/root/ca.pem"