Add ability to override cluster_uuid to be used in monitoring data (e…

…lastic#13182)

* Add ability to override cluster_uuid to be used in monitoring data

* Removing changes committed by accident

* Updating setting name and inline comments in reference config file

* Updating setting name in code

* Use struct tags pattern instead of repeating magic string

* Fixing logic after variable rename

* Remove unnecessary check and override

* Fixing duplicate comment

* Fixing comment

* Updating comment

auditbeat/auditbeat.reference.yml

@@ -1222,6 +1222,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

auditbeat/auditbeat.yml

@@ -168,6 +168,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

filebeat/filebeat.reference.yml

@@ -1923,6 +1923,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

filebeat/filebeat.yml

@@ -196,6 +196,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

heartbeat/heartbeat.reference.yml

@@ -1366,6 +1366,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Heartbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

heartbeat/heartbeat.yml

@@ -145,6 +145,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

journalbeat/journalbeat.reference.yml

@@ -1167,6 +1167,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Journalbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

journalbeat/journalbeat.yml

@@ -165,6 +165,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

libbeat/_meta/config.reference.yml.tmpl

@@ -1110,6 +1110,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# {{.BeatName | title}} instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

libbeat/_meta/config.yml.tmpl

@@ -123,6 +123,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

libbeat/cmd/instance/beat.go

@@ -394,6 +394,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error {
 		settings := report.Settings{
 			DefaultUsername: settings.Monitoring.DefaultUsername,
 			Format:          reporterSettings.Format,
+			ClusterUUID:     reporterSettings.ClusterUUID,
 		}
 		reporter, err := report.New(b.Info, settings, monitoringCfg, b.Config.Output)
 		if err != nil {

libbeat/monitoring/monitoring.go

@@ -97,7 +97,13 @@ func SelectConfig(beatCfg BeatConfig) (*common.Config, *report.Settings, error)
 		return monitoringCfg, &report.Settings{Format: report.FormatXPackMonitoringBulk}, nil
 	case beatCfg.Monitoring.Enabled():
 		monitoringCfg := beatCfg.Monitoring
-		return monitoringCfg, &report.Settings{Format: report.FormatBulk}, nil
+		var info struct {
+			ClusterUUID string `config:"cluster_uuid"`
+		}
+		if err := monitoringCfg.Unpack(&info); err != nil {
+			return nil, nil, err
+		}
+		return monitoringCfg, &report.Settings{Format: report.FormatBulk, ClusterUUID: info.ClusterUUID}, nil
 	default:
 		return nil, nil, nil
 	}

libbeat/monitoring/report/elasticsearch/config.go

@@ -46,6 +46,7 @@ type config struct {
 	Tags             []string          `config:"tags"`
 	Backoff          backoff           `config:"backoff"`
 	Format           report.Format     `config:"_format"`
+	ClusterUUID      string            `config:"cluster_uuid"`
 }
 
 type backoff struct {

libbeat/monitoring/report/elasticsearch/elasticsearch.go

@@ -98,7 +98,8 @@ func defaultConfig(settings report.Settings) config {
 			Init: 1 * time.Second,
 			Max:  60 * time.Second,
 		},
-		Format: report.FormatXPackMonitoringBulk,
+		Format:      report.FormatXPackMonitoringBulk,
+		ClusterUUID: settings.ClusterUUID,
 	}
 
 	if settings.DefaultUsername != "" {
@@ -261,12 +262,12 @@ func (r *reporter) initLoop(c config) {
 	log.Info("Successfully connected to X-Pack Monitoring endpoint.")
 
 	// Start collector and send loop if monitoring endpoint has been found.
-	go r.snapshotLoop("state", "state", c.StatePeriod)
+	go r.snapshotLoop("state", "state", c.StatePeriod, c.ClusterUUID)
 	// For backward compatibility stats is named to metrics.
-	go r.snapshotLoop("stats", "metrics", c.MetricsPeriod)
+	go r.snapshotLoop("stats", "metrics", c.MetricsPeriod, c.ClusterUUID)
 }
 
-func (r *reporter) snapshotLoop(namespace, prefix string, period time.Duration) {
+func (r *reporter) snapshotLoop(namespace, prefix string, period time.Duration, clusterUUID string) {
 	ticker := time.NewTicker(period)
 	defer ticker.Stop()
 
@@ -305,7 +306,9 @@ func (r *reporter) snapshotLoop(namespace, prefix string, period time.Duration)
 			"params": map[string]string{"interval": strconv.Itoa(int(period/time.Second)) + "s"},
 		}
 
-		clusterUUID := getClusterUUID()
+		if clusterUUID == "" {
+			clusterUUID = getClusterUUID()
+		}
 		if clusterUUID != "" {
 			meta.Put("cluster_uuid", clusterUUID)
 		}

libbeat/monitoring/report/report.go

@@ -50,6 +50,7 @@ type config struct {
 type Settings struct {
 	DefaultUsername string
 	Format          Format
+	ClusterUUID     string
 }
 
 type Reporter interface {

metricbeat/metricbeat.reference.yml

@@ -1881,6 +1881,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

metricbeat/metricbeat.yml

@@ -140,6 +140,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

packetbeat/packetbeat.reference.yml

@@ -1599,6 +1599,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Packetbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

packetbeat/packetbeat.yml

@@ -222,6 +222,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

winlogbeat/winlogbeat.reference.yml

@@ -1143,6 +1143,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Winlogbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

winlogbeat/winlogbeat.yml

@@ -147,6 +147,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/auditbeat/auditbeat.reference.yml

@@ -1273,6 +1273,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/auditbeat/auditbeat.yml

@@ -190,6 +190,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/filebeat/filebeat.reference.yml

@@ -2204,6 +2204,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/filebeat/filebeat.yml

@@ -196,6 +196,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/functionbeat/functionbeat.reference.yml

@@ -1060,6 +1060,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Functionbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/functionbeat/functionbeat.yml

@@ -364,6 +364,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/metricbeat/metricbeat.reference.yml

@@ -1978,6 +1978,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/metricbeat/metricbeat.yml

@@ -140,6 +140,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/winlogbeat/winlogbeat.reference.yml

@@ -1155,6 +1155,11 @@ logging.files:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Winlogbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.

x-pack/winlogbeat/winlogbeat.yml

@@ -159,6 +159,11 @@ processors:
 # Set to true to enable the monitoring reporter.
 #monitoring.enabled: false
 
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Auditbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.override_cluster_uuid:
+
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
 # Elasticsearch output are accepted here as well.
 # Note that the settings should point to your Elasticsearch *monitoring* cluster.