Add support for multiple sets of hints on autodiscover (#18883)

Allows defining multiple sets of annotations similar to how processors are defined. This functionality already exists in heartbeat. This change standardizes the utility and adds it to logs and metrics hints builder as well.
elastic · Jun 23, 2020 · a5da85c · a5da85c
1 parent 3beb956
commit a5da85c
Show file tree

Hide file tree

Showing 11 changed files with 734 additions and 402 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -306,6 +306,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808]
 - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607]
 - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
+- Add support for multiple sets of hints on autodiscover {pull}18883[18883]
 
 *Auditbeat*
 

diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
@@ -79,24 +79,20 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*comm
 		hints, _ = hIface.(common.MapStr)
 	}
 
-	inputConfig := l.getInputs(hints)
+	inputConfig := l.getInputsConfigs(hints)
 
 	// If default config is disabled return nothing unless it's explicty enabled
 	if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) {
 		logp.Debug("hints.builder", "default config is disabled: %+v", event)
 		return []*common.Config{}
 	}
 
-	// If explicty disabled, return nothing
+	// If explictly disabled, return nothing
 	if builder.IsDisabled(hints, l.config.Key) {
 		logp.Debug("hints.builder", "logs disabled by hint: %+v", event)
 		return []*common.Config{}
 	}
 
-	// Clone original config, enable it if disabled
-	config, _ := common.NewConfigFrom(l.config.DefaultConfig)
-	config.Remove("enabled", -1)
-
 	host, _ := event["host"].(string)
 	if host == "" {
 		return []*common.Config{}
@@ -114,58 +110,67 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*comm
 		return template.ApplyConfigTemplate(event, configs)
 	}
 
-	tempCfg := common.MapStr{}
-	mline := l.getMultiline(hints)
-	if len(mline) != 0 {
-		tempCfg.Put(multiline, mline)
-	}
-	if ilines := l.getIncludeLines(hints); len(ilines) != 0 {
-		tempCfg.Put(includeLines, ilines)
-	}
-	if elines := l.getExcludeLines(hints); len(elines) != 0 {
-		tempCfg.Put(excludeLines, elines)
-	}
-
-	if procs := l.getProcessors(hints); len(procs) != 0 {
-		tempCfg.Put(processors, procs)
-	}
-
-	if jsonOpts := l.getJSONOptions(hints); len(jsonOpts) != 0 {
-		tempCfg.Put(json, jsonOpts)
-	}
-	// Merge config template with the configs from the annotations
-	if err := config.Merge(tempCfg); err != nil {
-		logp.Debug("hints.builder", "config merge failed with error: %v", err)
-		return []*common.Config{config}
-	}
+	var configs []*common.Config
+	inputs := l.getInputs(hints)
+	for _, h := range inputs {
+		// Clone original config, enable it if disabled
+		config, _ := common.NewConfigFrom(l.config.DefaultConfig)
+		config.Remove("enabled", -1)
+
+		tempCfg := common.MapStr{}
+		mline := l.getMultiline(h)
+		if len(mline) != 0 {
+			tempCfg.Put(multiline, mline)
+		}
+		if ilines := l.getIncludeLines(h); len(ilines) != 0 {
+			tempCfg.Put(includeLines, ilines)
+		}
+		if elines := l.getExcludeLines(h); len(elines) != 0 {
+			tempCfg.Put(excludeLines, elines)
+		}
 
-	module := l.getModule(hints)
-	if module != "" {
-		moduleConf := map[string]interface{}{
-			"module": module,
+		if procs := l.getProcessors(h); len(procs) != 0 {
+			tempCfg.Put(processors, procs)
 		}
 
-		filesets := l.getFilesets(hints, module)
-		for fileset, conf := range filesets {
-			filesetConf, _ := common.NewConfigFrom(config)
+		if jsonOpts := l.getJSONOptions(h); len(jsonOpts) != 0 {
+			tempCfg.Put(json, jsonOpts)
+		}
+		// Merge config template with the configs from the annotations
+		if err := config.Merge(tempCfg); err != nil {
+			logp.Debug("hints.builder", "config merge failed with error: %v", err)
+			continue
+		}
 
-			if inputType, _ := filesetConf.String("type", -1); inputType == harvester.ContainerType {
-				filesetConf.SetString("stream", -1, conf.Stream)
-			} else {
-				filesetConf.SetString("containers.stream", -1, conf.Stream)
+		module := l.getModule(hints)
+		if module != "" {
+			moduleConf := map[string]interface{}{
+				"module": module,
 			}
 
-			moduleConf[fileset+".enabled"] = conf.Enabled
-			moduleConf[fileset+".input"] = filesetConf
+			filesets := l.getFilesets(hints, module)
+			for fileset, conf := range filesets {
+				filesetConf, _ := common.NewConfigFrom(config)
+
+				if inputType, _ := filesetConf.String("type", -1); inputType == harvester.ContainerType {
+					filesetConf.SetString("stream", -1, conf.Stream)
+				} else {
+					filesetConf.SetString("containers.stream", -1, conf.Stream)
+				}
 
-			logp.Debug("hints.builder", "generated config %+v", moduleConf)
+				moduleConf[fileset+".enabled"] = conf.Enabled
+				moduleConf[fileset+".input"] = filesetConf
+
+				logp.Debug("hints.builder", "generated config %+v", moduleConf)
+			}
+			config, _ = common.NewConfigFrom(moduleConf)
 		}
-		config, _ = common.NewConfigFrom(moduleConf)
+		logp.Debug("hints.builder", "generated config %+v", config)
+		configs = append(configs, config)
 	}
-	logp.Debug("hints.builder", "generated config %+v", config)
 
 	// Apply information in event to the template to generate the final config
-	return template.ApplyConfigTemplate(event, []*common.Config{config})
+	return template.ApplyConfigTemplate(event, configs)
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {
@@ -186,7 +191,7 @@ func (l *logHints) getModule(hints common.MapStr) string {
 	return validModuleNames.ReplaceAllString(module, "")
 }
 
-func (l *logHints) getInputs(hints common.MapStr) []common.MapStr {
+func (l *logHints) getInputsConfigs(hints common.MapStr) []common.MapStr {
 	return builder.GetHintAsConfigs(hints, l.config.Key)
 }
 
@@ -248,3 +253,23 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 
 	return filesets
 }
+
+func (l *logHints) getInputs(hints common.MapStr) []common.MapStr {
+	modules := builder.GetHintsAsList(hints, l.config.Key)
+	var output []common.MapStr
+
+	for _, mod := range modules {
+		output = append(output, common.MapStr{
+			l.config.Key: mod,
+		})
+	}
+
+	// Generate this so that no hints with completely valid templates work
+	if len(output) == 0 {
+		output = append(output, common.MapStr{
+			l.config.Key: common.MapStr{},
+		})
+	}
+
+	return output
+}