Generate configuration from mage for all Beats (elastic#12618)

From now on all Beats use `mage` when generating configuration files. This comes with the following changes: * Heartbeat does not use `post_process_config.py` to modify processors, instead it is added using the templating * Functionbeat does not include unsupported outputs * All Beats can alter the common configuration coming from `libbeat`. ### What does it mean for Community Beats? When adopting this version of `libbeat`, you need to make sure that you have a target named `Config` in your `magefile.go`. An example target which generates a sample configuration, a reference configuration and a Docker configuration from `{yourbeat}/_meta/beat.yml` and the common configuration from `libbeat` is the following: ```golang import devtools "github.com/elastic/beats/dev-tools/mage" func Config() error { return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".") } ``` If the configuration of your Beat consists of multiple files, you can pass your own instance of `ConfigFileParams`: ```golang func ConfigFileParams() devtools.ConfigFileParams { return devtools.ConfigFileParams{ ShortParts: []string{ devtools.OSSBeatDir("_meta/beat.yml"), devtools.OSSBeatDir("_meta/beat.custom.yml"), devtools.LibbeatDir("_meta/config.yml.tmpl"), }, ReferenceParts: []string{ devtools.OSSBeatDir("_meta/beat.reference.yml"), devtools.OSSBeatDir("_meta/beat.custom.referenceyml"), devtools.LibbeatDir("_meta/config.reference.yml.tmpl"), }, DockerParts: []string{ devtools.OSSBeatDir("_meta/beat.docker.yml"), devtools.LibbeatDir("_meta/config.docker.yml"), }, } } ``` It is also possible to pass variables to the templates using the `ExtraVars` attribute of `ConfigFileParams` struct.
ChrsMark · Jun 25, 2019 · 055dec8 · 055dec8
1 parent a33a8e1
commit 055dec8
Show file tree

Hide file tree

Showing 49 changed files with 593 additions and 810 deletions.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -41,3 +41,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add new option `IgnoreAllErrors` to `libbeat.common.schema` for skipping fields that failed while converting. {pull}12089[12089]
 - Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]
 - Use the go-lookslike library for testing in heartbeat. Eventually the mapval package will be replaced with it. {pull}12540[12540]
+- Generate configuration from `mage` for all Beats. {pull}12618[12618]
diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -368,7 +368,7 @@ auditbeat.modules:
 
 #============================= Elastic Cloud ==================================
 
-# These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
+# These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
 
 # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
 # `setup.kibana.host` options.
@@ -494,7 +494,6 @@ output.elasticsearch:
   # never, once, and freely. Default is never.
   #ssl.renegotiation: never
 
-
 #----------------------------- Logstash output ---------------------------------
 #output.logstash:
   # Boolean flag to enable or disable the output module.
@@ -638,7 +637,7 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
-  # Kafka version auditbeat is assumed to run against. Defaults to the "1.0.0".
+  # Kafka version Auditbeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 
   # Configure JSON encoding
@@ -895,7 +894,7 @@ output.elasticsearch:
   #filename: auditbeat
 
   # Maximum size in kilobytes of each file. When this size is reached, and on
-  # every auditbeat restart, the files are rotated. The default value is 10240
+  # every Auditbeat restart, the files are rotated. The default value is 10240
   # kB.
   #rotate_every_kb: 10000
 
@@ -907,7 +906,6 @@ output.elasticsearch:
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
 
-
 #----------------------------- Console output ---------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.
@@ -923,26 +921,26 @@ output.elasticsearch:
 
 #================================= Paths ======================================
 
-# The home path for the auditbeat installation. This is the default base path
+# The home path for the Auditbeat installation. This is the default base path
 # for all other path settings and for miscellaneous files that come with the
 # distribution (for example, the sample dashboards).
 # If not set by a CLI flag or in the configuration file, the default for the
 # home path is the location of the binary.
 #path.home:
 
-# The configuration path for the auditbeat installation. This is the default
+# The configuration path for the Auditbeat installation. This is the default
 # base path for configuration files, including the main YAML configuration file
 # and the Elasticsearch template file. If not set by a CLI flag or in the
 # configuration file, the default for the configuration path is the home path.
 #path.config: ${path.home}
 
-# The data path for the auditbeat installation. This is the default base path
-# for all the files in which auditbeat needs to store its data. If not set by a
+# The data path for the Auditbeat installation. This is the default base path
+# for all the files in which Auditbeat needs to store its data. If not set by a
 # CLI flag or in the configuration file, the default for the data path is a data
 # subdirectory inside the home path.
 #path.data: ${path.home}/data
 
-# The logs path for a auditbeat installation. This is the default location for
+# The logs path for a Auditbeat installation. This is the default location for
 # the Beat's log files. If not set by a CLI flag or in the configuration file,
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
@@ -1147,7 +1145,7 @@ setup.kibana:
 # Send all logging output to Windows Event Logs. The default is false.
 #logging.to_eventlog: false
 
-# If enabled, auditbeat periodically logs its internal metrics that have changed
+# If enabled, Auditbeat periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
 # all non-zero internal metrics are logged on shutdown. The default is true.
@@ -1194,7 +1192,7 @@ logging.files:
 
 
 #============================== Xpack Monitoring ===============================
-# auditbeat can export internal metrics to a central Elasticsearch monitoring
+# Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
 # reporting is disabled by default.
 

diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
@@ -101,7 +101,7 @@ setup.kibana:
 
 #============================= Elastic Cloud ==================================
 
-# These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
+# These settings simplify using Auditbeat with the Elastic Cloud (https://cloud.elastic.co/).
 
 # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
 # `setup.kibana.host` options.

diff --git a/auditbeat/scripts/mage/config.go b/auditbeat/scripts/mage/config.go
@@ -67,12 +67,12 @@ func configFileParams(dirs ...string) (devtools.ConfigFileParams, error) {
 			devtools.OSSBeatDir("_meta/common.p1.yml"),
 			configFiles,
 			devtools.OSSBeatDir("_meta/common.p2.yml"),
-			devtools.LibbeatDir("_meta/config.yml"),
+			devtools.LibbeatDir("_meta/config.yml.tmpl"),
 		),
 		ReferenceParts: join(
 			devtools.OSSBeatDir("_meta/common.reference.yml"),
 			configFiles,
-			devtools.LibbeatDir("_meta/config.reference.yml"),
+			devtools.LibbeatDir("_meta/config.reference.yml.tmpl"),
 		),
 		DockerParts: []string{
 			devtools.OSSBeatDir("_meta/beat.docker.yml"),

diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go
@@ -38,6 +38,21 @@ var (
 	shortTemplate     = filepath.Join("build", BeatName+".yml.tmpl")
 	referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl")
 	dockerTemplate    = filepath.Join("build", BeatName+".docker.yml.tmpl")
+
+	defaultConfigFileParams = ConfigFileParams{
+		ShortParts: []string{
+			OSSBeatDir("_meta/beat.yml"),
+			LibbeatDir("_meta/config.yml.tmpl"),
+		},
+		ReferenceParts: []string{
+			OSSBeatDir("_meta/beat.reference.yml"),
+			LibbeatDir("_meta/config.reference.yml.tmpl"),
+		},
+		DockerParts: []string{
+			OSSBeatDir("_meta/beat.docker.yml"),
+			LibbeatDir("_meta/config.docker.yml"),
+		},
+	}
 )
 
 // ConfigFileType is a bitset that indicates what types of config files to
@@ -73,18 +88,33 @@ type ConfigFileParams struct {
 	ExtraVars      map[string]interface{}
 }
 
+// Empty checks if configuration files are set.
+func (c ConfigFileParams) Empty() bool {
+	return len(c.ShortParts) == len(c.ReferenceDeps) && len(c.ReferenceParts) == len(c.DockerParts) && len(c.DockerParts) == 0
+}
+
 // Config generates config files. Set DEV_OS and DEV_ARCH to change the target
 // host for the generated configs. Defaults to linux/amd64.
 func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error {
+	if args.Empty() {
+		args = defaultConfigFileParams
+	}
+
 	if err := makeConfigTemplates(types, args); err != nil {
 		return errors.Wrap(err, "failed making config templates")
 	}
 
 	params := map[string]interface{}{
-		"GOOS":      EnvOr("DEV_OS", "linux"),
-		"GOARCH":    EnvOr("DEV_ARCH", "amd64"),
-		"Reference": false,
-		"Docker":    false,
+		"GOOS":                 EnvOr("DEV_OS", "linux"),
+		"GOARCH":               EnvOr("DEV_ARCH", "amd64"),
+		"Reference":            false,
+		"Docker":               false,
+		"ExcludeConsole":       false,
+		"ExcludeFileOutput":    false,
+		"ExcludeKafka":         false,
+		"ExcludeLogstash":      false,
+		"ExcludeRedis":         false,
+		"UseObserverProcessor": false,
 	}
 	for k, v := range args.ExtraVars {
 		params[k] = v

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1069,7 +1069,7 @@ filebeat.inputs:
 
 #============================= Elastic Cloud ==================================
 
-# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
 
 # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
 # `setup.kibana.host` options.
@@ -1195,7 +1195,6 @@ output.elasticsearch:
   # never, once, and freely. Default is never.
   #ssl.renegotiation: never
 
-
 #----------------------------- Logstash output ---------------------------------
 #output.logstash:
   # Boolean flag to enable or disable the output module.
@@ -1339,7 +1338,7 @@ output.elasticsearch:
   #username: ''
   #password: ''
 
-  # Kafka version filebeat is assumed to run against. Defaults to the "1.0.0".
+  # Kafka version Filebeat is assumed to run against. Defaults to the "1.0.0".
   #version: '1.0.0'
 
   # Configure JSON encoding
@@ -1596,7 +1595,7 @@ output.elasticsearch:
   #filename: filebeat
 
   # Maximum size in kilobytes of each file. When this size is reached, and on
-  # every filebeat restart, the files are rotated. The default value is 10240
+  # every Filebeat restart, the files are rotated. The default value is 10240
   # kB.
   #rotate_every_kb: 10000
 
@@ -1608,7 +1607,6 @@ output.elasticsearch:
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
 
-
 #----------------------------- Console output ---------------------------------
 #output.console:
   # Boolean flag to enable or disable the output module.
@@ -1624,26 +1622,26 @@ output.elasticsearch:
 
 #================================= Paths ======================================
 
-# The home path for the filebeat installation. This is the default base path
+# The home path for the Filebeat installation. This is the default base path
 # for all other path settings and for miscellaneous files that come with the
 # distribution (for example, the sample dashboards).
 # If not set by a CLI flag or in the configuration file, the default for the
 # home path is the location of the binary.
 #path.home:
 
-# The configuration path for the filebeat installation. This is the default
+# The configuration path for the Filebeat installation. This is the default
 # base path for configuration files, including the main YAML configuration file
 # and the Elasticsearch template file. If not set by a CLI flag or in the
 # configuration file, the default for the configuration path is the home path.
 #path.config: ${path.home}
 
-# The data path for the filebeat installation. This is the default base path
-# for all the files in which filebeat needs to store its data. If not set by a
+# The data path for the Filebeat installation. This is the default base path
+# for all the files in which Filebeat needs to store its data. If not set by a
 # CLI flag or in the configuration file, the default for the data path is a data
 # subdirectory inside the home path.
 #path.data: ${path.home}/data
 
-# The logs path for a filebeat installation. This is the default location for
+# The logs path for a Filebeat installation. This is the default location for
 # the Beat's log files. If not set by a CLI flag or in the configuration file,
 # the default for the logs path is a logs subdirectory inside the home path.
 #path.logs: ${path.home}/logs
@@ -1848,7 +1846,7 @@ setup.kibana:
 # Send all logging output to Windows Event Logs. The default is false.
 #logging.to_eventlog: false
 
-# If enabled, filebeat periodically logs its internal metrics that have changed
+# If enabled, Filebeat periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
 # all non-zero internal metrics are logged on shutdown. The default is true.
@@ -1895,7 +1893,7 @@ logging.files:
 
 
 #============================== Xpack Monitoring ===============================
-# filebeat can export internal metrics to a central Elasticsearch monitoring
+# Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
 # reporting is disabled by default.
 

diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -129,7 +129,7 @@ setup.kibana:
 
 #============================= Elastic Cloud ==================================
 
-# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
 
 # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
 # `setup.kibana.host` options.

diff --git a/filebeat/scripts/mage/config.go b/filebeat/scripts/mage/config.go
@@ -32,15 +32,15 @@ func configFileParams(moduleDirs ...string) devtools.ConfigFileParams {
 		ShortParts: []string{
 			devtools.OSSBeatDir("_meta/common.p1.yml"),
 			devtools.OSSBeatDir("_meta/common.p2.yml"),
-			devtools.LibbeatDir("_meta/config.yml"),
+			devtools.LibbeatDir("_meta/config.yml.tmpl"),
 		},
 		ReferenceDeps: []interface{}{collectModuleConfig},
 		ReferenceParts: []string{
 			devtools.OSSBeatDir("_meta/common.reference.p1.yml"),
 			modulesConfigYml,
 			devtools.OSSBeatDir("_meta/common.reference.inputs.yml"),
 			devtools.OSSBeatDir("_meta/common.reference.p2.yml"),
-			devtools.LibbeatDir("_meta/config.reference.yml"),
+			devtools.LibbeatDir("_meta/config.reference.yml.tmpl"),
 		},
 		DockerParts: []string{
 			devtools.OSSBeatDir("_meta/beat.docker.yml"),
@@ -65,7 +65,7 @@ func XPackConfigFileParams() devtools.ConfigFileParams {
 		devtools.OSSBeatDir("_meta/common.reference.inputs.yml"),
 		"_meta/common.reference.inputs.yml", // Added only to X-Pack.
 		devtools.OSSBeatDir("_meta/common.reference.p2.yml"),
-		devtools.LibbeatDir("_meta/config.reference.yml"),
+		devtools.LibbeatDir("_meta/config.reference.yml.tmpl"),
 	}
 	return args
 }
diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go
@@ -109,3 +109,8 @@ func GoTestUnit(ctx context.Context) error {
 func GoTestIntegration(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
 }
+
+// Config generates both the short/reference/docker configs.
+func Config() error {
+	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+}
diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go
@@ -122,3 +122,8 @@ func GoTestUnit(ctx context.Context) error {
 func GoTestIntegration(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
 }
+
+// Config generates both the short/reference/docker configs.
+func Config() error {
+	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+}