Allow rollover alias to use reference like agent.version or agent.name (

#12285)

* Allow rollover alias to use reference like agent.version or agent.name

Previously you were not able to defined fields in the roll over alias,
this commit allow to use some of the global values. We also update the
docs to remove the prefix part.

Fixes #12233

CHANGELOG.asciidoc

@@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-rc1...v7.0.0-rc2[Check the HEAD
 *Affecting all Beats*
 
 - Fixed OS family classification in `add_host_metadata` for Amazon Linux, Raspbian, and RedHat Linux. {issue}9134[9134] {pull}11494[11494]
+- Allow 'ilm.rollover_alias' to expand global fields like `agent.version`. {issue}12233[12233]
 
 *Auditbeat*
 

libbeat/common/fmtstr/formatevents.go

@@ -43,9 +43,10 @@ import (
 // Default values are given defined by the colon operator. For example:
 // `%{[field.name]:default value}`.
 type EventFormatString struct {
-	formatter StringFormatter
-	fields    []fieldInfo
-	timestamp bool
+	expression string
+	formatter  StringFormatter
+	fields     []fieldInfo
+	timestamp  bool
 }
 
 type eventFieldEvaler struct {
@@ -142,9 +143,10 @@ func CompileEvent(in string) (*EventFormatString, error) {
 
 	ctx.keys = make([]string, len(keys))
 	efs := &EventFormatString{
-		formatter: sf,
-		fields:    keys,
-		timestamp: efComp.timestamp,
+		expression: in,
+		formatter:  sf,
+		fields:     keys,
+		timestamp:  efComp.timestamp,
 	}
 	return efs, nil
 }
@@ -266,6 +268,12 @@ func (fs *EventFormatString) collectFields(
 	return nil
 }
 
+// IsEmpty returns true if the format string expression is an empty string, can be used
+// when validating to make defining a string mandatory.
+func (fs *EventFormatString) IsEmpty() bool {
+	return len(fs.expression) == 0
+}
+
 func (e *eventFieldCompiler) compileExpression(
 	s string,
 	opts []VariableOp,

libbeat/common/fmtstr/formatevents_test.go

@@ -35,6 +35,13 @@ func TestEventFormatString(t *testing.T) {
 		expected string
 		fields   []string
 	}{
+		{
+			"empty string",
+			"",
+			beat.Event{},
+			"",
+			nil,
+		},
 		{
 			"no fields configured",
 			"format string",
@@ -261,3 +268,15 @@ func TestEventFormatStringFromConfig(t *testing.T) {
 		assert.Equal(t, test.expected, actual)
 	}
 }
+
+func TestIsEmpty(t *testing.T) {
+	t.Run("when string is Empty", func(t *testing.T) {
+		fs := MustCompileEvent("")
+		assert.True(t, fs.IsEmpty())
+	})
+	t.Run("when string is not Empty", func(t *testing.T) {
+		fs := MustCompileEvent("hello")
+		assert.False(t, fs.IsEmpty())
+	})
+
+}

libbeat/docs/shared-ilm.asciidoc

@@ -51,8 +51,7 @@ required license; otherwise, {beatname_uc} creates daily indices.
 ==== `setup.ilm.rollover_alias`
 
 The index lifecycle write alias name. The default is
-+{beatname_lc}-\{{beat_version_key}\}+. Setting this option changes the prefix
-in the alias name. It doesn't remove +{beat_version_key}+ from the alias name. 
++{beatname_lc}-\{{beat_version_key}\}+. Setting this option changes the alias name.
 
 NOTE: If you modify this setting after loading the index template, you must
 overwrite the template to apply the changes.

libbeat/idxmgmt/ilm/config.go

@@ -32,7 +32,7 @@ type Config struct {
 	Mode          Mode                     `config:"enabled"`
 	PolicyName    fmtstr.EventFormatString `config:"policy_name"`
 	PolicyFile    string                   `config:"policy_file"`
-	RolloverAlias string                   `config:"rollover_alias"`
+	RolloverAlias fmtstr.EventFormatString `config:"rollover_alias"`
 	Pattern       string                   `config:"pattern"`
 
 	// CheckExists can disable the check for an existing policy. Check required
@@ -103,20 +103,20 @@ func (m *Mode) Unpack(in string) error {
 
 //Validate verifies that expected config options are given and valid
 func (cfg *Config) Validate() error {
-	if cfg.RolloverAlias == "" && cfg.Mode != ModeDisabled {
+	if cfg.RolloverAlias.IsEmpty() && cfg.Mode != ModeDisabled {
 		return fmt.Errorf("rollover_alias must be set when ILM is not disabled")
 	}
 	return nil
 }
 
 func defaultConfig(info beat.Info) Config {
-	name := fmt.Sprintf("%s-%s", info.Beat, info.Version)
+	name := info.Beat + "-%{[agent.version]}"
 	nameFmt := fmtstr.MustCompileEvent(name)
 
 	return Config{
 		Mode:          ModeAuto,
 		PolicyName:    *nameFmt,
-		RolloverAlias: name,
+		RolloverAlias: *nameFmt,
 		Pattern:       ilmDefaultPattern,
 		PolicyFile:    "",
 		CheckExists:   true,

libbeat/idxmgmt/ilm/ilm.go

@@ -104,8 +104,13 @@ func StdSupport(log *logp.Logger, info beat.Info, config *common.Config) (Suppor
 		return nil, errors.Wrap(err, "failed to read ilm policy name")
 	}
 
+	rolloverAlias, err := applyStaticFmtstr(info, &cfg.RolloverAlias)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read the ilm rollover alias")
+	}
+
 	alias := Alias{
-		Name:    cfg.RolloverAlias,
+		Name:    rolloverAlias,
 		Pattern: cfg.Pattern,
 	}
 

libbeat/idxmgmt/ilm/ilm_test.go

@@ -52,6 +52,19 @@ func TestDefaultSupport_Init(t *testing.T) {
 		}
 	})
 
+	t.Run("with an empty rollover_alias", func(t *testing.T) {
+		_, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":        true,
+				"rollover_alias": "",
+				"pattern":        "01",
+				"check_exists":   false,
+				"overwrite":      true,
+			},
+		))
+		require.Error(t, err)
+	})
+
 	t.Run("with custom config", func(t *testing.T) {
 		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
 			map[string]interface{}{
@@ -74,6 +87,47 @@ func TestDefaultSupport_Init(t *testing.T) {
 		assert.Equal(Alias{Name: "alias", Pattern: "01"}, s.Alias())
 	})
 
+	t.Run("with custom alias config with fieldref", func(t *testing.T) {
+		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":        true,
+				"rollover_alias": "alias-%{[agent.version]}",
+				"pattern":        "01",
+				"check_exists":   false,
+				"overwrite":      true,
+			},
+		))
+		require.NoError(t, err)
+
+		s := tmp.(*stdSupport)
+		assert := assert.New(t)
+		assert.Equal(true, s.overwrite)
+		assert.Equal(false, s.checkExists)
+		assert.Equal(ModeEnabled, s.Mode())
+		assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body))
+		assert.Equal(Alias{Name: "alias-9.9.9", Pattern: "01"}, s.Alias())
+	})
+
+	t.Run("with default alias", func(t *testing.T) {
+		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":      true,
+				"pattern":      "01",
+				"check_exists": false,
+				"overwrite":    true,
+			},
+		))
+		require.NoError(t, err)
+
+		s := tmp.(*stdSupport)
+		assert := assert.New(t)
+		assert.Equal(true, s.overwrite)
+		assert.Equal(false, s.checkExists)
+		assert.Equal(ModeEnabled, s.Mode())
+		assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body))
+		assert.Equal(Alias{Name: "test-9.9.9", Pattern: "01"}, s.Alias())
+	})
+
 	t.Run("load external policy", func(t *testing.T) {
 		s, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
 			common.MapStr{"policy_file": "testfiles/custom.json"},

libbeat/tests/system/test_cmd_setup_index_management.py

@@ -144,6 +144,26 @@ def test_setup_rollover_alias(self):
         self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
         self.idxmgmt.assert_alias_created(self.custom_alias)
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_setup_rollover_alias_with_fieldref(self):
+        """
+        Test setup --index-management when ilm.rollover_alias is configured and using field reference.
+        """
+        aliasFieldRef = "%{[agent.name]}-myalias"
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.cmd,
+                                              "-E", "setup.ilm.rollover_alias=" + aliasFieldRef])
+
+        self.custom_alias = self.beat_name + "-myalias"
+
+        assert exit_code == 0
+        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
+        self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
+        self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
+        self.idxmgmt.assert_alias_created(self.custom_alias)
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_setup_template_name_and_pattern(self):