Disable migration aliases in index pattern

The migration aliases should not show up in the index pattern if `migration.enabled: false`. For this to happen, the Kibana index pattern must be generated on the fly instead of packaging it with each Beat. This PR introduces the generation of the index pattern when Kibana data is loaded.

APM still needs the index pattern as file. For this the export command `index-pattern` was added. It will print the index pattern to the standard out:

```
./metricbeat export index-pattern > pattern.json
```

The commands to generate the index pattern in the dev environment were removed.

For checking if aliases are supported, the Kibana version is checked. Fully accurate would be to check the Elasticsearch version as it depends on the ES version in the end and not Kibana. But it's assume that in general the same minor version is used. The reason not Elasticsearch is checked as it would potentially require additional config options and adds unnecessary complexity.

For the index pattern the internal fields.go are used. Even if fields.yml is configured still fields.go is used. This is the same behavior as we had so far when the index pattern was generated. It could be improved in the future to also support a fields.yml for the generation if needed.

In general this PR tried to change as little code as possible. The code and tests around the Kibana dashboard generation and index pattern generation is not very nice. One reason is that it also contains old logic which was used for previous versions but also has the ability to read dashboards from a zip file. Because of this all the old capabilities have to stay in the code for now. The code should be cleaned up at a later stage.

Further changes:

* Added system tests to Filebeat to check for correct content in index pattern when migration is enabled.
* Fix double generation of common.yml in Metricbeat. It seems some changes in the past caused that the common.yml file was contained in two fields.go files in Metricbeat

CHANGELOG-developer.next.asciidoc

@@ -21,6 +21,7 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 ==== Breaking changes
 - Outputs receive Index Manager as additional parameter. The index manager can
   be used to create an index selector. {pull}10347[10347]
+- Remove support for loading dashboards to Elasticsearch 5. {pull}10451[10451]
 
 ==== Bugfixes
 
@@ -31,3 +32,4 @@ The list below covers the major changes between 7.0.0-alpha2 and master only.
 - Add (*common.Config).Has and (*common.Config).Remove. {pull}10363[10363]
 - Introduce ILM and IndexManagment support to beat.Settings. {pull}10347[10347]
 - Introduce ILM and IndexManagement support to beat.Settings. {pull}10347[10347]
+- Generating index pattern on demand instead of shipping them in the packages. {pull}10478[10478]

CHANGELOG.next.asciidoc

@@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add ILM mode `auto` to setup.ilm.enabled setting. This new default value detects if ILM is available {pull}10347[10347]
 - Add support to read ILM policy from external JSON file. {pull}10347[10347]
 - Add `overwrite` and `check_exists` settings to ILM support. {pull}10347[10347]
+- Generate Kibana index pattern on demand instead of using a local file. {pull}10478[10478]
 
 *Auditbeat*
 

dev-tools/mage/kibana.go

@@ -74,22 +74,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 		return err
 	}
 
-	beatVersion, err := BeatQualifiedVersion()
-	if err != nil {
-		return err
-	}
-
-	// Generate Kibana index pattern files from fields.yml.
-	indexPatternCmd := sh.RunCmd("go", "run",
-		filepath.Join(esBeatsDir, "dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go"),
-		"-beat", BeatName,
-		"-version", beatVersion,
-		"-index", BeatIndexPrefix+"-*",
-		"-fields", "fields.yml",
-		"-out", kibanaBuildDir,
-	)
-
-	return indexPatternCmd()
+	return nil
 }
 
 // PackageKibanaDashboardsFromBuildDir reconfigures the packaging configuration

filebeat/tests/system/test_index_pattern.py

@@ -0,0 +1,32 @@
+import os
+import unittest
+from filebeat import BaseTest
+
+
+class Test(BaseTest):
+
+    def test_export_index_pattern(self):
+        """
+        Test export index pattern
+        """
+        self.render_config_template()
+        exit_code = self.run_beat(
+            logging_args=[],
+            extra_args=["export", "index-pattern"])
+
+        assert exit_code == 0
+        assert self.log_contains('"objects": [')
+        assert self.log_contains('beat.name') == False
+
+    def test_export_index_pattern_migration(self):
+        """
+        Test export index pattern with migration flag enabled
+        """
+        self.render_config_template()
+        exit_code = self.run_beat(
+            logging_args=[],
+            extra_args=["export", "index-pattern", "-E", "migration.enabled:true"])
+
+        assert exit_code == 0
+        assert self.log_contains('"objects": [')
+        assert self.log_contains('beat.name')

libbeat/cmd/export.go

@@ -32,6 +32,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin
 
 	exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion))
+	exportCmd.AddCommand(export.GenIndexPatternConfigCmd(settings, name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion))
 	exportCmd.AddCommand(export.GenGetILMPolicyCmd(settings, name, idxPrefix, beatVersion))
 

libbeat/cmd/export/index_pattern.go

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package export
+
+import (
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/beats/libbeat/cmd/instance"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/kibana"
+)
+
+// GenIndexPatternConfigCmd generates an index pattern for Kibana
+func GenIndexPatternConfigCmd(settings instance.Settings, name, idxPrefix, beatVersion string) *cobra.Command {
+	genTemplateConfigCmd := &cobra.Command{
+		Use:   "index-pattern",
+		Short: "Export kibana index pattern to stdout",
+		Run: func(cmd *cobra.Command, args []string) {
+			version, _ := cmd.Flags().GetString("es.version")
+
+			b, err := instance.NewBeat(name, idxPrefix, beatVersion)
+			if err != nil {
+				fatalf("Error initializing beat: %+v", err)
+			}
+			err = b.InitWithSettings(settings)
+			if err != nil {
+				fatalf("Error initializing beat: %+v", err)
+			}
+
+			if version == "" {
+				version = b.Info.Version
+			}
+
+			var withMigration bool
+			if b.RawConfig.HasField("migration") {
+				sub, err := b.RawConfig.Child("migration", -1)
+				if err != nil {
+					fatalf("Failed to read migration setting: %+v", err)
+				}
+				withMigration = sub.Enabled()
+			}
+
+			// Index pattern generation
+			v, err := common.NewVersion(version)
+			if err != nil {
+				fatalf("Error creating version: %+v", err)
+			}
+			indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, beatVersion, *v, withMigration)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			pattern, err := indexPattern.Generate()
+			if err != nil {
+				log.Fatalf("ERROR: %s", err)
+			}
+
+			_, err = os.Stdout.WriteString(pattern.StringToPrint() + "\n")
+			if err != nil {
+				fatalf("Error writing index pattern: %+v", err)
+			}
+		},
+	}
+
+	genTemplateConfigCmd.Flags().String("es.version", beatVersion, "Elasticsearch version")
+	genTemplateConfigCmd.Flags().String("index", idxPrefix, "Base index name")
+
+	return genTemplateConfigCmd
+}

libbeat/cmd/instance/beat.go

@@ -32,6 +32,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/elastic/beats/libbeat/kibana"
+
 	"github.com/gofrs/uuid"
 	errw "github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -679,8 +681,42 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 	}
 
 	if b.Config.Dashboards.Enabled() {
-		err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
-			b.Config.Kibana, b.Config.Dashboards, nil)
+
+		var withMigration bool
+		if b.RawConfig.HasField("migration") {
+			sub, err := b.RawConfig.Child("migration", -1)
+			if err != nil {
+				return fmt.Errorf("Failed to read migration setting: %+v", err)
+			}
+			withMigration = sub.Enabled()
+		}
+
+		// init kibana config object
+		kibanaConfig := b.Config.Kibana
+		if kibanaConfig == nil {
+			kibanaConfig = common.NewConfig()
+		}
+
+		client, err := kibana.NewKibanaClient(kibanaConfig)
+		if err != nil {
+			return fmt.Errorf("error connecting to Kibana: %v", err)
+		}
+		// This fetches the version for Kibana. For the alias feature the version of ES would be needed
+		// but it's assumed that KB and ES have the same minor version.
+		v := client.GetVersion()
+
+		indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, b.Info.Version, v, withMigration)
+		if err != nil {
+			return fmt.Errorf("error creating index pattern generator: %v", err)
+		}
+
+		pattern, err := indexPattern.Generate()
+		if err != nil {
+			return fmt.Errorf("error generating index pattern: %v", err)
+		}
+
+		err = dashboards.ImportDashboards(ctx, b.Info, paths.Resolve(paths.Home, ""),
+			kibanaConfig, b.Config.Dashboards, nil, pattern)
 		if err != nil {
 			return errw.Wrap(err, "Error importing Kibana dashboards")
 		}

libbeat/common/field.go

@@ -119,7 +119,7 @@ func (f *Field) Validate() error {
 }
 
 func LoadFieldsYaml(path string) (Fields, error) {
-	keys := []Field{}
+	var keys []Field
 
 	cfg, err := yaml.NewConfigWithFile(path)
 	if err != nil {
@@ -135,6 +135,23 @@ func LoadFieldsYaml(path string) (Fields, error) {
 	return fields, nil
 }
 
+// LoadFields loads fields from a byte array
+func LoadFields(f []byte) (Fields, error) {
+	var keys []Field
+
+	cfg, err := yaml.NewConfig(f)
+	if err != nil {
+		return nil, err
+	}
+	cfg.Unpack(&keys)
+
+	fields := Fields{}
+	for _, key := range keys {
+		fields = append(fields, key.Fields...)
+	}
+	return fields, nil
+}
+
 // HasKey checks if inside fields the given key exists
 // The key can be in the form of a.b.c and it will check if the nested field exist
 // In case the key is `a` and there is a value `a.b` false is return as it only