Add support for loading the raw json template file (elastic#7039)

* Add support for loading the raw json template file

Current in 6.x templates can only be loaded from the fields.yml and are generated or have to be loaded manually. To allow users to create their own template.json file and still use the Beat to load it, `setup.template.json.path` can now be used.

The configuration options look as following:

```
setup.template.json.enabled: true
setup.template.json.path: "template.json"
setup.template.json.name: "template-name
```

CHANGELOG.asciidoc

@@ -136,6 +136,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add Indexer indexing by pod uid. Enable pod uid metadata gathering in add_kubernetes_metadata. Extended Matcher log_path matching to support volume mounts {pull}7072[7072]
 - Add Kibana module with log fileset. {pull}7052[7052]
 - Add default_fields to Elasticsearch template when connecting to Elasticsearch >= 7.0. {pull}7015[7015]
+- Add support for loading a template.json file directly instead of using fields.yml. {pull}7039[7039]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -854,6 +854,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

filebeat/filebeat.reference.yml

@@ -1420,6 +1420,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

heartbeat/heartbeat.reference.yml

@@ -963,6 +963,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

libbeat/_meta/config.reference.yml

@@ -749,6 +749,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

libbeat/docs/template-config.asciidoc

@@ -86,4 +86,17 @@ ifeval::["{beatname_lc}"!="apm-server"]
 *`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[]
 
 NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern.
+
+*`setup.template.json.enabled`*:: Set to true to load a json based template file. Specify the path to your Elasticsearch
+index template file and set the name of the template. experimental[]
+
+["source","yaml",subs="attributes"]
+----------------------------------------------------------------------
+setup.template.json.enabled: true
+setup.template.json.path: "template.json"
+setup.template.json.name: "template-name
+----------------------------------------------------------------------
+
+NOTE: If the JSON template is used, the fields.yml is skipped for the template generation.
+
 endif::[]

libbeat/template/config.go

@@ -3,10 +3,15 @@ package template
 import "github.com/elastic/beats/libbeat/common"
 
 type TemplateConfig struct {
-	Enabled      bool             `config:"enabled"`
-	Name         string           `config:"name"`
-	Pattern      string           `config:"pattern"`
-	Fields       string           `config:"fields"`
+	Enabled bool   `config:"enabled"`
+	Name    string `config:"name"`
+	Pattern string `config:"pattern"`
+	Fields  string `config:"fields"`
+	JSON    struct {
+		Enabled bool   `config:"enabled"`
+		Path    string `config:"path"`
+		Name    string `config:"name"`
+	} `config:"json"`
 	AppendFields common.Fields    `config:"append_fields"`
 	Overwrite    bool             `config:"overwrite"`
 	Settings     TemplateSettings `config:"settings"`

libbeat/template/load.go

@@ -1,7 +1,10 @@
 package template
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
@@ -51,40 +54,60 @@ func (l *Loader) Load() error {
 		return fmt.Errorf("error creating template instance: %v", err)
 	}
 
+	templateName := tmpl.GetName()
+	if l.config.JSON.Enabled {
+		templateName = l.config.JSON.Name
+	}
 	// Check if template already exist or should be overwritten
-	exists := l.CheckTemplate(tmpl.GetName())
+	exists := l.CheckTemplate(templateName)
 	if !exists || l.config.Overwrite {
 
 		logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion())
-
 		if l.config.Overwrite {
 			logp.Info("Existing template will be overwritten, as overwrite is enabled.")
 		}
 
-		var output common.MapStr
+		var template map[string]interface{}
+		if l.config.JSON.Enabled {
+			jsonPath := paths.Resolve(paths.Config, l.config.JSON.Path)
+			if _, err := os.Stat(jsonPath); err != nil {
+				return fmt.Errorf("error checking for json template: %s", err)
+			}
+
+			logp.Info("Loading json template from file %s", jsonPath)
+
+			content, err := ioutil.ReadFile(jsonPath)
+			if err != nil {
+				return fmt.Errorf("error reading file. Path: %s, Error: %s", jsonPath, err)
 
-		// Load fields from path
-		if l.config.Fields != "" {
+			}
+			err = json.Unmarshal(content, &template)
+			if err != nil {
+				return fmt.Errorf("could not unmarshal json template: %s", err)
+			}
+			// Load fields from path
+		} else if l.config.Fields != "" {
 			logp.Debug("template", "Load fields.yml from file: %s", l.config.Fields)
 
 			fieldsPath := paths.Resolve(paths.Config, l.config.Fields)
 
-			output, err = tmpl.LoadFile(fieldsPath)
+			template, err = tmpl.LoadFile(fieldsPath)
 			if err != nil {
 				return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err)
 			}
 		} else {
 			logp.Debug("template", "Load default fields.yml")
-			output, err = tmpl.LoadBytes(l.fields)
+			template, err = tmpl.LoadBytes(l.fields)
 			if err != nil {
 				return fmt.Errorf("error creating template: %v", err)
 			}
 		}
 
-		err = l.LoadTemplate(tmpl.GetName(), output)
+		err = l.LoadTemplate(templateName, template)
 		if err != nil {
-			return fmt.Errorf("could not load template. Elasticsearh returned: %v. Template is: %s", err, output)
+			return fmt.Errorf("could not load template. Elasticsearh returned: %v. Template is: %s", err, template)
 		}
+
 	} else {
 		logp.Info("Template already exists and will not be overwritten.")
 	}

libbeat/tests/files/template.json

@@ -0,0 +1,22 @@
+{
+    "index_patterns": ["bla"],
+    "settings": {
+        "number_of_shards": 1
+    },
+    "mappings": {
+        "example": {
+            "_source": {
+                "enabled": false
+            },
+            "properties": {
+                "host_name": {
+                    "type": "keyword"
+                },
+                "created_at": {
+                    "type": "date",
+                    "format": "EEE MMM dd HH:mm:ss Z YYYY"
+                }
+            }
+        }
+    }
+}

libbeat/tests/system/config/mockbeat.yml.j2

@@ -73,6 +73,12 @@ setup.template:
     index.codec: best_compression
   name: {{ es_template_name }}
   pattern: {{ es_template_pattern }}
+  overwrite: {{ template_overwrite|default("false") }}
+  json:
+    enabled: {{ template_json_enabled|default("false") }}
+    path: {{ template_json_path }}
+    name: {{ template_json_name }}
+
 #================================ Logging =====================================
 
 {% if metrics_period -%}

libbeat/tests/system/test_template.py

@@ -1,4 +1,10 @@
 from base import BaseTest
+import os
+from elasticsearch import Elasticsearch, TransportError
+from nose.plugins.attrib import attr
+import unittest
+
+INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
 
 class Test(BaseTest):
@@ -72,3 +78,36 @@ def test_index_with_pattern_name(self):
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
         proc.check_kill_and_wait()
+
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_json_template(self):
+        """
+        Test loading of json based template
+        """
+
+        self.copy_files(["template.json"])
+
+        path = os.path.join(self.working_dir, "template.json")
+
+        print path
+        self.render_config_template(
+            elasticsearch={"hosts": self.get_host()},
+            template_overwrite="true",
+            template_json_enabled="true",
+            template_json_path=path,
+            template_json_name="bla",
+        )
+
+        proc = self.start_beat()
+        self.wait_until(lambda: self.log_contains("mockbeat start running."))
+        self.wait_until(lambda: self.log_contains("Loading json template from file"))
+        self.wait_until(lambda: self.log_contains("Elasticsearch template with name 'bla' loaded"))
+        proc.check_kill_and_wait()
+
+        es = Elasticsearch([self.get_elasticsearch_url()])
+        result = es.transport.perform_request('GET', '/_template/bla')
+        assert len(result) == 1
+
+    def get_host(self):
+        return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')

metricbeat/metricbeat.reference.yml

@@ -1363,6 +1363,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

packetbeat/packetbeat.reference.yml

@@ -1226,6 +1226,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false
 

winlogbeat/winlogbeat.reference.yml

@@ -778,6 +778,15 @@ output.elasticsearch:
 # Path to fields.yml file to generate the template
 #setup.template.fields: "${path.config}/fields.yml"
 
+# Enable json template loading. If this is enabled, the fields.yml is ignored.
+#setup.template.json.enabled: false
+
+# Path to the json template file
+#setup.template.json.path: "${path.config}/template.json"
+
+# Name under which the template is stored in Elasticsearch
+#setup.template.json.name: ""
+
 # Overwrite existing template
 #setup.template.overwrite: false