Create ES index templates instead of indices (#1627)

* Create ES index templates instead of indices

Signed-off-by: Pavol Loffay <ploffay@redhat.com>

* Add flag to create index templates

Signed-off-by: Pavol Loffay <ploffay@redhat.com>

* Add changelog

Signed-off-by: Pavol Loffay <ploffay@redhat.com>

* Fix lint

Signed-off-by: Pavol Loffay <ploffay@redhat.com>

CHANGELOG.md

@@ -8,6 +8,11 @@ Changes by Version
 
 ##### Breaking Changes
 
+* Create ES index templates instead of indices ([#1627](https://github.com/jaegertracing/jaeger/pull/1627), [@pavolloffay](https://github.com/pavolloffay))
+
+  This can break existing Elasticsearch deployments if security policies are applied. 
+  For instance Jaeger `X-Pack` configuration now requires permission to create index templates - `manage_index_templates`.
+
 ##### New Features
 
 ##### Bug fixes, Minor Improvements
@@ -25,7 +30,7 @@ Changes by Version
 
 ##### Bug fixes, Minor Improvements
 
-Change default for bearer-token-propagation to false ([#1642](https://github.com/jaegertracing/jaeger/pull/1642), [@wsoula](https://github.com/wsoula))
+* Change default for bearer-token-propagation to false ([#1642](https://github.com/jaegertracing/jaeger/pull/1642), [@wsoula](https://github.com/wsoula))
 
 #### UI Changes
 

Makefile

@@ -355,7 +355,7 @@ generate-zipkin-swagger: idl-submodule
 
 .PHONY: install-mockery
 install-mockery:
-	go get -u github.com/vektra/mockery
+	go get -u github.com/vektra/mockery/.../
 
 .PHONY: generate-mocks
 generate-mocks: install-mockery

pkg/es/client.go

@@ -25,6 +25,7 @@ import (
 type Client interface {
 	IndexExists(index string) IndicesExistsService
 	CreateIndex(index string) IndicesCreateService
+	CreateTemplate(id string) TemplateCreateService
 	Index() IndexService
 	Search(indices ...string) SearchService
 	MultiSearch() MultiSearchService
@@ -42,6 +43,12 @@ type IndicesCreateService interface {
 	Do(ctx context.Context) (*elastic.IndicesCreateResult, error)
 }
 
+// TemplateCreateService is an abstraction for creating a mapping
+type TemplateCreateService interface {
+	Body(mapping string) TemplateCreateService
+	Do(ctx context.Context) (*elastic.IndicesPutTemplateResponse, error)
+}
+
 // IndexService is an abstraction for elastic BulkService
 type IndexService interface {
 	Index(index string) IndexService

pkg/es/config/config.go

@@ -60,6 +60,7 @@ type Configuration struct {
 	Enabled               bool
 	TLS                   TLSConfig
 	UseReadWriteAliases   bool
+	CreateIndexTemplates  bool
 }
 
 // TLSConfig describes the configuration properties to connect tls enabled ElasticSearch cluster
@@ -85,6 +86,7 @@ type ClientBuilder interface {
 	GetUseReadWriteAliases() bool
 	GetTokenFilePath() string
 	IsEnabled() bool
+	IsCreateIndexTemplates() bool
 }
 
 // NewClient creates a new ElasticSearch client
@@ -248,6 +250,11 @@ func (c *Configuration) IsEnabled() bool {
 	return c.Enabled
 }
 
+// IsCreateIndexTemplates determines whether index templates should be created or not
+func (c *Configuration) IsCreateIndexTemplates() bool {
+	return c.CreateIndexTemplates
+}
+
 // getConfigOptions wraps the configs to feed to the ElasticSearch client init
 func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOptionFunc, error) {
 

pkg/es/wrapper/wrapper.go

@@ -45,6 +45,11 @@ func (c ClientWrapper) CreateIndex(index string) es.IndicesCreateService {
 	return WrapESIndicesCreateService(c.client.CreateIndex(index))
 }
 
+// CreateTemplate calls this function to internal client.
+func (c ClientWrapper) CreateTemplate(ttype string) es.TemplateCreateService {
+	return WrapESTemplateCreateService(c.client.IndexPutTemplate(ttype))
+}
+
 // Index calls this function to internal client.
 func (c ClientWrapper) Index() es.IndexService {
 	r := elastic.NewBulkIndexRequest()
@@ -105,6 +110,26 @@ func (c IndicesCreateServiceWrapper) Do(ctx context.Context) (*elastic.IndicesCr
 	return c.indicesCreateService.Do(ctx)
 }
 
+// TemplateCreateServiceWrapper is a wrapper around elastic.IndicesPutTemplateService.
+type TemplateCreateServiceWrapper struct {
+	mappingCreateService *elastic.IndicesPutTemplateService
+}
+
+// WrapESTemplateCreateService creates an TemplateCreateService out of *elastic.IndicesPutTemplateService.
+func WrapESTemplateCreateService(mappingCreateService *elastic.IndicesPutTemplateService) TemplateCreateServiceWrapper {
+	return TemplateCreateServiceWrapper{mappingCreateService: mappingCreateService}
+}
+
+// Body calls this function to internal service.
+func (c TemplateCreateServiceWrapper) Body(mapping string) es.TemplateCreateService {
+	return WrapESTemplateCreateService(c.mappingCreateService.BodyString(mapping))
+}
+
+// Do calls this function to internal service.
+func (c TemplateCreateServiceWrapper) Do(ctx context.Context) (*elastic.IndicesPutTemplateResponse, error) {
+	return c.mappingCreateService.Do(ctx)
+}
+
 // ---
 
 // IndexServiceWrapper is a wrapper around elastic.ESIndexService.

plugin/storage/es/factory.go

@@ -126,20 +126,18 @@ func loadTagsFromFile(filePath string) ([]string, error) {
 
 // CreateArchiveSpanReader implements storage.ArchiveFactory
 func (f *Factory) CreateArchiveSpanReader() (spanstore.Reader, error) {
-	cfg := f.Options.Get(archiveNamespace)
-	if !cfg.Enabled {
+	if !f.archiveConfig.IsEnabled() {
 		return nil, nil
 	}
-	return createSpanReader(f.metricsFactory, f.logger, f.archiveClient, cfg, true)
+	return createSpanReader(f.metricsFactory, f.logger, f.archiveClient, f.archiveConfig, true)
 }
 
 // CreateArchiveSpanWriter implements storage.ArchiveFactory
 func (f *Factory) CreateArchiveSpanWriter() (spanstore.Writer, error) {
-	cfg := f.Options.Get(archiveNamespace)
-	if !cfg.Enabled {
+	if !f.archiveConfig.IsEnabled() {
 		return nil, nil
 	}
-	return createSpanWriter(f.metricsFactory, f.logger, f.archiveClient, cfg, true)
+	return createSpanWriter(f.metricsFactory, f.logger, f.archiveClient, f.archiveConfig, true)
 }
 
 func createSpanReader(
@@ -179,7 +177,7 @@ func createSpanWriter(
 	}
 
 	spanMapping, serviceMapping := GetMappings(cfg.GetNumShards(), cfg.GetNumReplicas())
-	return esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{
+	writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{
 		Client:              client,
 		Logger:              logger,
 		MetricsFactory:      mFactory,
@@ -189,9 +187,14 @@ func createSpanWriter(
 		TagDotReplacement:   cfg.GetTagDotReplacement(),
 		Archive:             archive,
 		UseReadWriteAliases: cfg.GetUseReadWriteAliases(),
-		SpanMapping:         spanMapping,
-		ServiceMapping:      serviceMapping,
-	}), nil
+	})
+	if cfg.IsCreateIndexTemplates() {
+		err := writer.CreateTemplates(spanMapping, serviceMapping)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return writer, nil
 }
 
 // GetMappings returns span and service mappings