Skip to content

Commit e5d6fd4

Browse files
dakronedakrone
authored
Use V2 templates when reading duplicate aliases and ingest pip… (#54902)
When a new index is rolled over, we check to see whether there are any duplicate alias configurations in the index template configuration. Additionally, when a new index is created from a bulk action, we check the templates to see if there are any ingest pipelines that need to be applied to the index that will be newly created. Both of these actions previously checked the v1 templates for their settings, they now also check the v2 index templates, with the v2 index templates taking precendence similar to the way they do when creating an index. Relates to #53101
1 parent 73b5f58 commit e5d6fd4

File tree

7 files changed

+172
-23
lines changed

7 files changed

+172
-23
lines changed

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,22 +25,25 @@
2525
import org.elasticsearch.cluster.ClusterState;
2626
import org.elasticsearch.cluster.metadata.AliasAction;
2727
import org.elasticsearch.cluster.metadata.AliasMetadata;
28+
import org.elasticsearch.cluster.metadata.IndexAbstraction;
2829
import org.elasticsearch.cluster.metadata.IndexMetadata;
2930
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
30-
import org.elasticsearch.cluster.metadata.IndexAbstraction;
3131
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
3232
import org.elasticsearch.cluster.metadata.Metadata;
3333
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
3434
import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
35+
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
3536
import org.elasticsearch.common.Nullable;
3637
import org.elasticsearch.common.inject.Inject;
3738
import org.elasticsearch.threadpool.ThreadPool;
3839

3940
import java.util.List;
4041
import java.util.Locale;
42+
import java.util.Map;
4143
import java.util.regex.Pattern;
4244

4345
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
46+
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
4447

4548
public class MetadataRolloverService {
4649
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@@ -171,6 +174,18 @@ static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String roll
171174
rolloverRequestAlias, template.aliases().keys(), template.name()));
172175
}
173176
}
177+
178+
final String matchedV2Template = findV2Template(metadata, rolloverIndexName, isHidden);
179+
if (matchedV2Template != null) {
180+
List<Map<String, AliasMetadata>> aliases = MetadataIndexTemplateService.resolveAliases(metadata, matchedV2Template);
181+
for (Map<String, AliasMetadata> aliasConfig : aliases) {
182+
if (aliasConfig.containsKey(rolloverRequestAlias)) {
183+
throw new IllegalArgumentException(String.format(Locale.ROOT,
184+
"Rollover alias [%s] can point to multiple indices, found duplicated alias [%s] in index template [%s]",
185+
rolloverRequestAlias, aliasConfig.keySet(), matchedV2Template));
186+
}
187+
}
188+
}
174189
}
175190

176191
static void validate(Metadata metadata, String aliasName) {

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -306,12 +306,12 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
306306
indexRequest.setFinalPipeline(finalPipeline);
307307
}
308308
} else if (indexRequest.index() != null) {
309-
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
310-
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
311-
assert (templates != null);
312-
// order of templates are highest order first
313-
for (final IndexTemplateMetadata template : templates) {
314-
final Settings settings = template.settings();
309+
// the index does not exist yet (and this is a valid request), so match index
310+
// templates to look for pipelines in either a matching V2 template (which takes
311+
// precedence), or if a V2 template does not match, any V1 templates
312+
String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), null);
313+
if (v2Template != null) {
314+
Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template);
315315
if (defaultPipeline == null && IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
316316
defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
317317
// we can not break in case a lower-order template has a final pipeline that we need to collect
@@ -320,13 +320,31 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
320320
finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
321321
// we can not break in case a lower-order template has a default pipeline that we need to collect
322322
}
323-
if (defaultPipeline != null && finalPipeline != null) {
324-
// we can break if we have already collected a default and final pipeline
325-
break;
323+
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
324+
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
325+
} else {
326+
List<IndexTemplateMetadata> templates =
327+
MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
328+
assert (templates != null);
329+
// order of templates are highest order first
330+
for (final IndexTemplateMetadata template : templates) {
331+
final Settings settings = template.settings();
332+
if (defaultPipeline == null && IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
333+
defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
334+
// we can not break in case a lower-order template has a final pipeline that we need to collect
335+
}
336+
if (finalPipeline == null && IndexSettings.FINAL_PIPELINE.exists(settings)) {
337+
finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
338+
// we can not break in case a lower-order template has a default pipeline that we need to collect
339+
}
340+
if (defaultPipeline != null && finalPipeline != null) {
341+
// we can break if we have already collected a default and final pipeline
342+
break;
343+
}
326344
}
345+
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
346+
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
327347
}
328-
indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, IngestService.NOOP_PIPELINE_NAME));
329-
indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, IngestService.NOOP_PIPELINE_NAME));
330348
}
331349

332350
if (requestPipeline != null) {

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,14 +458,15 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu
458458
MetadataIndexTemplateService.resolveMappings(currentState, templateName), xContentRegistry));
459459

460460
final Settings aggregatedIndexSettings =
461-
aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState, templateName),
461+
aggregateIndexSettings(currentState, request,
462+
MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName),
462463
mappings, null, settings, indexScopedSettings);
463464
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
464465
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
465466

466467
return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings,
467468
indexService -> resolveAndValidateAliases(request.index(), request.aliases(),
468-
MetadataIndexTemplateService.resolveAliases(currentState, templateName), currentState.metadata(), aliasValidator,
469+
MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator,
469470
// the context is only used for validation so it's fine to pass fake values for the
470471
// shard id and the current timestamp
471472
xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -715,14 +715,14 @@ public static Settings resolveSettings(final List<IndexTemplateMetadata> templat
715715
/**
716716
* Resolve the given v2 template into a collected {@link Settings} object
717717
*/
718-
public static Settings resolveSettings(final ClusterState state, final String templateName) {
719-
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
718+
public static Settings resolveSettings(final Metadata metadata, final String templateName) {
719+
final IndexTemplateV2 template = metadata.templatesV2().get(templateName);
720720
assert template != null : "attempted to resolve settings for a template [" + templateName +
721721
"] that did not exist in the cluster state";
722722
if (template == null) {
723723
return Settings.EMPTY;
724724
}
725-
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
725+
final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
726726
List<Settings> componentSettings = template.composedOf().stream()
727727
.map(componentTemplates::get)
728728
.filter(Objects::nonNull)
@@ -760,14 +760,14 @@ public static List<Map<String, AliasMetadata>> resolveAliases(final List<IndexTe
760760
/**
761761
* Resolve the given v2 template into an ordered list of aliases
762762
*/
763-
public static List<Map<String, AliasMetadata>> resolveAliases(final ClusterState state, final String templateName) {
764-
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
763+
public static List<Map<String, AliasMetadata>> resolveAliases(final Metadata metadata, final String templateName) {
764+
final IndexTemplateV2 template = metadata.templatesV2().get(templateName);
765765
assert template != null : "attempted to resolve aliases for a template [" + templateName +
766766
"] that did not exist in the cluster state";
767767
if (template == null) {
768768
return List.of();
769769
}
770-
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
770+
final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
771771
List<Map<String, AliasMetadata>> aliases = template.composedOf().stream()
772772
.map(componentTemplates::get)
773773
.filter(Objects::nonNull)

‎server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,16 @@
2828
import org.elasticsearch.cluster.metadata.AliasAction;
2929
import org.elasticsearch.cluster.metadata.AliasMetadata;
3030
import org.elasticsearch.cluster.metadata.AliasValidator;
31+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
32+
import org.elasticsearch.cluster.metadata.IndexAbstraction;
3133
import org.elasticsearch.cluster.metadata.IndexMetadata;
3234
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
33-
import org.elasticsearch.cluster.metadata.IndexAbstraction;
3435
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
36+
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
3537
import org.elasticsearch.cluster.metadata.Metadata;
3638
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
3739
import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
40+
import org.elasticsearch.cluster.metadata.Template;
3841
import org.elasticsearch.cluster.routing.allocation.AllocationService;
3942
import org.elasticsearch.cluster.service.ClusterService;
4043
import org.elasticsearch.common.CheckedFunction;
@@ -53,8 +56,10 @@
5356

5457
import java.util.Arrays;
5558
import java.util.Collections;
59+
import java.util.HashMap;
5660
import java.util.List;
5761
import java.util.Locale;
62+
import java.util.Map;
5863

5964
import static org.hamcrest.Matchers.containsString;
6065
import static org.hamcrest.Matchers.equalTo;
@@ -271,6 +276,41 @@ public void testRejectDuplicateAlias() {
271276
assertThat(ex.getMessage(), containsString("index template [test-template]"));
272277
}
273278

279+
public void testRejectDuplicateAliasV2() {
280+
Map<String, AliasMetadata> aliases = new HashMap<>();
281+
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
282+
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
283+
final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases),
284+
null, null, null, null);
285+
286+
final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
287+
.put("test-template", template).build();
288+
String indexName = randomFrom("foo-123", "bar-xyz");
289+
String aliasName = randomFrom("foo-write", "bar-write");
290+
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
291+
() -> MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomBoolean()));
292+
assertThat(ex.getMessage(), containsString("index template [test-template]"));
293+
}
294+
295+
public void testRejectDuplicateAliasV2UsingComponentTemplates() {
296+
Map<String, AliasMetadata> aliases = new HashMap<>();
297+
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
298+
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
299+
final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null);
300+
final IndexTemplateV2 template = new IndexTemplateV2(Arrays.asList("foo-*", "bar-*"), null,
301+
Collections.singletonList("ct"), null, null, null);
302+
303+
final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
304+
.put("ct", ct)
305+
.put("test-template", template)
306+
.build();
307+
String indexName = randomFrom("foo-123", "bar-xyz");
308+
String aliasName = randomFrom("foo-write", "bar-write");
309+
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
310+
() -> MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomBoolean()));
311+
assertThat(ex.getMessage(), containsString("index template [test-template]"));
312+
}
313+
274314
public void testHiddenAffectsResolvedTemplates() {
275315
final IndexTemplateMetadata template = IndexTemplateMetadata.builder("test-template")
276316
.patterns(Collections.singletonList("*"))
@@ -288,6 +328,49 @@ public void testHiddenAffectsResolvedTemplates() {
288328
assertThat(ex.getMessage(), containsString("index template [test-template]"));
289329
}
290330

331+
public void testHiddenAffectsResolvedV2Templates() {
332+
Map<String, AliasMetadata> aliases = new HashMap<>();
333+
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
334+
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
335+
final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), new Template(null, null, aliases),
336+
null, null, null, null);
337+
338+
final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
339+
.put("test-template", template).build();
340+
String indexName = randomFrom("foo-123", "bar-xyz");
341+
String aliasName = randomFrom("foo-write", "bar-write");
342+
343+
// hidden shouldn't throw
344+
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, Boolean.TRUE);
345+
// not hidden will throw
346+
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
347+
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomFrom(Boolean.FALSE, null)));
348+
assertThat(ex.getMessage(), containsString("index template [test-template]"));
349+
}
350+
351+
public void testHiddenAffectsResolvedV2ComponentTemplates() {
352+
Map<String, AliasMetadata> aliases = new HashMap<>();
353+
aliases.put("foo-write", AliasMetadata.builder("foo-write").build());
354+
aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build());
355+
final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null);
356+
final IndexTemplateV2 template = new IndexTemplateV2(Collections.singletonList("*"), null,
357+
Collections.singletonList("ct"), null, null, null);
358+
359+
final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false)
360+
.put("ct", ct)
361+
.put("test-template", template)
362+
.build();
363+
String indexName = randomFrom("foo-123", "bar-xyz");
364+
String aliasName = randomFrom("foo-write", "bar-write");
365+
366+
// hidden shouldn't throw
367+
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, Boolean.TRUE);
368+
// not hidden will throw
369+
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
370+
MetadataRolloverService.checkNoDuplicatedAliasInIndexTemplate(metadata, indexName, aliasName, randomFrom(Boolean.FALSE, null)));
371+
assertThat(ex.getMessage(), containsString("index template [test-template]"));
372+
}
373+
291374
/**
292375
* Test the main rolloverClusterState method. This does not validate every detail to depth, rather focuses on observing that each
293376
* parameter is used for the purpose intended.

‎server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@
3737
import org.elasticsearch.cluster.metadata.IndexMetadata;
3838
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
3939
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
40+
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
4041
import org.elasticsearch.cluster.metadata.Metadata;
42+
import org.elasticsearch.cluster.metadata.Template;
4143
import org.elasticsearch.cluster.node.DiscoveryNode;
4244
import org.elasticsearch.cluster.node.DiscoveryNodes;
4345
import org.elasticsearch.cluster.service.ClusterService;
@@ -573,6 +575,36 @@ public void testFindDefaultPipelineFromTemplateMatch(){
573575
completionHandler.capture(), any());
574576
}
575577

578+
public void testFindDefaultPipelineFromV2TemplateMatch() {
579+
Exception exception = new Exception("fake exception");
580+
581+
IndexTemplateV2 t1 = new IndexTemplateV2(Collections.singletonList("missing_*"),
582+
new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null),
583+
null, null, null, null);
584+
585+
ClusterState state = clusterService.state();
586+
Metadata metadata = Metadata.builder()
587+
.put("my-template", t1)
588+
.build();
589+
when(state.metadata()).thenReturn(metadata);
590+
when(state.getMetadata()).thenReturn(metadata);
591+
592+
IndexRequest indexRequest = new IndexRequest("missing_index").id("id");
593+
indexRequest.source(Collections.emptyMap());
594+
AtomicBoolean responseCalled = new AtomicBoolean(false);
595+
AtomicBoolean failureCalled = new AtomicBoolean(false);
596+
ActionTestUtils.execute(singleItemBulkWriteAction, null, indexRequest, ActionListener.wrap(
597+
response -> responseCalled.set(true),
598+
e -> {
599+
assertThat(e, sameInstance(exception));
600+
failureCalled.set(true);
601+
}));
602+
603+
assertEquals("pipeline2", indexRequest.getPipeline());
604+
verify(ingestService).executeBulkRequest(eq(1), bulkDocsItr.capture(), failureHandler.capture(),
605+
completionHandler.capture(), any());
606+
}
607+
576608
private void validateDefaultPipeline(IndexRequest indexRequest) {
577609
Exception exception = new Exception("fake exception");
578610
indexRequest.source(Collections.emptyMap());

0 commit comments

Comments
 (0)