Skip to content

Commit 22d5465

Browse files
dakronedakrone
authored
Allocate new indices on "hot" or "content" tier depending on data stream inclusion (#62338)
This commit changes the default allocation on the "hot" tier to allocating the newly created index to the "hot" tier if it is part of a new or existing data stream, and to the "content" tier if it is not part of a data stream. Overriding any of the index.routing.allocation.(include|exclude|require).* settings continues to cause the initial allocation not to be set (no change in behavior). Relates to #60848
1 parent 86ba732 commit 22d5465

File tree

6 files changed

+123
-12
lines changed

6 files changed

+123
-12
lines changed

‎docs/reference/api-conventions.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ Returns:
396396
"index.creation_date": "1474389951325",
397397
"index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
398398
"index.version.created": ...,
399-
"index.routing.allocation.include._tier" : "data_hot",
399+
"index.routing.allocation.include._tier" : "data_content",
400400
"index.provided_name" : "my-index-000001"
401401
}
402402
}
@@ -433,7 +433,7 @@ Returns:
433433
"routing": {
434434
"allocation": {
435435
"include": {
436-
"_tier": "data_hot"
436+
"_tier": "data_content"
437437
}
438438
}
439439
},

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,10 +629,12 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
629629
.put(request.settings())
630630
.build();
631631

632+
final boolean isDataStreamIndex = request.dataStreamName() != null;
632633
// Loop through all the explicit index setting providers, adding them to the
633634
// additionalIndexSettings map
634635
for (IndexSettingProvider provider : indexSettingProviders) {
635-
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), templateAndRequestSettings));
636+
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(),
637+
isDataStreamIndex, templateAndRequestSettings));
636638
}
637639

638640
// For all the explicit settings, we go through the template and request level settings

‎server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public interface IndexSettingProvider {
3030
* Returns explicitly set default index {@link Settings} for the given index. This should not
3131
* return null.
3232
*/
33-
default Settings getAdditionalIndexSettings(String indexName, Settings templateAndRequestSettings) {
33+
default Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
3434
return Settings.EMPTY;
3535
}
3636
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java renamed to ‎x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,22 +33,27 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3333
return Collections.singleton(LocalStateCompositeXPackPlugin.class);
3434
}
3535

36-
public void testDefaultAllocateToHot() {
36+
public void testDefaultIndexAllocateToContent() {
3737
startWarmOnlyNode();
3838
startColdOnlyNode();
3939
ensureGreen();
4040

4141
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
4242

4343
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
44-
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
44+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
4545

4646
// index should be red
4747
assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
4848
equalTo(ClusterHealthStatus.RED));
4949

50-
logger.info("--> starting hot node");
51-
startHotOnlyNode();
50+
if (randomBoolean()) {
51+
logger.info("--> starting content node");
52+
startContentOnlyNode();
53+
} else {
54+
logger.info("--> starting hot node");
55+
startDataNode();
56+
}
5257

5358
logger.info("--> waiting for {} to be yellow", index);
5459
ensureYellow(index);
@@ -189,6 +194,20 @@ public void testTemplateOverridesDefaults() {
189194
ensureYellow(index);
190195
}
191196

197+
public void startDataNode() {
198+
Settings nodeSettings = Settings.builder()
199+
.putList("node.roles", Arrays.asList("master", "data", "ingest"))
200+
.build();
201+
internalCluster().startNode(nodeSettings);
202+
}
203+
204+
public void startContentOnlyNode() {
205+
Settings nodeSettings = Settings.builder()
206+
.putList("node.roles", Arrays.asList("master", "data_content", "ingest"))
207+
.build();
208+
internalCluster().startNode(nodeSettings);
209+
}
210+
192211
public void startHotOnlyNode() {
193212
Settings nodeSettings = Settings.builder()
194213
.putList("node.roles", Arrays.asList("master", "data_hot", "ingest"))

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP
154154
private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class);
155155

156156
@Override
157-
public Settings getAdditionalIndexSettings(String indexName, Settings indexSettings) {
157+
public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings indexSettings) {
158158
Set<String> settings = indexSettings.keySet();
159159
if (settings.contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) {
160160
// It's okay to put it, it will be removed or overridden by the template/request settings
@@ -163,11 +163,17 @@ public Settings getAdditionalIndexSettings(String indexName, Settings indexSetti
163163
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) ||
164164
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) {
165165
// A different index level require, include, or exclude has been specified, so don't put the setting
166-
logger.debug("index [{}] specifies custom index level routing filtering, skipping hot tier allocation", indexName);
166+
logger.debug("index [{}] specifies custom index level routing filtering, skipping tier allocation", indexName);
167167
return Settings.EMPTY;
168168
} else {
169-
// Otherwise, put the setting in place by default
170-
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
169+
// Otherwise, put the setting in place by default, the "hot"
170+
// tier if the index is part of a data stream, the "content"
171+
// tier if it is not.
172+
if (isDataStreamIndex) {
173+
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
174+
} else {
175+
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_CONTENT).build();
176+
}
171177
}
172178
}
173179
}

‎x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.datastreams;
8+
9+
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
10+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
11+
import org.elasticsearch.cluster.metadata.DataStream;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.test.ESIntegTestCase;
15+
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
16+
import org.elasticsearch.xpack.core.DataTier;
17+
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
18+
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
19+
20+
import java.util.Arrays;
21+
import java.util.Collection;
22+
import java.util.Collections;
23+
24+
import static org.hamcrest.Matchers.equalTo;
25+
26+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
27+
public class DataTierDataStreamIT extends ESIntegTestCase {
28+
private static final String index = "myindex";
29+
30+
@Override
31+
protected Collection<Class<? extends Plugin>> nodePlugins() {
32+
return Arrays.asList(LocalStateCompositeXPackPlugin.class, DataStreamsPlugin.class);
33+
}
34+
35+
public void testDefaultDataStreamAllocateToHot() {
36+
startHotOnlyNode();
37+
ensureGreen();
38+
39+
ComposableIndexTemplate template = new ComposableIndexTemplate(
40+
Collections.singletonList(index),
41+
null,
42+
null,
43+
null,
44+
null,
45+
null,
46+
new ComposableIndexTemplate.DataStreamTemplate()
47+
);
48+
client().execute(
49+
PutComposableIndexTemplateAction.INSTANCE,
50+
new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)
51+
).actionGet();
52+
client().prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").setWaitForActiveShards(0).get();
53+
54+
Settings idxSettings = client().admin()
55+
.indices()
56+
.prepareGetIndex()
57+
.addIndices(index)
58+
.get()
59+
.getSettings()
60+
.get(DataStream.getDefaultBackingIndexName(index, 1));
61+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
62+
63+
logger.info("--> waiting for {} to be yellow", index);
64+
ensureYellow(index);
65+
66+
// Roll over index and ensure the second index also went to the "hot" tier
67+
client().admin().indices().prepareRolloverIndex(index).get();
68+
idxSettings = client().admin()
69+
.indices()
70+
.prepareGetIndex()
71+
.addIndices(index)
72+
.get()
73+
.getSettings()
74+
.get(DataStream.getDefaultBackingIndexName(index, 2));
75+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
76+
77+
client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { index }));
78+
}
79+
80+
public void startHotOnlyNode() {
81+
Settings nodeSettings = Settings.builder().putList("node.roles", Arrays.asList("master", "data_hot", "ingest")).build();
82+
internalCluster().startNode(nodeSettings);
83+
}
84+
}

0 commit comments

Comments
 (0)