Allocate new indices on "hot" or "content" tier depending on data stream inclusion

docs/reference/api-conventions.asciidoc

@@ -396,7 +396,7 @@ Returns:
       "index.creation_date": "1474389951325",
       "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
       "index.version.created": ...,
-      "index.routing.allocation.include._tier" : "data_hot",
+      "index.routing.allocation.include._tier" : "data_content",
       "index.provided_name" : "my-index-000001"
     }
   }
@@ -433,7 +433,7 @@ Returns:
         "routing": {
           "allocation": {
             "include": {
-              "_tier": "data_hot"
+              "_tier": "data_content"
             }
           }
         },

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -629,10 +629,12 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
                 .put(request.settings())
                 .build();
 
+            final boolean isDataStreamIndex = request.dataStreamName() != null;
             // Loop through all the explicit index setting providers, adding them to the
             // additionalIndexSettings map
             for (IndexSettingProvider provider : indexSettingProviders) {
-                additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), templateAndRequestSettings));
+                additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(),
+                    isDataStreamIndex, templateAndRequestSettings));
             }
 
             // For all the explicit settings, we go through the template and request level settings

server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java

@@ -30,7 +30,7 @@ public interface IndexSettingProvider {
      * Returns explicitly set default index {@link Settings} for the given index. This should not
      * return null.
      */
-    default Settings getAdditionalIndexSettings(String indexName, Settings templateAndRequestSettings) {
+    default Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
         return Settings.EMPTY;
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java → x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

@@ -33,22 +33,27 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(LocalStateCompositeXPackPlugin.class);
     }
 
-    public void testDefaultAllocateToHot() {
+    public void testDefaultIndexAllocateToContent() {
         startWarmOnlyNode();
         startColdOnlyNode();
         ensureGreen();
 
         client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
 
         Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
 
         // index should be red
         assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
             equalTo(ClusterHealthStatus.RED));
 
-        logger.info("--> starting hot node");
-        startHotOnlyNode();
+        if (randomBoolean()) {
+            logger.info("--> starting content node");
+            startContentOnlyNode();
+        } else {
+            logger.info("--> starting hot node");
+            startDataNode();
+        }
 
         logger.info("--> waiting for {} to be yellow", index);
         ensureYellow(index);
@@ -189,6 +194,20 @@ public void testTemplateOverridesDefaults() {
         ensureYellow(index);
     }
 
+    public void startDataNode() {
+        Settings nodeSettings = Settings.builder()
+            .putList("node.roles", Arrays.asList("master", "data", "ingest"))
+            .build();
+        internalCluster().startNode(nodeSettings);
+    }
+
+    public void startContentOnlyNode() {
+        Settings nodeSettings = Settings.builder()
+            .putList("node.roles", Arrays.asList("master", "data_content", "ingest"))
+            .build();
+        internalCluster().startNode(nodeSettings);
+    }
+
     public void startHotOnlyNode() {
         Settings nodeSettings = Settings.builder()
             .putList("node.roles", Arrays.asList("master", "data_hot", "ingest"))

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

@@ -154,7 +154,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP
         private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class);
 
         @Override
-        public Settings getAdditionalIndexSettings(String indexName, Settings indexSettings) {
+        public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings indexSettings) {
             Set<String> settings = indexSettings.keySet();
             if (settings.contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) {
                 // It's okay to put it, it will be removed or overridden by the template/request settings
@@ -163,11 +163,17 @@ public Settings getAdditionalIndexSettings(String indexName, Settings indexSetti
                 settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) ||
                 settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) {
                 // A different index level require, include, or exclude has been specified, so don't put the setting
-                logger.debug("index [{}] specifies custom index level routing filtering, skipping hot tier allocation", indexName);
+                logger.debug("index [{}] specifies custom index level routing filtering, skipping tier allocation", indexName);
                 return Settings.EMPTY;
             } else {
-                // Otherwise, put the setting in place by default
-                return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
+                // Otherwise, put the setting in place by default, the "hot"
+                // tier if the index is part of a data stream, the "content"
+                // tier if it is not.
+                if (isDataStreamIndex) {
+                    return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
+                } else {
+                    return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_CONTENT).build();
+                }
             }
         }
     }

x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.datastreams;
+
+import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
+import org.elasticsearch.xpack.core.DataTier;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
+public class DataTierDataStreamIT extends ESIntegTestCase {
+    private static final String index = "myindex";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, DataStreamsPlugin.class);
+    }
+
+    public void testDefaultDataStreamAllocateToHot() {
+        startHotOnlyNode();
+        ensureGreen();
+
+        ComposableIndexTemplate template = new ComposableIndexTemplate(
+            Collections.singletonList(index),
+            null,
+            null,
+            null,
+            null,
+            null,
+            new ComposableIndexTemplate.DataStreamTemplate()
+        );
+        client().execute(
+            PutComposableIndexTemplateAction.INSTANCE,
+            new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)
+        ).actionGet();
+        client().prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").setWaitForActiveShards(0).get();
+
+        Settings idxSettings = client().admin()
+            .indices()
+            .prepareGetIndex()
+            .addIndices(index)
+            .get()
+            .getSettings()
+            .get(DataStream.getDefaultBackingIndexName(index, 1));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
+
+        logger.info("--> waiting for {} to be yellow", index);
+        ensureYellow(index);
+
+        // Roll over index and ensure the second index also went to the "hot" tier
+        client().admin().indices().prepareRolloverIndex(index).get();
+        idxSettings = client().admin()
+            .indices()
+            .prepareGetIndex()
+            .addIndices(index)
+            .get()
+            .getSettings()
+            .get(DataStream.getDefaultBackingIndexName(index, 2));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
+
+        client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { index }));
+    }
+
+    public void startHotOnlyNode() {
+        Settings nodeSettings = Settings.builder().putList("node.roles", Arrays.asList("master", "data_hot", "ingest")).build();
+        internalCluster().startNode(nodeSettings);
+    }
+}