Add index.routing.allocation.include._tier_preference setting

docs/reference/api-conventions.asciidoc

@@ -396,7 +396,7 @@ Returns:
       "index.creation_date": "1474389951325",
       "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
       "index.version.created": ...,
-      "index.routing.allocation.include._tier" : "data_content",
+      "index.routing.allocation.include._tier_preference" : "data_content",
       "index.provided_name" : "my-index-000001"
     }
   }
@@ -433,7 +433,7 @@ Returns:
         "routing": {
           "allocation": {
             "include": {
-              "_tier": "data_content"
+              "_tier_preference": "data_content"
             }
           }
         },

docs/reference/cluster/allocation-explain.asciidoc

@@ -116,7 +116,7 @@ PUT /my-index-000001?master_timeout=1s&timeout=1s
 {
   "settings": {
     "index.routing.allocation.include._name": "non_existent_node",
-    "index.routing.allocation.include._tier": null
+    "index.routing.allocation.include._tier_preference": null
   }
 }
 

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -224,7 +224,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
                     // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
                     // before timing out
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
-                    .put("index.routing.allocation.include._tier", "")
+                    .put("index.routing.allocation.include._tier_preference", "")
                     .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
                 createIndex(index, settings.build());
                 indexDocs(index, 0, 10);
@@ -241,7 +241,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
                     .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
                     .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null)
                     .put("index.routing.allocation.include._id", oldNode)
-                    .putNull("index.routing.allocation.include._tier")
+                    .putNull("index.routing.allocation.include._tier_preference")
                 );
                 ensureGreen(index); // wait for the primary to be assigned
                 ensureNoInitializingShards(); // wait for all other shard activity to finish
@@ -264,7 +264,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
                 updateIndexSettings(index, Settings.builder()
                     .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
                     .put("index.routing.allocation.include._id", (String)null)
-                    .putNull("index.routing.allocation.include._tier")
+                    .putNull("index.routing.allocation.include._tier_preference")
                 );
                 asyncIndexDocs(index, 60, 45).get();
                 ensureGreen(index);

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java

@@ -181,7 +181,7 @@ public boolean match(DiscoveryNode node) {
                         }
                     }
                 }
-            } else if ("_tier".equals(attr)) {
+            } else if (attr != null && attr.startsWith("_tier")) {
                 // Always allow _tier as an attribute, will be handled elsewhere
                 return true;
             } else {

x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java

@@ -49,7 +49,7 @@ public void testDoNotAllocateFollowerPrimaryToNodesWithoutRemoteClusterClientRol
         final PutFollowAction.Request putFollowRequest = putFollow(leaderIndex, followerIndex);
         putFollowRequest.setSettings(Settings.builder()
             .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))
-            .putNull("index.routing.allocation.include._tier")
+            .putNull("index.routing.allocation.include._tier_preference")
             .build());
         putFollowRequest.waitForActiveShards(ActiveShardCount.ONE);
         putFollowRequest.timeout(TimeValue.timeValueSeconds(2));
@@ -83,7 +83,7 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro
             .put("index.routing.rebalance.enable", "none")
             .put("index.routing.allocation.include._name",
                 Stream.concat(dataOnlyNodes.stream(), dataAndRemoteNodes.stream()).collect(Collectors.joining(",")))
-            .putNull("index.routing.allocation.include._tier")
+            .putNull("index.routing.allocation.include._tier_preference")
             .build());
         final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get();
         assertTrue(response.isFollowIndexShardsAcked());
@@ -107,8 +107,8 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro
         followerClient().admin().indices().prepareUpdateSettings(followerIndex)
             .setMasterNodeTimeout(TimeValue.MAX_VALUE)
             .setSettings(Settings.builder()
-                .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))
-                .putNull("index.routing.allocation.include._tier"))
+                .putNull("index.routing.allocation.include._tier_preference")
+                .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)))
             .get();
         assertBusy(() -> {
             final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState();

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

@@ -41,7 +41,7 @@ public void testDefaultIndexAllocateToContent() {
         client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
 
         Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
 
         // index should be red
         assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
@@ -51,7 +51,7 @@ public void testDefaultIndexAllocateToContent() {
             logger.info("--> starting content node");
             startContentOnlyNode();
         } else {
-            logger.info("--> starting hot node");
+            logger.info("--> starting data node");
             startDataNode();
         }
 
@@ -65,7 +65,7 @@ public void testOverrideDefaultAllocation() {
         ensureGreen();
 
         String setting = randomBoolean() ? DataTierAllocationDecider.INDEX_ROUTING_REQUIRE :
-            DataTierAllocationDecider.INDEX_ROUTING_INCLUDE;
+            DataTierAllocationDecider.INDEX_ROUTING_PREFER;
 
         client().admin().indices().prepareCreate(index)
             .setWaitForActiveShards(0)
@@ -89,31 +89,31 @@ public void testRequestSettingOverridesAllocation() {
         client().admin().indices().prepareCreate(index)
             .setWaitForActiveShards(0)
             .setSettings(Settings.builder()
-                .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE))
+                .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER))
             .get();
 
         Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(""));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(""));
         // Even the key shouldn't exist if it has been nulled out
-        assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE));
+        assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER));
 
         // index should be yellow
         logger.info("--> waiting for {} to be yellow", index);
         ensureYellow(index);
 
         client().admin().indices().prepareDelete(index).get();
 
-        // Now test it overriding the "require" setting, in which case the include should be skipped
+        // Now test it overriding the "require" setting, in which case the preference should be skipped
         client().admin().indices().prepareCreate(index)
             .setWaitForActiveShards(0)
             .setSettings(Settings.builder()
                 .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_COLD))
             .get();
 
         idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(""));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(""));
         // The key should not be put in place since it was overridden
-        assertFalse(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE));
+        assertFalse(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER));
         assertThat(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE_SETTING.get(idxSettings), equalTo(DataTier.DATA_COLD));
 
         // index should be yellow
@@ -134,7 +134,7 @@ public void testShrinkStaysOnTier() {
             .setSettings(Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, "data_warm"))
+                .put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_warm"))
             .get();
 
         client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.READ_ONLY, index).get();
@@ -150,7 +150,7 @@ public void testShrinkStaysOnTier() {
         Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index + "-shrunk")
             .get().getSettings().get(index + "-shrunk");
         // It should inherit the setting of its originator
-        assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_WARM));
+        assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(DataTier.DATA_WARM));
 
         // Required or else the test cleanup fails because it can't delete the indices
         client().admin().indices().prepareUpdateSettings(index, index + "-shrunk")
@@ -172,15 +172,15 @@ public void testTemplateOverridesDefaults() {
         client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
 
         Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false));
+        assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER), equalTo(false));
 
         // index should be yellow
         ensureYellow(index);
 
         client().admin().indices().prepareDelete(index).get();
 
         t = new Template(Settings.builder()
-            .putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)
+            .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER)
             .build(), null, null);
         ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null);
         client().execute(PutComposableIndexTemplateAction.INSTANCE,
@@ -189,7 +189,7 @@ public void testTemplateOverridesDefaults() {
         client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
 
         idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
-        assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false));
+        assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER), equalTo(false));
 
         ensureYellow(index);
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java

@@ -6,9 +6,11 @@
 
 package org.elasticsearch.xpack.cluster.routing.allocation;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -21,6 +23,7 @@
 import org.elasticsearch.xpack.core.DataTier;
 
 import java.util.Arrays;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -38,6 +41,7 @@ public class DataTierAllocationDecider extends AllocationDecider {
     public static final String CLUSTER_ROUTING_EXCLUDE = "cluster.routing.allocation.exclude._tier";
     public static final String INDEX_ROUTING_REQUIRE = "index.routing.allocation.require._tier";
     public static final String INDEX_ROUTING_INCLUDE = "index.routing.allocation.include._tier";
+    public static final String INDEX_ROUTING_PREFER = "index.routing.allocation.include._tier_preference";
     public static final String INDEX_ROUTING_EXCLUDE = "index.routing.allocation.exclude._tier";
 
     public static final Setting<String> CLUSTER_ROUTING_REQUIRE_SETTING = Setting.simpleString(CLUSTER_ROUTING_REQUIRE,
@@ -52,6 +56,8 @@ public class DataTierAllocationDecider extends AllocationDecider {
         DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
     public static final Setting<String> INDEX_ROUTING_EXCLUDE_SETTING = Setting.simpleString(INDEX_ROUTING_EXCLUDE,
         DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
+    public static final Setting<String> INDEX_ROUTING_PREFER_SETTING = Setting.simpleString(INDEX_ROUTING_PREFER,
+        DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
 
     private static void validateTierSetting(String setting) {
         if (Strings.hasText(setting)) {
@@ -101,7 +107,12 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod
             return decision;
         }
 
-        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
+        decision = shouldIndexPreferTier(indexMetadata, node, allocation);
+        if (decision != null) {
+            return decision;
+        }
+
+        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
     }
 
     private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) {
@@ -115,7 +126,12 @@ private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, Rou
             return decision;
         }
 
-        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
+        decision = shouldIndexPreferTier(allocation.metadata().getIndexSafe(shardRouting.index()), node, allocation);
+        if (decision != null) {
+            return decision;
+        }
+
+        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
     }
 
     private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) {
@@ -129,7 +145,37 @@ private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, Routing
             return decision;
         }
 
-        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
+        decision = shouldIndexPreferTier(indexMd, node, allocation);
+        if (decision != null) {
+            return decision;
+        }
+
+        return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
+    }
+
+    private Decision shouldIndexPreferTier(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) {
+        Settings indexSettings = indexMetadata.getSettings();
+        String tierPreference = INDEX_ROUTING_PREFER_SETTING.get(indexSettings);
+
+        if (Strings.hasText(tierPreference)) {
+            Optional<String> tier = preferredAvailableTier(tierPreference, allocation.nodes());
+            if (tier.isPresent()) {
+                String tierName = tier.get();
+                // The OpType doesn't actually matter here, because we have
+                // selected only a single tier as our "preferred" tier
+                if (allocationAllowed(OpType.AND, tierName, node)) {
+                    return allocation.decision(Decision.YES, NAME,
+                        "index has a preference for tiers [%s] and node has tier [%s]", tierPreference, tierName);
+                } else {
+                    return allocation.decision(Decision.NO, NAME,
+                        "index has a preference for tiers [%s] and node does not meet the required [%s] tier", tierPreference, tierName);
+                }
+            } else {
+                return allocation.decision(Decision.NO, NAME, "index has a preference for tiers [%s], " +
+                    "but no nodes for any of those tiers are available in the cluster", tierPreference);
+            }
+        }
+        return null;
     }
 
     private Decision shouldIndexFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) {
@@ -186,6 +232,31 @@ private enum OpType {
         OR
     }
 
+    /**
+     * Given a string of comma-separated prioritized tiers (highest priority
+     * first) and an allocation, find the highest priority tier for which nodes
+     * exist. If no nodes for any of the tiers are available, returns an empty
+     * {@code Optional<String>}.
+     */
+    static Optional<String> preferredAvailableTier(String prioritizedTiers, DiscoveryNodes nodes) {
+        String[] tiers = Strings.tokenizeToStringArray(prioritizedTiers, ",");
+        return Arrays.stream(tiers).filter(tier -> tierNodesPresent(tier, nodes)).findFirst();
+    }
+
+    static boolean tierNodesPresent(String singleTier, DiscoveryNodes nodes) {
+        assert singleTier.equals(DiscoveryNodeRole.DATA_ROLE.roleName()) || DataTier.validTierName(singleTier) :
+            "tier " + singleTier + " is an invalid tier name";
+        for (ObjectCursor<DiscoveryNode> node : nodes.getNodes().values()) {
+            if (node.value.getRoles().stream()
+                .map(DiscoveryNodeRole::roleName)
+                .anyMatch(s -> s.equals(DiscoveryNodeRole.DATA_ROLE.roleName()) || s.equals(singleTier))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
     private static boolean allocationAllowed(OpType opType, String tierSetting, DiscoveryNode node) {
         String[] values = Strings.tokenizeToStringArray(tierSetting, ",");
         for (String value : values) {