[WIP] Various Speedups to Cluster State Updates (ILM + Allocation)

server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java

@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -72,9 +71,8 @@ protected ClusterBlockException checkBlock(ClusterAllocationExplainRequest reque
     @Override
     protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
                                    final ActionListener<ClusterAllocationExplainResponse> listener) {
-        final RoutingNodes routingNodes = state.getRoutingNodes();
         final ClusterInfo clusterInfo = clusterInfoService.getClusterInfo();
-        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
+        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state::getRoutingNodes, state,
                 clusterInfo, snapshotsInfoService.snapshotShardSizes(), System.nanoTime());
 
         ShardRouting shardRouting = findShardToExplain(request, allocation);

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -172,12 +172,12 @@ default boolean isPrivate() {
 
     public ClusterState(long version, String stateUUID, ClusterState state) {
         this(state.clusterName, version, stateUUID, state.metadata(), state.routingTable(), state.nodes(), state.blocks(),
-                state.customs(), -1, false);
+            state.customs(), -1, false, state.routingNodes);
     }
 
-    public ClusterState(ClusterName clusterName, long version, String stateUUID, Metadata metadata, RoutingTable routingTable,
+    private ClusterState(ClusterName clusterName, long version, String stateUUID, Metadata metadata, RoutingTable routingTable,
                         DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap<String, Custom> customs,
-                        int minimumMasterNodesOnPublishingMaster, boolean wasReadFromDiff) {
+                        int minimumMasterNodesOnPublishingMaster, boolean wasReadFromDiff, RoutingNodes routingNodes) {
         this.version = version;
         this.stateUUID = stateUUID;
         this.clusterName = clusterName;
@@ -187,6 +187,7 @@ public ClusterState(ClusterName clusterName, long version, String stateUUID, Met
         this.blocks = blocks;
         this.customs = customs;
         this.minimumMasterNodesOnPublishingMaster = minimumMasterNodesOnPublishingMaster;
+        this.routingNodes = routingNodes;
         this.wasReadFromDiff = wasReadFromDiff;
     }
 
@@ -302,15 +303,35 @@ public boolean wasReadFromDiff() {
         return wasReadFromDiff;
     }
 
+    public RoutingNodes mutableRoutingNodes() {
+        final RoutingNodes nodes = this.routingNodes;
+        if (nodes != null) {
+            return nodes.mutableCopy();
+        }
+        return new RoutingNodes(routingTable, this.nodes, false);
+    }
+
     /**
      * Returns a built (on demand) routing nodes view of the routing table.
      */
     public RoutingNodes getRoutingNodes() {
-        if (routingNodes != null) {
+        final RoutingNodes nodes = this.routingNodes;
+        if (nodes != null) {
+            return nodes;
+        }
+        return buildRoutingNodes();
+    }
+
+    private RoutingNodes buildRoutingNodes() {
+        synchronized (this) {
+            RoutingNodes routingNodes = this.routingNodes;
+            if (routingNodes != null) {
+                return routingNodes;
+            }
+            routingNodes = new RoutingNodes(routingTable, this.nodes);
+            this.routingNodes = routingNodes;
             return routingNodes;
         }
-        routingNodes = new RoutingNodes(this);
-        return routingNodes;
     }
 
     @Override
@@ -550,18 +571,22 @@ public static Builder builder(ClusterState state) {
 
     public static class Builder {
 
+        private ClusterState previous;
+
         private final ClusterName clusterName;
         private long version = 0;
         private String uuid = UNKNOWN_UUID;
         private Metadata metadata = Metadata.EMPTY_METADATA;
         private RoutingTable routingTable = RoutingTable.EMPTY_ROUTING_TABLE;
+        private RoutingNodes routingNodes = null;
         private DiscoveryNodes nodes = DiscoveryNodes.EMPTY_NODES;
         private ClusterBlocks blocks = ClusterBlocks.EMPTY_CLUSTER_BLOCK;
         private final ImmutableOpenMap.Builder<String, Custom> customs;
         private boolean fromDiff;
         private int minimumMasterNodesOnPublishingMaster = -1;
 
         public Builder(ClusterState state) {
+            this.previous = state;
             this.clusterName = state.clusterName;
             this.version = state.version();
             this.uuid = state.stateUUID();
@@ -592,6 +617,11 @@ public DiscoveryNodes nodes() {
             return nodes;
         }
 
+        public Builder routingNodes(RoutingNodes routingNodes) {
+            this.routingNodes = routingNodes;
+            return this;
+        }
+
         public Builder routingTable(RoutingTable routingTable) {
             this.routingTable = routingTable;
             return this;
@@ -661,8 +691,11 @@ public ClusterState build() {
             if (UNKNOWN_UUID.equals(uuid)) {
                 uuid = UUIDs.randomBase64UUID();
             }
+            if (previous != null && routingTable.indicesRouting() == previous.routingTable.indicesRouting() && nodes == previous.nodes) {
+                routingNodes = previous.routingNodes;
+            }
             return new ClusterState(clusterName, version, uuid, metadata, routingTable, nodes, blocks, customs.build(),
-                minimumMasterNodesOnPublishingMaster, fromDiff);
+                minimumMasterNodesOnPublishingMaster, fromDiff, routingNodes);
         }
 
         public static byte[] toBytes(ClusterState state) throws IOException {
@@ -818,6 +851,9 @@ public ClusterState apply(ClusterState state) {
             builder.blocks(blocks.apply(state.blocks));
             builder.customs(customs.apply(state.customs));
             builder.minimumMasterNodesOnPublishingMaster(minimumMasterNodesOnPublishingMaster);
+            if (builder.routingTable == state.routingTable && builder.nodes == state.nodes) {
+                builder.routingNodes(state.routingNodes);
+            }
             builder.fromDiff(true);
             return builder.build();
         }

server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java

@@ -104,20 +104,18 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT
         this.numberOfShards = indexMetadata.getNumberOfShards();
         this.numberOfReplicas = indexMetadata.getNumberOfReplicas();
 
-        shards = new HashMap<>();
-        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
-            int shardId = shardRoutingTable.shardId().id();
-            shards.put(shardId, new ClusterShardHealth(shardId, shardRoutingTable));
-        }
-
+        shards = new HashMap<>(this.numberOfShards);
         // update the index status
         ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
         int computeActivePrimaryShards = 0;
         int computeActiveShards = 0;
         int computeRelocatingShards = 0;
         int computeInitializingShards = 0;
         int computeUnassignedShards = 0;
-        for (ClusterShardHealth shardHealth : shards.values()) {
+        for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
+            int shardId = shardRoutingTable.shardId().id();
+            final ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, shardRoutingTable);
+            shards.put(shardId, shardHealth);
             if (shardHealth.isPrimaryActive()) {
                 computeActivePrimaryShards++;
             }
@@ -133,6 +131,7 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT
                 computeStatus = ClusterHealthStatus.YELLOW;
             }
         }
+
         if (shards.isEmpty()) { // might be since none has been created yet (two phase index creation)
             computeStatus = ClusterHealthStatus.RED;
         }

server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java

@@ -56,6 +56,12 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre
         numberOfNodes = clusterState.nodes().getSize();
         numberOfDataNodes = clusterState.nodes().getDataNodes().size();
         indices = new HashMap<>();
+        ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
+        int computeActivePrimaryShards = 0;
+        int computeActiveShards = 0;
+        int computeRelocatingShards = 0;
+        int computeInitializingShards = 0;
+        int computeUnassignedShards = 0;
         for (String index : concreteIndices) {
             IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(index);
             IndexMetadata indexMetadata = clusterState.metadata().index(index);
@@ -66,16 +72,6 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre
             ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetadata, indexRoutingTable);
 
             indices.put(indexHealth.getIndex(), indexHealth);
-        }
-
-        ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
-        int computeActivePrimaryShards = 0;
-        int computeActiveShards = 0;
-        int computeRelocatingShards = 0;
-        int computeInitializingShards = 0;
-        int computeUnassignedShards = 0;
-
-        for (ClusterIndexHealth indexHealth : indices.values()) {
             computeActivePrimaryShards += indexHealth.getActivePrimaryShards();
             computeActiveShards += indexHealth.getActiveShards();
             computeRelocatingShards += indexHealth.getRelocatingShards();

server/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java

@@ -142,7 +142,7 @@ public static Map<Integer, List<String>> getAutoExpandReplicaChanges(Metadata me
 
         for (final IndexMetadata indexMetadata : metadata) {
             if (indexMetadata.getState() == IndexMetadata.State.OPEN || isIndexVerifiedBeforeClosed(indexMetadata)) {
-                AutoExpandReplicas autoExpandReplicas = SETTING.get(indexMetadata.getSettings());
+                AutoExpandReplicas autoExpandReplicas = indexMetadata.getAutoExpandReplicas();
                 autoExpandReplicas.getDesiredNumberOfReplicas(indexMetadata, allocation).ifPresent(numberOfReplicas -> {
                     if (numberOfReplicas != indexMetadata.getNumberOfReplicas()) {
                         nrReplicasChanged.computeIfAbsent(numberOfReplicas, ArrayList::new).add(indexMetadata.getIndex().getName());

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java

@@ -32,6 +32,9 @@
  */
 public class ComposableIndexTemplateMetadata implements Metadata.Custom {
     public static final String TYPE = "index_template";
+
+    public static final ComposableIndexTemplateMetadata EMPTY = new ComposableIndexTemplateMetadata(org.elasticsearch.core.Map.of());
+
     private static final ParseField INDEX_TEMPLATE = new ParseField("index_template");
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<ComposableIndexTemplateMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE,

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

@@ -182,6 +182,23 @@ public boolean isSystem() {
         public List<String> getAliases() {
             return aliases;
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ConcreteIndex that = (ConcreteIndex) o;
+            return isHidden == that.isHidden &&
+                isSystem == that.isSystem &&
+                concreteIndexName.equals(that.concreteIndexName) &&
+                Objects.equals(aliases, that.aliases) &&
+                Objects.equals(dataStream, that.dataStream);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(concreteIndexName, isHidden, isSystem, aliases, dataStream);
+        }
     }
 
     /**
@@ -302,6 +319,24 @@ private void validateAliasProperties(List<IndexMetadata> referenceIndexMetadatas
         private boolean isNonEmpty(List<IndexMetadata> idxMetas) {
             return (Objects.isNull(idxMetas) || idxMetas.isEmpty()) == false;
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Alias alias = (Alias) o;
+            return isHidden == alias.isHidden &&
+                isSystem == alias.isSystem &&
+                dataStreamAlias == alias.dataStreamAlias &&
+                aliasName.equals(alias.aliasName) &&
+                referenceIndexMetadatas.equals(alias.referenceIndexMetadatas) &&
+                Objects.equals(writeIndex, alias.writeIndex);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(aliasName, referenceIndexMetadatas, writeIndex, isHidden, isSystem, dataStreamAlias);
+        }
     }
 
     class DataStream implements IndexAbstraction {
@@ -363,6 +398,20 @@ public List<String> getAliases() {
         public org.elasticsearch.cluster.metadata.DataStream getDataStream() {
             return dataStream;
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DataStream that = (DataStream) o;
+            return dataStream.equals(that.dataStream) &&
+                Objects.equals(referencedByDataStreamAliases, that.referencedByDataStreamAliases);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(dataStream, referencedByDataStreamAliases);
+        }
     }
 
 }