Simplify IndicesShardStoresAction (#94507)

- No need to use an `AsyncShardFetch` here, there is no caching
- Response may be very large, introduce chunking
- Fan-out may be very large, introduce throttling
- Processing time may be nontrivial, introduce cancellability
- Eliminate many unnecessary intermediate data structures
- Do shard-level response processing more eagerly
- Determine allocation from `RoutingTable` not `RoutingNodes`
- Add tests

Relates #81081

Author: DaveCTurner

docs/reference/indices/shard-stores.asciidoc

@@ -93,6 +93,16 @@ regardless of health status.
 Defaults to `yellow,red`.
 --
 
+`max_concurrent_shard_requests`::
++
+--
+(Optional, integer)
+Maximum number of concurrent shard-level requests sent by the coordinating
+node. Defaults to `100`. Larger values may yield a quicker response to requests
+that target many shards, but may also cause a larger impact on other cluster
+operations.
+--
+
 [[index-shard-stores-api-example]]
 ==== {api-examples-title}
 

server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresRequest.java

@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.action.admin.indices.shards;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -15,18 +16,25 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Map;
 
 /**
  * Request for {@link IndicesShardStoresAction}
  */
 public class IndicesShardStoresRequest extends MasterNodeReadRequest<IndicesShardStoresRequest> implements IndicesRequest.Replaceable {
 
+    static final int DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS = 100;
+
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = IndicesOptions.strictExpand();
     private EnumSet<ClusterHealthStatus> statuses = EnumSet.of(ClusterHealthStatus.YELLOW, ClusterHealthStatus.RED);
+    private int maxConcurrentShardRequests = DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS;
 
     /**
      * Create a request for shard stores info for <code>indices</code>
@@ -46,6 +54,12 @@ public IndicesShardStoresRequest(StreamInput in) throws IOException {
             statuses.add(ClusterHealthStatus.readFrom(in));
         }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
+            maxConcurrentShardRequests = in.readVInt();
+        } else {
+            // earlier versions had unlimited concurrency
+            maxConcurrentShardRequests = Integer.MAX_VALUE;
+        }
     }
 
     @Override
@@ -54,6 +68,17 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArrayNullable(indices);
         out.writeCollection(statuses, (o, v) -> o.writeByte(v.value()));
         indicesOptions.writeIndicesOptions(out);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
+            out.writeVInt(maxConcurrentShardRequests);
+        } else if (maxConcurrentShardRequests != DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS) {
+            throw new IllegalArgumentException(
+                "support for maxConcurrentShardRequests=["
+                    + maxConcurrentShardRequests
+                    + "] was added in version [8.8.0], cannot send this request using transport version ["
+                    + out.getTransportVersion()
+                    + "]"
+            );
+        } // else just drop the value and use the default behaviour
     }
 
     /**
@@ -114,8 +139,21 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    public void maxConcurrentShardRequests(int maxConcurrentShardRequests) {
+        this.maxConcurrentShardRequests = maxConcurrentShardRequests;
+    }
+
+    public int maxConcurrentShardRequests() {
+        return maxConcurrentShardRequests;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         return null;
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers);
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java

@@ -12,13 +12,19 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -28,7 +34,7 @@
  * Consists of {@link StoreStatus}s for requested indices grouped by
  * indices and shard ids and a list of encountered node {@link Failure}s
  */
-public class IndicesShardStoresResponse extends ActionResponse implements ToXContentFragment {
+public class IndicesShardStoresResponse extends ActionResponse implements ChunkedToXContentObject {
 
     /**
      * Shard store information from a node
@@ -196,7 +202,7 @@ public int compareTo(StoreStatus other) {
      * Single node failure while retrieving shard store information
      */
     public static class Failure extends DefaultShardOperationFailedException {
-        private String nodeId;
+        private final String nodeId;
 
         public Failure(String nodeId, String index, int shardId, Throwable reason) {
             super(index, shardId, reason);
@@ -273,38 +279,45 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (failures.size() > 0) {
-            builder.startArray(Fields.FAILURES);
-            for (Failure failure : failures) {
-                failure.toXContent(builder, params);
-            }
-            builder.endArray();
-        }
-
-        builder.startObject(Fields.INDICES);
-        for (Map.Entry<String, Map<Integer, List<StoreStatus>>> indexShards : storeStatuses.entrySet()) {
-            builder.startObject(indexShards.getKey());
-
-            builder.startObject(Fields.SHARDS);
-            for (Map.Entry<Integer, List<StoreStatus>> shardStatusesEntry : indexShards.getValue().entrySet()) {
-                builder.startObject(String.valueOf(shardStatusesEntry.getKey()));
-                builder.startArray(Fields.STORES);
-                for (StoreStatus storeStatus : shardStatusesEntry.getValue()) {
-                    builder.startObject();
-                    storeStatus.toXContent(builder, params);
-                    builder.endObject();
-                }
-                builder.endArray();
-
-                builder.endObject();
-            }
-            builder.endObject();
-
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.startObject(),
+
+            failures.isEmpty()
+                ? Collections.emptyIterator()
+                : Iterators.concat(
+                    ChunkedToXContentHelper.startArray(Fields.FAILURES),
+                    failures.iterator(),
+                    ChunkedToXContentHelper.endArray()
+                ),
+
+            ChunkedToXContentHelper.startObject(Fields.INDICES),
+
+            Iterators.flatMap(
+                storeStatuses.entrySet().iterator(),
+                indexShards -> Iterators.concat(
+                    ChunkedToXContentHelper.startObject(indexShards.getKey()),
+                    ChunkedToXContentHelper.startObject(Fields.SHARDS),
+                    Iterators.flatMap(
+                        indexShards.getValue().entrySet().iterator(),
+                        shardStatusesEntry -> Iterators.single((ToXContent) (builder, params) -> {
+                            builder.startObject(String.valueOf(shardStatusesEntry.getKey())).startArray(Fields.STORES);
+                            for (StoreStatus storeStatus : shardStatusesEntry.getValue()) {
+                                builder.startObject();
+                                storeStatus.toXContent(builder, params);
+                                builder.endObject();
+                            }
+                            return builder.endArray().endObject();
+                        })
+                    ),
+                    ChunkedToXContentHelper.endObject(),
+                    ChunkedToXContentHelper.endObject()
+                )
+            ),
+
+            ChunkedToXContentHelper.endObject(),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     static final class Fields {