Skip to content

Commit 7dbc1ea

Browse files
original-brownbearoriginal-brownbear
authored
Use chunked encoding for indices stats response (#91760)
These responses can become huge, lets chunk them by index.
1 parent c66ac71 commit 7dbc1ea

File tree

6 files changed

+188
-94
lines changed

6 files changed

+188
-94
lines changed

‎server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java‎

Lines changed: 62 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,13 @@
1313
import org.apache.lucene.search.SortedNumericSortField;
1414
import org.apache.lucene.search.SortedSetSortField;
1515
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
16-
import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
16+
import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
1717
import org.elasticsearch.common.collect.Iterators;
1818
import org.elasticsearch.common.io.stream.StreamInput;
1919
import org.elasticsearch.common.io.stream.StreamOutput;
2020
import org.elasticsearch.common.unit.ByteSizeValue;
21-
import org.elasticsearch.common.xcontent.ChunkedToXContent;
2221
import org.elasticsearch.core.RestApiVersion;
2322
import org.elasticsearch.index.engine.Segment;
24-
import org.elasticsearch.rest.action.RestActions;
2523
import org.elasticsearch.xcontent.ToXContent;
2624
import org.elasticsearch.xcontent.XContentBuilder;
2725

@@ -33,7 +31,7 @@
3331
import java.util.Locale;
3432
import java.util.Map;
3533

36-
public class IndicesSegmentResponse extends BaseBroadcastResponse implements ChunkedToXContent {
34+
public class IndicesSegmentResponse extends ChunkedBroadcastResponse {
3735

3836
private final ShardSegments[] shards;
3937

@@ -79,72 +77,72 @@ public void writeTo(StreamOutput out) throws IOException {
7977
}
8078

8179
@Override
82-
public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
83-
return Iterators.concat(Iterators.single(((builder, params) -> {
84-
builder.startObject();
85-
RestActions.buildBroadcastShardsHeader(builder, params, this);
86-
return builder.startObject(Fields.INDICES);
87-
})), getIndices().values().stream().map(indexSegments -> (ToXContent) (builder, params) -> {
88-
builder.startObject(indexSegments.getIndex());
89-
90-
builder.startObject(Fields.SHARDS);
91-
for (IndexShardSegments indexSegment : indexSegments) {
92-
builder.startArray(Integer.toString(indexSegment.shardId().id()));
93-
for (ShardSegments shardSegments : indexSegment) {
94-
builder.startObject();
95-
96-
builder.startObject(Fields.ROUTING);
97-
builder.field(Fields.STATE, shardSegments.getShardRouting().state());
98-
builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary());
99-
builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId());
100-
if (shardSegments.getShardRouting().relocatingNodeId() != null) {
101-
builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId());
102-
}
103-
builder.endObject();
104-
105-
builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted());
106-
builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch());
107-
108-
builder.startObject(Fields.SEGMENTS);
109-
for (Segment segment : shardSegments) {
110-
builder.startObject(segment.getName());
111-
builder.field(Fields.GENERATION, segment.getGeneration());
112-
builder.field(Fields.NUM_DOCS, segment.getNumDocs());
113-
builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
114-
builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize());
115-
if (builder.getRestApiVersion() == RestApiVersion.V_7) {
116-
builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO);
117-
}
118-
builder.field(Fields.COMMITTED, segment.isCommitted());
119-
builder.field(Fields.SEARCH, segment.isSearch());
120-
if (segment.getVersion() != null) {
121-
builder.field(Fields.VERSION, segment.getVersion());
122-
}
123-
if (segment.isCompound() != null) {
124-
builder.field(Fields.COMPOUND, segment.isCompound());
80+
protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
81+
return Iterators.concat(
82+
Iterators.single((builder, p) -> builder.startObject(Fields.INDICES)),
83+
getIndices().values().stream().map(indexSegments -> (ToXContent) (builder, p) -> {
84+
builder.startObject(indexSegments.getIndex());
85+
86+
builder.startObject(Fields.SHARDS);
87+
for (IndexShardSegments indexSegment : indexSegments) {
88+
builder.startArray(Integer.toString(indexSegment.shardId().id()));
89+
for (ShardSegments shardSegments : indexSegment) {
90+
builder.startObject();
91+
92+
builder.startObject(Fields.ROUTING);
93+
builder.field(Fields.STATE, shardSegments.getShardRouting().state());
94+
builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary());
95+
builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId());
96+
if (shardSegments.getShardRouting().relocatingNodeId() != null) {
97+
builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId());
12598
}
126-
if (segment.getMergeId() != null) {
127-
builder.field(Fields.MERGE_ID, segment.getMergeId());
128-
}
129-
if (segment.getSegmentSort() != null) {
130-
toXContent(builder, segment.getSegmentSort());
131-
}
132-
if (segment.attributes != null && segment.attributes.isEmpty() == false) {
133-
builder.field("attributes", segment.attributes);
99+
builder.endObject();
100+
101+
builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted());
102+
builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch());
103+
104+
builder.startObject(Fields.SEGMENTS);
105+
for (Segment segment : shardSegments) {
106+
builder.startObject(segment.getName());
107+
builder.field(Fields.GENERATION, segment.getGeneration());
108+
builder.field(Fields.NUM_DOCS, segment.getNumDocs());
109+
builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
110+
builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize());
111+
if (builder.getRestApiVersion() == RestApiVersion.V_7) {
112+
builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO);
113+
}
114+
builder.field(Fields.COMMITTED, segment.isCommitted());
115+
builder.field(Fields.SEARCH, segment.isSearch());
116+
if (segment.getVersion() != null) {
117+
builder.field(Fields.VERSION, segment.getVersion());
118+
}
119+
if (segment.isCompound() != null) {
120+
builder.field(Fields.COMPOUND, segment.isCompound());
121+
}
122+
if (segment.getMergeId() != null) {
123+
builder.field(Fields.MERGE_ID, segment.getMergeId());
124+
}
125+
if (segment.getSegmentSort() != null) {
126+
toXContent(builder, segment.getSegmentSort());
127+
}
128+
if (segment.attributes != null && segment.attributes.isEmpty() == false) {
129+
builder.field("attributes", segment.attributes);
130+
}
131+
builder.endObject();
134132
}
135133
builder.endObject();
136-
}
137-
builder.endObject();
138134

139-
builder.endObject();
135+
builder.endObject();
136+
}
137+
builder.endArray();
140138
}
141-
builder.endArray();
142-
}
143-
builder.endObject();
139+
builder.endObject();
144140

145-
builder.endObject();
146-
return builder;
147-
}).iterator(), Iterators.single((builder, params) -> builder.endObject().endObject()));
141+
builder.endObject();
142+
return builder;
143+
}).iterator(),
144+
Iterators.single((builder, p) -> builder.endObject())
145+
);
148146
}
149147

150148
private static void toXContent(XContentBuilder builder, Sort sort) throws IOException {

‎server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,23 @@
1111
import org.elasticsearch.Version;
1212
import org.elasticsearch.action.admin.indices.stats.IndexStats.IndexStatsBuilder;
1313
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
14-
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
14+
import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
1515
import org.elasticsearch.cluster.ClusterState;
1616
import org.elasticsearch.cluster.health.ClusterHealthStatus;
1717
import org.elasticsearch.cluster.health.ClusterIndexHealth;
1818
import org.elasticsearch.cluster.metadata.IndexMetadata;
1919
import org.elasticsearch.cluster.routing.ShardRouting;
2020
import org.elasticsearch.common.Strings;
21+
import org.elasticsearch.common.collect.Iterators;
2122
import org.elasticsearch.common.io.stream.StreamInput;
2223
import org.elasticsearch.common.io.stream.StreamOutput;
2324
import org.elasticsearch.index.Index;
25+
import org.elasticsearch.xcontent.ToXContent;
2426
import org.elasticsearch.xcontent.XContentBuilder;
2527

2628
import java.io.IOException;
2729
import java.util.HashMap;
30+
import java.util.Iterator;
2831
import java.util.List;
2932
import java.util.Locale;
3033
import java.util.Map;
@@ -33,7 +36,7 @@
3336

3437
import static java.util.Collections.unmodifiableMap;
3538

36-
public class IndicesStatsResponse extends BroadcastResponse {
39+
public class IndicesStatsResponse extends ChunkedBroadcastResponse {
3740

3841
private final Map<String, ClusterHealthStatus> indexHealthMap;
3942

@@ -171,30 +174,19 @@ public void writeTo(StreamOutput out) throws IOException {
171174
}
172175

173176
@Override
174-
protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException {
177+
protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
175178
final String level = params.param("level", "indices");
176179
final boolean isLevelValid = "cluster".equalsIgnoreCase(level)
177180
|| "indices".equalsIgnoreCase(level)
178181
|| "shards".equalsIgnoreCase(level);
179182
if (isLevelValid == false) {
180183
throw new IllegalArgumentException("level parameter must be one of [cluster] or [indices] or [shards] but was [" + level + "]");
181184
}
182-
183-
builder.startObject("_all");
184-
185-
builder.startObject("primaries");
186-
getPrimaries().toXContent(builder, params);
187-
builder.endObject();
188-
189-
builder.startObject("total");
190-
getTotal().toXContent(builder, params);
191-
builder.endObject();
192-
193-
builder.endObject();
194-
195185
if ("indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level)) {
196-
builder.startObject(Fields.INDICES);
197-
for (IndexStats indexStats : getIndices().values()) {
186+
return Iterators.concat(Iterators.single(((builder, p) -> {
187+
commonStats(builder, p);
188+
return builder.startObject(Fields.INDICES);
189+
})), getIndices().values().stream().<ToXContent>map(indexStats -> (builder, p) -> {
198190
builder.startObject(indexStats.getIndex());
199191
builder.field("uuid", indexStats.getUuid());
200192
if (indexStats.getHealth() != null) {
@@ -204,11 +196,11 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
204196
builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT));
205197
}
206198
builder.startObject("primaries");
207-
indexStats.getPrimaries().toXContent(builder, params);
199+
indexStats.getPrimaries().toXContent(builder, p);
208200
builder.endObject();
209201

210202
builder.startObject("total");
211-
indexStats.getTotal().toXContent(builder, params);
203+
indexStats.getTotal().toXContent(builder, p);
212204
builder.endObject();
213205

214206
if ("shards".equalsIgnoreCase(level)) {
@@ -217,17 +209,34 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
217209
builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
218210
for (ShardStats shardStats : indexShardStats) {
219211
builder.startObject();
220-
shardStats.toXContent(builder, params);
212+
shardStats.toXContent(builder, p);
221213
builder.endObject();
222214
}
223215
builder.endArray();
224216
}
225217
builder.endObject();
226218
}
227-
builder.endObject();
228-
}
229-
builder.endObject();
219+
return builder.endObject();
220+
}).iterator(), Iterators.single((b, p) -> b.endObject()));
230221
}
222+
return Iterators.single((b, p) -> {
223+
commonStats(b, p);
224+
return b;
225+
});
226+
}
227+
228+
private void commonStats(XContentBuilder builder, ToXContent.Params p) throws IOException {
229+
builder.startObject("_all");
230+
231+
builder.startObject("primaries");
232+
getPrimaries().toXContent(builder, p);
233+
builder.endObject();
234+
235+
builder.startObject("total");
236+
getTotal().toXContent(builder, p);
237+
builder.endObject();
238+
239+
builder.endObject();
231240
}
232241

233242
static final class Fields {

‎server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
package org.elasticsearch.action.support.broadcast;
9+
10+
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
11+
import org.elasticsearch.common.collect.Iterators;
12+
import org.elasticsearch.common.io.stream.StreamInput;
13+
import org.elasticsearch.common.xcontent.ChunkedToXContent;
14+
import org.elasticsearch.rest.action.RestActions;
15+
import org.elasticsearch.xcontent.ToXContent;
16+
17+
import java.io.IOException;
18+
import java.util.Iterator;
19+
import java.util.List;
20+
21+
public abstract class ChunkedBroadcastResponse extends BaseBroadcastResponse implements ChunkedToXContent {
22+
public ChunkedBroadcastResponse(StreamInput in) throws IOException {
23+
super(in);
24+
}
25+
26+
public ChunkedBroadcastResponse(
27+
int totalShards,
28+
int successfulShards,
29+
int failedShards,
30+
List<DefaultShardOperationFailedException> shardFailures
31+
) {
32+
super(totalShards, successfulShards, failedShards, shardFailures);
33+
}
34+
35+
@Override
36+
public final Iterator<ToXContent> toXContentChunked(ToXContent.Params params) {
37+
return Iterators.concat(Iterators.single((b, p) -> {
38+
b.startObject();
39+
RestActions.buildBroadcastShardsHeader(b, p, this);
40+
return b;
41+
}), customXContentChunks(params), Iterators.single((builder, p) -> builder.endObject()));
42+
}
43+
44+
protected abstract Iterator<ToXContent> customXContentChunks(ToXContent.Params params);
45+
}

‎server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import org.elasticsearch.rest.BaseRestHandler;
2020
import org.elasticsearch.rest.RestRequest;
2121
import org.elasticsearch.rest.action.RestCancellableNodeClient;
22-
import org.elasticsearch.rest.action.RestToXContentListener;
22+
import org.elasticsearch.rest.action.RestChunkedToXContentListener;
2323
import org.elasticsearch.rest.action.document.RestMultiTermVectorsAction;
2424

2525
import java.io.IOException;
@@ -140,7 +140,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
140140

141141
return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
142142
.indices()
143-
.stats(indicesStatsRequest, new RestToXContentListener<>(channel));
143+
.stats(indicesStatsRequest, new RestChunkedToXContentListener<>(channel));
144144
}
145145

146146
@Override

‎server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,6 @@ public void testSerializesOneChunkPerIndex() {
7474
iterator.next();
7575
chunks++;
7676
}
77-
assertEquals(indices + 2, chunks);
77+
assertEquals(indices + 4, chunks);
7878
}
7979
}

0 commit comments

Comments
 (0)