Skip to content

Commit 3f267ad

Browse files
author
Christoph Büscher
authored
Add runtime field section to Field Capabilities API (#68904)
Currently runtime fields from search requests don't appear in the output of the field capabilities API, but some consumer of runtime fields would like to see runtime section just like they are defined in search requests reflected and merged into the field capabilities output. This change adds parsing of a "runtime_mappings" section equivallent to the one on search requests to the `_field_caps` endpoint, passes this section down to the shard level where any runtime fields defined here overwrite the mapping of the targetet indices. Closes #68117
1 parent 742facd commit 3f267ad

File tree

8 files changed

+207
-6
lines changed

8 files changed

+207
-6
lines changed

‎docs/reference/search/field-caps.asciidoc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
8888
(Optional, <<query-dsl,query object>> Allows to filter indices if the provided
8989
query rewrites to `match_none` on every shard.
9090

91+
`runtime_mappings`::
92+
(Optional, object)
93+
Defines ad-hoc <<runtime-search-request,runtime fields>> in the request similar
94+
to the way it is done in <<search-api-body-runtime, search requests>>. These fields
95+
exist only as part of the query and take precedence over fields defined with the
96+
same name in the index mappings.
97+
9198
[[search-field-caps-api-response-body]]
9299
==== {api-response-body-title}
93100

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
import org.elasticsearch.index.shard.ShardId;
2121

2222
import java.io.IOException;
23+
import java.util.Collections;
24+
import java.util.Map;
2325
import java.util.Objects;
2426

2527
public class FieldCapabilitiesIndexRequest extends ActionRequest implements IndicesRequest {
@@ -31,6 +33,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
3133
private final OriginalIndices originalIndices;
3234
private final QueryBuilder indexFilter;
3335
private final long nowInMillis;
36+
private Map<String, Object> runtimeFields;
3437

3538
private ShardId shardId;
3639

@@ -43,13 +46,15 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
4346
originalIndices = OriginalIndices.readOriginalIndices(in);
4447
indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
4548
nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readLong() : 0L;
49+
runtimeFields = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readMap() : Collections.emptyMap();
4650
}
4751

4852
FieldCapabilitiesIndexRequest(String[] fields,
4953
String index,
5054
OriginalIndices originalIndices,
5155
QueryBuilder indexFilter,
52-
long nowInMillis) {
56+
long nowInMillis,
57+
Map<String, Object> runtimeFields) {
5358
if (fields == null || fields.length == 0) {
5459
throw new IllegalArgumentException("specified fields can't be null or empty");
5560
}
@@ -58,6 +63,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
5863
this.originalIndices = originalIndices;
5964
this.indexFilter = indexFilter;
6065
this.nowInMillis = nowInMillis;
66+
this.runtimeFields = runtimeFields;
6167
}
6268

6369
public String[] fields() {
@@ -82,6 +88,10 @@ public QueryBuilder indexFilter() {
8288
return indexFilter;
8389
}
8490

91+
public Map<String, Object> runtimeFields() {
92+
return runtimeFields;
93+
}
94+
8595
public ShardId shardId() {
8696
return shardId;
8797
}
@@ -106,6 +116,9 @@ public void writeTo(StreamOutput out) throws IOException {
106116
out.writeOptionalNamedWriteable(indexFilter);
107117
out.writeLong(nowInMillis);
108118
}
119+
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
120+
out.writeMap(runtimeFields);
121+
}
109122
}
110123

111124
@Override

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323

2424
import java.io.IOException;
2525
import java.util.Arrays;
26+
import java.util.Collections;
2627
import java.util.HashSet;
28+
import java.util.Map;
2729
import java.util.Objects;
2830
import java.util.Set;
2931

@@ -37,6 +39,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
3739
// pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
3840
private boolean mergeResults = true;
3941
private QueryBuilder indexFilter;
42+
private Map<String, Object> runtimeFields = Collections.emptyMap();
4043
private Long nowInMillis;
4144

4245
public FieldCapabilitiesRequest(StreamInput in) throws IOException {
@@ -52,6 +55,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
5255
}
5356
indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
5457
nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalLong() : null;
58+
runtimeFields = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readMap() : Collections.emptyMap();
5559
}
5660

5761
public FieldCapabilitiesRequest() {
@@ -90,6 +94,9 @@ public void writeTo(StreamOutput out) throws IOException {
9094
out.writeOptionalNamedWriteable(indexFilter);
9195
out.writeOptionalLong(nowInMillis);
9296
}
97+
if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
98+
out.writeMap(runtimeFields);
99+
}
93100
}
94101

95102
@Override
@@ -98,6 +105,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
98105
if (indexFilter != null) {
99106
builder.field("index_filter", indexFilter);
100107
}
108+
if (runtimeFields.isEmpty() == false) {
109+
builder.field("runtime_mappings", runtimeFields);
110+
}
101111
builder.endObject();
102112
return builder;
103113
}
@@ -121,6 +131,7 @@ public String[] fields() {
121131
/**
122132
* The list of indices to lookup
123133
*/
134+
@Override
124135
public FieldCapabilitiesRequest indices(String... indices) {
125136
this.indices = Objects.requireNonNull(indices, "indices must not be null");
126137
return this;
@@ -166,6 +177,17 @@ public FieldCapabilitiesRequest indexFilter(QueryBuilder indexFilter) {
166177
public QueryBuilder indexFilter() {
167178
return indexFilter;
168179
}
180+
/**
181+
* Allows adding search runtime fields if provided.
182+
*/
183+
public FieldCapabilitiesRequest runtimeFields(Map<String, Object> runtimeFieldsSection) {
184+
this.runtimeFields = runtimeFieldsSection;
185+
return this;
186+
}
187+
188+
public Map<String, Object> runtimeFields() {
189+
return this.runtimeFields;
190+
}
169191

170192
Long nowInMillis() {
171193
return nowInMillis;
@@ -195,12 +217,13 @@ public boolean equals(Object o) {
195217
indicesOptions.equals(that.indicesOptions) &&
196218
Arrays.equals(fields, that.fields) &&
197219
Objects.equals(indexFilter, that.indexFilter) &&
198-
Objects.equals(nowInMillis, that.nowInMillis);
220+
Objects.equals(nowInMillis, that.nowInMillis) &&
221+
Objects.equals(runtimeFields, that.runtimeFields);
199222
}
200223

201224
@Override
202225
public int hashCode() {
203-
int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis);
226+
int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields);
204227
result = 31 * result + Arrays.hashCode(indices);
205228
result = 31 * result + Arrays.hashCode(fields);
206229
return result;

‎server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public void onFailure(Exception e) {
101101
};
102102
for (String index : concreteIndices) {
103103
client.executeLocally(TransportFieldCapabilitiesIndexAction.TYPE, new FieldCapabilitiesIndexRequest(request.fields(),
104-
index, localIndices, request.indexFilter(), nowInMillis), innerListener);
104+
index, localIndices, request.indexFilter(), nowInMillis, request.runtimeFields()), innerListener);
105105
}
106106

107107
// this is the cross cluster part of this API - we force the other cluster to not merge the results but instead

‎server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd
107107
try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
108108

109109
final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0,
110-
searcher, request::nowInMillis, null, Collections.emptyMap());
110+
searcher, request::nowInMillis, null, request.runtimeFields());
111111

112112
if (canMatchShard(request, searchExecutionContext) == false) {
113113
return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);

‎server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
1212
import org.elasticsearch.action.support.IndicesOptions;
1313
import org.elasticsearch.client.node.NodeClient;
14+
import org.elasticsearch.common.ParseField;
1415
import org.elasticsearch.common.Strings;
16+
import org.elasticsearch.common.xcontent.ObjectParser;
1517
import org.elasticsearch.rest.BaseRestHandler;
1618
import org.elasticsearch.rest.RestRequest;
1719

1820
import java.io.IOException;
1921
import java.util.List;
2022

23+
import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
2124
import static org.elasticsearch.rest.RestRequest.Method.GET;
2225
import static org.elasticsearch.rest.RestRequest.Method.POST;
2326

@@ -50,9 +53,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request,
5053
fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false));
5154
request.withContentOrSourceParamParserOrNull(parser -> {
5255
if (parser != null) {
53-
fieldRequest.indexFilter(RestActions.getQueryContent("index_filter", parser));
56+
PARSER.parse(parser, fieldRequest, null);
5457
}
5558
});
5659
return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel));
5760
}
61+
62+
private static ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
63+
private static ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings");
64+
65+
private static final ObjectParser<FieldCapabilitiesRequest, Void> PARSER = new ObjectParser<>("field_caps_request");
66+
67+
static {
68+
PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseInnerQueryBuilder(p), INDEX_FILTER_FIELD);
69+
PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD);
70+
}
5871
}

‎server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,27 @@
1010

1111
import org.elasticsearch.action.ActionRequestValidationException;
1212
import org.elasticsearch.action.support.IndicesOptions;
13+
import org.elasticsearch.common.bytes.BytesReference;
14+
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1315
import org.elasticsearch.common.io.stream.Writeable;
16+
import org.elasticsearch.common.settings.Settings;
1417
import org.elasticsearch.common.util.ArrayUtils;
18+
import org.elasticsearch.common.xcontent.ToXContent;
19+
import org.elasticsearch.common.xcontent.XContentBuilder;
20+
import org.elasticsearch.common.xcontent.XContentFactory;
21+
import org.elasticsearch.common.xcontent.XContentType;
22+
import org.elasticsearch.index.query.QueryBuilders;
23+
import org.elasticsearch.search.SearchModule;
1524
import org.elasticsearch.test.AbstractWireSerializingTestCase;
1625

1726
import java.io.IOException;
1827
import java.util.ArrayList;
28+
import java.util.Collections;
1929
import java.util.List;
2030
import java.util.function.Consumer;
2131

32+
import static java.util.Collections.singletonMap;
33+
2234
public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCase<FieldCapabilitiesRequest> {
2335

2436
@Override
@@ -41,9 +53,24 @@ protected FieldCapabilitiesRequest createTestInstance() {
4153
request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
4254
}
4355
request.includeUnmapped(randomBoolean());
56+
if (randomBoolean()) {
57+
request.nowInMillis(randomLong());
58+
}
59+
if (randomBoolean()) {
60+
request.indexFilter(QueryBuilders.termQuery("field", randomAlphaOfLength(5)));
61+
}
62+
if (randomBoolean()) {
63+
request.runtimeFields(Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)));
64+
}
4465
return request;
4566
}
4667

68+
@Override
69+
protected NamedWriteableRegistry getNamedWriteableRegistry() {
70+
SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
71+
return new NamedWriteableRegistry(searchModule.getNamedWriteables());
72+
}
73+
4774
@Override
4875
protected Writeable.Reader<FieldCapabilitiesRequest> instanceReader() {
4976
return FieldCapabilitiesRequest::new;
@@ -67,13 +94,44 @@ protected FieldCapabilitiesRequest mutateInstance(FieldCapabilitiesRequest insta
6794
});
6895
mutators.add(request -> request.setMergeResults(request.isMergeResults() == false));
6996
mutators.add(request -> request.includeUnmapped(request.includeUnmapped() == false));
97+
mutators.add(request -> request.nowInMillis(request.nowInMillis() != null ? request.nowInMillis() + 1 : 1L));
98+
mutators.add(
99+
request -> request.indexFilter(request.indexFilter() != null ? request.indexFilter().boost(2) : QueryBuilders.matchAllQuery())
100+
);
101+
mutators.add(request -> request.runtimeFields(Collections.singletonMap("other_key", "other_value")));
70102

71103
FieldCapabilitiesRequest mutatedInstance = copyInstance(instance);
72104
Consumer<FieldCapabilitiesRequest> mutator = randomFrom(mutators);
73105
mutator.accept(mutatedInstance);
74106
return mutatedInstance;
75107
}
76108

109+
public void testToXContent() throws IOException {
110+
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
111+
request.indexFilter(QueryBuilders.termQuery("field", "value"));
112+
request.runtimeFields(singletonMap("day_of_week", singletonMap("type", "keyword")));
113+
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
114+
String xContent = BytesReference.bytes(request.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
115+
assertEquals(
116+
("{"
117+
+ " \"index_filter\": {\n"
118+
+ " \"term\": {\n"
119+
+ " \"field\": {\n"
120+
+ " \"value\": \"value\",\n"
121+
+ " \"boost\": 1.0\n"
122+
+ " }\n"
123+
+ " }\n"
124+
+ " },\n"
125+
+ " \"runtime_mappings\": {\n"
126+
+ " \"day_of_week\": {\n"
127+
+ " \"type\": \"keyword\"\n"
128+
+ " }\n"
129+
+ " }\n"
130+
+ "}").replaceAll("\\s+", ""),
131+
xContent
132+
);
133+
}
134+
77135
public void testValidation() {
78136
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
79137
.indices("index2");

0 commit comments

Comments
 (0)