Add runtime field section to Field Capabilities API (#68904) (#68915)

Currently runtime fields from search requests don't appear in the output of the
field capabilities API, but some consumer of runtime fields would like to see
runtime section just like they are defined in search requests reflected and
merged into the field capabilities output.
This change adds parsing of a "runtime_mappings" section equivallent to the one
on search requests to the `_field_caps` endpoint, passes this section down to
the shard level where any runtime fields defined here overwrite the mapping of
the targetet indices.

Closes #68117

Author: Christoph Büscher

docs/reference/search/field-caps.asciidoc

@@ -88,6 +88,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
 (Optional,  <<query-dsl,query object>> Allows to filter indices if the provided
 query rewrites to `match_none` on every shard.
 
+`runtime_mappings`::
+(Optional, object)
+Defines ad-hoc <<runtime-search-request,runtime fields>> in the request similar
+to the way it is done in <<search-api-body-runtime, search requests>>. These fields
+exist only as part of the query and take precedence over fields defined with the
+same name in the index mappings.
+
 [[search-field-caps-api-response-body]]
 ==== {api-response-body-title}
 

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java

@@ -20,6 +20,8 @@
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 public class FieldCapabilitiesIndexRequest extends ActionRequest implements IndicesRequest {
@@ -31,6 +33,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
     private final OriginalIndices originalIndices;
     private final QueryBuilder indexFilter;
     private final long nowInMillis;
+    private Map<String, Object> runtimeFields;
 
     private ShardId shardId;
 
@@ -47,13 +50,15 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         }
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis =  in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readLong() : 0L;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap();
     }
 
     FieldCapabilitiesIndexRequest(String[] fields,
                                   String index,
                                   OriginalIndices originalIndices,
                                   QueryBuilder indexFilter,
-                                  long nowInMillis) {
+                                  long nowInMillis,
+                                  Map<String, Object> runtimeFields) {
         if (fields == null || fields.length == 0) {
             throw new IllegalArgumentException("specified fields can't be null or empty");
         }
@@ -62,6 +67,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         this.originalIndices = originalIndices;
         this.indexFilter = indexFilter;
         this.nowInMillis = nowInMillis;
+        this.runtimeFields = runtimeFields;
     }
 
     public String[] fields() {
@@ -86,6 +92,10 @@ public QueryBuilder indexFilter() {
         return indexFilter;
     }
 
+    public Map<String, Object> runtimeFields() {
+        return runtimeFields;
+    }
+
     public ShardId shardId() {
         return shardId;
     }
@@ -112,6 +122,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -23,7 +23,9 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -37,6 +39,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
+    private Map<String, Object> runtimeFields = Collections.emptyMap();
     private Long nowInMillis;
 
     public FieldCapabilitiesRequest(StreamInput in) throws IOException {
@@ -52,6 +55,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
         }
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalLong() : null;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap();
     }
 
     public FieldCapabilitiesRequest() {
@@ -90,6 +94,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeOptionalLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override
@@ -98,6 +105,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (indexFilter != null) {
             builder.field("index_filter", indexFilter);
         }
+        if (runtimeFields.isEmpty() == false) {
+            builder.field("runtime_mappings", runtimeFields);
+        }
         builder.endObject();
         return builder;
     }
@@ -121,6 +131,7 @@ public String[] fields() {
     /**
      * The list of indices to lookup
      */
+    @Override
     public FieldCapabilitiesRequest indices(String... indices) {
         this.indices = Objects.requireNonNull(indices, "indices must not be null");
         return this;
@@ -166,6 +177,17 @@ public FieldCapabilitiesRequest indexFilter(QueryBuilder indexFilter) {
     public QueryBuilder indexFilter() {
         return indexFilter;
     }
+    /**
+     * Allows adding search runtime fields if provided.
+     */
+    public FieldCapabilitiesRequest runtimeFields(Map<String, Object> runtimeFieldsSection) {
+        this.runtimeFields = runtimeFieldsSection;
+        return this;
+    }
+
+    public Map<String, Object> runtimeFields() {
+        return this.runtimeFields;
+    }
 
     Long nowInMillis() {
         return nowInMillis;
@@ -195,12 +217,13 @@ public boolean equals(Object o) {
             indicesOptions.equals(that.indicesOptions) &&
             Arrays.equals(fields, that.fields) &&
             Objects.equals(indexFilter, that.indexFilter) &&
-            Objects.equals(nowInMillis, that.nowInMillis);
+            Objects.equals(nowInMillis, that.nowInMillis) &&
+            Objects.equals(runtimeFields, that.runtimeFields);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis);
+        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         return result;

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -101,7 +101,7 @@ public void onFailure(Exception e) {
             };
             for (String index : concreteIndices) {
                 shardAction.execute(new FieldCapabilitiesIndexRequest(request.fields(), index, localIndices,
-                    request.indexFilter(), nowInMillis), innerListener);
+                    request.indexFilter(), nowInMillis, request.runtimeFields()), innerListener);
             }
 
             // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -107,7 +107,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd
         try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
 
             final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0,
-                searcher, request::nowInMillis, null, Collections.emptyMap());
+                searcher, request::nowInMillis, null, request.runtimeFields());
 
             if (canMatchShard(request, searchExecutionContext) == false) {
                 return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);

server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java

@@ -11,7 +11,9 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 
@@ -20,6 +22,7 @@
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -52,9 +55,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request,
         fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false));
         request.withContentOrSourceParamParserOrNull(parser -> {
             if (parser != null) {
-                fieldRequest.indexFilter(RestActions.getQueryContent("index_filter", parser));
+                PARSER.parse(parser, fieldRequest, null);
             }
         });
         return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel));
     }
+
+    private static ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
+    private static ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings");
+
+    private static final ObjectParser<FieldCapabilitiesRequest, Void> PARSER = new ObjectParser<>("field_caps_request");
+
+    static {
+        PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseInnerQueryBuilder(p), INDEX_FILTER_FIELD);
+        PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD);
+    }
 }

server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java

@@ -10,15 +10,27 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 
+import static java.util.Collections.singletonMap;
+
 public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCase<FieldCapabilitiesRequest> {
 
     @Override
@@ -41,9 +53,24 @@ protected FieldCapabilitiesRequest createTestInstance() {
             request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
         }
         request.includeUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            request.nowInMillis(randomLong());
+        }
+        if (randomBoolean()) {
+            request.indexFilter(QueryBuilders.termQuery("field", randomAlphaOfLength(5)));
+        }
+        if (randomBoolean()) {
+            request.runtimeFields(Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        }
         return request;
     }
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
     @Override
     protected Writeable.Reader<FieldCapabilitiesRequest> instanceReader() {
         return FieldCapabilitiesRequest::new;
@@ -67,13 +94,44 @@ protected FieldCapabilitiesRequest mutateInstance(FieldCapabilitiesRequest insta
         });
         mutators.add(request -> request.setMergeResults(request.isMergeResults() == false));
         mutators.add(request -> request.includeUnmapped(request.includeUnmapped() == false));
+        mutators.add(request -> request.nowInMillis(request.nowInMillis() != null ? request.nowInMillis() + 1 : 1L));
+        mutators.add(
+            request -> request.indexFilter(request.indexFilter() != null ? request.indexFilter().boost(2) : QueryBuilders.matchAllQuery())
+        );
+        mutators.add(request -> request.runtimeFields(Collections.singletonMap("other_key", "other_value")));
 
         FieldCapabilitiesRequest mutatedInstance = copyInstance(instance);
         Consumer<FieldCapabilitiesRequest> mutator = randomFrom(mutators);
         mutator.accept(mutatedInstance);
         return mutatedInstance;
     }
 
+    public void testToXContent() throws IOException {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indexFilter(QueryBuilders.termQuery("field", "value"));
+        request.runtimeFields(singletonMap("day_of_week", singletonMap("type", "keyword")));
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        String xContent = BytesReference.bytes(request.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
+        assertEquals(
+            ("{"
+                + "  \"index_filter\": {\n"
+                + "    \"term\": {\n"
+                + "      \"field\": {\n"
+                + "        \"value\": \"value\",\n"
+                + "        \"boost\": 1.0\n"
+                + "      }\n"
+                + "    }\n"
+                + "  },\n"
+                + "  \"runtime_mappings\": {\n"
+                + "    \"day_of_week\": {\n"
+                + "      \"type\": \"keyword\"\n"
+                + "    }\n"
+                + "  }\n"
+                + "}").replaceAll("\\s+", ""),
+            xContent
+        );
+    }
+
     public void testValidation() {
         FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
             .indices("index2");