[6.8] Backport: Search optimisation - add canMatch early aborts for queries on "_index" field

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml

@@ -0,0 +1,102 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: single_doc_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+---
+teardown:
+  - do:
+      indices.delete:
+        index: single_doc_index
+        ignore_unavailable: true
+
+---
+"Test that queries on _index match against the correct indices.":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "single_doc_index"}}'
+            - '{"f1": "local_cluster", "sort_field": 0}'
+
+  - do:
+     search:
+       rest_total_hits_as_int: true
+       index: "single_doc_index,my_remote_cluster:single_doc_index"
+       body:
+         query:
+            term:
+              "_index": "single_doc_index"
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "single_doc_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 0}
+  - match: { _shards.failed: 0 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: "single_doc_index,my_remote_cluster:single_doc_index"
+        body:
+          query:
+            term:
+              "_index": "my_remote_cluster:single_doc_index"
+
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"}
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 0}
+  - match: { _shards.failed: 0 }
+
+---
+"Test that queries on _index that don't match are skipped":
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "single_doc_index"}}'
+            - '{"f1": "local_cluster", "sort_field": 0}'
+
+  - do:
+     search:
+       ccs_minimize_roundtrips: false
+       track_total_hits: true
+       index: "single_doc_index,my_remote_cluster:single_doc_index"
+       pre_filter_shard_size: 1
+       body:
+         query:
+            term:
+              "_index": "does_not_match"
+
+  - match: { hits.total.value: 0 }
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 1}
+  - match: { _shards.failed: 0 }
+
+  - do:
+      search:
+        ccs_minimize_roundtrips: false
+        track_total_hits: true
+        index: "single_doc_index,my_remote_cluster:single_doc_index"
+        pre_filter_shard_size: 1
+        body:
+          query:
+            term:
+              "_index": "my_remote_cluster:does_not_match"
+
+  - match: { hits.total.value: 0 }
+  - match: { _shards.total: 2 }
+  - match: { _shards.successful: 2 }
+  - match: { _shards.skipped : 1}
+  - match: { _shards.failed: 0 }

server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java

@@ -169,6 +169,19 @@ public String getWriteableName() {
         return NAME;
     }
 
+    @Override
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        if ("_index".equals(fieldName)) {
+            // Special-case optimisation for canMatch phase:
+            // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
+            QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
+            if (shardContext != null && !shardContext.index().getName().startsWith(value)) {
+                return new MatchNoneQueryBuilder();
+            }
+        }
+        return super.doRewrite(queryRewriteContext);
+    }
+
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE);

server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java

@@ -130,6 +130,19 @@ public static TermQueryBuilder fromXContent(XContentParser parser) throws IOExce
         return termQuery;
     }
 
+    @Override
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        if ("_index".equals(fieldName)) {
+            // Special-case optimisation for canMatch phase:
+            // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
+            QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
+            if (shardContext != null && !shardContext.index().getName().equals(BytesRefs.toString(value))) {
+                return new MatchNoneQueryBuilder();
+            }
+        }
+        return super.doRewrite(queryRewriteContext);
+    }
+
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query query = null;

server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java

@@ -493,6 +493,21 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
             });
             return new TermsQueryBuilder(this.fieldName, supplier::get);
         }
+        if ("_index".equals(this.fieldName) && values != null) {
+            // Special-case optimisation for canMatch phase:
+            // We can skip querying this shard if the index name doesn't match any of the search terms.
+            QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
+            if (shardContext != null) {
+                for (Object localValue : values) {
+                    if (shardContext.index().getName().equals(BytesRefs.toString(localValue))) {
+                        // We can match - at least one index name matches
+                        return this;
+                    }
+                }
+                // all index names are invalid - no possibility of a match on this shard.
+                return new MatchNoneQueryBuilder();
+            }
+        }
         return this;
     }
 }

server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -179,6 +180,19 @@ public static WildcardQueryBuilder fromXContent(XContentParser parser) throws IO
                 .queryName(queryName);
     }
 
+    @Override
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        if ("_index".equals(fieldName)) {
+            // Special-case optimisation for canMatch phase:
+            // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field.
+            QueryShardContext shardContext = queryRewriteContext.convertToShardContext();
+            if (shardContext != null && !shardContext.index().getName().equals(BytesRefs.toString(value))) {
+                return new MatchNoneQueryBuilder();
+            }
+        }
+        return super.doRewrite(queryRewriteContext);
+    }
+
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         MappedFieldType fieldType = context.fieldMapper(fieldName);

server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java

@@ -143,4 +143,19 @@ public void testParseFailsWithMultipleFields() throws IOException {
         e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
         assertEquals("[prefix] query doesn't support multiple fields, found [user1] and [user2]", e.getMessage());
     }
+
+    public void testRewriteIndexQueryToMatchNone() throws Exception {
+        PrefixQueryBuilder query = prefixQuery("_index", "does_not_exist");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteIndexQueryToNotMatchNone() throws Exception {
+        PrefixQueryBuilder query = prefixQuery("_index", getIndex().getName());
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(PrefixQueryBuilder.class));
+    }
+
 }

server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java

@@ -171,6 +171,20 @@ public void testParseFailsWithMultipleFields() throws IOException {
         assertEquals("[term] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
     }
 
+
+    public void testRewriteIndexQueryToMatchNone() throws IOException {
+        TermQueryBuilder query = QueryBuilders.termQuery("_index", "does_not_exist");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteIndexQueryToNotMatchNone() throws IOException {
+        TermQueryBuilder query = QueryBuilders.termQuery("_index", getIndex().getName());
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(TermQueryBuilder.class));
+    }
     public void testParseAndSerializeBigInteger() throws IOException {
         String json = "{\n" +
                 "  \"term\" : {\n" +

server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java

@@ -311,5 +311,19 @@ public void testConversion() {
         assertEquals(Arrays.asList(5, 42d), TermsQueryBuilder.convert(list));
         assertEquals(Arrays.asList(5, 42d), TermsQueryBuilder.convertBack(TermsQueryBuilder.convert(list)));
     }
-}
 
+    public void testRewriteIndexQueryToMatchNone() throws IOException {
+        TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", "also_does_not_exist");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testRewriteIndexQueryToNotMatchNone() throws IOException {
+        // At least one name is good
+        TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", getIndex().getName());
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(TermsQueryBuilder.class));
+    }
+}

server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java

@@ -154,4 +154,20 @@ public void testIndexWildcard() throws IOException {
         query = new WildcardQueryBuilder("_index", "index_" + index + "*").doToQuery(context);
         assertThat(query instanceof MatchNoDocsQuery, equalTo(true));
     }
+
+    public void testRewriteIndexQueryToMatchNone() throws IOException {
+        WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
+    }   
+
+    public void testRewriteIndexQueryNotMatchNone() throws IOException {
+        String fullIndexName = getIndex().getName();
+        String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);
+        WildcardQueryBuilder query = new WildcardQueryBuilder("_index", firstHalfOfIndexName +"*");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(WildcardQueryBuilder.class));
+    }      
 }