Skip to content

Commit e3b7e5d

Browse files
martijnvgmartijnvg
committed
percolator: Replace percolate api with the new percolator query
Also replaced the PercolatorQueryRegistry with the new PercolatorQueryCache. The PercolatorFieldMapper stores the rewritten form of each percolator query's xcontext in a binary doc values field. This make sure that the query rewrite happens only during indexing (some queries for example fetch shapes, terms in remote indices) and the speed up the loading of the queries in the percolator query cache. Because the percolator now works inside the search infrastructure a number of features (sorting fields, pagination, fetch features) are available out of the box. The following feature requests are automatically implemented via this refactoring: Closes #10741 Closes #7297 Closes #13176 Closes #13978 Closes #11264 Closes #10741 Closes #4317
1 parent 032678f commit e3b7e5d

File tree

97 files changed

+3071
-5610
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+3071
-5610
lines changed

‎buildSrc/src/main/resources/checkstyle_suppressions.xml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -663,10 +663,7 @@
663663
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]process[/\\]ProcessService.java" checks="LineLength" />
664664
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
665665
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
666-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateContext.java" checks="LineLength" />
667-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateDocumentParser.java" checks="LineLength" />
668666
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQuery.java" checks="LineLength" />
669-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorService.java" checks="LineLength" />
670667
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]DummyPluginInfo.java" checks="LineLength" />
671668
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]InstallPluginCommand.java" checks="LineLength" />
672669
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
@@ -1326,14 +1323,9 @@
13261323
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
13271324
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]nodesinfo[/\\]NodeInfoStreamingTests.java" checks="LineLength" />
13281325
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]options[/\\]detailederrors[/\\]DetailedErrorsEnabledIT.java" checks="LineLength" />
1329-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]ConcurrentPercolatorIT.java" checks="LineLength" />
13301326
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
1331-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateDocumentParserTests.java" checks="LineLength" />
13321327
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
13331328
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQueryTests.java" checks="LineLength" />
1334-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorServiceTests.java" checks="LineLength" />
1335-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]RecoveryPercolatorIT.java" checks="LineLength" />
1336-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TTLPercolatorIT.java" checks="LineLength" />
13371329
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
13381330
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
13391331
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />

‎core/src/main/java/org/apache/lucene/index/memory/ExtendedMemoryIndex.java‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎core/src/main/java/org/elasticsearch/ElasticsearchException.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -645,8 +645,6 @@ enum ElasticsearchExceptionHandle {
645645
// 87 used to be for MergeMappingException
646646
INVALID_INDEX_TEMPLATE_EXCEPTION(org.elasticsearch.indices.InvalidIndexTemplateException.class,
647647
org.elasticsearch.indices.InvalidIndexTemplateException::new, 88),
648-
PERCOLATE_EXCEPTION(org.elasticsearch.percolator.PercolateException.class,
649-
org.elasticsearch.percolator.PercolateException::new, 89),
650648
REFRESH_FAILED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.RefreshFailedEngineException.class,
651649
org.elasticsearch.index.engine.RefreshFailedEngineException::new, 90),
652650
AGGREGATION_INITIALIZATION_EXCEPTION(org.elasticsearch.search.aggregations.AggregationInitializationException.class,

‎core/src/main/java/org/elasticsearch/action/ActionModule.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,6 @@
165165
import org.elasticsearch.action.percolate.PercolateAction;
166166
import org.elasticsearch.action.percolate.TransportMultiPercolateAction;
167167
import org.elasticsearch.action.percolate.TransportPercolateAction;
168-
import org.elasticsearch.action.percolate.TransportShardMultiPercolateAction;
169168
import org.elasticsearch.action.search.ClearScrollAction;
170169
import org.elasticsearch.action.search.MultiSearchAction;
171170
import org.elasticsearch.action.search.SearchAction;
@@ -331,7 +330,7 @@ protected void configure() {
331330
registerAction(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
332331
registerAction(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
333332
registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
334-
registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class, TransportShardMultiPercolateAction.class);
333+
registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
335334
registerAction(ExplainAction.INSTANCE, TransportExplainAction.class);
336335
registerAction(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
337336
registerAction(RecoveryAction.INSTANCE, TransportRecoveryAction.class);

‎core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import org.elasticsearch.index.cache.query.QueryCacheStats;
3232
import org.elasticsearch.index.engine.SegmentsStats;
3333
import org.elasticsearch.index.fielddata.FieldDataStats;
34-
import org.elasticsearch.index.percolator.PercolateStats;
34+
import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
3535
import org.elasticsearch.index.shard.DocsStats;
3636
import org.elasticsearch.index.store.StoreStats;
3737
import org.elasticsearch.search.suggest.completion.CompletionStats;
@@ -48,7 +48,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
4848
private QueryCacheStats queryCache;
4949
private CompletionStats completion;
5050
private SegmentsStats segments;
51-
private PercolateStats percolate;
51+
private PercolatorQueryCacheStats percolatorCache;
5252

5353
private ClusterStatsIndices() {
5454
}
@@ -62,7 +62,7 @@ public ClusterStatsIndices(ClusterStatsNodeResponse[] nodeResponses) {
6262
this.queryCache = new QueryCacheStats();
6363
this.completion = new CompletionStats();
6464
this.segments = new SegmentsStats();
65-
this.percolate = new PercolateStats();
65+
this.percolatorCache = new PercolatorQueryCacheStats();
6666

6767
for (ClusterStatsNodeResponse r : nodeResponses) {
6868
for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
@@ -85,7 +85,7 @@ public ClusterStatsIndices(ClusterStatsNodeResponse[] nodeResponses) {
8585
queryCache.add(shardCommonStats.queryCache);
8686
completion.add(shardCommonStats.completion);
8787
segments.add(shardCommonStats.segments);
88-
percolate.add(shardCommonStats.percolate);
88+
percolatorCache.add(shardCommonStats.percolatorCache);
8989
}
9090
}
9191

@@ -128,8 +128,8 @@ public SegmentsStats getSegments() {
128128
return segments;
129129
}
130130

131-
public PercolateStats getPercolate() {
132-
return percolate;
131+
public PercolatorQueryCacheStats getPercolatorCache() {
132+
return percolatorCache;
133133
}
134134

135135
@Override
@@ -142,7 +142,7 @@ public void readFrom(StreamInput in) throws IOException {
142142
queryCache = QueryCacheStats.readQueryCacheStats(in);
143143
completion = CompletionStats.readCompletionStats(in);
144144
segments = SegmentsStats.readSegmentsStats(in);
145-
percolate = PercolateStats.readPercolateStats(in);
145+
percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
146146
}
147147

148148
@Override
@@ -155,7 +155,7 @@ public void writeTo(StreamOutput out) throws IOException {
155155
queryCache.writeTo(out);
156156
completion.writeTo(out);
157157
segments.writeTo(out);
158-
percolate.writeTo(out);
158+
percolatorCache.writeTo(out);
159159
}
160160

161161
public static ClusterStatsIndices readIndicesStats(StreamInput in) throws IOException {
@@ -178,7 +178,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
178178
queryCache.toXContent(builder, params);
179179
completion.toXContent(builder, params);
180180
segments.toXContent(builder, params);
181-
percolate.toXContent(builder, params);
181+
percolatorCache.toXContent(builder, params);
182182
return builder;
183183
}
184184

‎core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
5656

5757
private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store,
5858
CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments,
59-
CommonStatsFlags.Flag.Percolate);
59+
CommonStatsFlags.Flag.PercolatorCache);
6060

6161
private final NodeService nodeService;
6262
private final IndicesService indicesService;
@@ -105,7 +105,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
105105
for (IndexShard indexShard : indexService) {
106106
if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
107107
// only report on fully started shards
108-
shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
108+
shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
109109
}
110110
}
111111
}

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java‎

Lines changed: 22 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,10 @@
3232
import org.elasticsearch.index.fielddata.FieldDataStats;
3333
import org.elasticsearch.index.flush.FlushStats;
3434
import org.elasticsearch.index.get.GetStats;
35+
import org.elasticsearch.index.percolator.PercolatorQueryCache;
3536
import org.elasticsearch.index.shard.IndexingStats;
3637
import org.elasticsearch.index.merge.MergeStats;
37-
import org.elasticsearch.index.percolator.PercolateStats;
38+
import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
3839
import org.elasticsearch.index.recovery.RecoveryStats;
3940
import org.elasticsearch.index.refresh.RefreshStats;
4041
import org.elasticsearch.index.search.stats.SearchStats;
@@ -101,8 +102,8 @@ public CommonStats(CommonStatsFlags flags) {
101102
case Segments:
102103
segments = new SegmentsStats();
103104
break;
104-
case Percolate:
105-
percolate = new PercolateStats();
105+
case PercolatorCache:
106+
percolatorCache = new PercolatorQueryCacheStats();
106107
break;
107108
case Translog:
108109
translog = new TranslogStats();
@@ -123,7 +124,8 @@ public CommonStats(CommonStatsFlags flags) {
123124
}
124125

125126

126-
public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
127+
public CommonStats(IndicesQueryCache indicesQueryCache, PercolatorQueryCache percolatorQueryCache,
128+
IndexShard indexShard, CommonStatsFlags flags) {
127129

128130
CommonStatsFlags.Flag[] setFlags = flags.getFlags();
129131

@@ -168,8 +170,8 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
168170
case Segments:
169171
segments = indexShard.segmentStats(flags.includeSegmentFileSizes());
170172
break;
171-
case Percolate:
172-
percolate = indexShard.percolateStats();
173+
case PercolatorCache:
174+
percolatorCache = percolatorQueryCache.getStats(indexShard.shardId());
173175
break;
174176
case Translog:
175177
translog = indexShard.translogStats();
@@ -223,7 +225,7 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
223225
public FieldDataStats fieldData;
224226

225227
@Nullable
226-
public PercolateStats percolate;
228+
public PercolatorQueryCacheStats percolatorCache;
227229

228230
@Nullable
229231
public CompletionStats completion;
@@ -333,13 +335,13 @@ public void add(CommonStats stats) {
333335
} else {
334336
fieldData.add(stats.getFieldData());
335337
}
336-
if (percolate == null) {
337-
if (stats.getPercolate() != null) {
338-
percolate = new PercolateStats();
339-
percolate.add(stats.getPercolate());
338+
if (percolatorCache == null) {
339+
if (stats.getPercolatorCache() != null) {
340+
percolatorCache = new PercolatorQueryCacheStats();
341+
percolatorCache.add(stats.getPercolatorCache());
340342
}
341343
} else {
342-
percolate.add(stats.getPercolate());
344+
percolatorCache.add(stats.getPercolatorCache());
343345
}
344346
if (completion == null) {
345347
if (stats.getCompletion() != null) {
@@ -447,8 +449,8 @@ public FieldDataStats getFieldData() {
447449
}
448450

449451
@Nullable
450-
public PercolateStats getPercolate() {
451-
return percolate;
452+
public PercolatorQueryCacheStats getPercolatorCache() {
453+
return percolatorCache;
452454
}
453455

454456
@Nullable
@@ -489,7 +491,7 @@ public static CommonStats readCommonStats(StreamInput in) throws IOException {
489491

490492
/**
491493
* Utility method which computes total memory by adding
492-
* FieldData, Percolate, Segments (memory, index writer, version map)
494+
* FieldData, PercolatorCache, Segments (memory, index writer, version map)
493495
*/
494496
public ByteSizeValue getTotalMemory() {
495497
long size = 0;
@@ -499,9 +501,6 @@ public ByteSizeValue getTotalMemory() {
499501
if (this.getQueryCache() != null) {
500502
size += this.getQueryCache().getMemorySizeInBytes();
501503
}
502-
if (this.getPercolate() != null) {
503-
size += this.getPercolate().getMemorySizeInBytes();
504-
}
505504
if (this.getSegments() != null) {
506505
size += this.getSegments().getMemoryInBytes() +
507506
this.getSegments().getIndexWriterMemoryInBytes() +
@@ -547,7 +546,7 @@ public void readFrom(StreamInput in) throws IOException {
547546
fieldData = FieldDataStats.readFieldDataStats(in);
548547
}
549548
if (in.readBoolean()) {
550-
percolate = PercolateStats.readPercolateStats(in);
549+
percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
551550
}
552551
if (in.readBoolean()) {
553552
completion = CompletionStats.readCompletionStats(in);
@@ -629,11 +628,11 @@ public void writeTo(StreamOutput out) throws IOException {
629628
out.writeBoolean(true);
630629
fieldData.writeTo(out);
631630
}
632-
if (percolate == null) {
631+
if (percolatorCache == null) {
633632
out.writeBoolean(false);
634633
} else {
635634
out.writeBoolean(true);
636-
percolate.writeTo(out);
635+
percolatorCache.writeTo(out);
637636
}
638637
if (completion == null) {
639638
out.writeBoolean(false);
@@ -689,8 +688,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
689688
if (fieldData != null) {
690689
fieldData.toXContent(builder, params);
691690
}
692-
if (percolate != null) {
693-
percolate.toXContent(builder, params);
691+
if (percolatorCache != null) {
692+
percolatorCache.toXContent(builder, params);
694693
}
695694
if (completion != null) {
696695
completion.toXContent(builder, params);

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ public static enum Flag {
240240
FieldData("fielddata"),
241241
Docs("docs"),
242242
Warmer("warmer"),
243-
Percolate("percolate"),
243+
PercolatorCache("percolator_cache"),
244244
Completion("completion"),
245245
Segments("segments"),
246246
Translog("translog"),

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,12 +185,12 @@ public boolean fieldData() {
185185
}
186186

187187
public IndicesStatsRequest percolate(boolean percolate) {
188-
flags.set(Flag.Percolate, percolate);
188+
flags.set(Flag.PercolatorCache, percolate);
189189
return this;
190190
}
191191

192192
public boolean percolate() {
193-
return flags.isSet(Flag.Percolate);
193+
return flags.isSet(Flag.PercolatorCache);
194194
}
195195

196196
public IndicesStatsRequest segments(boolean segments) {

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh
140140
flags.fieldDataFields(request.fieldDataFields());
141141
}
142142
if (request.percolate()) {
143-
flags.set(CommonStatsFlags.Flag.Percolate);
143+
flags.set(CommonStatsFlags.Flag.PercolatorCache);
144144
}
145145
if (request.segments()) {
146146
flags.set(CommonStatsFlags.Flag.Segments);
@@ -163,6 +163,6 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh
163163
flags.set(CommonStatsFlags.Flag.Recovery);
164164
}
165165

166-
return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), indexShard.commitStats());
166+
return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats());
167167
}
168168
}

0 commit comments

Comments
 (0)