Skip to content

Commit f18b1a0

Browse files
nik9000nik9000
authored
Use a mapping snapshot for fetching nested docs (backport of #66877) (#67451)
This uses the mapping snapshot that we built for the search phase in #66295 for fetching nested documents. This is simpler to reason about because the mapping snapshot is immutable.
1 parent 2c564a2 commit f18b1a0

File tree

13 files changed

+151
-158
lines changed

13 files changed

+151
-158
lines changed

‎server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@
4747
import org.elasticsearch.index.IndexSettings;
4848
import org.elasticsearch.index.IndexWarmer;
4949
import org.elasticsearch.index.IndexWarmer.TerminationHandle;
50-
import org.elasticsearch.index.mapper.DocumentMapper;
5150
import org.elasticsearch.index.mapper.MapperService;
51+
import org.elasticsearch.index.mapper.MappingLookup;
5252
import org.elasticsearch.index.mapper.ObjectMapper;
5353
import org.elasticsearch.index.shard.IndexShard;
5454
import org.elasticsearch.index.shard.ShardId;
@@ -236,12 +236,10 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin
236236

237237
final Set<Query> warmUp = new HashSet<>();
238238
final MapperService mapperService = indexShard.mapperService();
239-
DocumentMapper docMapper = mapperService.documentMapper();
240-
if (docMapper != null) {
241-
if (docMapper.hasNestedObjects()) {
242-
warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated()));
243-
docMapper.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
244-
}
239+
MappingLookup lookup = mapperService.mappingLookup();
240+
if (lookup.hasNested()) {
241+
warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated()));
242+
lookup.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
245243
}
246244

247245
final CountDownLatch latch = new CountDownLatch(reader.leaves().size() * warmUp.size());

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

Lines changed: 2 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,8 @@
3434
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3535

3636
import java.io.IOException;
37-
import java.util.ArrayList;
3837
import java.util.Arrays;
3938
import java.util.Collection;
40-
import java.util.List;
4139
import java.util.Map;
4240
import java.util.Objects;
4341
import java.util.function.Function;
@@ -179,10 +177,6 @@ public IndexFieldMapper IndexFieldMapper() {
179177
return metadataMapper(IndexFieldMapper.class);
180178
}
181179

182-
public boolean hasNestedObjects() {
183-
return mappers().hasNested();
184-
}
185-
186180
public MappingLookup mappers() {
187181
return this.mappingLookup;
188182
}
@@ -207,91 +201,6 @@ public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws
207201
return parsedDoc;
208202
}
209203

210-
/**
211-
* Given an object path, checks to see if any of its parents are non-nested objects
212-
*/
213-
public boolean hasNonNestedParent(String path) {
214-
ObjectMapper mapper = mappers().objectMappers().get(path);
215-
if (mapper == null) {
216-
return false;
217-
}
218-
while (mapper != null) {
219-
if (mapper.nested().isNested() == false) {
220-
return true;
221-
}
222-
if (path.contains(".") == false) {
223-
return false;
224-
}
225-
path = path.substring(0, path.lastIndexOf("."));
226-
mapper = mappers().objectMappers().get(path);
227-
}
228-
return false;
229-
}
230-
231-
/**
232-
* Returns all nested object mappers
233-
*/
234-
public List<ObjectMapper> getNestedMappers() {
235-
List<ObjectMapper> childMappers = new ArrayList<>();
236-
for (ObjectMapper mapper : mappers().objectMappers().values()) {
237-
if (mapper.nested().isNested() == false) {
238-
continue;
239-
}
240-
childMappers.add(mapper);
241-
}
242-
return childMappers;
243-
}
244-
245-
/**
246-
* Returns all nested object mappers which contain further nested object mappers
247-
*
248-
* Used by BitSetProducerWarmer
249-
*/
250-
public List<ObjectMapper> getNestedParentMappers() {
251-
List<ObjectMapper> parents = new ArrayList<>();
252-
for (ObjectMapper mapper : mappers().objectMappers().values()) {
253-
String nestedParentPath = getNestedParent(mapper.fullPath());
254-
if (nestedParentPath == null) {
255-
continue;
256-
}
257-
ObjectMapper parent = mappers().objectMappers().get(nestedParentPath);
258-
if (parent.nested().isNested()) {
259-
parents.add(parent);
260-
}
261-
}
262-
return parents;
263-
}
264-
265-
/**
266-
* Given a nested object path, returns the path to its nested parent
267-
*
268-
* In particular, if a nested field `foo` contains an object field
269-
* `bar.baz`, then calling this method with `foo.bar.baz` will return
270-
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
271-
*/
272-
public String getNestedParent(String path) {
273-
ObjectMapper mapper = mappers().objectMappers().get(path);
274-
if (mapper == null) {
275-
return null;
276-
}
277-
if (path.contains(".") == false) {
278-
return null;
279-
}
280-
do {
281-
path = path.substring(0, path.lastIndexOf("."));
282-
mapper = mappers().objectMappers().get(path);
283-
if (mapper == null) {
284-
return null;
285-
}
286-
if (mapper.nested().isNested()) {
287-
return path;
288-
}
289-
if (path.contains(".") == false) {
290-
return null;
291-
}
292-
} while(true);
293-
}
294-
295204
public DocumentMapper merge(Mapping mapping, MergeReason reason) {
296205
Mapping merged = this.mapping().merge(mapping, reason);
297206
return new DocumentMapper(mappingLookup.getIndexSettings(), mappingLookup.getIndexAnalyzers(), documentParser, merged);
@@ -305,7 +214,7 @@ public void validate(IndexSettings settings, boolean checkLimits) {
305214
+ "required for partitioned index [" + settings.getIndex().getName() + "]");
306215
}
307216
}
308-
if (settings.getIndexSortConfig().hasIndexSort() && hasNestedObjects()) {
217+
if (settings.getIndexSortConfig().hasIndexSort() && mappers().hasNested()) {
309218
throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
310219
}
311220
if (checkLimits) {
@@ -327,7 +236,7 @@ public String toString() {
327236
", documentParser=" + documentParser +
328237
", mappingLookup=" + mappingLookup +
329238
", objectMappers=" + mappers().objectMappers() +
330-
", hasNestedObjects=" + hasNestedObjects() +
239+
", hasNestedObjects=" + mappingLookup.hasNested() +
331240
", deleteTombstoneMetadataFieldMappers=" + Arrays.toString(deleteTombstoneMetadataFieldMappers) +
332241
", noopTombstoneMetadataFieldMappers=" + Arrays.toString(noopTombstoneMetadataFieldMappers) +
333242
'}';

‎server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,4 +341,89 @@ IndexSettings getIndexSettings() {
341341
IndexAnalyzers getIndexAnalyzers() {
342342
return indexAnalyzers;
343343
}
344+
345+
/**
346+
* Given an object path, checks to see if any of its parents are non-nested objects
347+
*/
348+
public boolean hasNonNestedParent(String path) {
349+
ObjectMapper mapper = objectMappers().get(path);
350+
if (mapper == null) {
351+
return false;
352+
}
353+
while (mapper != null) {
354+
if (mapper.nested().isNested() == false) {
355+
return true;
356+
}
357+
if (path.contains(".") == false) {
358+
return false;
359+
}
360+
path = path.substring(0, path.lastIndexOf("."));
361+
mapper = objectMappers().get(path);
362+
}
363+
return false;
364+
}
365+
366+
/**
367+
* Returns all nested object mappers
368+
*/
369+
public List<ObjectMapper> getNestedMappers() {
370+
List<ObjectMapper> childMappers = new ArrayList<>();
371+
for (ObjectMapper mapper : objectMappers().values()) {
372+
if (mapper.nested().isNested() == false) {
373+
continue;
374+
}
375+
childMappers.add(mapper);
376+
}
377+
return childMappers;
378+
}
379+
380+
/**
381+
* Returns all nested object mappers which contain further nested object mappers
382+
*
383+
* Used by BitSetProducerWarmer
384+
*/
385+
public List<ObjectMapper> getNestedParentMappers() {
386+
List<ObjectMapper> parents = new ArrayList<>();
387+
for (ObjectMapper mapper : objectMappers().values()) {
388+
String nestedParentPath = getNestedParent(mapper.fullPath());
389+
if (nestedParentPath == null) {
390+
continue;
391+
}
392+
ObjectMapper parent = objectMappers().get(nestedParentPath);
393+
if (parent.nested().isNested()) {
394+
parents.add(parent);
395+
}
396+
}
397+
return parents;
398+
}
399+
400+
/**
401+
* Given a nested object path, returns the path to its nested parent
402+
*
403+
* In particular, if a nested field `foo` contains an object field
404+
* `bar.baz`, then calling this method with `foo.bar.baz` will return
405+
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
406+
*/
407+
public String getNestedParent(String path) {
408+
ObjectMapper mapper = objectMappers().get(path);
409+
if (mapper == null) {
410+
return null;
411+
}
412+
if (path.contains(".") == false) {
413+
return null;
414+
}
415+
do {
416+
path = path.substring(0, path.lastIndexOf("."));
417+
mapper = objectMappers().get(path);
418+
if (mapper == null) {
419+
return null;
420+
}
421+
if (mapper.nested().isNested()) {
422+
return path;
423+
}
424+
if (path.contains(".") == false) {
425+
return null;
426+
}
427+
} while(true);
428+
}
344429
}

‎server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import org.elasticsearch.script.ScriptContext;
6868
import org.elasticsearch.script.ScriptFactory;
6969
import org.elasticsearch.script.ScriptService;
70+
import org.elasticsearch.search.NestedDocuments;
7071
import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
7172
import org.elasticsearch.search.lookup.SearchLookup;
7273
import org.elasticsearch.transport.RemoteClusterAware;
@@ -677,4 +678,8 @@ private static Map<String, MappedFieldType> parseRuntimeMappings(Map<String, Obj
677678
public MappingLookup.CacheKey mappingCacheKey() {
678679
return mappingLookup.cacheKey();
679680
}
681+
682+
public NestedDocuments getNestedDocuments() {
683+
return new NestedDocuments(mappingLookup, indexVersionCreated(), bitsetFilterCache::getBitSetProducer);
684+
}
680685
}

‎server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -711,11 +711,6 @@ public QuerySearchResult queryResult() {
711711
return queryResult;
712712
}
713713

714-
@Override
715-
public NestedDocuments getNestedDocuments() {
716-
return new NestedDocuments(indexService.mapperService(), bitsetFilterCache()::getBitSetProducer);
717-
}
718-
719714
@Override
720715
public FetchPhase fetchPhase() {
721716
return fetchPhase;

‎server/src/main/java/org/elasticsearch/search/NestedDocuments.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import org.apache.lucene.util.BitSet;
3232
import org.elasticsearch.Version;
3333
import org.elasticsearch.common.lucene.search.Queries;
34-
import org.elasticsearch.index.mapper.MapperService;
34+
import org.elasticsearch.index.mapper.MappingLookup;
3535
import org.elasticsearch.index.mapper.ObjectMapper;
3636

3737
import java.io.IOException;
@@ -48,26 +48,26 @@ public class NestedDocuments {
4848
private final Map<String, Weight> childObjectFilters = new HashMap<>();
4949
private final Map<String, ObjectMapper> childObjectMappers = new HashMap<>();
5050
private final BitSetProducer parentDocumentFilter;
51-
private final MapperService mapperService;
52-
private final Version version;
51+
private final MappingLookup mappingLookup;
52+
private final Version indexVersionCreated;
5353

5454
/**
5555
* Create a new NestedDocuments object for an index
56-
* @param mapperService the index's MapperService
56+
* @param mappingLookup the index's mapping
5757
* @param filterProducer a function to build BitSetProducers from filter queries
5858
*/
59-
public NestedDocuments(MapperService mapperService, Function<Query, BitSetProducer> filterProducer) {
60-
this.mapperService = mapperService;
61-
this.version = mapperService.getIndexSettings().getIndexVersionCreated();
62-
if (mapperService.hasNested() == false) {
59+
public NestedDocuments(MappingLookup mappingLookup, Version indexVersionCreated, Function<Query, BitSetProducer> filterProducer) {
60+
this.mappingLookup = mappingLookup;
61+
this.indexVersionCreated = indexVersionCreated;
62+
if (mappingLookup.hasNested() == false) {
6363
this.parentDocumentFilter = null;
6464
} else {
65-
this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter(version));
66-
for (ObjectMapper mapper : mapperService.documentMapper().getNestedParentMappers()) {
65+
this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter(indexVersionCreated));
66+
for (ObjectMapper mapper : mappingLookup.getNestedParentMappers()) {
6767
parentObjectFilters.put(mapper.name(),
6868
filterProducer.apply(mapper.nestedTypeFilter()));
6969
}
70-
for (ObjectMapper mapper : mapperService.documentMapper().getNestedMappers()) {
70+
for (ObjectMapper mapper : mappingLookup.getNestedMappers()) {
7171
childObjectFilters.put(mapper.name(), null);
7272
childObjectMappers.put(mapper.name(), mapper);
7373
}
@@ -101,7 +101,7 @@ private Weight getNestedChildWeight(LeafReaderContext ctx, String path) throws I
101101
* Given an object path, returns whether or not any of its parents are plain objects
102102
*/
103103
public boolean hasNonNestedParent(String path) {
104-
return mapperService.documentMapper().hasNonNestedParent(path);
104+
return mappingLookup.hasNonNestedParent(path);
105105
}
106106

107107
private class HasNestedDocuments implements LeafNestedDocuments {
@@ -188,7 +188,7 @@ private SearchHit.NestedIdentity loadNestedIdentity() throws IOException {
188188
int parentNameLength;
189189
String path = findObjectPath(doc);
190190
while (path != null) {
191-
String parent = mapperService.documentMapper().getNestedParent(path);
191+
String parent = mappingLookup.getNestedParent(path);
192192
// We have to pull a new scorer for each document here, because we advance from
193193
// the last parent which will be behind the doc
194194
Scorer childScorer = getNestedChildWeight(ctx, path).scorer(ctx);
@@ -208,7 +208,7 @@ private SearchHit.NestedIdentity loadNestedIdentity() throws IOException {
208208
}
209209
int offset = 0;
210210
DocIdSetIterator childIt = childScorer.iterator();
211-
if (version.onOrAfter(Version.V_6_5_0)) {
211+
if (indexVersionCreated.onOrAfter(Version.V_6_5_0)) {
212212
/*
213213
* Starts from the previous parent and finds the offset of the
214214
* <code>nestedSubDocID</code> within the nested children. Nested documents

‎server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public void execute(SearchContext context) {
113113
SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
114114

115115
List<FetchSubPhaseProcessor> processors = getProcessors(context.shardTarget(), fetchContext);
116-
NestedDocuments nestedDocuments = context.getNestedDocuments();
116+
NestedDocuments nestedDocuments = context.getQueryShardContext().getNestedDocuments();
117117

118118
int currentReaderIndex = -1;
119119
LeafReaderContext currentReaderContext = null;

‎server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.elasticsearch.index.query.ParsedQuery;
3030
import org.elasticsearch.index.query.QueryShardContext;
3131
import org.elasticsearch.index.shard.IndexShard;
32-
import org.elasticsearch.search.NestedDocuments;
3332
import org.elasticsearch.search.SearchExtBuilder;
3433
import org.elasticsearch.search.SearchShardTarget;
3534
import org.elasticsearch.search.aggregations.SearchContextAggregations;
@@ -390,11 +389,6 @@ public FetchSearchResult fetchResult() {
390389
return in.fetchResult();
391390
}
392391

393-
@Override
394-
public NestedDocuments getNestedDocuments() {
395-
return in.getNestedDocuments();
396-
}
397-
398392
@Override
399393
public FetchPhase fetchPhase() {
400394
return in.fetchPhase();

‎server/src/main/java/org/elasticsearch/search/internal/SearchContext.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import org.elasticsearch.index.query.ParsedQuery;
3333
import org.elasticsearch.index.query.QueryShardContext;
3434
import org.elasticsearch.index.shard.IndexShard;
35-
import org.elasticsearch.search.NestedDocuments;
3635
import org.elasticsearch.search.RescoreDocIds;
3736
import org.elasticsearch.search.SearchExtBuilder;
3837
import org.elasticsearch.search.SearchShardTarget;
@@ -311,8 +310,6 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) {
311310

312311
public abstract QuerySearchResult queryResult();
313312

314-
public abstract NestedDocuments getNestedDocuments();
315-
316313
public abstract FetchPhase fetchPhase();
317314

318315
public abstract FetchSearchResult fetchResult();

0 commit comments

Comments
 (0)