Skip to content

Commit

Permalink
SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsW…
Browse files Browse the repository at this point in the history 
…henCollapsing parameters to query elevation.

Closes apache#2134
  • Loading branch information
@tkaessmann @bruno-roustant
tkaessmann authored and bruno-roustant committed Feb 17, 2021
1 parent 2ae45cc commit f142bf9
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 31 deletions.
2 changes: 1 addition & 1 deletion gradle/testing/randomization/policies/solr-tests.policy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
grant {
// 3rd party jar resources (where symlinks are not supported), test-files/ resources
permission java.io.FilePermission "${common.dir}${/}-", "read";
permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read";
permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read,write";

// system jar resources
permission java.io.FilePermission "${java.home}${/}-", "read";
Expand Down
3 changes: 3 additions & 0 deletions solr/CHANGES.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,9 @@ Improvements

* SOLR-15101: Add "list" and "delete" APIs for managing incremental backups (Jason Gerlowski, shalin, Cao Manh Dat)

* SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.
(Dennis Berger, Tobias Kässmann via Bruno Roustant)

Optimizations
---------------------
* SOLR-15079: Block Collapse - Faster collapse code when groups are co-located via Block Join style nested doc indexing.
Expand Down
5 changes: 4 additions & 1 deletion solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,10 @@ private void setQuery(ResponseBuilder rb, Elevation elevation) {
rb.setQuery(new BoostQuery(elevation.includeQuery, 0f));
} else {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
queryBuilder.add(rb.getQuery(), BooleanClause.Occur.SHOULD);
BooleanClause.Occur queryOccurrence =
params.getBool(QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, false) ?
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
queryBuilder.add(rb.getQuery(), queryOccurrence);
queryBuilder.add(new BoostQuery(elevation.includeQuery, 0f), BooleanClause.Occur.SHOULD);
if (elevation.excludeQueries != null) {
if (params.getBool(QueryElevationParams.MARK_EXCLUDES, false)) {
Expand Down
92 changes: 63 additions & 29 deletions solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
*/
public static final String HINT_BLOCK = "block";

/**
* If elevation is used in combination with the collapse query parser, we can define that we only want to return the
* representative and not all elevated docs by setting this parameter to false (true by default).
*/
public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing";

/**
* @deprecated use {@link NullPolicy} instead.
*/
Expand Down Expand Up @@ -585,6 +591,7 @@ static class OrdScoreCollector extends DelegatingCollector {
private int nullPolicy;
private float nullScore = -Float.MAX_VALUE;
private int nullDoc = -1;
private boolean collectElevatedDocsWhenCollapsing;
private FloatArrayList nullScores;

private final BoostedDocsCollector boostedDocsCollector;
Expand All @@ -594,9 +601,11 @@ public OrdScoreCollector(int maxDoc,
DocValuesProducer collapseValuesProducer,
int nullPolicy,
IntIntHashMap boostDocsMap,
IndexSearcher searcher) throws IOException {
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
for(int i=0; i<con.size(); i++) {
contexts[i] = con.get(i);
Expand Down Expand Up @@ -653,12 +662,14 @@ public void collect(int contextDoc) throws IOException {
ord = -1;
}
}

// Check to see if we have documents boosted by the QueryElevationComponent
if (0 <= ord) {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
} else {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;

if (collectElevatedDocsWhenCollapsing) {
// Check to see if we have documents boosted by the QueryElevationComponent
if (0 <= ord) {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
} else {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}
}

if(ord > -1) {
Expand Down Expand Up @@ -785,6 +796,7 @@ static class IntScoreCollector extends DelegatingCollector {
private int nullDoc = -1;
private FloatArrayList nullScores;
private String field;
private boolean collectElevatedDocsWhenCollapsing;

private final BoostedDocsCollector boostedDocsCollector;

Expand All @@ -794,9 +806,11 @@ public IntScoreCollector(int maxDoc,
int size,
String field,
IntIntHashMap boostDocsMap,
IndexSearcher searcher) {
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
for(int i=0; i<con.size(); i++) {
contexts[i] = con.get(i);
Expand Down Expand Up @@ -827,8 +841,11 @@ public void collect(int contextDoc) throws IOException {
final int globalDoc = docBase+contextDoc;
if (collapseValues.advanceExact(contextDoc)) {
final int collapseValue = (int) collapseValues.longValue();
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;

if (collectElevatedDocsWhenCollapsing) {
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;
}

float score = scorer.score();
final int idx;
Expand All @@ -847,9 +864,11 @@ public void collect(int contextDoc) throws IOException {
}

} else { // Null Group...

// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;

if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}

if(nullPolicy == NullPolicy.COLLAPSE.getCode()) {
float score = scorer.score();
Expand Down Expand Up @@ -958,7 +977,9 @@ static class OrdFieldValueCollector extends DelegatingCollector {
private OrdFieldValueStrategy collapseStrategy;
private boolean needsScores4Collapsing;
private boolean needsScores;


private boolean collectElevatedDocsWhenCollapsing;

private final BoostedDocsCollector boostedDocsCollector;

public OrdFieldValueCollector(int maxDoc,
Expand All @@ -971,10 +992,12 @@ public OrdFieldValueCollector(int maxDoc,
boolean needsScores,
FieldType fieldType,
IntIntHashMap boostDocsMap,
FunctionQuery funcQuery, IndexSearcher searcher) throws IOException{
FunctionQuery funcQuery, IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException{

assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);

this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
Expand Down Expand Up @@ -1053,12 +1076,14 @@ public void collect(int contextDoc) throws IOException {
ord = segmentValues.ordValue();
}
}

// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (-1 == ord) {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
} else {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;

if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (-1 == ord) {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
} else {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
}
}

collapseStrategy.collapse(ord, contextDoc, globalDoc);
Expand Down Expand Up @@ -1166,6 +1191,7 @@ static class IntFieldValueCollector extends DelegatingCollector {
private String collapseField;

private final BoostedDocsCollector boostedDocsCollector;
private boolean collectElevatedDocsWhenCollapsing;

public IntFieldValueCollector(int maxDoc,
int size,
Expand All @@ -1179,7 +1205,9 @@ public IntFieldValueCollector(int maxDoc,
FieldType fieldType,
IntIntHashMap boostDocsMap,
FunctionQuery funcQuery,
IndexSearcher searcher) throws IOException{
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException{
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;

assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);

Expand Down Expand Up @@ -1243,10 +1271,11 @@ public void collect(int contextDoc) throws IOException {
collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);

} else { // Null Group...

// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;

if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}
if (NullPolicy.IGNORE.getCode() != nullPolicy) {
collapseStrategy.collapseNullGroup(contextDoc, globalDoc);
}
Expand Down Expand Up @@ -1894,20 +1923,23 @@ public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
int maxDoc = searcher.maxDoc();
int leafCount = searcher.getTopReaderContext().leaves().size();

SolrRequestInfo req = SolrRequestInfo.getRequestInfo();
boolean collectElevatedDocsWhenCollapsing = req != null && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true);

if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {

if (collapseFieldType instanceof StrField) {
if (blockCollapse) {
return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs);
}
return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher);
return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher, collectElevatedDocsWhenCollapsing);

} else if (isNumericCollapsible(collapseFieldType)) {
if (blockCollapse) {
return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs);
}

return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher);
return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher, collectElevatedDocsWhenCollapsing);

} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
Expand Down Expand Up @@ -1937,7 +1969,8 @@ public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
minMaxFieldType,
boostDocs,
funcQuery,
searcher);
searcher,
collectElevatedDocsWhenCollapsing);

} else if (isNumericCollapsible(collapseFieldType)) {

Expand All @@ -1962,7 +1995,8 @@ public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
minMaxFieldType,
boostDocs,
funcQuery,
searcher);
searcher,
collectElevatedDocsWhenCollapsing);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");
Expand Down
101 changes: 101 additions & 0 deletions solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.CollapsingQParserPlugin;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.FileUtils;
import org.junit.Before;
Expand Down Expand Up @@ -932,6 +933,106 @@ public void testElevatedIds() throws Exception {
}
}

@Test
public void testOnlyDocsInSearchResultsWillBeElevated() throws Exception {
try {
init("schema12.xml");
assertU(adoc("id", "1", "title", "XXXX", "str_s1", "a"));
assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));
assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));

assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));
assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));
assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));
assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));

assertU(commit());

// default behaviour
assertQ("", req(
CommonParams.Q, "YYYY",
CommonParams.QT, "/elevate",
QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "false",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='1']",
"//result/doc[2]/str[@name='id'][.='2']",
"//result/doc[3]/str[@name='id'][.='5']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
);

// only docs that matches q
assertQ("", req(
CommonParams.Q, "YYYY",
CommonParams.QT, "/elevate",
QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "true",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='2']",
"//result/doc[1]/str[@name='id'][.='2']",
"//result/doc[2]/str[@name='id'][.='5']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='false']"
);



} finally {
delete();
}
}

@Test
public void testOnlyRepresentativeIsVisibleWhenCollapsing() throws Exception {
try {
init("schema12.xml");
assertU(adoc("id", "1", "title", "ZZZZ", "str_s1", "a"));
assertU(adoc("id", "2", "title", "ZZZZ", "str_s1", "b"));
assertU(adoc("id", "3", "title", "ZZZZ ZZZZ", "str_s1", "a"));
assertU(adoc("id", "4", "title", "ZZZZ ZZZZ", "str_s1", "c"));

assertU(commit());

// default behaviour - all elevated docs are visible
assertQ("", req(
CommonParams.Q, "ZZZZ",
CommonParams.QT, "/elevate",
CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "true",
CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='1']",
"//result/doc[2]/str[@name='id'][.='2']",
"//result/doc[3]/str[@name='id'][.='3']",
"//result/doc[4]/str[@name='id'][.='4']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='true']",
"//result/doc[4]/bool[@name='[elevated]'][.='false']"
);

// only representative elevated doc visible
assertQ("", req(
CommonParams.Q, "ZZZZ",
CommonParams.QT, "/elevate",
CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "false",
CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='2']",
"//result/doc[2]/str[@name='id'][.='3']",
"//result/doc[3]/str[@name='id'][.='4']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
);

} finally {
delete();
}
}

private static Set<BytesRef> toIdSet(String... ids) {
return Arrays.stream(ids).map(BytesRef::new).collect(Collectors.toSet());
}
Expand Down
4 changes: 4 additions & 0 deletions solr/solr-ref-guide/src/collapse-and-expand-results.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ The data structures used for collapsing grow dynamically when collapsing on nume
+
The default is 100,000.

`collectElevatedDocsWhenCollapsing`::
In combination with the <<collapse-and-expand-results.adoc#collapsing-query-parser,Collapse Query Parser>> all elevated docs are visible at the beginning of the result set.
If this parameter is `false`, only the representative is visible if the elevated docs has the same collapse key (default is `true`).


=== Sample Usage Syntax

Expand Down
4 changes: 4 additions & 0 deletions solr/solr-ref-guide/src/the-query-elevation-component.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ they be subject to whatever the sort criteria is? True by default.
This is also a request parameter, which will override the config.
The effect is most apparent when forceElevation is true and there is sorting on fields.

`elevateOnlyDocsMatchingQuery`::
By default, the component will also elevate docs that aren't part of the search result (matching the query).
If you only want to elevate the docs that are part of the search result, set this to `true` (default is `false`).

=== The elevate.xml File

Elevated query results can be configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this:
Expand Down
Loading

0 comments on commit f142bf9

Please sign in to comment.