Skip to content

Integrate "fields" API into QL #68467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 4, 2021
Merged

Integrate "fields" API into QL #68467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
72b1135
Integrate "fields" API into QL
astefan Aug 13, 2020
8960cb4
Fix yml test
astefan Feb 3, 2021
af7934c
Address review
astefan Feb 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions docs/reference/sql/endpoints/translate.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,22 @@ Which returns:
--------------------------------------------------
{
"size": 10,
"docvalue_fields": [
"_source": false,
"fields": [
{
"field": "author"
},
{
"field": "name"
},
{
"field": "page_count"
},
{
"field": "release_date",
"format": "epoch_millis"
}
],
"_source": {
"includes": [
"author",
"name",
"page_count"
],
"excludes": []
},
"sort": [
{
"page_count": {
Expand Down
12 changes: 6 additions & 6 deletions ...k/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public static List<HitExtractor> createExtractor(List<FieldExtraction> fields, E
public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration cfg) {
if (ref instanceof SearchHitFieldRef) {
SearchHitFieldRef f = (SearchHitFieldRef) ref;
return new FieldHitExtractor(f.name(), f.fullFieldName(), f.getDataType(), cfg.zoneId(), f.useDocValue(), f.hitName(), false);
return new FieldHitExtractor(f.name(), f.getDataType(), cfg.zoneId(), f.hitName(), false);
}

if (ref instanceof ComputedRef) {
Expand Down Expand Up @@ -150,11 +150,11 @@ public static SearchRequest prepareRequest(Client client,
boolean includeFrozen,
String... indices) {
return client.prepareSearch(indices)
.setSource(source)
.setAllowPartialSearchResults(false)
.setIndicesOptions(
includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
.request();
.setSource(source)
.setAllowPartialSearchResults(false)
.setIndicesOptions(
includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
.request();
}

public static List<SearchHit> searchHits(SearchResponse response) {
Expand Down
11 changes: 2 additions & 9 deletions ...lugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@
*/
package org.elasticsearch.xpack.eql.execution.search;

import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.NestedSortBuilder;
import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
Expand Down Expand Up @@ -60,13 +58,8 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB

sorting(container, source);

// disable the source if there are no includes
if (source.fetchSource() == null || CollectionUtils.isEmpty(source.fetchSource().includes())) {
source.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE);
} else {
// use true to fetch only the needed bits from the source
source.fetchSource(true);
}
// disable the source, as we rely on "fields" API
source.fetchSource(false);

if (container.limit() != null) {
// add size and from
Expand Down
4 changes: 2 additions & 2 deletions ...c/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public FieldHitExtractor(StreamInput in) throws IOException {
super(in);
}

public FieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName,
public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, String hitName,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

boolean arrayLeniency) {
super(name, fullFieldName, dataType, zoneId, useDocValue, hitName, arrayLeniency);
super(name, dataType, zoneId, hitName, arrayLeniency);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...va/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
public class TimestampFieldHitExtractor extends FieldHitExtractor {

public TimestampFieldHitExtractor(FieldHitExtractor target) {
super(target.fieldName(), target.fullFieldName(), target.dataType(), target.zoneId(), target.useDocValues(), target.hitName(),
super(target.fieldName(), target.dataType(), target.zoneId(), target.hitName(),
target.arrayLeniency());
}

Expand Down
36 changes: 1 addition & 35 deletions .../src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput;
import org.elasticsearch.xpack.ql.type.DataTypes;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -46,39 +45,6 @@ private FieldExtraction createFieldExtractionFor(Expression expression) {
}

private FieldExtraction topHitFieldExtractor(FieldAttribute fieldAttr) {
FieldAttribute actualField = fieldAttr;
FieldAttribute rootField = fieldAttr;
StringBuilder fullFieldName = new StringBuilder(fieldAttr.field().getName());

// Only if the field is not an alias (in which case it will be taken out from docvalue_fields if it's isAggregatable()),
// go up the tree of parents until a non-object (and non-nested) type of field is found and use that specific parent
// as the field to extract data from, from _source. We do it like this because sub-fields are not in the _source, only
// the root field to which those sub-fields belong to, are. Instead of "text_field.keyword_subfield" for _source extraction,
// we use "text_field", because there is no source for "keyword_subfield".
/*
* "text_field": {
* "type": "text",
* "fields": {
* "keyword_subfield": {
* "type": "keyword"
* }
* }
* }
*/
if (fieldAttr.field().isAlias() == false) {
while (actualField.parent() != null
&& actualField.parent().field().getDataType() != DataTypes.OBJECT
&& actualField.parent().field().getDataType() != DataTypes.NESTED
&& actualField.field().getDataType().hasDocValues() == false) {
actualField = actualField.parent();
}
}
while (rootField.parent() != null) {
fullFieldName.insert(0, ".").insert(0, rootField.parent().field().getName());
rootField = rootField.parent();
}

return new SearchHitFieldRef(actualField.name(), fullFieldName.toString(), fieldAttr.field().getDataType(),
fieldAttr.field().isAggregatable(), fieldAttr.field().isAlias());
return new SearchHitFieldRef(fieldAttr.name(), fieldAttr.field().getDataType(), fieldAttr.field().isAlias());
}
}
32 changes: 4 additions & 28 deletions ...n/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,23 @@

import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME_NANOS;
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;

// NB: this class is taken from SQL - it hasn't been ported over to QL
// since at this stage is unclear whether the whole FieldExtraction infrastructure
// needs porting or just the field extraction
public class SearchHitFieldRef implements FieldExtraction {

private final String name;
private final String fullFieldName; // path included. If field full path is a.b.c, full field name is "a.b.c" and name is "c"
private final DataType dataType;
private final boolean docValue;
private final String hitName;

public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias) {
this(name, fullFieldName, dataType, useDocValueInsteadOfSource, isAlias, null);
public SearchHitFieldRef(String name, DataType dataType, boolean isAlias) {
this(name, dataType, isAlias, null);
}

public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias,
String hitName) {
public SearchHitFieldRef(String name, DataType dataType, boolean isAlias, String hitName) {
this.name = name;
this.fullFieldName = fullFieldName;
this.dataType = dataType;
// these field types can only be extracted from docvalue_fields (ie, values already computed by Elasticsearch)
// because, for us to be able to extract them from _source, we would need the mapping of those fields (which we don't have)
this.docValue = isAlias ? useDocValueInsteadOfSource : (hasDocValues(dataType) ? useDocValueInsteadOfSource : false);
this.hitName = hitName;
}

Expand All @@ -49,29 +41,17 @@ public String name() {
return name;
}

public String fullFieldName() {
return fullFieldName;
}

public DataType getDataType() {
return dataType;
}

public boolean useDocValue() {
return docValue;
}

@Override
public void collectFields(QlSourceBuilder sourceBuilder) {
// nested fields are handled by inner hits
if (hitName != null) {
return;
}
if (docValue) {
sourceBuilder.addDocField(name, format(dataType));
} else {
sourceBuilder.addSourceField(name);
}
sourceBuilder.addFetchField(name, format(dataType));
}

@Override
Expand All @@ -84,10 +64,6 @@ public String toString() {
return name;
}

private static boolean hasDocValues(DataType dataType) {
return dataType == KEYWORD || dataType == DATETIME || dataType == DATETIME_NANOS;
}

private static String format(DataType dataType) {
if (dataType == DATETIME_NANOS) {
return "strict_date_optional_time_nanos";
Expand Down
6 changes: 3 additions & 3 deletions ...st/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ public class CriterionOrdinalExtractionTests extends ESTestCase {
private String tsField = "timestamp";
private String tbField = "tiebreaker";

private HitExtractor tsExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.LONG, null, true, null, false);
private HitExtractor tbExtractor = new FieldHitExtractor(tbField, tbField, DataTypes.LONG, null, true, null, false);
private HitExtractor tsExtractor = new FieldHitExtractor(tsField, DataTypes.LONG, null, null, false);
private HitExtractor tbExtractor = new FieldHitExtractor(tbField, DataTypes.LONG, null, null, false);

public void testTimeOnly() throws Exception {
long time = randomLong();
Expand All @@ -56,7 +56,7 @@ public void testTimeAndTiebreakerNull() throws Exception {
}

public void testTimeNotComparable() throws Exception {
HitExtractor badExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.BINARY, null, true, null, false);
HitExtractor badExtractor = new FieldHitExtractor(tsField, DataTypes.BINARY, null, null, false);
SearchHit hit = searchHit(randomAlphaOfLength(10), null);
Criterion<BoxedQueryRequest> criterion = new Criterion<BoxedQueryRequest>(0, null, emptyList(), badExtractor, null, false);
EqlIllegalArgumentException exception = expectThrows(EqlIllegalArgumentException.class, () -> criterion.ordinal(hit));
Expand Down
26 changes: 5 additions & 21 deletions .../plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/
package org.elasticsearch.xpack.ql.execution.search;

import org.elasticsearch.common.Strings;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
Expand All @@ -23,8 +22,7 @@
*/
public class QlSourceBuilder {
// The LinkedHashMaps preserve the order of the fields in the response
private final Set<String> sourceFields = new LinkedHashSet<>();
private final Set<FieldAndFormat> docFields = new LinkedHashSet<>();
private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();
private final Map<String, Script> scriptFields = new LinkedHashMap<>();

boolean trackScores = false;
Expand All @@ -40,17 +38,10 @@ public void trackScores() {
}

/**
* Retrieve the requested field from the {@code _source} of the document
* Retrieve the requested field using the "fields" API
*/
public void addSourceField(String field) {
sourceFields.add(field);
}

/**
* Retrieve the requested field from doc values (or fielddata) of the document
*/
public void addDocField(String field, String format) {
docFields.add(new FieldAndFormat(field, format));
public void addFetchField(String field, String format) {
fetchFields.add(new FieldAndFormat(field, format));
}

/**
Expand All @@ -66,14 +57,7 @@ public void addScriptField(String name, Script script) {
*/
public void build(SearchSourceBuilder sourceBuilder) {
sourceBuilder.trackScores(this.trackScores);
if (!sourceFields.isEmpty()) {
sourceBuilder.fetchSource(sourceFields.toArray(Strings.EMPTY_ARRAY), null);
}
docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format));
fetchFields.forEach(field -> sourceBuilder.fetchField(new FieldAndFormat(field.field, field.format, null)));
scriptFields.forEach(sourceBuilder::scriptField);
}

public boolean noSource() {
return sourceFields.isEmpty();
}
}
Loading