Skip to content

Commit f2892f1

Browse files
jpountzjpountz
authored
Add a doc value format to binary fields. (#30860)
This will be necessary for the `docvalue_fields` option to work correctly once we use the field's doc-value format to format doc-value fields. Binary values are formatted as base64-encoded strings.
1 parent 0fad7cc commit f2892f1

File tree

5 files changed

+77
-3
lines changed

5 files changed

+77
-3
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
4141
import org.elasticsearch.index.query.QueryShardContext;
4242
import org.elasticsearch.index.query.QueryShardException;
43+
import org.elasticsearch.search.DocValueFormat;
44+
import org.joda.time.DateTimeZone;
4345

4446
import java.io.IOException;
4547
import java.util.Base64;
@@ -104,6 +106,10 @@ public String typeName() {
104106
return CONTENT_TYPE;
105107
}
106108

109+
@Override
110+
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
111+
return DocValueFormat.BINARY;
112+
}
107113

108114
@Override
109115
public BytesReference valueForDisplay(Object value) {

‎server/src/main/java/org/elasticsearch/search/DocValueFormat.java

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.text.NumberFormat;
4040
import java.text.ParseException;
4141
import java.util.Arrays;
42+
import java.util.Base64;
4243
import java.util.Locale;
4344
import java.util.Objects;
4445
import java.util.function.LongSupplier;
@@ -121,6 +122,50 @@ public BytesRef parseBytesRef(String value) {
121122
}
122123
};
123124

125+
DocValueFormat BINARY = new DocValueFormat() {
126+
127+
@Override
128+
public String getWriteableName() {
129+
return "binary";
130+
}
131+
132+
@Override
133+
public void writeTo(StreamOutput out) throws IOException {
134+
}
135+
136+
@Override
137+
public Object format(long value) {
138+
throw new UnsupportedOperationException();
139+
}
140+
141+
@Override
142+
public Object format(double value) {
143+
throw new UnsupportedOperationException();
144+
}
145+
146+
@Override
147+
public String format(BytesRef value) {
148+
return Base64.getEncoder()
149+
.withoutPadding()
150+
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
151+
}
152+
153+
@Override
154+
public long parseLong(String value, boolean roundUp, LongSupplier now) {
155+
throw new UnsupportedOperationException();
156+
}
157+
158+
@Override
159+
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
160+
throw new UnsupportedOperationException();
161+
}
162+
163+
@Override
164+
public BytesRef parseBytesRef(String value) {
165+
return new BytesRef(Base64.getDecoder().decode(value));
166+
}
167+
};
168+
124169
final class DateTime implements DocValueFormat {
125170

126171
public static final String NAME = "date_time";

‎server/src/main/java/org/elasticsearch/search/SearchModule.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,6 +645,7 @@ private void registerValueFormats() {
645645
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
646646
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
647647
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
648+
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
648649
}
649650

650651
/**

‎server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public void testSerialization() throws Exception {
4444
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
4545
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
4646
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
47+
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
4748
NamedWriteableRegistry registry = new NamedWriteableRegistry(entries);
4849

4950
BytesStreamOutput out = new BytesStreamOutput();
@@ -82,6 +83,11 @@ public void testSerialization() throws Exception {
8283
out.writeNamedWriteable(DocValueFormat.RAW);
8384
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
8485
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
86+
87+
out = new BytesStreamOutput();
88+
out.writeNamedWriteable(DocValueFormat.BINARY);
89+
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
90+
assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class));
8591
}
8692

8793
public void testRawFormat() {
@@ -96,6 +102,14 @@ public void testRawFormat() {
96102
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
97103
}
98104

105+
public void testBinaryFormat() {
106+
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
107+
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));
108+
109+
assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
110+
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
111+
}
112+
99113
public void testBooleanFormat() {
100114
assertEquals(false, DocValueFormat.BOOLEAN.format(0));
101115
assertEquals(true, DocValueFormat.BOOLEAN.format(1));

‎server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.search.fields;
2121

22+
import org.apache.lucene.util.BytesRef;
2223
import org.elasticsearch.action.index.IndexRequestBuilder;
2324
import org.elasticsearch.action.search.SearchRequestBuilder;
2425
import org.elasticsearch.action.search.SearchResponse;
@@ -700,7 +701,7 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup
700701
assertThat(fields.get("test_field").getValue(), equalTo("foobar"));
701702
}
702703

703-
public void testFieldsPulledFromFieldData() throws Exception {
704+
public void testDocValueFields() throws Exception {
704705
createIndex("test");
705706

706707
String mapping = Strings
@@ -744,6 +745,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
744745
.endObject()
745746
.startObject("binary_field")
746747
.field("type", "binary")
748+
.field("doc_values", true) // off by default on binary fields
747749
.endObject()
748750
.startObject("ip_field")
749751
.field("type", "ip")
@@ -766,6 +768,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
766768
.field("double_field", 6.0d)
767769
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
768770
.field("boolean_field", true)
771+
.field("binary_field", new byte[] {42, 100})
769772
.field("ip_field", "::1")
770773
.endObject()).execute().actionGet();
771774

@@ -782,6 +785,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
782785
.addDocValueField("double_field")
783786
.addDocValueField("date_field")
784787
.addDocValueField("boolean_field")
788+
.addDocValueField("binary_field")
785789
.addDocValueField("ip_field");
786790
SearchResponse searchResponse = builder.execute().actionGet();
787791

@@ -790,7 +794,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
790794
Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
791795
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
792796
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
793-
"ip_field")));
797+
"binary_field", "ip_field")));
794798

795799
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
796800
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@@ -802,6 +806,8 @@ public void testFieldsPulledFromFieldData() throws Exception {
802806
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
803807
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
804808
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
809+
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
810+
equalTo(new BytesRef(new byte[] {42, 100})));
805811
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
806812

807813
builder = client().prepareSearch().setQuery(matchAllQuery())
@@ -815,6 +821,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
815821
.addDocValueField("double_field", "use_field_mapping")
816822
.addDocValueField("date_field", "use_field_mapping")
817823
.addDocValueField("boolean_field", "use_field_mapping")
824+
.addDocValueField("binary_field", "use_field_mapping")
818825
.addDocValueField("ip_field", "use_field_mapping");
819826
searchResponse = builder.execute().actionGet();
820827

@@ -823,7 +830,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
823830
fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
824831
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
825832
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
826-
"ip_field")));
833+
"binary_field", "ip_field")));
827834

828835
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
829836
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@@ -836,6 +843,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
836843
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
837844
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
838845
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
846+
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
839847
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
840848

841849
builder = client().prepareSearch().setQuery(matchAllQuery())

0 commit comments

Comments
 (0)