Skip to content

Commit

Permalink
Introduce new 'unsigned_long' numeric field type support
Browse files Browse the repository at this point in the history 
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
  • Loading branch information
@reta
reta committed Apr 21, 2023
1 parent 1ab22e1 commit f4f136f
Show file tree
Hide file tree
Showing 97 changed files with 7,269 additions and 92 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
## [Unreleased 2.x]
### Added
- [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
- Introduce new 'unsigned_long' numeric field type support ([#6237](https://github.com/opensearch-project/OpenSearch/pull/6237))

### Dependencies

Expand Down
3 changes: 3 additions & 0 deletions client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,10 @@
import org.opensearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
import org.opensearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.opensearch.search.aggregations.bucket.terms.ParsedUnsignedLongTerms;
import org.opensearch.search.aggregations.bucket.terms.StringRareTerms;
import org.opensearch.search.aggregations.bucket.terms.StringTerms;
import org.opensearch.search.aggregations.bucket.terms.UnsignedLongTerms;
import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
Expand Down Expand Up @@ -2274,6 +2276,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
map.put(VariableWidthHistogramAggregationBuilder.NAME, (p, c) -> ParsedVariableWidthHistogram.fromXContent(p, (String) c));
map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
map.put(UnsignedLongTerms.NAME, (p, c) -> ParsedUnsignedLongTerms.fromXContent(p, (String) c));
map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
map.put(LongRareTerms.NAME, (p, c) -> ParsedLongRareTerms.fromXContent(p, (String) c));
Expand Down
114 changes: 114 additions & 0 deletions client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
import org.opensearch.search.aggregations.bucket.range.Range;
import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
import org.opensearch.search.aggregations.bucket.terms.RareTerms;
import org.opensearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
import org.opensearch.search.aggregations.bucket.terms.Terms;
Expand All @@ -89,6 +90,7 @@
import org.opensearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
import org.opensearch.search.aggregations.metrics.WeightedAvg;
import org.opensearch.search.aggregations.metrics.WeightedAvgAggregationBuilder;
import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
import org.opensearch.search.aggregations.support.MultiValuesSourceFieldConfig;
import org.opensearch.search.aggregations.support.ValueType;
import org.opensearch.search.builder.PointInTimeBuilder;
Expand All @@ -105,6 +107,7 @@
import org.junit.Before;

import java.io.IOException;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -297,6 +300,67 @@ public void testSearchWithTermsAgg() throws IOException {
assertEquals(0, type2.getAggregations().asList().size());
}

public void testSearchWithMultiTermsAgg() throws IOException {
final String indexName = "multi_aggs";
Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName);
createIndex.setJsonEntity(
"{\n"
+ " \"mappings\": {\n"
+ " \"properties\" : {\n"
+ " \"username\" : {\n"
+ " \"type\" : \"keyword\"\n"
+ " },\n"
+ " \"rating\" : {\n"
+ " \"type\" : \"unsigned_long\"\n"
+ " }\n"
+ " }\n"
+ " }"
+ "}"
);
client().performRequest(createIndex);

{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/1");
doc1.setJsonEntity("{\"username\":\"bob\", \"rating\": 18446744073709551615}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/2");
doc2.setJsonEntity("{\"username\":\"tom\", \"rating\": 10223372036854775807}");
client().performRequest(doc2);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/3");
doc3.setJsonEntity("{\"username\":\"john\"}");
client().performRequest(doc3);
}
client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));

SearchRequest searchRequest = new SearchRequest().indices(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(
new MultiTermsAggregationBuilder("agg1").terms(
Arrays.asList(
new MultiTermsValuesSourceConfig.Builder().setFieldName("username").build(),
new MultiTermsValuesSourceConfig.Builder().setFieldName("rating").build()
)
)
);
searchSourceBuilder.size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
assertSearchHeader(searchResponse);
assertNull(searchResponse.getSuggest());
assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
assertEquals(0, searchResponse.getHits().getHits().length);
assertEquals(Float.NaN, searchResponse.getHits().getMaxScore(), 0f);
Terms termsAgg = searchResponse.getAggregations().get("agg1");
assertEquals("agg1", termsAgg.getName());
assertEquals(2, termsAgg.getBuckets().size());
Terms.Bucket bucket1 = termsAgg.getBucketByKey("bob|18446744073709551615");
assertEquals(1, bucket1.getDocCount());
assertEquals(0, bucket1.getAggregations().asList().size());
Terms.Bucket bucket2 = termsAgg.getBucketByKey("tom|10223372036854775807");
assertEquals(1, bucket2.getDocCount());
assertEquals(0, bucket2.getAggregations().asList().size());
}

public void testSearchWithRareTermsAgg() throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
Expand Down Expand Up @@ -847,6 +911,56 @@ public void testMultiSearch() throws Exception {
assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(1).getId(), Matchers.equalTo("6"));
}

public void testSearchWithSort() throws Exception {
final String indexName = "search_sort";
Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName);
createIndex.setJsonEntity(
"{\n"
+ " \"mappings\": {\n"
+ " \"properties\" : {\n"
+ " \"username\" : {\n"
+ " \"type\" : \"keyword\"\n"
+ " },\n"
+ " \"rating\" : {\n"
+ " \"type\" : \"unsigned_long\"\n"
+ " }\n"
+ " }\n"
+ " }"
+ "}"
);
client().performRequest(createIndex);

{
Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/1");
doc1.setJsonEntity("{\"username\":\"bob\", \"rating\": 18446744073709551610}");
client().performRequest(doc1);
Request doc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/2");
doc2.setJsonEntity("{\"username\":\"tom\", \"rating\": 10223372036854775807}");
client().performRequest(doc2);
Request doc3 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/3");
doc3.setJsonEntity("{\"username\":\"john\"}");
client().performRequest(doc3);
}
client().performRequest(new Request(HttpPost.METHOD_NAME, "/search_sort/_refresh"));

SearchRequest searchRequest = new SearchRequest("search_sort");
searchRequest.source().sort("rating", SortOrder.ASC);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);

assertThat(searchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
assertThat(searchResponse.getHits().getTotalHits().value, Matchers.equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), Matchers.equalTo("2"));
assertThat(searchResponse.getHits().getAt(1).getId(), Matchers.equalTo("1"));
assertThat(searchResponse.getHits().getAt(2).getId(), Matchers.equalTo("3"));

assertThat(searchResponse.getHits().getAt(0).getSortValues().length, equalTo(1));
assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(new BigInteger("10223372036854775807")));
assertThat(searchResponse.getHits().getAt(1).getSortValues().length, equalTo(1));
assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(new BigInteger("18446744073709551610")));
assertThat(searchResponse.getHits().getAt(2).getSortValues().length, equalTo(1));
assertThat(searchResponse.getHits().getAt(2).getSortValues()[0], equalTo(new BigInteger("18446744073709551615")));
}

public void testMultiSearch_withAgg() throws Exception {
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
SearchRequest searchRequest1 = new SearchRequest("index1");
Expand Down
109 changes: 99 additions & 10 deletions ...n/java/org/opensearch/common/Numbers.java → ...n/java/org/opensearch/common/Numbers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@

package org.opensearch.common;

import org.apache.lucene.util.BytesRef;

import java.math.BigDecimal;
import java.math.BigInteger;

Expand All @@ -43,20 +41,17 @@
* @opensearch.internal
*/
public final class Numbers {
public static final BigInteger MAX_UNSIGNED_LONG_VALUE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE);
public static final BigInteger MIN_UNSIGNED_LONG_VALUE = BigInteger.ZERO;

public static final long MIN_UNSIGNED_LONG_VALUE_AS_LONG = MIN_UNSIGNED_LONG_VALUE.longValue();
public static final long MAX_UNSIGNED_LONG_VALUE_AS_LONG = MAX_UNSIGNED_LONG_VALUE.longValue();

private static final BigInteger MAX_LONG_VALUE = BigInteger.valueOf(Long.MAX_VALUE);
private static final BigInteger MIN_LONG_VALUE = BigInteger.valueOf(Long.MIN_VALUE);

private Numbers() {}

public static long bytesToLong(BytesRef bytes) {
int high = (bytes.bytes[bytes.offset + 0] << 24) | ((bytes.bytes[bytes.offset + 1] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 2]
& 0xff) << 8) | (bytes.bytes[bytes.offset + 3] & 0xff);
int low = (bytes.bytes[bytes.offset + 4] << 24) | ((bytes.bytes[bytes.offset + 5] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 6]
& 0xff) << 8) | (bytes.bytes[bytes.offset + 7] & 0xff);
return (((long) high) << 32) | (low & 0x0ffffffffL);
}

/**
* Converts a long to a byte array.
*
Expand Down Expand Up @@ -107,9 +102,46 @@ public static long toLongExact(Number n) {
}
}

/** Return the {@link BigInteger} that {@code n} stores, or throws an exception if the
* stored value cannot be converted to a {@link BigInteger} that stores the exact same
* value. */
public static BigInteger toBigIntegerExact(Number n) {
if (n instanceof Byte || n instanceof Short || n instanceof Integer || n instanceof Long) {
return BigInteger.valueOf(n.longValue());
} else if (n instanceof Float || n instanceof Double) {
double d = n.doubleValue();
if (d != Math.round(d)) {
throw new IllegalArgumentException(n + " is not an integer value");
}
return BigInteger.valueOf(n.longValue());
} else if (n instanceof BigDecimal) {
return ((BigDecimal) n).toBigIntegerExact();
} else if (n instanceof BigInteger) {
return ((BigInteger) n);
} else {
throw new IllegalArgumentException(
"Cannot check whether [" + n + "] of class [" + n.getClass().getName() + "] is actually a long"
);
}
}

/** Return the unsigned long (as {@link BigInteger}) that {@code n} stores, or throws an exception if the
* stored value cannot be converted to an unsigned long that stores the exact same
* value. */
public static BigInteger toUnsignedLongExact(Number value) {
final BigInteger v = Numbers.toBigIntegerExact(value);

if (v.compareTo(Numbers.MAX_UNSIGNED_LONG_VALUE) > 0 || v.compareTo(BigInteger.ZERO) < 0) {
throw new IllegalArgumentException("Value [" + value + "] is out of range for an unsigned long");
}

return v;
}

// weak bounds on the BigDecimal representation to allow for coercion
private static BigDecimal BIGDECIMAL_GREATER_THAN_LONG_MAX_VALUE = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE);
private static BigDecimal BIGDECIMAL_LESS_THAN_LONG_MIN_VALUE = BigDecimal.valueOf(Long.MIN_VALUE).subtract(BigDecimal.ONE);
private static BigDecimal BIGDECIMAL_GREATER_THAN_USIGNED_LONG_MAX_VALUE = new BigDecimal(MAX_UNSIGNED_LONG_VALUE).add(BigDecimal.ONE);

/** Return the long that {@code stringValue} stores or throws an exception if the
* stored value cannot be converted to a long that stores the exact same
Expand Down Expand Up @@ -142,6 +174,31 @@ public static long toLong(String stringValue, boolean coerce) {
return bigIntegerValue.longValue();
}

/** Return the long that {@code stringValue} stores or throws an exception if the
* stored value cannot be converted to a long that stores the exact same
* value and {@code coerce} is false. */
public static BigInteger toUnsignedLong(String stringValue, boolean coerce) {
final BigInteger bigIntegerValue;
try {
BigDecimal bigDecimalValue = new BigDecimal(stringValue);
if (bigDecimalValue.compareTo(BIGDECIMAL_GREATER_THAN_USIGNED_LONG_MAX_VALUE) >= 0
|| bigDecimalValue.compareTo(BigDecimal.ZERO) <= 0) {
throw new IllegalArgumentException("Value [" + stringValue + "] is out of range for an unsigned long");
}
bigIntegerValue = coerce ? bigDecimalValue.toBigInteger() : bigDecimalValue.toBigIntegerExact();
} catch (ArithmeticException e) {
throw new IllegalArgumentException("Value [" + stringValue + "] has a decimal part");
} catch (NumberFormatException e) {
throw new IllegalArgumentException("For input string: \"" + stringValue + "\"");
}

if (bigIntegerValue.compareTo(MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(BigInteger.ZERO) < 0) {
throw new IllegalArgumentException("Value [" + stringValue + "] is out of range for an unsigned long");
}

return bigIntegerValue;
}

/** Return the int that {@code n} stores, or throws an exception if the
* stored value cannot be converted to an int that stores the exact same
* value. */
Expand Down Expand Up @@ -170,4 +227,36 @@ public static byte toByteExact(Number n) {
}
return (byte) l;
}

/**
* Return a BigInteger equal to the unsigned value of the
* argument.
*/
public static BigInteger toUnsignedBigInteger(long i) {
if (i >= 0L) return BigInteger.valueOf(i);
else {
int upper = (int) (i >>> 32);
int lower = (int) i;

// return (upper << 32) + lower
return (BigInteger.valueOf(Integer.toUnsignedLong(upper))).shiftLeft(32).add(BigInteger.valueOf(Integer.toUnsignedLong(lower)));
}
}

/**
* Convert unsigned long to double value (see please Guava's com.google.common.primitives.UnsignedLong),
* this is faster then going through {@link #toUnsignedBigInteger(long)} conversion.
*/
public static double unsignedLongToDouble(long value) {
if (value >= 0) {
return (double) value;
}

// The top bit is set, which means that the double value is going to come from the top 53 bits.
// So we can ignore the bottom 11, except for rounding. We can unsigned-shift right 1, aka
// unsigned-divide by 2, and convert that. Then we'll get exactly half of the desired double
// value. But in the specific case where the bottom two bits of the original number are 01, we
// want to replace that with 1 in the shifted value for correct rounding.
return (double) ((value >>> 1) | (value & 1)) * 2.0;
}
}
Loading

0 comments on commit f4f136f

Please sign in to comment.