Introduce new 'unsigned_long' numeric field type support

CHANGELOG.md

@@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Extensions] Add IdentityPlugin into core to support Extension identities ([#7246](https://github.com/opensearch-project/OpenSearch/pull/7246))
 - Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
 - Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244))
+- Add 'unsigned_long' numeric field type ([#6237](https://github.com/opensearch-project/OpenSearch/pull/6237))
 
 ### Dependencies
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.0

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -153,8 +153,10 @@
 import org.opensearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedLongTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms;
+import org.opensearch.search.aggregations.bucket.terms.ParsedUnsignedLongTerms;
 import org.opensearch.search.aggregations.bucket.terms.StringRareTerms;
 import org.opensearch.search.aggregations.bucket.terms.StringTerms;
+import org.opensearch.search.aggregations.bucket.terms.UnsignedLongTerms;
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
@@ -2274,6 +2276,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
         map.put(VariableWidthHistogramAggregationBuilder.NAME, (p, c) -> ParsedVariableWidthHistogram.fromXContent(p, (String) c));
         map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
+        map.put(UnsignedLongTerms.NAME, (p, c) -> ParsedUnsignedLongTerms.fromXContent(p, (String) c));
         map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
         map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
         map.put(LongRareTerms.NAME, (p, c) -> ParsedLongRareTerms.fromXContent(p, (String) c));

client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java

@@ -81,6 +81,7 @@
 import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
 import org.opensearch.search.aggregations.bucket.range.Range;
 import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.RareTerms;
 import org.opensearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.Terms;
@@ -89,6 +90,7 @@
 import org.opensearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.WeightedAvg;
 import org.opensearch.search.aggregations.metrics.WeightedAvgAggregationBuilder;
+import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
 import org.opensearch.search.aggregations.support.MultiValuesSourceFieldConfig;
 import org.opensearch.search.aggregations.support.ValueType;
 import org.opensearch.search.builder.PointInTimeBuilder;
@@ -105,6 +107,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -297,6 +300,67 @@ public void testSearchWithTermsAgg() throws IOException {
         assertEquals(0, type2.getAggregations().asList().size());
     }
 
+    public void testSearchWithMultiTermsAgg() throws IOException {
+        final String indexName = "multi_aggs";
+        Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName);
+        createIndex.setJsonEntity(
+            "{\n"
+                + "    \"mappings\": {\n"
+                + "        \"properties\" : {\n"
+                + "            \"username\" : {\n"
+                + "                \"type\" : \"keyword\"\n"
+                + "            },\n"
+                + "            \"rating\" : {\n"
+                + "                \"type\" : \"unsigned_long\"\n"
+                + "            }\n"
+                + "        }\n"
+                + "    }"
+                + "}"
+        );
+        client().performRequest(createIndex);
+
+        {
+            Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/1");
+            doc1.setJsonEntity("{\"username\":\"bob\", \"rating\": 18446744073709551615}");
+            client().performRequest(doc1);
+            Request doc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/2");
+            doc2.setJsonEntity("{\"username\":\"tom\", \"rating\": 10223372036854775807}");
+            client().performRequest(doc2);
+            Request doc3 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/3");
+            doc3.setJsonEntity("{\"username\":\"john\"}");
+            client().performRequest(doc3);
+        }
+        client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
+
+        SearchRequest searchRequest = new SearchRequest().indices(indexName);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.aggregation(
+            new MultiTermsAggregationBuilder("agg1").terms(
+                Arrays.asList(
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("username").build(),
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("rating").build()
+                )
+            )
+        );
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(Float.NaN, searchResponse.getHits().getMaxScore(), 0f);
+        Terms termsAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", termsAgg.getName());
+        assertEquals(2, termsAgg.getBuckets().size());
+        Terms.Bucket bucket1 = termsAgg.getBucketByKey("bob|18446744073709551615");
+        assertEquals(1, bucket1.getDocCount());
+        assertEquals(0, bucket1.getAggregations().asList().size());
+        Terms.Bucket bucket2 = termsAgg.getBucketByKey("tom|10223372036854775807");
+        assertEquals(1, bucket2.getDocCount());
+        assertEquals(0, bucket2.getAggregations().asList().size());
+    }
+
     public void testSearchWithRareTermsAgg() throws IOException {
         SearchRequest searchRequest = new SearchRequest();
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@@ -847,6 +911,56 @@ public void testMultiSearch() throws Exception {
         assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(1).getId(), Matchers.equalTo("6"));
     }
 
+    public void testSearchWithSort() throws Exception {
+        final String indexName = "search_sort";
+        Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName);
+        createIndex.setJsonEntity(
+            "{\n"
+                + "    \"mappings\": {\n"
+                + "        \"properties\" : {\n"
+                + "            \"username\" : {\n"
+                + "                \"type\" : \"keyword\"\n"
+                + "            },\n"
+                + "            \"rating\" : {\n"
+                + "                \"type\" : \"unsigned_long\"\n"
+                + "            }\n"
+                + "        }\n"
+                + "    }"
+                + "}"
+        );
+        client().performRequest(createIndex);
+
+        {
+            Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/1");
+            doc1.setJsonEntity("{\"username\":\"bob\", \"rating\": 18446744073709551610}");
+            client().performRequest(doc1);
+            Request doc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/2");
+            doc2.setJsonEntity("{\"username\":\"tom\", \"rating\": 10223372036854775807}");
+            client().performRequest(doc2);
+            Request doc3 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/3");
+            doc3.setJsonEntity("{\"username\":\"john\"}");
+            client().performRequest(doc3);
+        }
+        client().performRequest(new Request(HttpPost.METHOD_NAME, "/search_sort/_refresh"));
+
+        SearchRequest searchRequest = new SearchRequest("search_sort");
+        searchRequest.source().sort("rating", SortOrder.ASC);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+
+        assertThat(searchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(searchResponse.getHits().getTotalHits().value, Matchers.equalTo(3L));
+        assertThat(searchResponse.getHits().getAt(0).getId(), Matchers.equalTo("2"));
+        assertThat(searchResponse.getHits().getAt(1).getId(), Matchers.equalTo("1"));
+        assertThat(searchResponse.getHits().getAt(2).getId(), Matchers.equalTo("3"));
+
+        assertThat(searchResponse.getHits().getAt(0).getSortValues().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getSortValues()[0], equalTo(new BigInteger("10223372036854775807")));
+        assertThat(searchResponse.getHits().getAt(1).getSortValues().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(1).getSortValues()[0], equalTo(new BigInteger("18446744073709551610")));
+        assertThat(searchResponse.getHits().getAt(2).getSortValues().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(2).getSortValues()[0], equalTo(new BigInteger("18446744073709551615")));
+    }
+
     public void testMultiSearch_withAgg() throws Exception {
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
         SearchRequest searchRequest1 = new SearchRequest("index1");

server/src/main/java/org/opensearch/common/Numbers.java → libs/common/src/main/java/org/opensearch/common/Numbers.java

@@ -32,8 +32,6 @@
 
 package org.opensearch.common;
 
-import org.apache.lucene.util.BytesRef;
-
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
@@ -43,20 +41,17 @@
  * @opensearch.internal
  */
 public final class Numbers {
+    public static final BigInteger MAX_UNSIGNED_LONG_VALUE = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE);
+    public static final BigInteger MIN_UNSIGNED_LONG_VALUE = BigInteger.ZERO;
+
+    public static final long MIN_UNSIGNED_LONG_VALUE_AS_LONG = MIN_UNSIGNED_LONG_VALUE.longValue();
+    public static final long MAX_UNSIGNED_LONG_VALUE_AS_LONG = MAX_UNSIGNED_LONG_VALUE.longValue();
 
     private static final BigInteger MAX_LONG_VALUE = BigInteger.valueOf(Long.MAX_VALUE);
     private static final BigInteger MIN_LONG_VALUE = BigInteger.valueOf(Long.MIN_VALUE);
 
     private Numbers() {}
 
-    public static long bytesToLong(BytesRef bytes) {
-        int high = (bytes.bytes[bytes.offset + 0] << 24) | ((bytes.bytes[bytes.offset + 1] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 2]
-            & 0xff) << 8) | (bytes.bytes[bytes.offset + 3] & 0xff);
-        int low = (bytes.bytes[bytes.offset + 4] << 24) | ((bytes.bytes[bytes.offset + 5] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 6]
-            & 0xff) << 8) | (bytes.bytes[bytes.offset + 7] & 0xff);
-        return (((long) high) << 32) | (low & 0x0ffffffffL);
-    }
-
     /**
      * Converts a long to a byte array.
      *
@@ -107,9 +102,44 @@ public static long toLongExact(Number n) {
         }
     }
 
+    /** Return the {@link BigInteger} that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to a {@link BigInteger} that stores the exact same
+     *  value. */
+    public static BigInteger toBigIntegerExact(Number n) {
+        if (n instanceof Byte || n instanceof Short || n instanceof Integer || n instanceof Long) {
+            return BigInteger.valueOf(n.longValue());
+        } else if (n instanceof Float || n instanceof Double) {
+            double d = n.doubleValue();
+            if (d != Math.round(d)) {
+                throw new IllegalArgumentException(n + " is not an integer value");
+            }
+            return BigInteger.valueOf(n.longValue());
+        } else if (n instanceof BigDecimal) {
+            return ((BigDecimal) n).toBigIntegerExact();
+        } else if (n instanceof BigInteger) {
+            return ((BigInteger) n);
+        } else {
+            throw new IllegalArgumentException("Cannot convert [" + n + "] of class [" + n.getClass().getName() + "] to a BigInteger");
+        }
+    }
+
+    /** Return the unsigned long (as {@link BigInteger}) that {@code n} stores, or throws an exception if the
+     *  stored value cannot be converted to an unsigned long that stores the exact same
+     *  value. */
+    public static BigInteger toUnsignedLongExact(Number value) {
+        final BigInteger v = Numbers.toBigIntegerExact(value);
+
+        if (v.compareTo(MAX_UNSIGNED_LONG_VALUE) > 0 || v.compareTo(MIN_UNSIGNED_LONG_VALUE) < 0) {
+            throw new IllegalArgumentException("Value [" + value + "] is out of range for an unsigned long");
+        }
+
+        return v;
+    }
+
     // weak bounds on the BigDecimal representation to allow for coercion
     private static BigDecimal BIGDECIMAL_GREATER_THAN_LONG_MAX_VALUE = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE);
     private static BigDecimal BIGDECIMAL_LESS_THAN_LONG_MIN_VALUE = BigDecimal.valueOf(Long.MIN_VALUE).subtract(BigDecimal.ONE);
+    private static BigDecimal BIGDECIMAL_GREATER_THAN_USIGNED_LONG_MAX_VALUE = new BigDecimal(MAX_UNSIGNED_LONG_VALUE).add(BigDecimal.ONE);
 
     /** Return the long that {@code stringValue} stores or throws an exception if the
      *  stored value cannot be converted to a long that stores the exact same
@@ -142,6 +172,31 @@ public static long toLong(String stringValue, boolean coerce) {
         return bigIntegerValue.longValue();
     }
 
+    /** Return the long that {@code stringValue} stores or throws an exception if the
+     *  stored value cannot be converted to a long that stores the exact same
+     *  value and {@code coerce} is false. */
+    public static BigInteger toUnsignedLong(String stringValue, boolean coerce) {
+        final BigInteger bigIntegerValue;
+        try {
+            BigDecimal bigDecimalValue = new BigDecimal(stringValue);
+            if (bigDecimalValue.compareTo(BIGDECIMAL_GREATER_THAN_USIGNED_LONG_MAX_VALUE) >= 0
+                || bigDecimalValue.compareTo(BigDecimal.ZERO) <= 0) {
+                throw new IllegalArgumentException("Value [" + stringValue + "] is out of range for an unsigned long");
+            }
+            bigIntegerValue = coerce ? bigDecimalValue.toBigInteger() : bigDecimalValue.toBigIntegerExact();
+        } catch (ArithmeticException e) {
+            throw new IllegalArgumentException("Value [" + stringValue + "] has a decimal part");
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("For input string: \"" + stringValue + "\"");
+        }
+
+        if (bigIntegerValue.compareTo(MAX_UNSIGNED_LONG_VALUE) > 0 || bigIntegerValue.compareTo(MIN_UNSIGNED_LONG_VALUE) < 0) {
+            throw new IllegalArgumentException("Value [" + stringValue + "] is out of range for an unsigned long");
+        }
+
+        return bigIntegerValue;
+    }
+
     /** Return the int that {@code n} stores, or throws an exception if the
      *  stored value cannot be converted to an int that stores the exact same
      *  value. */
@@ -170,4 +225,36 @@ public static byte toByteExact(Number n) {
         }
         return (byte) l;
     }
+
+    /**
+     * Return a BigInteger equal to the unsigned value of the
+     * argument.
+     */
+    public static BigInteger toUnsignedBigInteger(long i) {
+        if (i >= 0L) return BigInteger.valueOf(i);
+        else {
+            int upper = (int) (i >>> 32);
+            int lower = (int) i;
+
+            // return (upper << 32) + lower
+            return (BigInteger.valueOf(Integer.toUnsignedLong(upper))).shiftLeft(32).add(BigInteger.valueOf(Integer.toUnsignedLong(lower)));
+        }
+    }
+
+    /**
+     * Convert unsigned long to double value (see please Guava's com.google.common.primitives.UnsignedLong),
+     * this is faster then going through {@link #toUnsignedBigInteger(long)} conversion.
+     */
+    public static double unsignedLongToDouble(long value) {
+        if (value >= 0) {
+            return (double) value;
+        }
+
+        // The top bit is set, which means that the double value is going to come from the top 53 bits.
+        // So we can ignore the bottom 11, except for rounding. We can unsigned-shift right 1, aka
+        // unsigned-divide by 2, and convert that. Then we'll get exactly half of the desired double
+        // value. But in the specific case where the bottom two bits of the original number are 01, we
+        // want to replace that with 1 in the shifted value for correct rounding.
+        return (double) ((value >>> 1) | (value & 1)) * 2.0;
+    }
 }

libs/core/src/main/java/org/opensearch/core/util/BytesRefUtils.java

@@ -102,4 +102,13 @@ public static int sortAndDedup(final BytesRefArray bytes, final int[] indices) {
         }
         return uniqueCount;
     }
+
+    public static long bytesToLong(BytesRef bytes) {
+        int high = (bytes.bytes[bytes.offset + 0] << 24) | ((bytes.bytes[bytes.offset + 1] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 2]
+            & 0xff) << 8) | (bytes.bytes[bytes.offset + 3] & 0xff);
+        int low = (bytes.bytes[bytes.offset + 4] << 24) | ((bytes.bytes[bytes.offset + 5] & 0xff) << 16) | ((bytes.bytes[bytes.offset + 6]
+            & 0xff) << 8) | (bytes.bytes[bytes.offset + 7] & 0xff);
+        return (((long) high) << 32) | (low & 0x0ffffffffL);
+    }
+
 }