Introduce new 'unsigned_long' numeric field type support

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

CHANGELOG.md

@@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add a setting to control auto release of OpenSearch managed index creation block ([#6277](https://github.com/opensearch-project/OpenSearch/pull/6277))
 - Fix timeout error when adding a document to an index with extension running ([#6275](https://github.com/opensearch-project/OpenSearch/pull/6275))
 - Clean up temporary files created during segment merge incase segment merge fails ([#6324](https://github.com/opensearch-project/OpenSearch/pull/6324))
+- Introduce new 'unsigned_long' numeric field type support ([#6237](https://github.com/opensearch-project/OpenSearch/pull/6237))
 
 ### Dependencies
 - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704))

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -153,8 +153,10 @@
 import org.opensearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedLongTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedStringTerms;
+import org.opensearch.search.aggregations.bucket.terms.ParsedUnsignedLongTerms;
 import org.opensearch.search.aggregations.bucket.terms.StringRareTerms;
 import org.opensearch.search.aggregations.bucket.terms.StringTerms;
+import org.opensearch.search.aggregations.bucket.terms.UnsignedLongTerms;
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
@@ -2269,6 +2271,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
         map.put(VariableWidthHistogramAggregationBuilder.NAME, (p, c) -> ParsedVariableWidthHistogram.fromXContent(p, (String) c));
         map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
+        map.put(UnsignedLongTerms.NAME, (p, c) -> ParsedUnsignedLongTerms.fromXContent(p, (String) c));
         map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
         map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
         map.put(LongRareTerms.NAME, (p, c) -> ParsedLongRareTerms.fromXContent(p, (String) c));

client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java

@@ -81,6 +81,7 @@
 import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
 import org.opensearch.search.aggregations.bucket.range.Range;
 import org.opensearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.RareTerms;
 import org.opensearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.Terms;
@@ -89,6 +90,7 @@
 import org.opensearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.WeightedAvg;
 import org.opensearch.search.aggregations.metrics.WeightedAvgAggregationBuilder;
+import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig;
 import org.opensearch.search.aggregations.support.MultiValuesSourceFieldConfig;
 import org.opensearch.search.aggregations.support.ValueType;
 import org.opensearch.search.builder.PointInTimeBuilder;
@@ -297,6 +299,67 @@ public void testSearchWithTermsAgg() throws IOException {
         assertEquals(0, type2.getAggregations().asList().size());
     }
 
+    public void testSearchWithMultiTermsAgg() throws IOException {
+        final String indexName = "multi_aggs";
+        Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName);
+        createIndex.setJsonEntity(
+            "{\n"
+                + "    \"mappings\": {\n"
+                + "        \"properties\" : {\n"
+                + "            \"username\" : {\n"
+                + "                \"type\" : \"keyword\"\n"
+                + "            },\n"
+                + "            \"rating\" : {\n"
+                + "                \"type\" : \"unsigned_long\"\n"
+                + "            }\n"
+                + "        }\n"
+                + "    }"
+                + "}"
+        );
+        client().performRequest(createIndex);
+
+        {
+            Request doc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/1");
+            doc1.setJsonEntity("{\"username\":\"bob\", \"rating\": 18446744073709551615}");
+            client().performRequest(doc1);
+            Request doc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/2");
+            doc2.setJsonEntity("{\"username\":\"tom\", \"rating\": 10223372036854775807}");
+            client().performRequest(doc2);
+            Request doc3 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/_doc/3");
+            doc3.setJsonEntity("{\"username\":\"john\"}");
+            client().performRequest(doc3);
+        }
+        client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh"));
+
+        SearchRequest searchRequest = new SearchRequest().indices(indexName);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.aggregation(
+            new MultiTermsAggregationBuilder("agg1").terms(
+                Arrays.asList(
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("username").build(),
+                    new MultiTermsValuesSourceConfig.Builder().setFieldName("rating").build()
+                )
+            )
+        );
+        searchSourceBuilder.size(0);
+        searchRequest.source(searchSourceBuilder);
+        SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+        assertSearchHeader(searchResponse);
+        assertNull(searchResponse.getSuggest());
+        assertEquals(Collections.emptyMap(), searchResponse.getProfileResults());
+        assertEquals(0, searchResponse.getHits().getHits().length);
+        assertEquals(Float.NaN, searchResponse.getHits().getMaxScore(), 0f);
+        Terms termsAgg = searchResponse.getAggregations().get("agg1");
+        assertEquals("agg1", termsAgg.getName());
+        assertEquals(2, termsAgg.getBuckets().size());
+        Terms.Bucket bucket1 = termsAgg.getBucketByKey("bob|18446744073709551615");
+        assertEquals(1, bucket1.getDocCount());
+        assertEquals(0, bucket1.getAggregations().asList().size());
+        Terms.Bucket bucket2 = termsAgg.getBucketByKey("tom|10223372036854775807");
+        assertEquals(1, bucket2.getDocCount());
+        assertEquals(0, bucket2.getAggregations().asList().size());
+    }
+
     public void testSearchWithRareTermsAgg() throws IOException {
         SearchRequest searchRequest = new SearchRequest();
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentParser.java

@@ -36,6 +36,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.CharBuffer;
 import java.util.List;
 import java.util.Map;
@@ -218,6 +219,8 @@ <T> Map<String, T> map(Supplier<Map<String, T>> mapFactory, CheckedFunction<XCon
 
     double doubleValue(boolean coerce) throws IOException;
 
+    BigInteger bigIntegerValue(boolean coerce) throws IOException;
+
     short shortValue() throws IOException;
 
     int intValue() throws IOException;
@@ -228,6 +231,8 @@ <T> Map<String, T> map(Supplier<Map<String, T>> mapFactory, CheckedFunction<XCon
 
     double doubleValue() throws IOException;
 
+    BigInteger bigIntegerValue() throws IOException;
+
     /**
      * @return true iff the current value is either boolean (<code>true</code> or <code>false</code>) or one of "false", "true".
      */

libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentSubParser.java

@@ -35,6 +35,7 @@
 import org.opensearch.common.CheckedFunction;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.CharBuffer;
 import java.util.List;
 import java.util.Map;
@@ -219,7 +220,12 @@ public float floatValue(boolean coerce) throws IOException {
 
     @Override
     public double doubleValue(boolean coerce) throws IOException {
-        return parser.doubleValue();
+        return parser.doubleValue(coerce);
+    }
+
+    @Override
+    public BigInteger bigIntegerValue(boolean coerce) throws IOException {
+        return parser.bigIntegerValue(coerce);
     }
 
     @Override
@@ -247,6 +253,11 @@ public double doubleValue() throws IOException {
         return parser.doubleValue();
     }
 
+    @Override
+    public BigInteger bigIntegerValue() throws IOException {
+        return parser.bigIntegerValue();
+    }
+
     @Override
     public boolean isBooleanValue() throws IOException {
         return parser.isBooleanValue();

libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentParser.java

@@ -43,6 +43,7 @@
 import org.opensearch.core.internal.io.IOUtils;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.CharBuffer;
 
 public class JsonXContentParser extends AbstractXContentParser {
@@ -188,6 +189,11 @@ public double doDoubleValue() throws IOException {
         return parser.getDoubleValue();
     }
 
+    @Override
+    public BigInteger doBigIntegerValue() throws IOException {
+        return parser.getBigIntegerValue();
+    }
+
     @Override
     public byte[] binaryValue() throws IOException {
         return parser.getBinaryValue();

libs/x-content/src/main/java/org/opensearch/common/xcontent/support/AbstractXContentParser.java

@@ -257,6 +257,23 @@ public double doubleValue(boolean coerce) throws IOException {
 
     protected abstract double doDoubleValue() throws IOException;
 
+    @Override
+    public BigInteger bigIntegerValue() throws IOException {
+        return bigIntegerValue(DEFAULT_NUMBER_COERCE_POLICY);
+    }
+
+    @Override
+    public BigInteger bigIntegerValue(boolean coerce) throws IOException {
+        Token token = currentToken();
+        if (token == Token.VALUE_STRING) {
+            checkCoerceString(coerce, BigInteger.class);
+            return new BigInteger(text());
+        }
+        return doBigIntegerValue();
+    }
+
+    protected abstract BigInteger doBigIntegerValue() throws IOException;
+
     @Override
     public final String textOrNull() throws IOException {
         if (currentToken() == Token.VALUE_NULL) {

libs/x-content/src/main/java/org/opensearch/common/xcontent/support/MapXContentParser.java

@@ -99,6 +99,15 @@ protected double doDoubleValue() throws IOException {
         return numberValue().doubleValue();
     }
 
+    @Override
+    protected BigInteger doBigIntegerValue() throws IOException {
+        if (numberValue() instanceof BigInteger) {
+            return (BigInteger) numberValue();
+        } else {
+            return BigInteger.valueOf(numberValue().longValue());
+        }
+    }
+
     @Override
     public XContentType contentType() {
         return xContentType;

modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellIdSource.java

@@ -68,6 +68,11 @@ public boolean isFloatingPoint() {
         return false;
     }
 
+    @Override
+    public boolean isBigInteger() {
+        return false;
+    }
+
     @Override
     public SortedNumericDocValues longValues(LeafReaderContext ctx) {
         if (geoBoundingBox.isUnbounded()) {

modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellIdSource.java

@@ -64,6 +64,14 @@ public boolean isFloatingPoint() {
         return false;
     }
 
+    /**
+     * Whether the underlying data is big integer or not.
+     */
+    @Override
+    public boolean isBigInteger() {
+        return false;
+    }
+
     /**
      * Get the current {@link SortedNumericDocValues}.
      *

server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java

@@ -44,6 +44,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Numbers;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -65,6 +66,7 @@
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -577,6 +579,9 @@ public void testSimpleSorts() throws Exception {
                     .startObject("long_value")
                     .field("type", "long")
                     .endObject()
+                    .startObject("unsigned_long_value")
+                    .field("type", "unsigned_long")
+                    .endObject()
                     .startObject("float_value")
                     .field("type", "float")
                     .endObject()
@@ -588,6 +593,7 @@ public void testSimpleSorts() throws Exception {
             )
         );
         ensureGreen();
+        BigInteger UNSIGNED_LONG_BASE = Numbers.MAX_UNSIGNED_LONG_VALUE.subtract(BigInteger.valueOf(10));
         List<IndexRequestBuilder> builders = new ArrayList<>();
         for (int i = 0; i < 10; i++) {
             IndexRequestBuilder builder = client().prepareIndex("test")
@@ -600,6 +606,7 @@ public void testSimpleSorts() throws Exception {
                         .field("short_value", i)
                         .field("integer_value", i)
                         .field("long_value", i)
+                        .field("unsigned_long_value", UNSIGNED_LONG_BASE.add(BigInteger.valueOf(i)))
                         .field("float_value", 0.1 * i)
                         .field("double_value", 0.1 * i)
                         .endObject()
@@ -622,7 +629,6 @@ public void testSimpleSorts() throws Exception {
 
         // STRING
         int size = 1 + random.nextInt(10);
-
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(matchAllQuery())
             .setSize(size)
@@ -791,6 +797,45 @@ public void testSimpleSorts() throws Exception {
         }
 
         assertNoFailures(searchResponse);
+
+        // UNSIGNED_LONG
+        size = 1 + random.nextInt(10);
+        searchResponse = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .setSize(size)
+            .addSort("unsigned_long_value", SortOrder.ASC)
+            .get();
+
+        assertHitCount(searchResponse, 10);
+        assertThat(searchResponse.getHits().getHits().length, equalTo(size));
+        for (int i = 0; i < size; i++) {
+            assertThat(searchResponse.getHits().getAt(i).getId(), equalTo(Integer.toString(i)));
+            assertThat(
+                ((Number) searchResponse.getHits().getAt(i).getSortValues()[0]),
+                equalTo(UNSIGNED_LONG_BASE.add(BigInteger.valueOf(i)))
+            );
+        }
+
+        assertThat(searchResponse.toString(), not(containsString("error")));
+        size = 1 + random.nextInt(10);
+        searchResponse = client().prepareSearch()
+            .setQuery(matchAllQuery())
+            .setSize(size)
+            .addSort("unsigned_long_value", SortOrder.DESC)
+            .get();
+        assertHitCount(searchResponse, 10L);
+        assertHitCount(searchResponse, 10);
+        assertThat(searchResponse.getHits().getHits().length, equalTo(size));
+        for (int i = 0; i < size; i++) {
+            assertThat(searchResponse.getHits().getAt(i).getId(), equalTo(Integer.toString(9 - i)));
+            assertThat(
+                ((Number) searchResponse.getHits().getAt(i).getSortValues()[0]),
+                equalTo(UNSIGNED_LONG_BASE.add(BigInteger.valueOf(9 - i)))
+            );
+        }
+
+        assertThat(searchResponse.toString(), not(containsString("error")));
+        assertNoFailures(searchResponse);
     }
 
     public void testSortMissingNumbers() throws Exception {