Support for IndexSort in Nested Fields

rest-api-spec/src/main/resources/rest-api-spec/test/indices.sort/10_basic.yml

@@ -176,3 +176,106 @@
   # This should failed with 400 as half_float is not supported for index sort
   - match: { status: 400 }
   - match: { error.type: illegal_argument_exception }
+
+---
+"Index sort with nested fields":
+  - skip:
+        version: " - 3.1.0"
+        reason: "Index sort on nested field is only supported after 3.1.0"
+  - do:
+      indices.create:
+        index: test_nested_sort
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index.sort.field: foo
+            index.sort.order: desc
+          mappings:
+            properties:
+              foo:
+                type: integer
+              foo1:
+                type: keyword
+              contacts:
+                type: nested
+                properties:
+                  name:
+                    type: keyword
+                  age:
+                    type: integer
+
+  - do:
+      index:
+        index: test_nested_sort
+        id: "1"
+        body:
+          foo: 100
+          foo1: "A"
+          contacts:
+            - name: "Alice"
+              age: 30
+
+  - do:
+      index:
+        index: test_nested_sort
+        id: "2"
+        body:
+          foo: 200
+          foo1: "B"
+          contacts:
+            - name: "Bob"
+              age: 40
+
+  - do:
+      index:
+        index: test_nested_sort
+        id: "3"
+        body:
+          foo: 150
+          foo1: "C"
+          contacts:
+            - name: "Charlie"
+              age: 25
+
+  - do:
+      indices.refresh:
+        index: test_nested_sort
+
+  - do:
+      search:
+        index: test_nested_sort
+        body:
+          sort:
+            - foo: desc
+          size: 3
+
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.2._id: "1" }
+
+---
+"Index sort with nested field as sort field validation":
+  - skip:
+      version: " - 3.1.0"
+      reason: "Index sort on nested field is only supported after 3.1.0"
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_nested_sort
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index.sort.field: contacts.age
+          mappings:
+            properties:
+              contacts:
+                type: nested
+                properties:
+                  age:
+                    type: integer
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "index sorting on nested fields is not supported: found nested sort field [contacts.age] in [test_nested_sort]" }

server/src/internalClusterTest/java/org/opensearch/index/IndexSortIT.java

@@ -36,22 +36,34 @@
 
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.sort.SortOrder;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.hamcrest.Matchers.containsString;
 
 public class IndexSortIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
     private static final XContentBuilder TEST_MAPPING = createTestMapping();
+    private static final XContentBuilder NESTED_TEST_MAPPING = createNestedTestMapping();
 
     public IndexSortIT(Settings staticSettings) {
         super(staticSettings);
@@ -95,6 +107,49 @@ private static XContentBuilder createTestMapping() {
         }
     }
 
+    private static XContentBuilder createNestedTestMapping() {
+        try {
+            return jsonBuilder().startObject()
+                .startObject("properties")
+                .startObject("foo")
+                .field("type", "integer")
+                .endObject()
+                .startObject("foo1")
+                .field("type", "keyword")
+                .endObject()
+                .startObject("contacts")
+                .field("type", "nested")
+                .startObject("properties")
+                .startObject("name")
+                .field("type", "keyword")
+                .endObject()
+                .startObject("age")
+                .field("type", "integer")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    private static void addNestedDocuments(String id, int foo, String foo1, String name, int age) throws IOException {
+        XContentBuilder sourceBuilder = jsonBuilder().startObject()
+            .field("foo", foo)
+            .field("foo1", foo1)
+            .startArray("contacts")
+            .startObject()
+            .field("name", name)
+            .field("age", age)
+            .endObject()
+            .endArray()
+            .endObject();
+
+        client().prepareIndex("nested-test-index").setId(id).setSource(sourceBuilder).get();
+    }
+
     public void testIndexSort() {
         SortField dateSort = new SortedNumericSortField("date", SortField.Type.LONG, false);
         dateSort.setMissingValue(Long.MAX_VALUE);
@@ -146,4 +201,141 @@ public void testInvalidIndexSort() {
         );
         assertThat(exc.getMessage(), containsString("docvalues not found for index sort field:[keyword]"));
     }
+
+    public void testIndexSortOnNestedField() throws IOException {
+        boolean ascending = randomBoolean();
+        SortedNumericSelector.Type selector = ascending ? SortedNumericSelector.Type.MIN : SortedNumericSelector.Type.MAX;
+        SortField regularSort = new SortedNumericSortField("foo", SortField.Type.INT, !ascending, selector);
+        regularSort.setMissingValue(ascending ? Integer.MAX_VALUE : Integer.MIN_VALUE);
+
+        Sort indexSort = new Sort(regularSort);
+
+        prepareCreate("nested-test-index").setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("index.number_of_shards", "1")
+                .put("index.number_of_replicas", "0")
+                .putList("index.sort.field", "foo")
+                .putList("index.sort.order", ascending ? "asc" : "desc")
+        ).setMapping(NESTED_TEST_MAPPING).get();
+
+        int numDocs = randomIntBetween(10, 30);
+        List<Integer> fooValues = new ArrayList<>(numDocs);
+        List<String> ids = new ArrayList<>(numDocs);
+
+        for (int i = 0; i < numDocs; i++) {
+            String id = String.valueOf(i);
+            int fooValue = randomIntBetween(1, 100);
+            String name = UUID.randomUUID().toString().replace("-", "").substring(0, 5);
+
+            addNestedDocuments(id, fooValue, "", name, fooValue);
+            fooValues.add(fooValue);
+            ids.add(id);
+        }
+
+        flushAndRefresh("nested-test-index");
+        ensureGreen("nested-test-index");
+
+        assertSortedSegments("nested-test-index", indexSort);
+
+        SearchResponse response = client().prepareSearch("nested-test-index")
+            .addSort("foo", ascending ? SortOrder.ASC : SortOrder.DESC)
+            .setQuery(QueryBuilders.matchAllQuery())
+            .setSize(numDocs)
+            .get();
+
+        assertEquals(numDocs, response.getHits().getTotalHits().value());
+
+        Map<Integer, String> valueToId = new HashMap<>();
+        for (int i = 0; i < numDocs; i++) {
+            valueToId.put(fooValues.get(i), ids.get(i));
+        }
+
+        List<Integer> sortedValues = new ArrayList<>(fooValues);
+        if (ascending) {
+            Collections.sort(sortedValues);
+        } else {
+            sortedValues.sort(Collections.reverseOrder());
+        }
+
+        for (int i = 0; i < numDocs; i++) {
+            int expectedValue = sortedValues.get(i);
+            assertEquals(expectedValue, response.getHits().getAt(i).getSourceAsMap().get("foo"));
+        }
+    }
+
+    public void testIndexSortWithNestedField_MultiField() throws IOException {
+        boolean ascendingPrimary = randomBoolean();
+        boolean ascendingSecondary = randomBoolean();
+        prepareCreate("nested-test-index").setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("index.number_of_shards", "1")
+                .put("index.number_of_replicas", "0")
+                .putList("index.sort.field", "foo", "foo1")
+                .putList("index.sort.order", ascendingPrimary ? "asc" : "desc", ascendingSecondary ? "asc" : "desc")
+        ).setMapping(NESTED_TEST_MAPPING).get();
+
+        int numDocs = randomIntBetween(10, 30);
+        List<Tuple<Integer, String>> docValues = new ArrayList<>(numDocs);
+        List<String> ids = new ArrayList<>(numDocs);
+
+        int duplicateValue = randomIntBetween(30, 50);
+        int numDuplicates = randomIntBetween(3, 5);
+
+        for (int i = 0; i < numDocs; i++) {
+            String id = String.valueOf(i);
+            int fooValue;
+            if (i < numDuplicates) {
+                fooValue = duplicateValue;
+            } else {
+                fooValue = randomIntBetween(1, 100);
+            }
+            String name = UUID.randomUUID().toString().replace("-", "").substring(0, 5);
+            addNestedDocuments(id, fooValue, name, name, fooValue);
+            docValues.add(new Tuple<>(fooValue, name));
+            ids.add(id);
+        }
+
+        flushAndRefresh("nested-test-index");
+        ensureGreen("nested-test-index");
+        SearchResponse response = client().prepareSearch("nested-test-index")
+            .addSort("foo", ascendingPrimary ? SortOrder.ASC : SortOrder.DESC)
+            .addSort("foo1", ascendingSecondary ? SortOrder.ASC : SortOrder.DESC)
+            .setQuery(QueryBuilders.matchAllQuery())
+            .setSize(numDocs)
+            .get();
+
+        assertEquals(numDocs, response.getHits().getTotalHits().value());
+
+        List<Tuple<Integer, String>> sortedValues = new ArrayList<>(docValues);
+        sortedValues.sort((a, b) -> {
+            int primaryCompare = ascendingPrimary ? Integer.compare(a.v1(), b.v1()) : Integer.compare(b.v1(), a.v1());
+            if (primaryCompare != 0) {
+                return primaryCompare;
+            }
+            return ascendingSecondary ? a.v2().compareTo(b.v2()) : b.v2().compareTo(a.v2());
+        });
+
+        for (int i = 0; i < numDocs; i++) {
+            assertEquals(sortedValues.get(i).v1(), response.getHits().getAt(i).getSourceAsMap().get("foo"));
+            assertEquals(sortedValues.get(i).v2(), response.getHits().getAt(i).getSourceAsMap().get("foo1"));
+        }
+    }
+
+    public void testIndexSortWithSortFieldInsideDocBlock() {
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> prepareCreate("nested-sort-test").setSettings(
+                Settings.builder()
+                    .put(indexSettings())
+                    .put("index.number_of_shards", "1")
+                    .put("index.number_of_replicas", "0")
+                    .putList("index.sort.field", "contacts.age")
+                    .putList("index.sort.order", "desc")
+            ).setMapping(NESTED_TEST_MAPPING).get()
+        );
+
+        assertThat(exception.getMessage(), containsString("index sorting on nested fields is not supported"));
+    }
 }

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -113,6 +113,7 @@ public class Lucene {
     public static final String LATEST_CODEC = "Lucene101";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
+    public static final String PARENT_FIELD = "__nested_parent";
 
     public static final NamedAnalyzer STANDARD_ANALYZER = new NamedAnalyzer("_standard", AnalyzerScope.GLOBAL, new StandardAnalyzer());
     public static final NamedAnalyzer KEYWORD_ANALYZER = new NamedAnalyzer("_keyword", AnalyzerScope.GLOBAL, new KeywordAnalyzer());

server/src/main/java/org/opensearch/index/engine/InternalEngine.java

@@ -66,6 +66,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InfoStream;
 import org.opensearch.ExceptionsHelper;
+import org.opensearch.Version;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.common.Booleans;
 import org.opensearch.common.Nullable;
@@ -2379,6 +2380,9 @@ private IndexWriterConfig getIndexWriterConfig() {
         iwc.setUseCompoundFile(engineConfig.useCompoundFile());
         if (config().getIndexSort() != null) {
             iwc.setIndexSort(config().getIndexSort());
+            if (config().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_3_2_0)) {
+                iwc.setParentField(Lucene.PARENT_FIELD);
+            }
         }
         if (config().getLeafSorter() != null) {
             iwc.setLeafSorter(config().getLeafSorter()); // The default segment search order