opensearch-project · ajleong623 · Jun 24, 2025 · Jun 24, 2025 · Jul 4, 2025 · Jul 4, 2025
@@ -5,6 +5,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.x]
 ### Added
+- Add search API tracker ([#18601](https://github.com/opensearch-project/OpenSearch/pull/18601))
+- Add support for linux riscv64 platform ([#18156](https://github.com/opensearch-project/OpenSearch/pull/18156))
+- [Rule based auto-tagging] Add get rule API ([#17336](https://github.com/opensearch-project/OpenSearch/pull/17336))
+- [Rule based auto-tagging] Add Delete Rule API ([#18184](https://github.com/opensearch-project/OpenSearch/pull/18184))
+- Implement parallel shard refresh behind cluster settings ([#17782](https://github.com/opensearch-project/OpenSearch/pull/17782))
+- Bump OpenSearch Core main branch to 3.0.0 ([#18039](https://github.com/opensearch-project/OpenSearch/pull/18039))
+- [Rule based Auto-tagging] Add wlm `ActionFilter` ([#17791](https://github.com/opensearch-project/OpenSearch/pull/17791))
+- Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/))
+- Add Warm Disk Threshold Allocation Decider for Warm shards ([#18082](https://github.com/opensearch-project/OpenSearch/pull/18082))
+- Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988))
+- Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088))
+- Adding support for derive source feature and implementing it for various type of field mappers ([#17759](https://github.com/opensearch-project/OpenSearch/pull/17759))
+- [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989))
+- Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139))
+- Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818))
+- Reject close index requests, while remote store migration is in progress.([#18327](https://github.com/opensearch-project/OpenSearch/pull/18327))
+- Improve sort-query performance by retaining the default `totalHitsThreshold` for approximated `match_all` queries ([#18189](https://github.com/opensearch-project/OpenSearch/pull/18189))
+- Enable testing for ExtensiblePlugins using classpath plugins ([#16908](https://github.com/opensearch-project/OpenSearch/pull/16908))
+- Introduce system generated ingest pipeline ([#17817](https://github.com/opensearch-project/OpenSearch/pull/17817)))
+- Apply cluster state metadata and routing table diff when building cluster state from remote([#18256](https://github.com/opensearch-project/OpenSearch/pull/18256))
+- Support create mode in pull-based ingestion and add retries for transient failures ([#18250](https://github.com/opensearch-project/OpenSearch/pull/18250)))
+- Decouple the init of Crypto Plugin and KeyProvider in CryptoRegistry ([18270](https://github.com/opensearch-project/OpenSearch/pull18270)))
+- Support cluster write block in pull-based ingestion ([#18280](https://github.com/opensearch-project/OpenSearch/pull/18280)))
+- Use QueryCoordinatorContext for the rewrite in validate API. ([#18272](https://github.com/opensearch-project/OpenSearch/pull/18272))
+- Upgrade crypto kms plugin dependencies for AWS SDK v2.x. ([#18268](https://github.com/opensearch-project/OpenSearch/pull/18268))
+- Add support for `matched_fields` with the unified highlighter ([#18164](https://github.com/opensearch-project/OpenSearch/issues/18164))
+- [repository-s3] Add support for SSE-KMS and S3 bucket owner verification ([#18312](https://github.com/opensearch-project/OpenSearch/pull/18312))
+- Optimize gRPC perf by passing by reference ([#18303](https://github.com/opensearch-project/OpenSearch/pull/18303))
+- Added File Cache Stats - Involves Block level as well as full file level stats ([#17538](https://github.com/opensearch-project/OpenSearch/issues/17479))
+- Added File Cache Pinning ([#17617](https://github.com/opensearch-project/OpenSearch/issues/13648))
+- Support consumer reset in Resume API for pull-based ingestion. This PR includes a breaking change for the experimental pull-based ingestion feature. ([#18332](https://github.com/opensearch-project/OpenSearch/pull/18332))
 - Add hierarchical routing processors for ingest and search pipelines ([#18826](https://github.com/opensearch-project/OpenSearch/pull/18826))
 - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375))
 - FS stats for warm nodes based on addressable space ([#18767](https://github.com/opensearch-project/OpenSearch/pull/18767))

@@ -536,6 +536,17 @@ public int getStatusFamilyCode() {
         return status / 100;
     }
 
+    public String getErrorType() {
+        int family = getStatusFamilyCode();
+        if (family <= 3) {
+            return "success";
+        } else if (family == 4) {
+            return "user_error";
+        } else {
+            return "system_failure";
+        }
+    }
+
     public static RestStatus readFrom(StreamInput in) throws IOException {
         return RestStatus.valueOf(in.readString());
     }

@@ -98,14 +98,14 @@
 ---
 "Help from 2.14.0 to 3.0.99":
   - skip:
-      version: " - 2.13.99, 3.2.0 - "
+      version: " - 2.13.99, 3.1.0 - "
       reason: search idle reactivate count total is only added in 3.0.0
       features: node_selector
   - do:
       cat.shards:
         help: true
       node_selector:
-        version: "2.14.0 - 3.1.99"
+        version: "2.14.0 - 3.2.99"
 
   - match:
       $body: |

@@ -25,6 +25,7 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - _all":
@@ -53,6 +54,7 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - indices _all":
@@ -81,6 +83,7 @@
   - is_true:  nodes.$node_id.indices.segments
   - is_true:  nodes.$node_id.indices.translog
   - is_true:  nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
 
 ---
 "Metric - one":
@@ -109,6 +112,7 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_true:   nodes.$node_id.indices.status_counter
 
 ---
 "Metric - multi":
@@ -137,6 +141,7 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_true:   nodes.$node_id.indices.status_counter
 
 ---
 "Metric - indexing doc_status":
@@ -153,8 +158,38 @@
 
   - is_false: nodes.$node_id.indices.docs
   - is_false: nodes.$node_id.indices.store
-  - is_true: nodes.$node_id.indices.indexing
-  - is_true: nodes.$node_id.indices.indexing.doc_status
+  - is_true:  nodes.$node_id.indices.indexing
+  - is_false: nodes.$node_id.indices.get
+  - is_false: nodes.$node_id.indices.search
+  - is_false: nodes.$node_id.indices.merges
+  - is_false: nodes.$node_id.indices.refresh
+  - is_false: nodes.$node_id.indices.flush
+  - is_false: nodes.$node_id.indices.warmer
+  - is_false: nodes.$node_id.indices.query_cache
+  - is_false: nodes.$node_id.indices.fielddata
+  - is_false: nodes.$node_id.indices.completion
+  - is_false: nodes.$node_id.indices.segments
+  - is_false: nodes.$node_id.indices.translog
+  - is_false: nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
+  - is_true:  nodes.$node_id.indices.status_counter.doc_status
+
+---
+"Metric - indexing search_response_status":
+  - skip:
+      version: " - 2.10.99"
+      reason: "Doc Status Stats were introduced in v2.11.0"
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: indices, index_metric: indexing }
+
+  - is_false: nodes.$node_id.indices.docs
+  - is_false: nodes.$node_id.indices.store
+  - is_true:  nodes.$node_id.indices.indexing
   - is_false: nodes.$node_id.indices.get
   - is_false: nodes.$node_id.indices.search
   - is_false: nodes.$node_id.indices.merges
@@ -167,6 +202,8 @@
   - is_false: nodes.$node_id.indices.segments
   - is_false: nodes.$node_id.indices.translog
   - is_false: nodes.$node_id.indices.recovery
+  - is_true:  nodes.$node_id.indices.status_counter
+  - is_true:  nodes.$node_id.indices.status_counter.search_response_status
 
 ---
 "Metric - recovery":
@@ -195,6 +232,7 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_true:   nodes.$node_id.indices.recovery
+  - is_true:   nodes.$node_id.indices.status_counter
 
 ---
 "Metric - _all include_segment_file_sizes":

@@ -13,6 +13,7 @@
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
+import org.opensearch.action.admin.indices.stats.DocStatusStats;
 import org.opensearch.action.bulk.BulkItemResponse;
 import org.opensearch.action.bulk.BulkRequest;
 import org.opensearch.action.bulk.BulkResponse;
@@ -32,7 +33,6 @@
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.engine.DocumentMissingException;
 import org.opensearch.index.engine.VersionConflictEngineException;
-import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats;
 import org.opensearch.indices.NodeIndicesStats;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
@@ -47,7 +47,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 
 import static java.util.Collections.singletonMap;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -281,15 +281,14 @@ public void testNodeIndicesStatsDocStatsWithAggregations() {
                 .getNodes()
                 .get(0)
                 .getIndices()
-                .getIndexing()
-                .getTotal()
+                .getStatusCounterStats()
                 .getDocStatusStats();
 
             assertTrue(
                 Arrays.equals(
                     docStatusStats.getDocStatusCounter(),
                     expectedDocStatusStats.getDocStatusCounter(),
-                    Comparator.comparingLong(AtomicLong::longValue)
+                    Comparator.comparingLong(LongAdder::longValue)
                 )
             );
         }
@@ -543,15 +542,14 @@ private void assertDocStatusStats() {
             .getNodes()
             .get(0)
             .getIndices()
-            .getIndexing()
-            .getTotal()
+            .getStatusCounterStats()
             .getDocStatusStats();
 
         assertTrue(
             Arrays.equals(
                 docStatusStats.getDocStatusCounter(),
                 expectedDocStatusStats.getDocStatusCounter(),
-                Comparator.comparingLong(AtomicLong::longValue)
+                Comparator.comparingLong(LongAdder::longValue)
             )
         );
     }

@@ -0,0 +1,111 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.search.msearch;
+
+import org.opensearch.ExceptionsHelper;
+import org.opensearch.action.admin.indices.stats.SearchResponseStatusStats;
+import org.opensearch.action.search.MultiSearchRequestBuilder;
+import org.opensearch.action.search.MultiSearchResponse;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.concurrent.atomic.LongAdder;
+
+@ClusterScope(scope = Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false)
+public class MultiSearchStatsIT extends OpenSearchIntegTestCase {
+    private final SearchResponseStatusStats expectedSearchResponseStatusStats = new SearchResponseStatusStats();
+
+    public void testNodeIndicesStatsDocStatusStatsIndexBulk() {
+        createIndex("test");
+        ensureGreen();
+        client().prepareIndex("test").setId("1").setSource("field", "xxx").get();
+        client().prepareIndex("test").setId("2").setSource("field", "yyy").get();
+        refresh();
+
+        int failureCount = randomIntBetween(0, 50);
+        int successCount = randomIntBetween(0, 50);
+
+        MultiSearchRequestBuilder multiSearchRequestBuilder = client().prepareMultiSearch();
+
+        for (int i = 0; i < failureCount; i++) {
+            multiSearchRequestBuilder.add(client().prepareSearch("noIndex").setQuery(QueryBuilders.termQuery("field", "yyy")));
+        }
+
+        for (int i = 0; i < successCount; i++) {
+            multiSearchRequestBuilder.add(client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()));
+        }
+
+        MultiSearchResponse response = multiSearchRequestBuilder.get();
+
+        for (MultiSearchResponse.Item item : response) {
+            if (item.isFailure()) {
+                updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getFailure());
+            } else {
+                updateExpectedDocStatusCounter(expectedSearchResponseStatusStats, item.getResponse());
+            }
+        }
+        assertSearchResponseStatusStats();
+    }
+
+    private void assertSearchResponseStatusStats() {
+        SearchResponseStatusStats searchResponseStatusStats = client().admin()
+            .cluster()
+            .prepareNodesStats()
+            .execute()
+            .actionGet()
+            .getNodes()
+            .get(0)
+            .getIndices()
+            .getStatusCounterStats()
+            .getSearchResponseStatusStats();
+
+        Arrays.equals(
+            searchResponseStatusStats.getSearchResponseStatusCounter(),
+            expectedSearchResponseStatusStats.getSearchResponseStatusCounter(),
+            Comparator.comparingLong(LongAdder::longValue)
+        );
+    }
+
+    private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, SearchResponse r) {
+        expectedSearchResponseStatusStats.inc(r.status());
+    }
+
+    private void updateExpectedDocStatusCounter(SearchResponseStatusStats expectedSearchResponseStatusStats, Exception e) {
+        expectedSearchResponseStatusStats.inc(ExceptionsHelper.status(e));
+    }
+}