Merge branch 'main' into sync-release-note

Signed-off-by: Louis Chu <clingzhi@amazon.com>

CHANGELOG.md

@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
+- Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -82,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 
 ### Dependencies
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298))
@@ -96,6 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
 - Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add the means to extract the contextual properties from HttpChannel, TcpCChannel and TrasportChannel without excessive typecasting ([#10562](https://github.com/opensearch-project/OpenSearch/pull/10562))
+- [Remote Store] Add Remote Store backpressure rejection stats to `_nodes/stats` ([#10524](https://github.com/opensearch-project/OpenSearch/pull/10524))
 
 ### Deprecated
 

buildSrc/version.properties

@@ -68,4 +68,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.30.1
+opentelemetry         = 1.31.0
+opentelemetrysemconv  = 1.21.0-alpha

plugins/telemetry-otel/build.gradle

@@ -28,7 +28,7 @@ dependencies {
   api "io.opentelemetry:opentelemetry-sdk-trace:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-sdk-metrics:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-logging:${versions.opentelemetry}"
-  api "io.opentelemetry:opentelemetry-semconv:${versions.opentelemetry}-alpha"
+  api "io.opentelemetry.semconv:opentelemetry-semconv:${versions.opentelemetrysemconv}"
   api "io.opentelemetry:opentelemetry-sdk-logs:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-otlp:${versions.opentelemetry}"
   api "io.opentelemetry:opentelemetry-exporter-common:${versions.opentelemetry}"

plugins/telemetry-otel/licenses/opentelemetry-api-1.31.0.jar.sha1

@@ -0,0 +1 @@
+bb24a44d73484c681c236aed84fe6c28d17f30e2

plugins/telemetry-otel/licenses/opentelemetry-context-1.31.0.jar.sha1

@@ -0,0 +1 @@
+b8004737f7a970124e36ac71fde8eb88423e8cee

plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.31.0.jar.sha1

@@ -0,0 +1 @@
+b7b4baf5f9af72d5eb8a231dfb114ae31c57150d

plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.31.0.jar.sha1

@@ -0,0 +1 @@
+260e5363dad83a0ae65c16ad6a3dd2914e0db201

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.31.0.jar.sha1

@@ -0,0 +1 @@
+b6454464425dfd81519070caeca3824558a2f1ae

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.31.0.jar.sha1

@@ -0,0 +1 @@
+d8c22b6851bbc3dbf5d2387b9bde158ed5416ba4

plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.31.0.jar.sha1

@@ -0,0 +1 @@
+dd209381d58cfe81a989e29c9ca26d97c8dabd7a

plugins/telemetry-otel/licenses/opentelemetry-extension-incubator-1.31.0-alpha.jar.sha1

@@ -0,0 +1 @@
+6c9f5c063309d92b6dd28bff0667f54b63afd36f

plugins/telemetry-otel/licenses/opentelemetry-sdk-1.31.0.jar.sha1

@@ -0,0 +1 @@
+2b2093be08a09ac536292bf6cecf8129cc7fb191

plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.31.0.jar.sha1

@@ -0,0 +1 @@
+f492528288236e097e12fc1c45963dd82c70d33c

plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.31.0.jar.sha1

@@ -0,0 +1 @@
+a63a203d3dc6f8875f8c26b9e3b522dc9a3f6280

plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.31.0.jar.sha1

@@ -0,0 +1 @@
+47cc23762fae728d68e4fda1dfb71986ae0b8b3e

plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.31.0.jar.sha1

@@ -0,0 +1 @@
+a3941197cfb8ae9eb9e482073480c0c3918b746c

plugins/telemetry-otel/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1

@@ -0,0 +1 @@
+207660e74d1e155272e9559fd4d27854b92fc6ac

plugins/telemetry-otel/src/main/java/org/opensearch/telemetry/tracing/OTelResourceProvider.java

@@ -30,7 +30,7 @@
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 import io.opentelemetry.sdk.trace.samplers.Sampler;
-import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
+import io.opentelemetry.semconv.ResourceAttributes;
 
 import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_BATCH_SIZE_SETTING;
 import static org.opensearch.telemetry.OTelTelemetrySettings.TRACER_EXPORTER_DELAY_SETTING;

server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java

@@ -215,6 +215,7 @@ public final void start() {
                     0,
                     0,
                     buildTookInMillis(),
+                    timeProvider.getPhaseTook(),
                     ShardSearchFailure.EMPTY_ARRAY,
                     clusters,
                     null
@@ -662,6 +663,7 @@ protected final SearchResponse buildSearchResponse(
             successfulOps.get(),
             skippedOps.get(),
             buildTookInMillis(),
+            timeProvider.getPhaseTook(),
             failures,
             clusters,
             searchContextId

server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java

@@ -277,6 +277,8 @@ public static void readMultiLineFormat(
                         } else if ("cancel_after_time_interval".equals(entry.getKey())
                             || "cancelAfterTimeInterval".equals(entry.getKey())) {
                                 searchRequest.setCancelAfterTimeInterval(nodeTimeValue(value, null));
+                            } else if ("phase_took".equals(entry.getKey())) {
+                                searchRequest.setPhaseTook(nodeBooleanValue(value));
                             } else {
                                 throw new IllegalArgumentException("key [" + entry.getKey() + "] is not supported in the metadata section");
                             }
@@ -374,6 +376,9 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild
         if (request.getCancelAfterTimeInterval() != null) {
             xContentBuilder.field("cancel_after_time_interval", request.getCancelAfterTimeInterval().getStringRep());
         }
+        if (request.isPhaseTook() != null) {
+            xContentBuilder.field("phase_took", request.isPhaseTook());
+        }
         xContentBuilder.endObject();
     }
 

server/src/main/java/org/opensearch/action/search/SearchRequest.java

@@ -117,6 +117,8 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla
 
     private String pipeline;
 
+    private Boolean phaseTook = null;
+
     public SearchRequest() {
         this.localClusterAlias = null;
         this.absoluteStartMillis = DEFAULT_ABSOLUTE_START_MILLIS;
@@ -209,6 +211,7 @@ private SearchRequest(
         this.absoluteStartMillis = absoluteStartMillis;
         this.finalReduce = finalReduce;
         this.cancelAfterTimeInterval = searchRequest.cancelAfterTimeInterval;
+        this.phaseTook = searchRequest.phaseTook;
     }
 
     /**
@@ -253,6 +256,9 @@ public SearchRequest(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_2_7_0)) {
             pipeline = in.readOptionalString();
         }
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            phaseTook = in.readOptionalBoolean();
+        }
     }
 
     @Override
@@ -284,6 +290,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_7_0)) {
             out.writeOptionalString(pipeline);
         }
+        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+            out.writeOptionalBoolean(phaseTook);
+        }
     }
 
     @Override
@@ -615,6 +624,20 @@ public void setPreFilterShardSize(int preFilterShardSize) {
         this.preFilterShardSize = preFilterShardSize;
     }
 
+    /**
+     * Returns value of user-provided phase_took query parameter for this search request.
+     */
+    public Boolean isPhaseTook() {
+        return phaseTook;
+    }
+
+    /**
+     * Sets value of phase_took query param if provided by user. Defaults to <code>null</code>.
+     */
+    public void setPhaseTook(Boolean phaseTook) {
+        this.phaseTook = phaseTook;
+    }
+
     /**
      * Returns a threshold that enforces a pre-filter roundtrip to pre-filter search shards based on query rewriting if the number of shards
      * the search request expands to exceeds the threshold, or <code>null</code> if the threshold is unspecified.
@@ -719,7 +742,8 @@ public boolean equals(Object o) {
             && absoluteStartMillis == that.absoluteStartMillis
             && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips
             && Objects.equals(cancelAfterTimeInterval, that.cancelAfterTimeInterval)
-            && Objects.equals(pipeline, that.pipeline);
+            && Objects.equals(pipeline, that.pipeline)
+            && Objects.equals(phaseTook, that.phaseTook);
     }
 
     @Override
@@ -740,7 +764,8 @@ public int hashCode() {
             localClusterAlias,
             absoluteStartMillis,
             ccsMinimizeRoundtrips,
-            cancelAfterTimeInterval
+            cancelAfterTimeInterval,
+            phaseTook
         );
     }
 
@@ -783,6 +808,8 @@ public String toString() {
             + cancelAfterTimeInterval
             + ", pipeline="
             + pipeline
+            + ", phaseTook="
+            + phaseTook
             + "}";
     }
 }