Add cluster manager task throttling stats in nodes stats API (opensea…

…rch-project#5790) * Add cluster manager task throttling stats in nodes stats API Signed-off-by: Dhwanil Patel <dhwanip@amazon.com>
SonalMahambrey1 · Jan 17, 2023 · 6da16e1 · 6da16e1
1 parent baa6e6e
commit 6da16e1
Show file tree

Hide file tree

Showing 13 changed files with 177 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -107,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Added cluster manager throttling stats in nodes/stats API ([#5790](https://github.com/opensearch-project/OpenSearch/pull/5790))
 
 ### Dependencies
 

diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.support.nodes.BaseNodeResponse;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.service.ClusterManagerThrottlingStats;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
@@ -122,6 +123,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private SearchBackpressureStats searchBackpressureStats;
 
+    @Nullable
+    private ClusterManagerThrottlingStats clusterManagerThrottlingStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -152,6 +156,12 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             searchBackpressureStats = null;
         }
+
+        if (in.getVersion().onOrAfter(Version.V_2_6_0)) {
+            clusterManagerThrottlingStats = in.readOptionalWriteable(ClusterManagerThrottlingStats::new);
+        } else {
+            clusterManagerThrottlingStats = null;
+        }
     }
 
     public NodeStats(
@@ -173,7 +183,8 @@ public NodeStats(
         @Nullable ScriptCacheStats scriptCacheStats,
         @Nullable IndexingPressureStats indexingPressureStats,
         @Nullable ShardIndexingPressureStats shardIndexingPressureStats,
-        @Nullable SearchBackpressureStats searchBackpressureStats
+        @Nullable SearchBackpressureStats searchBackpressureStats,
+        @Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats
     ) {
         super(node);
         this.timestamp = timestamp;
@@ -194,6 +205,7 @@ public NodeStats(
         this.indexingPressureStats = indexingPressureStats;
         this.shardIndexingPressureStats = shardIndexingPressureStats;
         this.searchBackpressureStats = searchBackpressureStats;
+        this.clusterManagerThrottlingStats = clusterManagerThrottlingStats;
     }
 
     public long getTimestamp() {
@@ -308,6 +320,11 @@ public SearchBackpressureStats getSearchBackpressureStats() {
         return searchBackpressureStats;
     }
 
+    @Nullable
+    public ClusterManagerThrottlingStats getClusterManagerThrottlingStats() {
+        return clusterManagerThrottlingStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -336,6 +353,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_4_0)) {
             out.writeOptionalWriteable(searchBackpressureStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_2_6_0)) {
+            out.writeOptionalWriteable(clusterManagerThrottlingStats);
+        }
     }
 
     @Override
@@ -411,6 +431,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getSearchBackpressureStats() != null) {
             getSearchBackpressureStats().toXContent(builder, params);
         }
+        if (getClusterManagerThrottlingStats() != null) {
+            getClusterManagerThrottlingStats().toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -207,7 +207,8 @@ public enum Metric {
         SCRIPT_CACHE("script_cache"),
         INDEXING_PRESSURE("indexing_pressure"),
         SHARD_INDEXING_PRESSURE("shard_indexing_pressure"),
-        SEARCH_BACKPRESSURE("search_backpressure");
+        SEARCH_BACKPRESSURE("search_backpressure"),
+        CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling");
 
         private String metricName;
 

diff --git a/...c/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/...c/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -119,7 +119,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.SCRIPT_CACHE.containedIn(metrics),
             NodesStatsRequest.Metric.INDEXING_PRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.SHARD_INDEXING_PRESSURE.containedIn(metrics),
-            NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics)
+            NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics),
+            NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics)
         );
     }
 

diff --git a/.../src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/.../src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -163,6 +163,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List<ShardStats> shardsStats = new ArrayList<>();

diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingStats.java
@@ -8,18 +8,29 @@
 
 package org.opensearch.cluster.service;
 
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.common.io.stream.Writeable;
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.xcontent.ToXContent;
+import org.opensearch.common.xcontent.ToXContentFragment;
+import org.opensearch.common.xcontent.XContentBuilder;
 
+import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Contains stats of Cluster Manager Task Throttling.
  * It stores the total cumulative count of throttled tasks per task type.
  */
-public class ClusterManagerThrottlingStats implements ClusterManagerTaskThrottlerListener {
+public class ClusterManagerThrottlingStats implements ClusterManagerTaskThrottlerListener, Writeable, ToXContentFragment {
 
-    private Map<String, CounterMetric> throttledTasksCount = new ConcurrentHashMap<>();
+    private Map<String, CounterMetric> throttledTasksCount;
+
+    public ClusterManagerThrottlingStats() {
+        throttledTasksCount = new ConcurrentHashMap<>();
+    }
 
     private void incrementThrottlingCount(String type, final int counts) {
         throttledTasksCount.computeIfAbsent(type, k -> new CounterMetric()).inc(counts);
@@ -39,4 +50,63 @@ public long getTotalThrottledTaskCount() {
     public void onThrottle(String type, int counts) {
         incrementThrottlingCount(type, counts);
     }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(throttledTasksCount.size());
+        for (Map.Entry<String, CounterMetric> entry : throttledTasksCount.entrySet()) {
+            out.writeString(entry.getKey());
+            out.writeVInt((int) entry.getValue().count());
+        }
+    }
+
+    public ClusterManagerThrottlingStats(StreamInput in) throws IOException {
+        int throttledTaskEntries = in.readVInt();
+        throttledTasksCount = new ConcurrentHashMap<>();
+        for (int i = 0; i < throttledTaskEntries; i++) {
+            String taskType = in.readString();
+            int throttledTaskCount = in.readVInt();
+            onThrottle(taskType, throttledTaskCount);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject("cluster_manager_throttling");
+        builder.startObject("stats");
+        builder.field("total_throttled_tasks", getTotalThrottledTaskCount());
+        builder.startObject("throttled_tasks_per_task_type");
+        for (Map.Entry<String, CounterMetric> entry : throttledTasksCount.entrySet()) {
+            builder.field(entry.getKey(), entry.getValue().count());
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder.endObject();
+    }
+
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        } else if (o != null && this.getClass() == o.getClass()) {
+            ClusterManagerThrottlingStats that = (ClusterManagerThrottlingStats) o;
+
+            if (this.throttledTasksCount.size() == that.throttledTasksCount.size()) {
+                for (Map.Entry<String, CounterMetric> entry : this.throttledTasksCount.entrySet()) {
+                    if (that.throttledTasksCount.get(entry.getKey()).count() != entry.getValue().count()) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public int hashCode() {
+        Map<String, Long> countMap = new ConcurrentHashMap<>();
+        for (Map.Entry<String, CounterMetric> entry : this.throttledTasksCount.entrySet()) {
+            countMap.put(entry.getKey(), entry.getValue().count());
+        }
+        return countMap.hashCode();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
@@ -606,6 +606,13 @@ public long numberOfThrottledPendingTasks() {
         return throttlingStats.getTotalThrottledTaskCount();
     }
 
+    /**
+     * Returns the stats of throttled pending tasks.
+     */
+    public ClusterManagerThrottlingStats getThrottlingStats() {
+        return throttlingStats;
+    }
+
     /**
      * Returns the min version of nodes in cluster
      */

diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -83,6 +83,7 @@ public class NodeService implements Closeable {
     private final IndexingPressureService indexingPressureService;
     private final AggregationUsageService aggregationUsageService;
     private final SearchBackpressureService searchBackpressureService;
+    private final ClusterService clusterService;
 
     private final Discovery discovery;
 
@@ -123,6 +124,7 @@ public class NodeService implements Closeable {
         this.indexingPressureService = indexingPressureService;
         this.aggregationUsageService = aggregationUsageService;
         this.searchBackpressureService = searchBackpressureService;
+        this.clusterService = clusterService;
         clusterService.addStateApplier(ingestService);
     }
 
@@ -174,7 +176,8 @@ public NodeStats stats(
         boolean scriptCache,
         boolean indexingPressure,
         boolean shardIndexingPressure,
-        boolean searchBackpressure
+        boolean searchBackpressure,
+        boolean clusterManagerThrottling
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -197,7 +200,8 @@ public NodeStats stats(
             scriptCache ? scriptService.cacheStats() : null,
             indexingPressure ? this.indexingPressureService.nodeStats() : null,
             shardIndexingPressure ? this.indexingPressureService.shardStats(indices) : null,
-            searchBackpressure ? this.searchBackpressureService.nodeStats() : null
+            searchBackpressure ? this.searchBackpressureService.nodeStats() : null,
+            clusterManagerThrottling ? this.clusterService.getClusterManagerService().getThrottlingStats() : null
         );
     }
 

diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.admin.cluster.node.stats;
 
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.service.ClusterManagerThrottlingStats;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.discovery.DiscoveryStats;
@@ -418,6 +419,21 @@ public void testSerialization() throws IOException {
                     assertEquals(limited, sum.getCompilationLimitTriggered());
                     assertEquals(compilations, sum.getCompilations());
                 }
+                ClusterManagerThrottlingStats clusterManagerThrottlingStats = nodeStats.getClusterManagerThrottlingStats();
+                ClusterManagerThrottlingStats deserializedClusterManagerThrottlingStats = deserializedNodeStats
+                    .getClusterManagerThrottlingStats();
+                if (clusterManagerThrottlingStats == null) {
+                    assertNull(deserializedClusterManagerThrottlingStats);
+                } else {
+                    assertEquals(
+                        clusterManagerThrottlingStats.getTotalThrottledTaskCount(),
+                        deserializedClusterManagerThrottlingStats.getTotalThrottledTaskCount()
+                    );
+                    assertEquals(
+                        clusterManagerThrottlingStats.getThrottlingCount("test-task"),
+                        deserializedClusterManagerThrottlingStats.getThrottlingCount("test-task")
+                    );
+                }
             }
         }
     }
@@ -689,6 +705,11 @@ public static NodeStats createNodeStats() {
             }
             adaptiveSelectionStats = new AdaptiveSelectionStats(nodeConnections, nodeStats);
         }
+        ClusterManagerThrottlingStats clusterManagerThrottlingStats = null;
+        if (frequently()) {
+            clusterManagerThrottlingStats = new ClusterManagerThrottlingStats();
+            clusterManagerThrottlingStats.onThrottle("test-task", randomInt());
+        }
         ScriptCacheStats scriptCacheStats = scriptStats != null ? scriptStats.toScriptCacheStats() : null;
         // TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet
         return new NodeStats(
@@ -710,7 +731,8 @@ public static NodeStats createNodeStats() {
             scriptCacheStats,
             null,
             null,
-            null
+            null,
+            clusterManagerThrottlingStats
         );
     }
 

diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -186,6 +186,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -207,6 +208,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -228,6 +230,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -280,6 +283,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -301,6 +305,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -322,6 +327,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );

diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterManagerThrottlingStatsTests.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.service;
+
+import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.test.AbstractWireSerializingTestCase;
+
+public class ClusterManagerThrottlingStatsTests extends AbstractWireSerializingTestCase<ClusterManagerThrottlingStats> {
+    @Override
+    protected Writeable.Reader<ClusterManagerThrottlingStats> instanceReader() {
+        return ClusterManagerThrottlingStats::new;
+    }
+
+    @Override
+    protected ClusterManagerThrottlingStats createTestInstance() {
+        return randomInstance();
+    }
+
+    public static ClusterManagerThrottlingStats randomInstance() {
+        ClusterManagerThrottlingStats randomStats = new ClusterManagerThrottlingStats();
+        randomStats.onThrottle(randomAlphaOfLengthBetween(3, 10), randomInt());
+        return randomStats;
+    }
+}
-Original file line number
+Diff line change
@@ Expand Up @@
                 false,
                 false,
                 false,
+                false,
                 false
             );
             List<ShardStats> shardsStats = new ArrayList<>();
@@ Expand Down @@