-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Integrate IO Usage Tracker to the Resource Usage Collector Service an…
…d Emit IO Usage Stats Signed-off-by: Ajay Kumar Movva <movvaam@amazon.com>
- Loading branch information
Ajay Kumar Movva
committed
Mar 4, 2024
1 parent
9814eb9
commit 22da423
Showing
16 changed files
with
376 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
server/src/main/java/org/opensearch/node/IoUsageStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.node; | ||
|
|
||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
| import org.opensearch.core.common.io.stream.Writeable; | ||
| import org.opensearch.core.xcontent.ToXContentFragment; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Locale; | ||
|
|
||
| /** | ||
| * This class is to store tne IO Usage Stats and used to return in node stats API. | ||
| */ | ||
| public class IoUsageStats implements Writeable, ToXContentFragment { | ||
|
|
||
| private double ioUtilisationPercent; | ||
|
|
||
| public IoUsageStats(double ioUtilisationPercent) { | ||
| this.ioUtilisationPercent = ioUtilisationPercent; | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param in the stream to read from | ||
| * @throws IOException if an error occurs while reading from the StreamOutput | ||
| */ | ||
| public IoUsageStats(StreamInput in) throws IOException { | ||
| this.ioUtilisationPercent = in.readDouble(); | ||
| } | ||
|
|
||
| /** | ||
| * Write this into the {@linkplain StreamOutput}. | ||
| * | ||
| * @param out the output stream to write entity content to | ||
| */ | ||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeDouble(this.ioUtilisationPercent); | ||
| } | ||
|
|
||
| public double getIoUtilisationPercent() { | ||
| return ioUtilisationPercent; | ||
| } | ||
|
|
||
| public void setIoUtilisationPercent(double ioUtilisationPercent) { | ||
| this.ioUtilisationPercent = ioUtilisationPercent; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| builder.field("io_utilization_percent", String.format(Locale.ROOT, "%.1f", this.ioUtilisationPercent)); | ||
| return builder.endObject(); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return "IO utilization percent: " + String.format(Locale.ROOT, "%.1f", this.ioUtilisationPercent); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
server/src/main/java/org/opensearch/node/resource/tracker/AverageIoUsageTracker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.node.resource.tracker; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.common.unit.TimeValue; | ||
| import org.opensearch.monitor.fs.FsInfo.DeviceStats; | ||
| import org.opensearch.monitor.fs.FsService; | ||
| import org.opensearch.node.IoUsageStats; | ||
| import org.opensearch.threadpool.ThreadPool; | ||
|
|
||
| import java.util.HashMap; | ||
|
|
||
| /** | ||
| * AverageIoUsageTracker tracks the IO usage by polling the FS Stats for IO metrics every (pollingInterval) | ||
| * and keeping track of the rolling average over a defined time window (windowDuration). | ||
| */ | ||
| public class AverageIoUsageTracker extends AbstractAverageUsageTracker { | ||
|
|
||
| private static final Logger LOGGER = LogManager.getLogger(AverageIoUsageTracker.class); | ||
| private final FsService fsService; | ||
| private final HashMap<String, Long> prevIoTimeDeviceMap; | ||
| private long prevTimeInMillis; | ||
| private final IoUsageStats ioUsageStats; | ||
|
|
||
| public AverageIoUsageTracker(FsService fsService, ThreadPool threadPool, TimeValue pollingInterval, TimeValue windowDuration) { | ||
| super(threadPool, pollingInterval, windowDuration); | ||
| this.fsService = fsService; | ||
| this.prevIoTimeDeviceMap = new HashMap<>(); | ||
| this.prevTimeInMillis = -1; | ||
| this.ioUsageStats = new IoUsageStats(-1); | ||
| } | ||
|
|
||
| /** | ||
| * Get current IO usage percentage calculated using fs stats | ||
| */ | ||
| @Override | ||
| public long getUsage() { | ||
| long usage = 0; | ||
| if (this.preValidateFsStats()) { | ||
| return usage; | ||
| } | ||
| // Currently even during the raid setup we have only one mount device and it is giving 0 io time from /proc/diskstats | ||
| DeviceStats[] devicesStats = fsService.stats().getIoStats().getDevicesStats(); | ||
| long latestTimeInMillis = fsService.stats().getTimestamp(); | ||
| for (DeviceStats devicesStat : devicesStats) { | ||
| long devicePreviousIoTime = prevIoTimeDeviceMap.getOrDefault(devicesStat.getDeviceName(), (long) -1); | ||
| long deviceCurrentIoTime = devicesStat.ioTimeInMillis(); | ||
| if (prevTimeInMillis > 0 && (latestTimeInMillis - this.prevTimeInMillis > 0) && devicePreviousIoTime > 0) { | ||
| long absIoTime = (deviceCurrentIoTime - devicePreviousIoTime); | ||
| long deviceCurrentIoUsage = absIoTime * 100 / (latestTimeInMillis - this.prevTimeInMillis); | ||
| // We are returning the maximum IO Usage for all the attached devices | ||
| usage = Math.max(usage, deviceCurrentIoUsage); | ||
| } | ||
| prevIoTimeDeviceMap.put(devicesStat.getDeviceName(), devicesStat.ioTimeInMillis()); | ||
| } | ||
| this.prevTimeInMillis = latestTimeInMillis; | ||
| return usage; | ||
| } | ||
|
|
||
| @Override | ||
| protected void doStart() { | ||
| scheduledFuture = threadPool.scheduleWithFixedDelay(() -> { | ||
| long usage = getUsage(); | ||
| recordUsage(usage); | ||
| updateIoUsageStats(); | ||
| }, pollingInterval, ThreadPool.Names.GENERIC); | ||
| } | ||
|
|
||
| private boolean preValidateFsStats() { | ||
| return fsService == null | ||
| || fsService.stats() == null | ||
| || fsService.stats().getIoStats() == null | ||
| || fsService.stats().getIoStats().getDevicesStats() == null; | ||
| } | ||
|
|
||
| private void updateIoUsageStats() { | ||
| this.ioUsageStats.setIoUtilisationPercent(this.isReady() ? this.getAverage() : -1); | ||
| } | ||
|
|
||
| public IoUsageStats getIoUsageStats() { | ||
| return this.ioUsageStats; | ||
| } | ||
| } |
Oops, something went wrong.