Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for Heap after GC stats #1265

Merged
merged 3 commits into from
Sep 28, 2021
Merged

Support for Heap after GC stats #1265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19d145a
Support for Heap after GC stats
reta Sep 21, 2021
e7155ca
Addressing code review comments
reta Sep 22, 2021
8dfe9a1
Using the right version 2.0.0 (instead of 1.2.0) for the change
reta Sep 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 82 additions & 4 deletions server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

package org.opensearch.monitor.jvm;

import org.opensearch.Version;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
Expand Down Expand Up @@ -85,6 +86,11 @@ public static JvmStats jvmStats() {
try {
MemoryUsage usage = memoryPoolMXBean.getUsage();
MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage();
MemoryUsage collectionUsage = memoryPoolMXBean.getCollectionUsage();
// For some pools, the collection usage may not be supported and the method returns null.
if (collectionUsage == null) {
collectionUsage = new MemoryUsage(0, 0, 0, 0);
}
String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), null);
if (name == null) { // if we can't resolve it, its not interesting.... (Per Gen, Code Cache)
continue;
Expand All @@ -93,8 +99,13 @@ public static JvmStats jvmStats() {
usage.getUsed() < 0 ? 0 : usage.getUsed(),
usage.getMax() < 0 ? 0 : usage.getMax(),
peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(),
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax()
));
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax(),
new MemoryPoolGcStats(
collectionUsage.getUsed() < 0 ? 0 : collectionUsage.getUsed(),
collectionUsage.getMax() < 0 ? 0 : collectionUsage.getMax()
)
));

} catch (final Exception ignored) {

}
Expand Down Expand Up @@ -223,6 +234,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.humanReadableField(Fields.PEAK_USED_IN_BYTES, Fields.PEAK_USED, new ByteSizeValue(pool.peakUsed));
builder.humanReadableField(Fields.PEAK_MAX_IN_BYTES, Fields.PEAK_MAX, new ByteSizeValue(pool.peakMax));

builder.startObject(Fields.LAST_GC_STATS);
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, new ByteSizeValue(pool.getLastGcStats().used));
builder.humanReadableField(Fields.MAX_IN_BYTES, Fields.MAX, new ByteSizeValue(pool.getLastGcStats().max));
builder.field(Fields.USAGE_PERCENT, pool.getLastGcStats().getUsagePercent());
builder.endObject();

builder.endObject();
}
builder.endObject();
Expand Down Expand Up @@ -299,7 +316,9 @@ static final class Fields {
static final String PEAK_USED_IN_BYTES = "peak_used_in_bytes";
static final String PEAK_MAX = "peak_max";
static final String PEAK_MAX_IN_BYTES = "peak_max_in_bytes";

static final String USAGE_PERCENT = "usage_percent";
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this be confused with current usage vs usage after GC?

Copy link
Collaborator Author

@reta reta Sep 22, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question, I think it should not since the field is under last_gc_stats object, in this context, it should not be confusing. Also, the semantic of this field stays consistent between current usage (if we add it there) and usage after GC. What do you think?

static final String LAST_GC_STATS = "last_gc_stats";

static final String THREADS = "threads";
static final String COUNT = "count";
static final String PEAK_COUNT = "peak_count";
Expand Down Expand Up @@ -414,6 +433,48 @@ public int getPeakCount() {
return peakCount;
}
}

/**
* Stores the memory usage after the Java virtual machine
* most recently expended effort in recycling unused objects
* in particular memory pool.
*/
public static class MemoryPoolGcStats implements Writeable {

private final long used;
private final long max;

public MemoryPoolGcStats(long used, long max) {
this.used = used;
this.max = max;
}

public MemoryPoolGcStats(StreamInput in) throws IOException {
used = in.readVLong();
max = in.readVLong();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(used);
out.writeVLong(max);
}

public ByteSizeValue getUsed() {
return new ByteSizeValue(used);
}

public ByteSizeValue getMax() {
return new ByteSizeValue(max);
}

public short getUsagePercent() {
if (max == 0) {
return -1;
}
return (short) (used * 100 / max);
}
}

public static class MemoryPool implements Writeable {

Expand All @@ -422,13 +483,15 @@ public static class MemoryPool implements Writeable {
private final long max;
private final long peakUsed;
private final long peakMax;
private final MemoryPoolGcStats lastGcStats;

public MemoryPool(String name, long used, long max, long peakUsed, long peakMax) {
public MemoryPool(String name, long used, long max, long peakUsed, long peakMax, MemoryPoolGcStats lastGcStats) {
this.name = name;
this.used = used;
this.max = max;
this.peakUsed = peakUsed;
this.peakMax = peakMax;
this.lastGcStats = lastGcStats;
}

public MemoryPool(StreamInput in) throws IOException {
Expand All @@ -437,6 +500,11 @@ public MemoryPool(StreamInput in) throws IOException {
max = in.readVLong();
peakUsed = in.readVLong();
peakMax = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_2_0_0)) {
lastGcStats = new MemoryPoolGcStats(in);
} else {
lastGcStats = new MemoryPoolGcStats(0, 0);
}
}

@Override
Expand All @@ -446,6 +514,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(max);
out.writeVLong(peakUsed);
out.writeVLong(peakMax);
if (out.getVersion().onOrAfter(Version.V_2_0_0)) {
lastGcStats.writeTo(out);
}
}

public String getName() {
Expand All @@ -467,6 +538,13 @@ public ByteSizeValue getPeakUsed() {
public ByteSizeValue getPeakMax() {
return new ByteSizeValue(peakMax);
}

/**
* Returns the heap usage after last garbage collection cycle
*/
public MemoryPoolGcStats getLastGcStats() {
return lastGcStats;
}
}

public static class Mem implements Writeable, Iterable<MemoryPool> {
Expand Down
38 changes: 36 additions & 2 deletions server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
import org.opensearch.test.VersionUtils;
import org.opensearch.threadpool.ThreadPoolStats;
import org.opensearch.transport.TransportStats;
import org.opensearch.action.admin.cluster.node.stats.NodeStats;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -63,9 +62,15 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.collection.IsEmptyCollection.empty;

public class NodeStatsTests extends OpenSearchTestCase {
public void testSerialization() throws IOException {
Expand Down Expand Up @@ -149,6 +154,34 @@ public void testSerialization() throws IOException {
assertEquals(mem.getNonHeapCommitted(), deserializedMem.getNonHeapCommitted());
assertEquals(mem.getNonHeapUsed(), deserializedMem.getNonHeapUsed());
assertEquals(mem.getHeapMax(), deserializedMem.getHeapMax());

final Map<String, JvmStats.MemoryPool> pools = StreamSupport
.stream(mem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));

final Map<String, JvmStats.MemoryPool> deserializedPools = StreamSupport
.stream(deserializedMem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));

assertThat(pools.keySet(), not(empty()));
assertThat(deserializedPools.keySet(), not(empty()));

for (final Map.Entry<String, JvmStats.MemoryPool> entry: pools.entrySet()) {
assertThat(deserializedPools.containsKey(entry.getKey()), is(true));
assertEquals(entry.getValue().getName(), deserializedPools.get(entry.getKey()).getName());
assertEquals(entry.getValue().getMax(), deserializedPools.get(entry.getKey()).getMax());
assertEquals(entry.getValue().getPeakMax(), deserializedPools.get(entry.getKey()).getPeakMax());
assertEquals(entry.getValue().getPeakUsed(), deserializedPools.get(entry.getKey()).getPeakUsed());
assertEquals(entry.getValue().getUsed(), deserializedPools.get(entry.getKey()).getUsed());

assertEquals(entry.getValue().getLastGcStats().getUsed(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsed());
assertEquals(entry.getValue().getLastGcStats().getMax(),
deserializedPools.get(entry.getKey()).getLastGcStats().getMax());
assertEquals(entry.getValue().getLastGcStats().getUsagePercent(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsagePercent());
}

JvmStats.Classes classes = jvm.getClasses();
assertEquals(classes.getLoadedClassCount(), deserializedJvm.getClasses().getLoadedClassCount());
assertEquals(classes.getTotalLoadedClassCount(), deserializedJvm.getClasses().getTotalLoadedClassCount());
Expand Down Expand Up @@ -397,7 +430,8 @@ public static NodeStats createNodeStats() {
List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
for (int i = 0; i < numMemoryPools; i++) {
memoryPools.add(new JvmStats.MemoryPool(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
new JvmStats.MemoryPoolGcStats(randomNonNegativeLong(), randomNonNegativeLong())));
}
JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
int numGarbageCollectors = randomIntBetween(0, 10);
Expand Down