Skip to content

Commit

Permalink
Support for Heap after GC stats (opensearch-project#1265)
Browse files Browse the repository at this point in the history 
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
  • Loading branch information
@reta
reta committed Sep 28, 2021
1 parent a80979c commit fbe0db6
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 6 deletions.
86 changes: 82 additions & 4 deletions server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

package org.opensearch.monitor.jvm;

import org.opensearch.Version;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
Expand Down Expand Up @@ -85,6 +86,11 @@ public static JvmStats jvmStats() {
try {
MemoryUsage usage = memoryPoolMXBean.getUsage();
MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage();
MemoryUsage collectionUsage = memoryPoolMXBean.getCollectionUsage();
// For some pools, the collection usage may not be supported and the method returns null.
if (collectionUsage == null) {
collectionUsage = new MemoryUsage(0, 0, 0, 0);
}
String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), null);
if (name == null) { // if we can't resolve it, its not interesting.... (Per Gen, Code Cache)
continue;
Expand All @@ -93,8 +99,13 @@ public static JvmStats jvmStats() {
usage.getUsed() < 0 ? 0 : usage.getUsed(),
usage.getMax() < 0 ? 0 : usage.getMax(),
peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(),
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax()
));
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax(),
new MemoryPoolGcStats(
collectionUsage.getUsed() < 0 ? 0 : collectionUsage.getUsed(),
collectionUsage.getMax() < 0 ? 0 : collectionUsage.getMax()
)
));

} catch (final Exception ignored) {

}
Expand Down Expand Up @@ -223,6 +234,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.humanReadableField(Fields.PEAK_USED_IN_BYTES, Fields.PEAK_USED, new ByteSizeValue(pool.peakUsed));
builder.humanReadableField(Fields.PEAK_MAX_IN_BYTES, Fields.PEAK_MAX, new ByteSizeValue(pool.peakMax));

builder.startObject(Fields.LAST_GC_STATS);
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, new ByteSizeValue(pool.getLastGcStats().used));
builder.humanReadableField(Fields.MAX_IN_BYTES, Fields.MAX, new ByteSizeValue(pool.getLastGcStats().max));
builder.field(Fields.USAGE_PERCENT, pool.getLastGcStats().getUsagePercent());
builder.endObject();

builder.endObject();
}
builder.endObject();
Expand Down Expand Up @@ -299,7 +316,9 @@ static final class Fields {
static final String PEAK_USED_IN_BYTES = "peak_used_in_bytes";
static final String PEAK_MAX = "peak_max";
static final String PEAK_MAX_IN_BYTES = "peak_max_in_bytes";

static final String USAGE_PERCENT = "usage_percent";
static final String LAST_GC_STATS = "last_gc_stats";

static final String THREADS = "threads";
static final String COUNT = "count";
static final String PEAK_COUNT = "peak_count";
Expand Down Expand Up @@ -414,6 +433,48 @@ public int getPeakCount() {
return peakCount;
}
}

/**
* Stores the memory usage after the Java virtual machine
* most recently expended effort in recycling unused objects
* in particular memory pool.
*/
public static class MemoryPoolGcStats implements Writeable {

private final long used;
private final long max;

public MemoryPoolGcStats(long used, long max) {
this.used = used;
this.max = max;
}

public MemoryPoolGcStats(StreamInput in) throws IOException {
used = in.readVLong();
max = in.readVLong();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(used);
out.writeVLong(max);
}

public ByteSizeValue getUsed() {
return new ByteSizeValue(used);
}

public ByteSizeValue getMax() {
return new ByteSizeValue(max);
}

public short getUsagePercent() {
if (max == 0) {
return -1;
}
return (short) (used * 100 / max);
}
}

public static class MemoryPool implements Writeable {

Expand All @@ -422,13 +483,15 @@ public static class MemoryPool implements Writeable {
private final long max;
private final long peakUsed;
private final long peakMax;
private final MemoryPoolGcStats lastGcStats;

public MemoryPool(String name, long used, long max, long peakUsed, long peakMax) {
public MemoryPool(String name, long used, long max, long peakUsed, long peakMax, MemoryPoolGcStats lastGcStats) {
this.name = name;
this.used = used;
this.max = max;
this.peakUsed = peakUsed;
this.peakMax = peakMax;
this.lastGcStats = lastGcStats;
}

public MemoryPool(StreamInput in) throws IOException {
Expand All @@ -437,6 +500,11 @@ public MemoryPool(StreamInput in) throws IOException {
max = in.readVLong();
peakUsed = in.readVLong();
peakMax = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_1_2_0)) {
lastGcStats = new MemoryPoolGcStats(in);
} else {
lastGcStats = new MemoryPoolGcStats(0, 0);
}
}

@Override
Expand All @@ -446,6 +514,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(max);
out.writeVLong(peakUsed);
out.writeVLong(peakMax);
if (out.getVersion().onOrAfter(Version.V_1_2_0)) {
lastGcStats.writeTo(out);
}
}

public String getName() {
Expand All @@ -467,6 +538,13 @@ public ByteSizeValue getPeakUsed() {
public ByteSizeValue getPeakMax() {
return new ByteSizeValue(peakMax);
}

/**
* Returns the heap usage after last garbage collection cycle
*/
public MemoryPoolGcStats getLastGcStats() {
return lastGcStats;
}
}

public static class Mem implements Writeable, Iterable<MemoryPool> {
Expand Down
38 changes: 36 additions & 2 deletions server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
import org.opensearch.test.VersionUtils;
import org.opensearch.threadpool.ThreadPoolStats;
import org.opensearch.transport.TransportStats;
import org.opensearch.action.admin.cluster.node.stats.NodeStats;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -63,9 +62,15 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.collection.IsEmptyCollection.empty;

public class NodeStatsTests extends OpenSearchTestCase {
public void testSerialization() throws IOException {
Expand Down Expand Up @@ -149,6 +154,34 @@ public void testSerialization() throws IOException {
assertEquals(mem.getNonHeapCommitted(), deserializedMem.getNonHeapCommitted());
assertEquals(mem.getNonHeapUsed(), deserializedMem.getNonHeapUsed());
assertEquals(mem.getHeapMax(), deserializedMem.getHeapMax());

final Map<String, JvmStats.MemoryPool> pools = StreamSupport
.stream(mem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));

final Map<String, JvmStats.MemoryPool> deserializedPools = StreamSupport
.stream(deserializedMem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));

assertThat(pools.keySet(), not(empty()));
assertThat(deserializedPools.keySet(), not(empty()));

for (final Map.Entry<String, JvmStats.MemoryPool> entry: pools.entrySet()) {
assertThat(deserializedPools.containsKey(entry.getKey()), is(true));
assertEquals(entry.getValue().getName(), deserializedPools.get(entry.getKey()).getName());
assertEquals(entry.getValue().getMax(), deserializedPools.get(entry.getKey()).getMax());
assertEquals(entry.getValue().getPeakMax(), deserializedPools.get(entry.getKey()).getPeakMax());
assertEquals(entry.getValue().getPeakUsed(), deserializedPools.get(entry.getKey()).getPeakUsed());
assertEquals(entry.getValue().getUsed(), deserializedPools.get(entry.getKey()).getUsed());

assertEquals(entry.getValue().getLastGcStats().getUsed(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsed());
assertEquals(entry.getValue().getLastGcStats().getMax(),
deserializedPools.get(entry.getKey()).getLastGcStats().getMax());
assertEquals(entry.getValue().getLastGcStats().getUsagePercent(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsagePercent());
}

JvmStats.Classes classes = jvm.getClasses();
assertEquals(classes.getLoadedClassCount(), deserializedJvm.getClasses().getLoadedClassCount());
assertEquals(classes.getTotalLoadedClassCount(), deserializedJvm.getClasses().getTotalLoadedClassCount());
Expand Down Expand Up @@ -397,7 +430,8 @@ public static NodeStats createNodeStats() {
List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
for (int i = 0; i < numMemoryPools; i++) {
memoryPools.add(new JvmStats.MemoryPool(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
new JvmStats.MemoryPoolGcStats(randomNonNegativeLong(), randomNonNegativeLong())));
}
JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
int numGarbageCollectors = randomIntBetween(0, 10);
Expand Down

0 comments on commit fbe0db6

Please sign in to comment.