Skip to content

Commit 7f06e56

Browse files
opensearch-trigger-bot[bot]github-actions[bot]
opensearch-trigger-bot[bot]
and
github-actions[bot]
authored
Add remote refresh lag in millis from local refresh (#7694) (#7752)
(cherry picked from commit ece27af) Signed-off-by: bansvaru <bansvaru@amazon.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent 7438886 commit 7f06e56

File tree

5 files changed

+17
-53
lines changed

5 files changed

+17
-53
lines changed

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
4343
builder.startObject()
4444
.field(Fields.SHARD_ID, remoteSegmentUploadShardStats.shardId)
4545

46-
.field(Fields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentUploadShardStats.localRefreshTimeMs)
47-
.field(Fields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentUploadShardStats.remoteRefreshTimeMs)
46+
.field(Fields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentUploadShardStats.refreshTimeLagMs)
4847
.field(Fields.REFRESH_LAG, remoteSegmentUploadShardStats.localRefreshNumber - remoteSegmentUploadShardStats.remoteRefreshNumber)
4948
.field(Fields.BYTES_LAG, remoteSegmentUploadShardStats.bytesLag)
5049

@@ -90,16 +89,6 @@ public void writeTo(StreamOutput out) throws IOException {
9089
static final class Fields {
9190
static final String SHARD_ID = "shard_id";
9291

93-
/**
94-
* Last successful local refresh timestamp in milliseconds
95-
*/
96-
static final String LOCAL_REFRESH_TIMESTAMP = "local_refresh_timestamp_in_millis";
97-
98-
/**
99-
* Last successful remote refresh timestamp in milliseconds
100-
*/
101-
static final String REMOTE_REFRESH_TIMESTAMP = "remote_refresh_timestamp_in_millis";
102-
10392
/**
10493
* Lag in terms of bytes b/w local and remote store
10594
*/
@@ -110,6 +99,11 @@ static final class Fields {
11099
*/
111100
static final String REFRESH_LAG = "refresh_lag";
112101

102+
/**
103+
* Time in millis remote refresh is behind local refresh
104+
*/
105+
static final String REFRESH_TIME_LAG_IN_MILLIS = "refresh_time_lag_in_millis";
106+
113107
/**
114108
* Total write rejections due to remote store backpressure kick in
115109
*/

‎server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -445,10 +445,9 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) {
445445
public RemoteRefreshSegmentTracker.Stats stats() {
446446
return new RemoteRefreshSegmentTracker.Stats(
447447
shardId,
448+
timeMsLag,
448449
localRefreshSeqNo,
449-
localRefreshTimeMs,
450450
remoteRefreshSeqNo,
451-
remoteRefreshTimeMs,
452451
uploadBytesStarted,
453452
uploadBytesSucceeded,
454453
uploadBytesFailed,
@@ -473,10 +472,9 @@ public RemoteRefreshSegmentTracker.Stats stats() {
473472
public static class Stats implements Writeable {
474473

475474
public final ShardId shardId;
475+
public final long refreshTimeLagMs;
476476
public final long localRefreshNumber;
477-
public final long localRefreshTimeMs;
478477
public final long remoteRefreshNumber;
479-
public final long remoteRefreshTimeMs;
480478
public final long uploadBytesStarted;
481479
public final long uploadBytesFailed;
482480
public final long uploadBytesSucceeded;
@@ -493,10 +491,9 @@ public static class Stats implements Writeable {
493491

494492
public Stats(
495493
ShardId shardId,
494+
long refreshTimeLagMs,
496495
long localRefreshNumber,
497-
long localRefreshTimeMs,
498496
long remoteRefreshNumber,
499-
long remoteRefreshTimeMs,
500497
long uploadBytesStarted,
501498
long uploadBytesSucceeded,
502499
long uploadBytesFailed,
@@ -512,10 +509,9 @@ public Stats(
512509
long bytesLag
513510
) {
514511
this.shardId = shardId;
512+
this.refreshTimeLagMs = refreshTimeLagMs;
515513
this.localRefreshNumber = localRefreshNumber;
516-
this.localRefreshTimeMs = localRefreshTimeMs;
517514
this.remoteRefreshNumber = remoteRefreshNumber;
518-
this.remoteRefreshTimeMs = remoteRefreshTimeMs;
519515
this.uploadBytesStarted = uploadBytesStarted;
520516
this.uploadBytesFailed = uploadBytesFailed;
521517
this.uploadBytesSucceeded = uploadBytesSucceeded;
@@ -534,10 +530,9 @@ public Stats(
534530
public Stats(StreamInput in) throws IOException {
535531
try {
536532
this.shardId = new ShardId(in);
533+
this.refreshTimeLagMs = in.readLong();
537534
this.localRefreshNumber = in.readLong();
538-
this.localRefreshTimeMs = in.readLong();
539535
this.remoteRefreshNumber = in.readLong();
540-
this.remoteRefreshTimeMs = in.readLong();
541536
this.uploadBytesStarted = in.readLong();
542537
this.uploadBytesFailed = in.readLong();
543538
this.uploadBytesSucceeded = in.readLong();
@@ -559,10 +554,9 @@ public Stats(StreamInput in) throws IOException {
559554
@Override
560555
public void writeTo(StreamOutput out) throws IOException {
561556
shardId.writeTo(out);
557+
out.writeLong(refreshTimeLagMs);
562558
out.writeLong(localRefreshNumber);
563-
out.writeLong(localRefreshTimeMs);
564559
out.writeLong(remoteRefreshNumber);
565-
out.writeLong(remoteRefreshTimeMs);
566560
out.writeLong(uploadBytesStarted);
567561
out.writeLong(uploadBytesFailed);
568562
out.writeLong(uploadBytesSucceeded);

‎server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java‎

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,43 +14,22 @@
1414
import java.util.Map;
1515

1616
import static org.opensearch.test.OpenSearchTestCase.assertEquals;
17-
import static org.opensearch.test.OpenSearchTestCase.randomIntBetween;
1817

1918
/**
2019
* Helper utilities for Remote Store stats tests
2120
*/
2221
public class RemoteStoreStatsTestHelper {
2322
static RemoteRefreshSegmentTracker.Stats createPressureTrackerStats(ShardId shardId) {
24-
return new RemoteRefreshSegmentTracker.Stats(
25-
shardId,
26-
3,
27-
System.nanoTime() / 1_000_000L + randomIntBetween(10, 100),
28-
2,
29-
System.nanoTime() / 1_000_000L + randomIntBetween(10, 100),
30-
10,
31-
5,
32-
5,
33-
10,
34-
5,
35-
5,
36-
3,
37-
2,
38-
5,
39-
2,
40-
3,
41-
4,
42-
9
43-
);
23+
return new RemoteRefreshSegmentTracker.Stats(shardId, 100, 3, 2, 10, 5, 5, 10, 5, 5, 3, 2, 5, 2, 3, 4, 9);
4424
}
4525

4626
static void compareStatsResponse(Map<String, Object> statsObject, RemoteRefreshSegmentTracker.Stats pressureTrackerStats) {
4727
assertEquals(statsObject.get(RemoteStoreStats.Fields.SHARD_ID), pressureTrackerStats.shardId.toString());
48-
assertEquals(statsObject.get(RemoteStoreStats.Fields.LOCAL_REFRESH_TIMESTAMP), (int) pressureTrackerStats.localRefreshTimeMs);
28+
assertEquals(statsObject.get(RemoteStoreStats.Fields.REFRESH_TIME_LAG_IN_MILLIS), (int) pressureTrackerStats.refreshTimeLagMs);
4929
assertEquals(
5030
statsObject.get(RemoteStoreStats.Fields.REFRESH_LAG),
5131
(int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber)
5232
);
53-
assertEquals(statsObject.get(RemoteStoreStats.Fields.REMOTE_REFRESH_TIMESTAMP), (int) pressureTrackerStats.remoteRefreshTimeMs);
5433
assertEquals(statsObject.get(RemoteStoreStats.Fields.BYTES_LAG), (int) pressureTrackerStats.bytesLag);
5534

5635
assertEquals(statsObject.get(RemoteStoreStats.Fields.BACKPRESSURE_REJECTION_COUNT), (int) pressureTrackerStats.rejectionCount);

‎server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,9 @@ public void testSerialization() throws Exception {
6262
try (StreamInput in = out.bytes().streamInput()) {
6363
RemoteStoreStats deserializedStats = new RemoteStoreStats(in);
6464
assertEquals(deserializedStats.getStats().shardId.toString(), stats.getStats().shardId.toString());
65+
assertEquals(deserializedStats.getStats().refreshTimeLagMs, stats.getStats().refreshTimeLagMs);
6566
assertEquals(deserializedStats.getStats().localRefreshNumber, stats.getStats().localRefreshNumber);
66-
assertEquals(deserializedStats.getStats().localRefreshTimeMs, stats.getStats().localRefreshTimeMs);
6767
assertEquals(deserializedStats.getStats().remoteRefreshNumber, stats.getStats().remoteRefreshNumber);
68-
assertEquals(deserializedStats.getStats().remoteRefreshTimeMs, stats.getStats().remoteRefreshTimeMs);
6968
assertEquals(deserializedStats.getStats().uploadBytesStarted, stats.getStats().uploadBytesStarted);
7069
assertEquals(deserializedStats.getStats().uploadBytesSucceeded, stats.getStats().uploadBytesSucceeded);
7170
assertEquals(deserializedStats.getStats().uploadBytesFailed, stats.getStats().uploadBytesFailed);

‎server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -411,9 +411,8 @@ public void testStatsObjectCreation() {
411411
pressureTracker = constructTracker();
412412
RemoteRefreshSegmentTracker.Stats pressureTrackerStats = pressureTracker.stats();
413413
assertEquals(pressureTracker.getShardId(), pressureTrackerStats.shardId);
414-
assertEquals(pressureTracker.getLocalRefreshTimeMs(), (int) pressureTrackerStats.localRefreshTimeMs);
414+
assertEquals(pressureTracker.getTimeMsLag(), (int) pressureTrackerStats.refreshTimeLagMs);
415415
assertEquals(pressureTracker.getLocalRefreshSeqNo(), (int) pressureTrackerStats.localRefreshNumber);
416-
assertEquals(pressureTracker.getRemoteRefreshTimeMs(), (int) pressureTrackerStats.remoteRefreshTimeMs);
417416
assertEquals(pressureTracker.getRemoteRefreshSeqNo(), (int) pressureTrackerStats.remoteRefreshNumber);
418417
assertEquals(pressureTracker.getBytesLag(), (int) pressureTrackerStats.bytesLag);
419418
assertEquals(pressureTracker.getRejectionCount(), (int) pressureTrackerStats.rejectionCount);
@@ -441,9 +440,8 @@ public void testStatsObjectCreationViaStream() throws IOException {
441440
try (StreamInput in = out.bytes().streamInput()) {
442441
RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteRefreshSegmentTracker.Stats(in);
443442
assertEquals(deserializedStats.shardId, pressureTrackerStats.shardId);
444-
assertEquals((int) deserializedStats.localRefreshTimeMs, (int) pressureTrackerStats.localRefreshTimeMs);
443+
assertEquals((int) deserializedStats.refreshTimeLagMs, (int) pressureTrackerStats.refreshTimeLagMs);
445444
assertEquals((int) deserializedStats.localRefreshNumber, (int) pressureTrackerStats.localRefreshNumber);
446-
assertEquals((int) deserializedStats.remoteRefreshTimeMs, (int) pressureTrackerStats.remoteRefreshTimeMs);
447445
assertEquals((int) deserializedStats.remoteRefreshNumber, (int) pressureTrackerStats.remoteRefreshNumber);
448446
assertEquals((int) deserializedStats.bytesLag, (int) pressureTrackerStats.bytesLag);
449447
assertEquals((int) deserializedStats.rejectionCount, (int) pressureTrackerStats.rejectionCount);

0 commit comments

Comments
 (0)