Skip to content

Commit da9273a

Browse files
author
Hendrik Muhs
authored
[Transform][Rollup] add processing stats to record the time sp… (#53770)
add 2 additional stats: processing time and processing total which capture the time spent for processing results and how often it ran. The 2 new stats correspond to the existing indexing and search stats. Together with indexing and search this now allows the user to see the full picture, all 3 stages.
1 parent e98b9c0 commit da9273a

File tree

27 files changed

+583
-201
lines changed

27 files changed

+583
-201
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@ public abstract class IndexerJobStats {
3131
public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
3232
public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
3333
public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
34+
public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
3435
public static ParseField INDEX_TOTAL = new ParseField("index_total");
3536
public static ParseField SEARCH_TOTAL = new ParseField("search_total");
37+
public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
3638
public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
3739
public static ParseField INDEX_FAILURES = new ParseField("index_failures");
3840

@@ -44,11 +46,14 @@ public abstract class IndexerJobStats {
4446
protected final long indexTotal;
4547
protected final long searchTime;
4648
protected final long searchTotal;
49+
protected final long processingTime;
50+
protected final long processingTotal;
4751
protected final long indexFailures;
4852
protected final long searchFailures;
4953

5054
public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
51-
long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {
55+
long indexTime, long searchTime, long processingTime, long indexTotal, long searchTotal, long processingTotal,
56+
long indexFailures, long searchFailures) {
5257
this.numPages = numPages;
5358
this.numInputDocuments = numInputDocuments;
5459
this.numOuputDocuments = numOutputDocuments;
@@ -57,6 +62,8 @@ public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocu
5762
this.indexTotal = indexTotal;
5863
this.searchTime = searchTime;
5964
this.searchTotal = searchTotal;
65+
this.processingTime = processingTime;
66+
this.processingTotal = processingTotal;
6067
this.indexFailures = indexFailures;
6168
this.searchFailures = searchFailures;
6269
}
@@ -117,6 +124,13 @@ public long getSearchTime() {
117124
return searchTime;
118125
}
119126

127+
/**
128+
* Returns the time spent processing (cumulative) in milliseconds
129+
*/
130+
public long getProcessingTime() {
131+
return processingTime;
132+
}
133+
120134
/**
121135
* Returns the total number of indexing requests that have been processed
122136
* (Note: this is not the number of _documents_ that have been indexed)
@@ -132,6 +146,14 @@ public long getSearchTotal() {
132146
return searchTotal;
133147
}
134148

149+
/**
150+
* Returns the total number of processing runs that have been made
151+
*/
152+
public long getProcessingTotal() {
153+
return processingTotal;
154+
}
155+
156+
135157
@Override
136158
public boolean equals(Object other) {
137159
if (this == other) {
@@ -149,16 +171,19 @@ public boolean equals(Object other) {
149171
&& Objects.equals(this.numInvocations, that.numInvocations)
150172
&& Objects.equals(this.indexTime, that.indexTime)
151173
&& Objects.equals(this.searchTime, that.searchTime)
174+
&& Objects.equals(this.processingTime, that.processingTime)
152175
&& Objects.equals(this.indexFailures, that.indexFailures)
153176
&& Objects.equals(this.searchFailures, that.searchFailures)
154177
&& Objects.equals(this.searchTotal, that.searchTotal)
178+
&& Objects.equals(this.processingTotal, that.processingTotal)
155179
&& Objects.equals(this.indexTotal, that.indexTotal);
156180
}
157181

158182
@Override
159183
public int hashCode() {
160184
return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
161-
indexTime, searchTime, indexFailures, searchFailures, searchTotal, indexTotal);
185+
indexTime, searchTime, processingTime, indexFailures, searchFailures, searchTotal,
186+
indexTotal, processingTotal);
162187
}
163188

164189
@Override
@@ -172,6 +197,8 @@ public final String toString() {
172197
+ ", index_time_in_ms=" + indexTime
173198
+ ", index_total=" + indexTotal
174199
+ ", search_time_in_ms=" + searchTime
175-
+ ", search_total=" + searchTotal+ "}";
200+
+ ", search_total=" + searchTotal
201+
+ ", processing_time_in_ms=" + processingTime
202+
+ ", processing_total=" + processingTotal + "}";
176203
}
177204
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,16 +177,18 @@ public final String toString() {
177177
public static class RollupIndexerJobStats extends IndexerJobStats {
178178

179179
RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
180-
long indexTime, long indexTotal, long searchTime, long searchTotal, long indexFailures, long searchFailures) {
180+
long indexTime, long indexTotal, long searchTime, long searchTotal, long processingTime,
181+
long processingTotal, long indexFailures, long searchFailures) {
181182
super(numPages, numInputDocuments, numOuputDocuments, numInvocations,
182-
indexTime, searchTime, indexTotal, searchTotal, indexFailures, searchFailures);
183+
indexTime, searchTime, processingTime, indexTotal, searchTotal, processingTotal, indexFailures, searchFailures);
183184
}
184185

185186
private static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER = new ConstructingObjectParser<>(
186187
STATS.getPreferredName(),
187188
true,
188189
args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
189-
(long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
190+
(long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
191+
(long) args[10], (long) args[11]));
190192
static {
191193
PARSER.declareLong(constructorArg(), NUM_PAGES);
192194
PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
@@ -196,6 +198,8 @@ public static class RollupIndexerJobStats extends IndexerJobStats {
196198
PARSER.declareLong(constructorArg(), INDEX_TOTAL);
197199
PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
198200
PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
201+
PARSER.declareLong(constructorArg(), PROCESSING_TIME_IN_MS);
202+
PARSER.declareLong(constructorArg(), PROCESSING_TOTAL);
199203
PARSER.declareLong(constructorArg(), INDEX_FAILURES);
200204
PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
201205
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java

Lines changed: 92 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import java.io.IOException;
2828
import java.util.Objects;
2929

30-
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
3130
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
3231

3332
public class TransformIndexerStats extends IndexerJobStats {
@@ -39,21 +38,38 @@ public class TransformIndexerStats extends IndexerJobStats {
3938
public static final ConstructingObjectParser<TransformIndexerStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
4039
NAME,
4140
true,
42-
args -> new TransformIndexerStats((long) args[0], (long) args[1], (long) args[2],
43-
(long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
44-
(Double) args[10], (Double) args[11], (Double) args[12]));
41+
args -> new TransformIndexerStats(
42+
unboxSafe(args[0], 0L),
43+
unboxSafe(args[1], 0L),
44+
unboxSafe(args[2], 0L),
45+
unboxSafe(args[3], 0L),
46+
unboxSafe(args[4], 0L),
47+
unboxSafe(args[5], 0L),
48+
unboxSafe(args[6], 0L),
49+
unboxSafe(args[7], 0L),
50+
unboxSafe(args[8], 0L),
51+
unboxSafe(args[9], 0L),
52+
unboxSafe(args[10], 0L),
53+
unboxSafe(args[11], 0L),
54+
unboxSafe(args[12], 0.0),
55+
unboxSafe(args[13], 0.0),
56+
unboxSafe(args[14], 0.0)
57+
)
58+
);
4559

4660
static {
47-
LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
48-
LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
49-
LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
50-
LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
51-
LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
52-
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
53-
LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL);
54-
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
55-
LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
56-
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
61+
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
62+
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
63+
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
64+
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
65+
LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
66+
LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
67+
LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
68+
LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL);
69+
LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL);
70+
LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL);
71+
LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_FAILURES);
72+
LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_FAILURES);
5773
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
5874
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
5975
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
@@ -67,16 +83,40 @@ public static TransformIndexerStats fromXContent(XContentParser parser) throws I
6783
private final double expAvgDocumentsIndexed;
6884
private final double expAvgDocumentsProcessed;
6985

70-
public TransformIndexerStats(long numPages, long numInputDocuments, long numOuputDocuments,
71-
long numInvocations, long indexTime, long searchTime,
72-
long indexTotal, long searchTotal, long indexFailures, long searchFailures,
73-
Double expAvgCheckpointDurationMs, Double expAvgDocumentsIndexed,
74-
Double expAvgDocumentsProcessed) {
75-
super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
76-
indexTotal, searchTotal, indexFailures, searchFailures);
77-
this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
78-
this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
79-
this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
86+
public TransformIndexerStats(
87+
long numPages,
88+
long numInputDocuments,
89+
long numOuputDocuments,
90+
long numInvocations,
91+
long indexTime,
92+
long searchTime,
93+
long processingTime,
94+
long indexTotal,
95+
long searchTotal,
96+
long processingTotal,
97+
long indexFailures,
98+
long searchFailures,
99+
double expAvgCheckpointDurationMs,
100+
double expAvgDocumentsIndexed,
101+
double expAvgDocumentsProcessed
102+
) {
103+
super(
104+
numPages,
105+
numInputDocuments,
106+
numOuputDocuments,
107+
numInvocations,
108+
indexTime,
109+
searchTime,
110+
processingTime,
111+
indexTotal,
112+
searchTotal,
113+
processingTotal,
114+
indexFailures,
115+
searchFailures
116+
);
117+
this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
118+
this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
119+
this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
80120
}
81121

82122
public double getExpAvgCheckpointDurationMs() {
@@ -109,19 +149,44 @@ public boolean equals(Object other) {
109149
&& Objects.equals(this.numInvocations, that.numInvocations)
110150
&& Objects.equals(this.indexTime, that.indexTime)
111151
&& Objects.equals(this.searchTime, that.searchTime)
152+
&& Objects.equals(this.processingTime, that.processingTime)
112153
&& Objects.equals(this.indexFailures, that.indexFailures)
113154
&& Objects.equals(this.searchFailures, that.searchFailures)
114155
&& Objects.equals(this.indexTotal, that.indexTotal)
115156
&& Objects.equals(this.searchTotal, that.searchTotal)
157+
&& Objects.equals(this.processingTotal, that.processingTotal)
116158
&& Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
117159
&& Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
118160
&& Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
119161
}
120162

121163
@Override
122164
public int hashCode() {
123-
return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
124-
indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal,
125-
expAvgCheckpointDurationMs, expAvgDocumentsIndexed, expAvgDocumentsProcessed);
165+
return Objects.hash(
166+
numPages,
167+
numInputDocuments,
168+
numOuputDocuments,
169+
numInvocations,
170+
indexTime,
171+
searchTime,
172+
processingTime,
173+
indexFailures,
174+
searchFailures,
175+
indexTotal,
176+
searchTotal,
177+
processingTotal,
178+
expAvgCheckpointDurationMs,
179+
expAvgDocumentsIndexed,
180+
expAvgDocumentsProcessed
181+
);
182+
}
183+
184+
@SuppressWarnings("unchecked")
185+
private static <T> T unboxSafe(Object l, T default_value) {
186+
if (l == null) {
187+
return default_value;
188+
} else {
189+
return (T) l;
190+
}
126191
}
127192
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,8 @@ public void testGetStats() throws Exception {
441441
0L,
442442
0L,
443443
0L,
444+
0L,
445+
0L,
444446
0.0,
445447
0.0,
446448
0.0);

‎client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupJobResponseTests.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@ private GetRollupJobResponse createTestInstance() {
6464

6565
private RollupIndexerJobStats randomStats() {
6666
return new RollupIndexerJobStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
67-
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
68-
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
67+
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
68+
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
69+
randomNonNegativeLong());
6970
}
7071

7172
private RollupJobStatus randomStatus() {
@@ -120,6 +121,8 @@ public void toXContent(RollupIndexerJobStats stats, XContentBuilder builder, ToX
120121
builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
121122
builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
122123
builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
124+
builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
125+
builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
123126
builder.endObject();
124127
}
125128

0 commit comments

Comments
 (0)