Skip to content

Commit 0ad67f9

Browse files
daniel-mohedanodaniel-mohedano
authored
Add session level metric for Failed Test Replay (#9531)
1 parent dacc5a5 commit 0ad67f9

File tree

14 files changed

+78
-42
lines changed

14 files changed

+78
-42
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static datadog.trace.civisibility.Constants.CI_VISIBILITY_INSTRUMENTATION_NAME;
55

66
import datadog.trace.api.Config;
7+
import datadog.trace.api.DDTags;
78
import datadog.trace.api.DDTraceId;
89
import datadog.trace.api.IdGenerationStrategy;
910
import datadog.trace.api.civisibility.CIConstants;
@@ -13,8 +14,10 @@
1314
import datadog.trace.api.civisibility.telemetry.TagValue;
1415
import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
1516
import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
17+
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
1618
import datadog.trace.api.civisibility.telemetry.tag.EventType;
1719
import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
20+
import datadog.trace.api.civisibility.telemetry.tag.FailedTestReplayEnabled;
1821
import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
1922
import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
2023
import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
@@ -26,13 +29,15 @@
2629
import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
2730
import datadog.trace.bootstrap.instrumentation.api.TagContext;
2831
import datadog.trace.bootstrap.instrumentation.api.Tags;
32+
import datadog.trace.civisibility.Constants;
2933
import datadog.trace.civisibility.codeowners.Codeowners;
3034
import datadog.trace.civisibility.decorator.TestDecorator;
3135
import datadog.trace.civisibility.source.LinesResolver;
3236
import datadog.trace.civisibility.source.SourcePathResolver;
3337
import java.util.ArrayList;
3438
import java.util.Collection;
35-
import java.util.Collections;
39+
import java.util.HashSet;
40+
import java.util.Set;
3641
import javax.annotation.Nullable;
3742

3843
public abstract class AbstractTestSession {
@@ -188,6 +193,13 @@ private TagValue[] telemetryTags() {
188193
}
189194

190195
protected Collection<TagValue> additionalTelemetryTags() {
191-
return Collections.emptyList();
196+
Set<TagValue> tags = new HashSet<>();
197+
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
198+
tags.add(EarlyFlakeDetectionAbortReason.FAULTY);
199+
}
200+
if (span.getTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY) != null) {
201+
tags.add(FailedTestReplayEnabled.SessionMetric.TRUE);
202+
}
203+
return tags;
192204
}
193205
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,11 @@ public void end(@Nullable Long endTime) {
284284
}
285285
}
286286

287+
boolean debugInfoCaptured = span.getTag(Tags.ERROR_DEBUG_INFO_CAPTURED) != null;
288+
if (debugInfoCaptured) {
289+
executionResults.setHasFailedTestReplayTests();
290+
}
291+
287292
AgentTracer.closeActive();
288293

289294
onSpanFinish.accept(span);
@@ -310,9 +315,7 @@ public void end(@Nullable Long endTime) {
310315
span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
311316
span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
312317
retryReason instanceof TagValue ? (TagValue) retryReason : null,
313-
span.getTag(Tags.ERROR_DEBUG_INFO_CAPTURED) != null
314-
? FailedTestReplayEnabled.TestMetric.TRUE
315-
: null,
318+
debugInfoCaptured ? FailedTestReplayEnabled.TestMetric.TRUE : null,
316319
span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
317320
CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))
318321
? BrowserDriver.SELENIUM

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,10 @@ private SignalResponse onModuleExecutionResultReceived(ModuleExecutionResult res
290290
setTag(Tags.TEST_TEST_MANAGEMENT_ENABLED, true);
291291
}
292292

293+
if (result.hasFailedTestReplayTests()) {
294+
setTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY, true);
295+
}
296+
293297
testsSkipped.add(result.getTestsSkippedTotal());
294298

295299
tagsPropagator.mergeTestFrameworks(result.getTestFrameworks());

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,9 @@
99
import datadog.trace.api.civisibility.domain.BuildSessionSettings;
1010
import datadog.trace.api.civisibility.domain.JavaAgent;
1111
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
12-
import datadog.trace.api.civisibility.telemetry.TagValue;
13-
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
1412
import datadog.trace.api.civisibility.telemetry.tag.Provider;
1513
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
1614
import datadog.trace.bootstrap.instrumentation.api.Tags;
17-
import datadog.trace.civisibility.Constants;
1815
import datadog.trace.civisibility.codeowners.Codeowners;
1916
import datadog.trace.civisibility.config.ExecutionSettings;
2017
import datadog.trace.civisibility.config.ExecutionSettingsFactory;
@@ -196,22 +193,15 @@ private void onModuleFinish(AgentSpan moduleSpan) {
196193
TagMergeSpec.of(Tags.TEST_ITR_TESTS_SKIPPING_TYPE),
197194
TagMergeSpec.of(Tags.TEST_ITR_TESTS_SKIPPING_COUNT, Long::sum),
198195
TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED, Boolean::logicalOr),
199-
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED, Boolean::logicalOr));
196+
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED, Boolean::logicalOr),
197+
TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY, Boolean::logicalOr));
200198
}
201199

202200
@Override
203201
public BuildSessionSettings getSettings() {
204202
return settings;
205203
}
206204

207-
@Override
208-
protected Collection<TagValue> additionalTelemetryTags() {
209-
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
210-
return Collections.singleton(EarlyFlakeDetectionAbortReason.FAULTY);
211-
}
212-
return Collections.emptySet();
213-
}
214-
215205
@Override
216206
public void end(@Nullable Long endTime) {
217207
signalServer.stop();

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ private void sendModuleExecutionResult() {
166166
earlyFlakeDetectionEnabled && executionStrategy.isEFDLimitReached();
167167
TestManagementSettings testManagementSettings = executionSettings.getTestManagementSettings();
168168
boolean testManagementEnabled = testManagementSettings.isEnabled();
169+
boolean hasFailedTestReplayTests = executionResults.hasFailedTestReplayTests();
169170
long testsSkippedTotal = executionResults.getTestsSkippedByItr();
170171

171172
signalClient.send(
@@ -177,6 +178,7 @@ private void sendModuleExecutionResult() {
177178
earlyFlakeDetectionEnabled,
178179
earlyFlakeDetectionFaulty,
179180
testManagementEnabled,
181+
hasFailedTestReplayTests,
180182
testsSkippedTotal,
181183
new TreeSet<>(testFrameworks)));
182184

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,10 @@ public void end(@Nullable Long endTime) {
153153
setTag(Tags.TEST_TEST_MANAGEMENT_ENABLED, true);
154154
}
155155

156+
if (executionResults.hasFailedTestReplayTests()) {
157+
setTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY, true);
158+
}
159+
156160
super.end(endTime);
157161
}
158162

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestSession.java‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,9 @@
77
import datadog.trace.api.civisibility.config.LibraryCapability;
88
import datadog.trace.api.civisibility.coverage.CoverageStore;
99
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
10-
import datadog.trace.api.civisibility.telemetry.TagValue;
11-
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
1210
import datadog.trace.api.civisibility.telemetry.tag.Provider;
1311
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
1412
import datadog.trace.bootstrap.instrumentation.api.Tags;
15-
import datadog.trace.civisibility.Constants;
1613
import datadog.trace.civisibility.codeowners.Codeowners;
1714
import datadog.trace.civisibility.decorator.TestDecorator;
1815
import datadog.trace.civisibility.domain.AbstractTestSession;
@@ -22,7 +19,6 @@
2219
import datadog.trace.civisibility.source.SourcePathResolver;
2320
import datadog.trace.civisibility.test.ExecutionStrategy;
2421
import java.util.Collection;
25-
import java.util.Collections;
2622
import javax.annotation.Nonnull;
2723
import javax.annotation.Nullable;
2824

@@ -97,15 +93,8 @@ private void onModuleFinish(AgentSpan moduleSpan) {
9793
TagMergeSpec.of(Tags.TEST_EARLY_FLAKE_ENABLED),
9894
TagMergeSpec.of(Tags.TEST_EARLY_FLAKE_ABORT_REASON),
9995
TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED),
100-
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED));
101-
}
102-
103-
@Override
104-
protected Collection<TagValue> additionalTelemetryTags() {
105-
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
106-
return Collections.singleton(EarlyFlakeDetectionAbortReason.FAULTY);
107-
}
108-
return Collections.emptySet();
96+
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED),
97+
TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY));
10998
}
11099

111100
@Override

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/ModuleExecutionResult.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ public class ModuleExecutionResult extends ModuleSignal {
1313
private static final int EARLY_FLAKE_DETECTION_ENABLED_FLAG = 4;
1414
private static final int EARLY_FLAKE_DETECTION_FAULTY_FLAG = 8;
1515
private static final int TEST_MANAGEMENT_ENABLED_FLAG = 16;
16+
private static final int HAS_FAILED_TEST_REPLAY_TESTS_FLAG = 32;
1617

1718
private final boolean coverageEnabled;
1819
private final boolean testSkippingEnabled;
1920
private final boolean earlyFlakeDetectionEnabled;
2021
private final boolean earlyFlakeDetectionFaulty;
2122
private final boolean testManagementEnabled;
23+
private final boolean hasFailedTestReplayTests;
2224
private final long testsSkippedTotal;
2325
private final Collection<TestFramework> testFrameworks;
2426

@@ -30,6 +32,7 @@ public ModuleExecutionResult(
3032
boolean earlyFlakeDetectionEnabled,
3133
boolean earlyFlakeDetectionFaulty,
3234
boolean testManagementEnabled,
35+
boolean hasFailedTestReplayTests,
3336
long testsSkippedTotal,
3437
Collection<TestFramework> testFrameworks) {
3538
super(sessionId, moduleId);
@@ -38,6 +41,7 @@ public ModuleExecutionResult(
3841
this.earlyFlakeDetectionEnabled = earlyFlakeDetectionEnabled;
3942
this.earlyFlakeDetectionFaulty = earlyFlakeDetectionFaulty;
4043
this.testManagementEnabled = testManagementEnabled;
44+
this.hasFailedTestReplayTests = hasFailedTestReplayTests;
4145
this.testsSkippedTotal = testsSkippedTotal;
4246
this.testFrameworks = testFrameworks;
4347
}
@@ -62,6 +66,10 @@ public boolean isTestManagementEnabled() {
6266
return testManagementEnabled;
6367
}
6468

69+
public boolean hasFailedTestReplayTests() {
70+
return hasFailedTestReplayTests;
71+
}
72+
6573
public long getTestsSkippedTotal() {
6674
return testsSkippedTotal;
6775
}
@@ -84,6 +92,7 @@ public boolean equals(Object o) {
8492
&& moduleId == that.moduleId
8593
&& coverageEnabled == that.coverageEnabled
8694
&& testSkippingEnabled == that.testSkippingEnabled
95+
&& hasFailedTestReplayTests == that.hasFailedTestReplayTests
8796
&& testsSkippedTotal == that.testsSkippedTotal
8897
&& Objects.equals(testFrameworks, that.testFrameworks);
8998
}
@@ -95,6 +104,7 @@ public int hashCode() {
95104
moduleId,
96105
coverageEnabled,
97106
testSkippingEnabled,
107+
hasFailedTestReplayTests,
98108
testsSkippedTotal,
99109
testFrameworks);
100110
}
@@ -108,6 +118,8 @@ public String toString() {
108118
+ moduleId
109119
+ ", coverageEnabled="
110120
+ coverageEnabled
121+
+ ", hasFailedTestReplayTests="
122+
+ hasFailedTestReplayTests
111123
+ ", testSkippingEnabled="
112124
+ testSkippingEnabled
113125
+ ", itrTestsSkipped="
@@ -142,6 +154,9 @@ public ByteBuffer serialize() {
142154
if (testManagementEnabled) {
143155
flags |= TEST_MANAGEMENT_ENABLED_FLAG;
144156
}
157+
if (hasFailedTestReplayTests) {
158+
flags |= HAS_FAILED_TEST_REPLAY_TESTS_FLAG;
159+
}
145160
s.write(flags);
146161

147162
s.write(testsSkippedTotal);
@@ -160,6 +175,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) {
160175
boolean earlyFlakeDetectionEnabled = (flags & EARLY_FLAKE_DETECTION_ENABLED_FLAG) != 0;
161176
boolean earlyFlakeDetectionFaulty = (flags & EARLY_FLAKE_DETECTION_FAULTY_FLAG) != 0;
162177
boolean testManagementEnabled = (flags & TEST_MANAGEMENT_ENABLED_FLAG) != 0;
178+
boolean hasFailedTestReplayTests = (flags & HAS_FAILED_TEST_REPLAY_TESTS_FLAG) != 0;
163179

164180
long testsSkippedTotal = Serializer.readLong(buffer);
165181
Collection<TestFramework> testFrameworks =
@@ -173,6 +189,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) {
173189
earlyFlakeDetectionEnabled,
174190
earlyFlakeDetectionFaulty,
175191
testManagementEnabled,
192+
hasFailedTestReplayTests,
176193
testsSkippedTotal,
177194
testFrameworks);
178195
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionResults.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package datadog.trace.civisibility.test;
22

3+
import java.util.concurrent.atomic.AtomicBoolean;
34
import java.util.concurrent.atomic.LongAdder;
45

56
public class ExecutionResults {
67

78
private final LongAdder testsSkippedByItr = new LongAdder();
9+
private final AtomicBoolean hasFailedTestReplayTests = new AtomicBoolean();
810

911
public void incrementTestsSkippedByItr() {
1012
testsSkippedByItr.increment();
@@ -13,4 +15,12 @@ public void incrementTestsSkippedByItr() {
1315
public long getTestsSkippedByItr() {
1416
return testsSkippedByItr.sum();
1517
}
18+
19+
public void setHasFailedTestReplayTests() {
20+
this.hasFailedTestReplayTests.set(true);
21+
}
22+
23+
public boolean hasFailedTestReplayTests() {
24+
return hasFailedTestReplayTests.get();
25+
}
1626
}

‎dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/ModuleExecutionResultTest.groovy‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ class ModuleExecutionResultTest extends Specification {
1616

1717
where:
1818
signal << [
19-
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, false, false, false, false, 0, Collections.emptyList()),
20-
new ModuleExecutionResult(DDTraceId.from(12345), 67890, true, false, true, true, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2"))),
21-
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, true, true, false, false, 2, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework("junit", "5.9.2"))),
22-
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, false, false, false, true, true, 3, Arrays.asList(new TestFramework("junit", null), new TestFramework("junit", "5.9.2"))),
23-
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, true, true, true, true, true, Integer.MAX_VALUE, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework(null, "5.9.2")))
19+
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, false, false, false, false, false, 0, Collections.emptyList()),
20+
new ModuleExecutionResult(DDTraceId.from(12345), 67890, true, false, true, true, true, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2"))),
21+
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, true, true, false, false, true, 2, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework("junit", "5.9.2"))),
22+
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, false, false, false, true, true, false, 3, Arrays.asList(new TestFramework("junit", null), new TestFramework("junit", "5.9.2"))),
23+
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, true, true, true, true, true, true, Integer.MAX_VALUE, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework(null, "5.9.2")))
2424
]
2525
}
2626
}

0 commit comments

Comments
 (0)