Skip to content

Commit 0382750

Browse files
randomandersonrandomanderson
authored
Make Crashtracking smoke test more resilient (#8685)
* Make crash tracking smoke test more resiliant * loop through messages until the correct event * some debug * configurable dogstatsd port * use sleep instead of park nanos * spotless * cleanup
1 parent e8762d2 commit 0382750

File tree

6 files changed

+413
-303
lines changed

6 files changed

+413
-303
lines changed

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashTelemetryData.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package datadog.smoketest;
2+
3+
import java.util.List;
4+
5+
public class CrashTelemetryData extends MinimalTelemetryData {
6+
List<LogMessage> payload;
7+
8+
public static class LogMessage {
9+
public String message;
10+
public String level;
11+
public String tags;
12+
}
13+
}

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java‎

Lines changed: 85 additions & 176 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,30 @@
33
import static org.hamcrest.CoreMatchers.containsString;
44
import static org.hamcrest.MatcherAssert.assertThat;
55
import static org.junit.jupiter.api.Assertions.assertNotEquals;
6+
import static org.junit.jupiter.api.Assertions.assertNotNull;
67
import static org.junit.jupiter.api.Assumptions.assumeFalse;
78

9+
import com.squareup.moshi.JsonAdapter;
10+
import com.squareup.moshi.Moshi;
811
import datadog.trace.api.Platform;
9-
import java.io.BufferedReader;
1012
import java.io.File;
11-
import java.io.InputStreamReader;
13+
import java.io.IOException;
14+
import java.nio.charset.StandardCharsets;
1215
import java.nio.file.FileSystems;
1316
import java.nio.file.Files;
1417
import java.nio.file.Path;
18+
import java.nio.file.Paths;
1519
import java.util.Arrays;
1620
import java.util.Comparator;
21+
import java.util.concurrent.BlockingQueue;
22+
import java.util.concurrent.LinkedBlockingQueue;
23+
import java.util.concurrent.TimeUnit;
1724
import java.util.stream.Stream;
1825
import okhttp3.mockwebserver.Dispatcher;
1926
import okhttp3.mockwebserver.MockResponse;
2027
import okhttp3.mockwebserver.MockWebServer;
2128
import okhttp3.mockwebserver.RecordedRequest;
29+
import org.junit.jupiter.api.AfterAll;
2230
import org.junit.jupiter.api.AfterEach;
2331
import org.junit.jupiter.api.BeforeAll;
2432
import org.junit.jupiter.api.BeforeEach;
@@ -29,7 +37,14 @@
2937
* that ships with OS X by default.
3038
*/
3139
public class CrashtrackingSmokeTest {
40+
private static final long DATA_TIMEOUT_MS = 10 * 1000;
41+
private static final OutputThreads OUTPUT = new OutputThreads();
42+
private static final Path LOG_FILE_DIR =
43+
Paths.get(System.getProperty("datadog.smoketest.builddir"), "reports");
44+
3245
private MockWebServer tracingServer;
46+
private TestUDPServer udpServer;
47+
private final BlockingQueue<CrashTelemetryData> crashEvents = new LinkedBlockingQueue<>();
3348

3449
@BeforeAll
3550
static void setupAll() {
@@ -43,27 +58,61 @@ static void setupAll() {
4358
void setup() throws Exception {
4459
tempDir = Files.createTempDirectory("dd-smoketest-");
4560

61+
crashEvents.clear();
62+
63+
Moshi moshi = new Moshi.Builder().build();
4664
tracingServer = new MockWebServer();
4765
tracingServer.setDispatcher(
4866
new Dispatcher() {
4967
@Override
50-
public MockResponse dispatch(final RecordedRequest request) throws InterruptedException {
68+
public MockResponse dispatch(final RecordedRequest request) {
69+
System.out.println("URL ====== " + request.getPath());
70+
71+
String data = request.getBody().readString(StandardCharsets.UTF_8);
72+
73+
if ("/telemetry/proxy/api/v2/apmtelemetry".equals(request.getPath())) {
74+
try {
75+
JsonAdapter<MinimalTelemetryData> adapter =
76+
moshi.adapter(MinimalTelemetryData.class);
77+
MinimalTelemetryData minimal = adapter.fromJson(data);
78+
if ("logs".equals(minimal.request_type)) {
79+
JsonAdapter<CrashTelemetryData> crashAdapter =
80+
moshi.adapter(CrashTelemetryData.class);
81+
crashEvents.add(crashAdapter.fromJson(data));
82+
}
83+
} catch (IOException e) {
84+
System.out.println("Unable to parse " + e);
85+
}
86+
}
87+
88+
System.out.println(data);
89+
5190
return new MockResponse().setResponseCode(200);
5291
}
5392
});
54-
// tracingServer.start(8126);
93+
94+
udpServer = new TestUDPServer();
95+
udpServer.start();
96+
97+
OUTPUT.clearMessages();
5598
}
5699

57100
@AfterEach
58101
void teardown() throws Exception {
59102
tracingServer.shutdown();
103+
udpServer.close();
60104

61105
try (Stream<Path> fileStream = Files.walk(tempDir)) {
62106
fileStream.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
63107
}
64108
Files.deleteIfExists(tempDir);
65109
}
66110

111+
@AfterAll
112+
static void shutdown() {
113+
OUTPUT.close();
114+
}
115+
67116
private static String javaPath() {
68117
final String separator = FileSystems.getDefault().getSeparator();
69118
return System.getProperty("java.home") + separator + "bin" + separator + "java";
@@ -108,52 +157,12 @@ void testCrashTracking() throws Exception {
108157
appShadowJar(),
109158
script.toString()));
110159
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
111-
StringBuilder stdoutStr = new StringBuilder();
112-
StringBuilder stderrStr = new StringBuilder();
113160

114161
Process p = pb.start();
115-
Thread stdout =
116-
new Thread(
117-
() -> {
118-
try (BufferedReader br =
119-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
120-
br.lines()
121-
.forEach(
122-
l -> {
123-
System.out.println(l);
124-
stdoutStr.append(l).append('\n');
125-
});
126-
} catch (Exception e) {
127-
throw new RuntimeException(e);
128-
}
129-
});
130-
Thread stderr =
131-
new Thread(
132-
() -> {
133-
try (BufferedReader br =
134-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
135-
br.lines()
136-
.forEach(
137-
l -> {
138-
System.err.println(l);
139-
stderrStr.append(l).append('\n');
140-
});
141-
} catch (Exception e) {
142-
throw new RuntimeException(e);
143-
}
144-
});
145-
stdout.setDaemon(true);
146-
stderr.setDaemon(true);
147-
stdout.start();
148-
stderr.start();
162+
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCrashTracking.log").toFile());
149163

150164
assertNotEquals(0, p.waitFor(), "Application should have crashed");
151-
152-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
153-
assertThat(
154-
stderrStr.toString(),
155-
containsString(
156-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
165+
assertCrashData();
157166
}
158167

159168
/*
@@ -183,52 +192,14 @@ void testCrashTrackingLegacy() throws Exception {
183192
appShadowJar(),
184193
script.toString()));
185194
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
186-
StringBuilder stdoutStr = new StringBuilder();
187-
StringBuilder stderrStr = new StringBuilder();
188195

189196
Process p = pb.start();
190-
Thread stdout =
191-
new Thread(
192-
() -> {
193-
try (BufferedReader br =
194-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
195-
br.lines()
196-
.forEach(
197-
l -> {
198-
System.out.println(l);
199-
stdoutStr.append(l).append('\n');
200-
});
201-
} catch (Exception e) {
202-
throw new RuntimeException(e);
203-
}
204-
});
205-
Thread stderr =
206-
new Thread(
207-
() -> {
208-
try (BufferedReader br =
209-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
210-
br.lines()
211-
.forEach(
212-
l -> {
213-
System.err.println(l);
214-
stderrStr.append(l).append('\n');
215-
});
216-
} catch (Exception e) {
217-
throw new RuntimeException(e);
218-
}
219-
});
220-
stdout.setDaemon(true);
221-
stderr.setDaemon(true);
222-
stdout.start();
223-
stderr.start();
197+
OUTPUT.captureOutput(
198+
p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingLegacy.log").toFile());
224199

225200
assertNotEquals(0, p.waitFor(), "Application should have crashed");
226201

227-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
228-
assertThat(
229-
stderrStr.toString(),
230-
containsString(
231-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
202+
assertCrashData();
232203
}
233204

234205
/*
@@ -255,51 +226,13 @@ void testOomeTracking() throws Exception {
255226
"-jar",
256227
appShadowJar(),
257228
script.toString()));
258-
StringBuilder stdoutStr = new StringBuilder();
259-
StringBuilder stderrStr = new StringBuilder();
229+
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));
260230

261231
Process p = pb.start();
262-
Thread stdout =
263-
new Thread(
264-
() -> {
265-
try (BufferedReader br =
266-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
267-
br.lines()
268-
.forEach(
269-
l -> {
270-
System.out.println(l);
271-
stdoutStr.append(l).append('\n');
272-
});
273-
} catch (Exception e) {
274-
throw new RuntimeException(e);
275-
}
276-
});
277-
Thread stderr =
278-
new Thread(
279-
() -> {
280-
try (BufferedReader br =
281-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
282-
br.lines()
283-
.forEach(
284-
l -> {
285-
System.err.println(l);
286-
stderrStr.append(l).append('\n');
287-
});
288-
} catch (Exception e) {
289-
throw new RuntimeException(e);
290-
}
291-
});
292-
stdout.setDaemon(true);
293-
stderr.setDaemon(true);
294-
stdout.start();
295-
stderr.start();
232+
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testOomeTracking.log").toFile());
296233

297234
assertNotEquals(0, p.waitFor(), "Application should have crashed");
298-
299-
assertThat(
300-
stderrStr.toString(),
301-
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
302-
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
235+
assertOOMEvent();
303236
}
304237

305238
@Test
@@ -326,58 +259,34 @@ void testCombineTracking() throws Exception {
326259
appShadowJar(),
327260
oomeScript.toString()));
328261
pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort()));
329-
StringBuilder stdoutStr = new StringBuilder();
330-
StringBuilder stderrStr = new StringBuilder();
262+
pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort()));
331263

332264
Process p = pb.start();
333-
Thread stdout =
334-
new Thread(
335-
() -> {
336-
try (BufferedReader br =
337-
new BufferedReader(new InputStreamReader(p.getInputStream()))) {
338-
br.lines()
339-
.forEach(
340-
l -> {
341-
System.out.println(l);
342-
stdoutStr.append(l).append('\n');
343-
});
344-
} catch (Exception e) {
345-
throw new RuntimeException(e);
346-
}
347-
});
348-
Thread stderr =
349-
new Thread(
350-
() -> {
351-
try (BufferedReader br =
352-
new BufferedReader(new InputStreamReader(p.getErrorStream()))) {
353-
br.lines()
354-
.forEach(
355-
l -> {
356-
System.err.println(l);
357-
stderrStr.append(l).append('\n');
358-
});
359-
} catch (Exception e) {
360-
throw new RuntimeException(e);
361-
}
362-
});
363-
stdout.setDaemon(true);
364-
stderr.setDaemon(true);
365-
stdout.start();
366-
stderr.start();
265+
OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCombineTracking.log").toFile());
367266

368267
assertNotEquals(0, p.waitFor(), "Application should have crashed");
369268

370-
// Crash uploader did get triggered
371-
assertThat(stdoutStr.toString(), containsString(" was uploaded successfully"));
372-
assertThat(
373-
stderrStr.toString(),
374-
containsString(
375-
"com.datadog.crashtracking.CrashUploader - Successfully uploaded the crash files"));
376-
377-
// OOME notifier did get triggered
378-
assertThat(
379-
stderrStr.toString(),
380-
containsString("com.datadog.crashtracking.OOMENotifier - OOME event sent"));
381-
assertThat(stdoutStr.toString(), containsString("OOME Event generated successfully"));
269+
assertCrashData();
270+
assertOOMEvent();
271+
}
272+
273+
private void assertCrashData() throws InterruptedException {
274+
CrashTelemetryData crashData = crashEvents.poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
275+
assertNotNull(crashData, "Crash data not uploaded");
276+
assertThat(crashData.payload.get(0).message, containsString("OutOfMemory"));
277+
assertThat(crashData.payload.get(0).tags, containsString("severity:crash"));
278+
}
279+
280+
private void assertOOMEvent() throws InterruptedException {
281+
String event;
282+
do {
283+
event = udpServer.getMessages().poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS);
284+
} while (event != null && !event.startsWith("_e"));
285+
286+
assertNotNull(event, "OOM Event not received");
287+
288+
assertThat(event, containsString(":OutOfMemoryError"));
289+
assertThat(event, containsString("t:error"));
290+
assertThat(event, containsString("s:java"));
382291
}
383292
}

‎dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/MinimalTelemetryData.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package datadog.smoketest;
2+
3+
public class MinimalTelemetryData {
4+
String request_type;
5+
}

0 commit comments

Comments
 (0)