Skip to content

Commit 3d7db54

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Truncate span stack traces when Test Optimization is enabled (#8903)
1 parent 2ba8c6a commit 3d7db54

File tree

5 files changed

+352
-5
lines changed

5 files changed

+352
-5
lines changed

‎dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,5 +101,7 @@ public final class GeneralConfig {
101101
public static final String APM_TRACING_ENABLED = "apm.tracing.enabled";
102102
public static final String JDK_SOCKET_ENABLED = "jdk.socket.enabled";
103103

104+
public static final String STACK_TRACE_LENGTH_LIMIT = "stack.trace.length.limit";
105+
104106
private GeneralConfig() {}
105107
}

‎dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@
2727
import datadog.trace.bootstrap.instrumentation.api.ErrorPriorities;
2828
import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities;
2929
import datadog.trace.bootstrap.instrumentation.api.Tags;
30-
import java.io.PrintWriter;
31-
import java.io.StringWriter;
30+
import datadog.trace.core.util.StackTraces;
3231
import java.util.Collections;
3332
import java.util.List;
3433
import java.util.Map;
@@ -350,9 +349,9 @@ public DDSpan addThrowable(Throwable error, byte errorPriority) {
350349
// or warming up - capturing the stack trace and keeping
351350
// the trace may exacerbate existing problems.
352351
setError(true, errorPriority);
353-
final StringWriter errorString = new StringWriter();
354-
error.printStackTrace(new PrintWriter(errorString));
355-
setTag(DDTags.ERROR_STACK, errorString.toString());
352+
setTag(
353+
DDTags.ERROR_STACK,
354+
StackTraces.getStackTrace(error, Config.get().getStackTraceLengthLimit()));
356355
}
357356

358357
setTag(DDTags.ERROR_MSG, message);

‎dd-trace-core/src/main/java/datadog/trace/core/util/StackTraces.java‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package datadog.trace.core.util;
2+
3+
import java.io.BufferedReader;
4+
import java.io.PrintWriter;
5+
import java.io.StringReader;
6+
import java.io.StringWriter;
7+
import java.util.ArrayList;
8+
import java.util.List;
9+
import java.util.regex.Matcher;
10+
import java.util.regex.Pattern;
11+
import java.util.stream.Collectors;
12+
13+
public final class StackTraces {
14+
private StackTraces() {}
15+
16+
public static String getStackTrace(Throwable t, int maxChars) {
17+
StringWriter sw = new StringWriter();
18+
t.printStackTrace(new PrintWriter(sw));
19+
String trace = sw.toString();
20+
try {
21+
return truncate(trace, maxChars);
22+
} catch (Exception e) {
23+
// If something goes wrong, return the original trace
24+
return trace;
25+
}
26+
}
27+
28+
static String truncate(String trace, int maxChars) {
29+
if (trace.length() <= maxChars) {
30+
return trace;
31+
}
32+
33+
trace = abbreviatePackageNames(trace);
34+
if (trace.length() <= maxChars) {
35+
return trace;
36+
}
37+
38+
trace = removeStackTraceMiddleForEachException(trace);
39+
if (trace.length() <= maxChars) {
40+
return trace;
41+
}
42+
43+
/* last-ditch centre cut to guarantee the limit */
44+
String cutMessage = "\t... trace centre-cut to " + maxChars + " chars ...";
45+
int retainedLength = maxChars - cutMessage.length() - 2; // 2 for the newlines
46+
int half = retainedLength / 2;
47+
return trace.substring(0, half)
48+
+ System.lineSeparator()
49+
+ cutMessage
50+
+ System.lineSeparator()
51+
+ trace.substring(trace.length() - (retainedLength - half));
52+
}
53+
54+
private static final Pattern FRAME = Pattern.compile("^\\s*at ([^(]+)(\\(.*)$");
55+
56+
private static String abbreviatePackageNames(String trace) {
57+
StringBuilder sb = new StringBuilder(trace.length());
58+
new BufferedReader(new StringReader(trace))
59+
.lines()
60+
.forEach(
61+
line -> {
62+
Matcher m = FRAME.matcher(line);
63+
if (m.matches()) {
64+
sb.append("\tat ").append(abbreviatePackageName(m.group(1))).append(m.group(2));
65+
} else {
66+
sb.append(line);
67+
}
68+
sb.append(System.lineSeparator());
69+
});
70+
return sb.toString();
71+
}
72+
73+
/**
74+
* Abbreviates only the package part of a fully qualified class name with member. For example,
75+
* "com.myorg.MyClass.myMethod" to "c.m.MyClass.myMethod". If there is no package (e.g.
76+
* "MyClass.myMethod"), returns the input unchanged.
77+
*/
78+
private static String abbreviatePackageName(String fqcnWithMember) {
79+
int lastDot = fqcnWithMember.lastIndexOf('.');
80+
if (lastDot < 0) {
81+
return fqcnWithMember;
82+
}
83+
int preClassDot = fqcnWithMember.lastIndexOf('.', lastDot - 1);
84+
if (preClassDot < 0) {
85+
return fqcnWithMember;
86+
}
87+
String packagePart = fqcnWithMember.substring(0, preClassDot);
88+
String classAndAfter = fqcnWithMember.substring(preClassDot + 1);
89+
90+
StringBuilder sb = new StringBuilder(fqcnWithMember.length());
91+
int segmentStart = 0;
92+
for (int i = 0; i <= packagePart.length(); i++) {
93+
if (i == packagePart.length() || packagePart.charAt(i) == '.') {
94+
sb.append(packagePart.charAt(segmentStart)).append('.');
95+
segmentStart = i + 1;
96+
}
97+
}
98+
sb.append(classAndAfter);
99+
return sb.toString();
100+
}
101+
102+
private static final int HEAD_LINES = 8, TAIL_LINES = 4;
103+
104+
/**
105+
* Removes lines from the middle of each exception stack trace, leaving {@link
106+
* StackTraces#HEAD_LINES} lines at the beginning and {@link StackTraces#TAIL_LINES} lines at the
107+
* end
108+
*/
109+
private static String removeStackTraceMiddleForEachException(String trace) {
110+
List<String> lines =
111+
new BufferedReader(new StringReader(trace)).lines().collect(Collectors.toList());
112+
List<String> out = new ArrayList<>(lines.size());
113+
int i = 0;
114+
while (i < lines.size()) {
115+
out.add(lines.get(i++)); // "Exception ..." / "Caused by: ..."
116+
int start = i;
117+
while (i < lines.size() && lines.get(i).startsWith("\tat")) {
118+
i++;
119+
}
120+
121+
int total = i - start;
122+
123+
int keepHead = Math.min(HEAD_LINES, total);
124+
for (int j = 0; j < keepHead; j++) {
125+
out.add(lines.get(start + j));
126+
}
127+
128+
int keepTail = Math.min(TAIL_LINES, total - keepHead);
129+
int skipped = total - keepHead - keepTail;
130+
if (skipped > 0) {
131+
out.add("\t... " + skipped + " trimmed ...");
132+
}
133+
134+
for (int j = total - keepTail; j < total; j++) {
135+
out.add(lines.get(start + j));
136+
}
137+
138+
// "... n more" continuation markers
139+
if (i < lines.size() && lines.get(i).startsWith("\t...")) {
140+
out.add(lines.get(i++));
141+
}
142+
}
143+
return String.join(System.lineSeparator(), out) + System.lineSeparator();
144+
}
145+
}

‎dd-trace-core/src/test/groovy/datadog/trace/core/util/StackTracesTest.groovy‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
package datadog.trace.core.util
2+
3+
4+
import spock.lang.Specification
5+
6+
class StackTracesTest extends Specification {
7+
8+
def "test stack trace truncation: #limit"() {
9+
given:
10+
def trace = """
11+
Exception in thread "main" com.example.app.MainException: Unexpected application failure
12+
at com.example.app.Application\$Runner.run(Application.java:102)
13+
at com.example.app.Application.lambda\$start\$0(Application.java:75)
14+
at java.base/java.util.Optional.ifPresent(Optional.java:178)
15+
at com.example.app.Application.start(Application.java:74)
16+
at com.example.app.Main.main(Main.java:21)
17+
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
18+
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
19+
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
20+
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
21+
at com.example.launcher.Bootstrap.run(Bootstrap.java:39)
22+
at com.example.launcher.Bootstrap.main(Bootstrap.java:25)
23+
at com.example.internal.\$Proxy1.start(Unknown Source)
24+
at com.example.internal.Initializer\$1.run(Initializer.java:47)
25+
at com.example.internal.Initializer.lambda\$init\$0(Initializer.java:38)
26+
at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Executors.java:515)
27+
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
28+
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
29+
at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628)
30+
at java.base/java.lang.Thread.run(Thread.java:834)
31+
at com.example.synthetic.Helper.access\$100(Helper.java:14)
32+
Caused by: com.example.db.DatabaseException: Failed to load user data
33+
at com.example.db.UserDao.findUser(UserDao.java:88)
34+
at com.example.db.UserDao.lambda\$cacheLookup\$1(UserDao.java:64)
35+
at com.example.cache.Cache\$Entry.computeIfAbsent(Cache.java:111)
36+
at com.example.cache.Cache.get(Cache.java:65)
37+
at com.example.service.UserService.loadUser(UserService.java:42)
38+
at com.example.service.UserService.lambda\$loadUserAsync\$0(UserService.java:36)
39+
at com.example.util.SafeRunner.run(SafeRunner.java:27)
40+
at java.base/java.util.concurrent.Executors\$RunnableAdapter.call(Executors.java:515)
41+
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
42+
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
43+
at java.base/java.util.concurrent.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628)
44+
at java.base/java.lang.Thread.run(Thread.java:834)
45+
at com.example.synthetic.UserDao\$1.run(UserDao.java:94)
46+
at com.example.synthetic.UserDao\$1.run(UserDao.java:94)
47+
at com.example.db.ConnectionManager.getConnection(ConnectionManager.java:55)
48+
Suppressed: java.io.IOException: Resource cleanup failed
49+
at com.example.util.ResourceManager.close(ResourceManager.java:23)
50+
at com.example.service.UserService.lambda\$loadUserAsync\$0(UserService.java:38)
51+
... 3 more
52+
Caused by: java.nio.file.AccessDeniedException: /data/user/config.json
53+
at java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:90)
54+
at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)
55+
at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116)
56+
at java.base/sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219)
57+
at java.base/java.nio.file.Files.newByteChannel(Files.java:375)
58+
at java.base/java.nio.file.Files.newInputStream(Files.java:489)
59+
at com.example.util.FileUtils.readFile(FileUtils.java:22)
60+
at com.example.util.ResourceManager.close(ResourceManager.java:21)
61+
... 3 more
62+
"""
63+
64+
expect:
65+
StackTraces.truncate(trace, limit) == expected
66+
67+
where:
68+
limit | expected
69+
1000 | """
70+
Exception in thread "main" com.example.app.MainException: Unexpected application failure
71+
at c.e.a.Application\$Runner.run(Application.java:102)
72+
at c.e.a.Application.lambda\$start\$0(Application.java:75)
73+
at j.b.u.Optional.ifPresent(Optional.java:178)
74+
at c.e.a.Application.start(Application.java:74)
75+
at c.e.a.Main.main(Main.java:21)
76+
at s.r.NativeMethodAccessorImpl.invoke0(Native Method)
77+
at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
78+
at s.r.Delegat
79+
... trace centre-cut to 1000 chars ...
80+
ToIOException(UnixException.java:90)
81+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111)
82+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116)
83+
at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219)
84+
at j.b.n.f.Files.newByteChannel(Files.java:375)
85+
at j.b.n.f.Files.newInputStream(Files.java:489)
86+
at c.e.u.FileUtils.readFile(FileUtils.java:22)
87+
at c.e.u.ResourceManager.close(ResourceManager.java:21)
88+
... 3 more
89+
"""
90+
2500 | """
91+
Exception in thread "main" com.example.app.MainException: Unexpected application failure
92+
at c.e.a.Application\$Runner.run(Application.java:102)
93+
at c.e.a.Application.lambda\$start\$0(Application.java:75)
94+
at j.b.u.Optional.ifPresent(Optional.java:178)
95+
at c.e.a.Application.start(Application.java:74)
96+
at c.e.a.Main.main(Main.java:21)
97+
at s.r.NativeMethodAccessorImpl.invoke0(Native Method)
98+
at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
99+
at s.r.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
100+
... 8 trimmed ...
101+
at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
102+
at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628)
103+
at j.b.l.Thread.run(Thread.java:834)
104+
at c.e.s.Helper.access\$100(Helper.java:14)
105+
Caused by: com.example.db.DatabaseException: Failed to load user data
106+
at c.e.d.UserDao.findUser(UserDao.java:88)
107+
at c.e.d.UserDao.lambda\$cacheLookup\$1(UserDao.java:64)
108+
at c.e.c.Cache\$Entry.computeIfAbsent(Cache.java:111)
109+
at c.e.c.Cache.get(Cache.java:65)
110+
at c.e.s.UserService.loadUser(UserService.java:42)
111+
at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:36)
112+
at c.e.u.SafeRunner.run(SafeRunner.java:27)
113+
at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515)
114+
... 3 trimmed ...
115+
at j.b.l.Thread.run(Thread.java:834)
116+
at c.e.s.UserDao\$1.run(UserDao.java:94)
117+
at c.e.s.UserDao\$1.run(UserDao.java:94)
118+
at c.e.d.ConnectionManager.getConnection(ConnectionManager.java:55)
119+
Suppressed: java.io.IOException: Resource cleanup failed
120+
at c.e.u.ResourceManager.close(ResourceManager.java:23)
121+
at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:38)
122+
... 3 more
123+
Caused by: java.nio.file.AccessDeniedException: /data/user/config.json
124+
at j.b.n.f.UnixException.translateToIOException(UnixException.java:90)
125+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111)
126+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116)
127+
at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219)
128+
at j.b.n.f.Files.newByteChannel(Files.java:375)
129+
at j.b.n.f.Files.newInputStream(Files.java:489)
130+
at c.e.u.FileUtils.readFile(FileUtils.java:22)
131+
at c.e.u.ResourceManager.close(ResourceManager.java:21)
132+
... 3 more
133+
"""
134+
3000 | """
135+
Exception in thread "main" com.example.app.MainException: Unexpected application failure
136+
at c.e.a.Application\$Runner.run(Application.java:102)
137+
at c.e.a.Application.lambda\$start\$0(Application.java:75)
138+
at j.b.u.Optional.ifPresent(Optional.java:178)
139+
at c.e.a.Application.start(Application.java:74)
140+
at c.e.a.Main.main(Main.java:21)
141+
at s.r.NativeMethodAccessorImpl.invoke0(Native Method)
142+
at s.r.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
143+
at s.r.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
144+
at j.b.l.r.Method.invoke(Method.java:566)
145+
at c.e.l.Bootstrap.run(Bootstrap.java:39)
146+
at c.e.l.Bootstrap.main(Bootstrap.java:25)
147+
at c.e.i.\$Proxy1.start(Unknown Source)
148+
at c.e.i.Initializer\$1.run(Initializer.java:47)
149+
at c.e.i.Initializer.lambda\$init\$0(Initializer.java:38)
150+
at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515)
151+
at j.b.u.c.FutureTask.run(FutureTask.java:264)
152+
at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
153+
at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628)
154+
at j.b.l.Thread.run(Thread.java:834)
155+
at c.e.s.Helper.access\$100(Helper.java:14)
156+
Caused by: com.example.db.DatabaseException: Failed to load user data
157+
at c.e.d.UserDao.findUser(UserDao.java:88)
158+
at c.e.d.UserDao.lambda\$cacheLookup\$1(UserDao.java:64)
159+
at c.e.c.Cache\$Entry.computeIfAbsent(Cache.java:111)
160+
at c.e.c.Cache.get(Cache.java:65)
161+
at c.e.s.UserService.loadUser(UserService.java:42)
162+
at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:36)
163+
at c.e.u.SafeRunner.run(SafeRunner.java:27)
164+
at j.b.u.c.Executors\$RunnableAdapter.call(Executors.java:515)
165+
at j.b.u.c.FutureTask.run(FutureTask.java:264)
166+
at j.b.u.c.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
167+
at j.b.u.c.ThreadPoolExecutor\$Worker.run(ThreadPoolExecutor.java:628)
168+
at j.b.l.Thread.run(Thread.java:834)
169+
at c.e.s.UserDao\$1.run(UserDao.java:94)
170+
at c.e.s.UserDao\$1.run(UserDao.java:94)
171+
at c.e.d.ConnectionManager.getConnection(ConnectionManager.java:55)
172+
Suppressed: java.io.IOException: Resource cleanup failed
173+
at c.e.u.ResourceManager.close(ResourceManager.java:23)
174+
at c.e.s.UserService.lambda\$loadUserAsync\$0(UserService.java:38)
175+
... 3 more
176+
Caused by: java.nio.file.AccessDeniedException: /data/user/config.json
177+
at j.b.n.f.UnixException.translateToIOException(UnixException.java:90)
178+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:111)
179+
at j.b.n.f.UnixException.rethrowAsIOException(UnixException.java:116)
180+
at j.b.n.f.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219)
181+
at j.b.n.f.Files.newByteChannel(Files.java:375)
182+
at j.b.n.f.Files.newInputStream(Files.java:489)
183+
at c.e.u.FileUtils.readFile(FileUtils.java:22)
184+
at c.e.u.ResourceManager.close(ResourceManager.java:21)
185+
... 3 more
186+
"""
187+
}
188+
}

‎internal-api/src/main/java/datadog/trace/api/Config.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,8 @@ public static String getHostName() {
579579

580580
private final boolean jdkSocketEnabled;
581581

582+
private final int stackTraceLengthLimit;
583+
582584
// Read order: System Properties -> Env Variables, [-> properties file], [-> default value]
583585
private Config() {
584586
this(ConfigProvider.createDefault());
@@ -2037,6 +2039,13 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
20372039

20382040
this.jdkSocketEnabled = configProvider.getBoolean(JDK_SOCKET_ENABLED, true);
20392041

2042+
int defaultStackTraceLengthLimit =
2043+
instrumenterConfig.isCiVisibilityEnabled()
2044+
? 5000 // EVP limit
2045+
: Integer.MAX_VALUE; // no effective limit (old behavior)
2046+
this.stackTraceLengthLimit =
2047+
configProvider.getInteger(STACK_TRACE_LENGTH_LIMIT, defaultStackTraceLengthLimit);
2048+
20402049
log.debug("New instance: {}", this);
20412050
}
20422051

@@ -3659,6 +3668,10 @@ public boolean isJdkSocketEnabled() {
36593668
return jdkSocketEnabled;
36603669
}
36613670

3671+
public int getStackTraceLengthLimit() {
3672+
return stackTraceLengthLimit;
3673+
}
3674+
36623675
/** @return A map of tags to be applied only to the local application root span. */
36633676
public Map<String, Object> getLocalRootSpanTags() {
36643677
final Map<String, String> runtimeTags = getRuntimeTags();

0 commit comments

Comments
 (0)