Skip to content

Commit 9435a59

Browse files
jpbempeljpbempel
authored
Add support for filtering shaded third-party libs (#8612)
Add filtering on shading identifiers Add option to add more shading idententifiers
1 parent cc427a4 commit 9435a59

File tree

11 files changed

+166
-16
lines changed

11 files changed

+166
-16
lines changed

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ThirdPartyLibraries.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
import java.io.IOException;
88
import java.io.InputStream;
99
import java.io.InputStreamReader;
10+
import java.util.Arrays;
1011
import java.util.HashSet;
1112
import java.util.List;
1213
import java.util.Set;
1314
import java.util.stream.Collectors;
15+
import java.util.stream.Stream;
1416
import org.slf4j.Logger;
1517
import org.slf4j.LoggerFactory;
1618

@@ -22,6 +24,20 @@ public class ThirdPartyLibraries {
2224
private static final JsonAdapter<InternalConfig> ADAPTER =
2325
new Moshi.Builder().build().adapter(InternalConfig.class);
2426
private static final String FILE_NAME = "/third_party_libraries.json";
27+
private static final Set<String> DEFAULT_SHADING_IDENTIFIERS =
28+
new HashSet<>(
29+
Arrays.asList(
30+
"shaded",
31+
"thirdparty",
32+
"dependencies",
33+
"relocated",
34+
"bundled",
35+
"embedded",
36+
"vendor",
37+
"repackaged",
38+
"shadow",
39+
"shim",
40+
"wrapper"));
2541

2642
private ThirdPartyLibraries() {}
2743

@@ -46,6 +62,13 @@ public Set<String> getThirdPartyExcludes(Config config) {
4662
.collect(Collectors.toSet());
4763
}
4864

65+
public Set<String> getShadingIdentifiers(Config config) {
66+
Stream<String> configStream =
67+
config.getThirdPartyShadingIdentifiers().stream().filter(s -> !s.isEmpty());
68+
return Stream.concat(configStream, DEFAULT_SHADING_IDENTIFIERS.stream())
69+
.collect(Collectors.toSet());
70+
}
71+
4972
// Add a*, b*, c*, ..., z* to the exclude trie in ClassNameFiltering. Simply adding * does not
5073
// work.
5174
private static Set<String> getExcludeAll() {

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.datadog.debugger.symbol;
22

33
import datadog.trace.bootstrap.debugger.DebuggerContext.ClassNameFilter;
4+
import datadog.trace.util.Strings;
45
import java.lang.instrument.ClassFileTransformer;
56
import java.security.ProtectionDomain;
67
import org.slf4j.Logger;
@@ -34,7 +35,7 @@ public byte[] transform(
3435
// Don't parse our own classes to avoid duplicate class definition
3536
return null;
3637
}
37-
if (classNameFiltering.isExcluded(className)) {
38+
if (classNameFiltering.isExcluded(Strings.getClassName(className))) {
3839
return null;
3940
}
4041
symbolAggregator.parseClass(className, classfileBuffer, protectionDomain);

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ClassNameFiltering.java

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,36 +14,60 @@ public class ClassNameFiltering implements ClassNameFilter {
1414

1515
private final ClassNameTrie includeTrie;
1616
private final ClassNameTrie excludeTrie;
17+
private final ClassNameTrie shadingTrie;
1718

1819
public ClassNameFiltering(Config config) {
1920
this(
2021
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config),
21-
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config));
22+
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config),
23+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(config));
2224
}
2325

2426
public ClassNameFiltering(Set<String> excludes) {
25-
this(excludes, Collections.emptySet());
27+
this(excludes, Collections.emptySet(), Collections.emptySet());
2628
}
2729

28-
public ClassNameFiltering(Set<String> excludes, Set<String> includes) {
30+
public ClassNameFiltering(
31+
Set<String> excludes, Set<String> includes, Set<String> shadingIdentifiers) {
2932
ClassNameTrie.Builder excludeBuilder = new ClassNameTrie.Builder();
3033
excludes.forEach(s -> excludeBuilder.put(s + "*", 1));
3134
this.excludeTrie = excludeBuilder.buildTrie();
3235
ClassNameTrie.Builder includeBuilder = new ClassNameTrie.Builder();
3336
includes.forEach(s -> includeBuilder.put(s + "*", 1));
3437
this.includeTrie = includeBuilder.buildTrie();
38+
ClassNameTrie.Builder shadingBuilder = new ClassNameTrie.Builder();
39+
shadingIdentifiers.forEach(s -> shadingBuilder.put(s + "*", 1));
40+
this.shadingTrie = shadingBuilder.buildTrie();
3541
}
3642

43+
// className is the fully qualified class name with '.' (Java type) notation
3744
public boolean isExcluded(String className) {
38-
return (includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0)
45+
int shadedIdx = shadedIndexOf(className);
46+
shadedIdx = Math.max(shadedIdx, 0);
47+
return (includeTrie.apply(className, shadedIdx) < 0
48+
&& excludeTrie.apply(className, shadedIdx) > 0)
3949
|| isLambdaProxyClass(className);
4050
}
4151

4252
static boolean isLambdaProxyClass(String className) {
4353
return LAMBDA_PROXY_CLASS_PATTERN.matcher(className).matches();
4454
}
4555

56+
int shadedIndexOf(String className) {
57+
int idx = 0;
58+
int previousIdx = 0;
59+
while ((idx = className.indexOf('.', previousIdx)) > 0) {
60+
if (shadingTrie.apply(className, previousIdx) > 0) {
61+
return idx + 1;
62+
}
63+
idx++;
64+
previousIdx = idx;
65+
}
66+
return -1;
67+
}
68+
4669
public static ClassNameFiltering allowAll() {
47-
return new ClassNameFiltering(Collections.emptySet(), Collections.emptySet());
70+
return new ClassNameFiltering(
71+
Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
4872
}
4973
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ThirdPartyLibrariesTest.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.datadog.debugger.agent;
22

3+
import static org.junit.jupiter.api.Assertions.assertEquals;
34
import static org.junit.jupiter.api.Assertions.assertFalse;
45
import static org.junit.jupiter.api.Assertions.assertNotNull;
56
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -20,6 +21,7 @@ class ThirdPartyLibrariesTest {
2021
void setUp() {
2122
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
2223
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
24+
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
2325
}
2426

2527
@Test
@@ -29,7 +31,6 @@ void testGetExcludesContainsDefaultExclude() {
2931

3032
@Test
3133
void testGetExcludesWithExplicitExclude() {
32-
3334
when(mockConfig.getThirdPartyIncludes())
3435
.thenReturn(Collections.singleton("com.datadog.debugger"));
3536
assertTrue(
@@ -72,4 +73,22 @@ void testGetExcludeAll() {
7273
Set<String> excludeAll = ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(null);
7374
for (char c : ThirdPartyLibraries.ALPHABET) assertTrue(excludeAll.contains(String.valueOf(c)));
7475
}
76+
77+
@Test
78+
void testEmptyStrings() {
79+
int expectedIncludeDefaultSize =
80+
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size();
81+
int expectedShadingDefaultSize =
82+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size();
83+
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.singleton(""));
84+
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.singleton(""));
85+
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.singleton(""));
86+
assertEquals(
87+
expectedIncludeDefaultSize,
88+
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size());
89+
assertTrue(ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(mockConfig).isEmpty());
90+
assertEquals(
91+
expectedShadingDefaultSize,
92+
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size());
93+
}
7594
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymDBEnablementTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,8 @@ public void noDuplicateSymbolExtraction() {
169169
ClassNameFiltering classNameFiltering =
170170
new ClassNameFiltering(
171171
Collections.singleton("org.springframework."),
172-
Collections.singleton("com.datadog.debugger."));
172+
Collections.singleton("com.datadog.debugger."),
173+
Collections.emptySet());
173174
SymbolAggregator symbolAggregator = new SymbolAggregator(classNameFiltering, mockSymbolSink, 1);
174175
SymDBEnablement symDBEnablement =
175176
new SymDBEnablement(instr, config, symbolAggregator, classNameFiltering);

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -982,7 +982,8 @@ private SymbolExtractionTransformer createTransformer(
982982
return createTransformer(
983983
symbolSink,
984984
symbolFlushThreshold,
985-
new ClassNameFiltering(TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE)));
985+
new ClassNameFiltering(
986+
TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE), Collections.emptySet()));
986987
}
987988

988989
private SymbolExtractionTransformer createTransformer(

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/util/ClassNameFilteringTest.java

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
import static org.mockito.Mockito.mock;
77
import static org.mockito.Mockito.when;
88

9-
import com.datadog.debugger.agent.ThirdPartyLibraries;
109
import datadog.trace.api.Config;
10+
import java.util.Arrays;
1111
import java.util.Collections;
1212
import java.util.HashSet;
1313
import java.util.Set;
@@ -52,15 +52,18 @@ public void testIncludeOverridesExclude() {
5252
ClassNameFiltering classNameFiltering =
5353
new ClassNameFiltering(
5454
Collections.singleton("com.datadog.debugger"),
55-
Collections.singleton("com.datadog.debugger"));
55+
Collections.singleton("com.datadog.debugger"),
56+
Collections.emptySet());
5657
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
5758
}
5859

5960
@Test
6061
public void testIncludePrefixOverridesExclude() {
6162
ClassNameFiltering classNameFiltering =
6263
new ClassNameFiltering(
63-
Collections.singleton("com.datadog.debugger"), Collections.singleton("com.datadog"));
64+
Collections.singleton("com.datadog.debugger"),
65+
Collections.singleton("com.datadog"),
66+
Collections.emptySet());
6467
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
6568
}
6669

@@ -69,7 +72,8 @@ public void testIncludeSomeExcludeSome() {
6972
ClassNameFiltering classNameFiltering =
7073
new ClassNameFiltering(
7174
Stream.of("com.datadog.debugger", "org.junit").collect(Collectors.toSet()),
72-
Collections.singleton("com.datadog.debugger"));
75+
Collections.singleton("com.datadog.debugger"),
76+
Collections.emptySet());
7377
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
7478
assertTrue(classNameFiltering.isExcluded("org.junit.FooBar"));
7579
}
@@ -88,11 +92,25 @@ public void testExcludeDefaults(String input) {
8892
Config config = mock(Config.class);
8993
when(config.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
9094
when(config.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
91-
ClassNameFiltering classNameFiltering =
92-
new ClassNameFiltering(ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config));
95+
when(config.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
96+
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
9397
assertTrue(classNameFiltering.isExcluded(input));
9498
}
9599

100+
@Test
101+
public void testShaded() {
102+
Config config = mock(Config.class);
103+
when(config.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
104+
when(config.getThirdPartyIncludes())
105+
.thenReturn(new HashSet<>(Arrays.asList("com.google", "org.junit")));
106+
when(config.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
107+
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
108+
assertTrue(classNameFiltering.isExcluded("com.google.FooBar"));
109+
assertTrue(classNameFiltering.isExcluded("shaded.com.google.FooBar"));
110+
assertFalse(classNameFiltering.isExcluded("com.example.shaded.com.example.FooBar"));
111+
assertTrue(classNameFiltering.isExcluded("com.example.shaded.com.google.FooBar"));
112+
}
113+
96114
@Test
97115
void lambdaProxyClasses() {
98116
// jdk8: at

‎dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public final class DebuggerConfig {
6161
public static final String DISTRIBUTED_DEBUGGER_ENABLED = "distributed.debugger.enabled";
6262
public static final String THIRD_PARTY_INCLUDES = "third.party.includes";
6363
public static final String THIRD_PARTY_EXCLUDES = "third.party.excludes";
64+
public static final String THIRD_PARTY_SHADING_IDENTIFIERS = "third.party.shading.identifiers";
6465

6566
private DebuggerConfig() {}
6667
}

‎internal-api/src/main/java/datadog/trace/api/Config.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,7 @@ public static String getHostName() {
430430

431431
private final Set<String> debuggerThirdPartyIncludes;
432432
private final Set<String> debuggerThirdPartyExcludes;
433+
private final Set<String> debuggerShadingIdentifiers;
433434

434435
private final boolean awsPropagationEnabled;
435436
private final boolean sqsPropagationEnabled;
@@ -1735,6 +1736,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
17351736

17361737
debuggerThirdPartyIncludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_INCLUDES));
17371738
debuggerThirdPartyExcludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_EXCLUDES));
1739+
debuggerShadingIdentifiers =
1740+
tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_SHADING_IDENTIFIERS));
17381741

17391742
awsPropagationEnabled = isPropagationEnabled(true, "aws", "aws-sdk");
17401743
sqsPropagationEnabled = isPropagationEnabled(true, "sqs");
@@ -3300,6 +3303,10 @@ public Set<String> getThirdPartyExcludes() {
33003303
return debuggerThirdPartyExcludes;
33013304
}
33023305

3306+
public Set<String> getThirdPartyShadingIdentifiers() {
3307+
return debuggerShadingIdentifiers;
3308+
}
3309+
33033310
private String getFinalDebuggerBaseUrl() {
33043311
if (agentUrl.startsWith("unix:")) {
33053312
// provide placeholder agent URL, in practice we'll be tunnelling over UDS

‎internal-api/src/main/java/datadog/trace/util/ClassNameTrie.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,17 @@ public int apply(String key) {
117117
return apply(trieData, longJumps, key);
118118
}
119119

120+
public int apply(String key, int fromIndex) {
121+
return apply(trieData, longJumps, key, fromIndex);
122+
}
123+
120124
public static int apply(char[] data, int[] longJumps, String key) {
125+
return apply(data, longJumps, key, 0);
126+
}
127+
128+
public static int apply(char[] data, int[] longJumps, String key, int fromIndex) {
121129
int keyLength = key.length();
122-
int keyIndex = 0;
130+
int keyIndex = fromIndex;
123131
int dataIndex = 0;
124132
int result = -1;
125133

@@ -840,6 +848,14 @@ private static void generateJavaFile(
840848
}
841849
lines.add(" }");
842850
lines.add("");
851+
lines.add(" public static int apply(String key, int fromIndex) {");
852+
if (hasLongJumps) {
853+
lines.add(" return ClassNameTrie.apply(TRIE_DATA, LONG_JUMPS, key, fromIndex);");
854+
} else {
855+
lines.add(" return ClassNameTrie.apply(TRIE_DATA, null, key, fromIndex);");
856+
}
857+
lines.add(" }");
858+
lines.add("");
843859
lines.add(" private " + className + "() {}");
844860
lines.add("}");
845861
Files.write(javaPath, lines, StandardCharsets.UTF_8);

‎internal-api/src/test/groovy/datadog/trace/util/ClassNameTrieTest.groovy

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,45 @@ class ClassNameTrieTest extends DDSpecification {
4444
// spotless:on
4545
}
4646

47+
def 'test class name "#key" mapping with fromIndex'() {
48+
when:
49+
int value = TestClassNamesTrie.apply(key, "garbage.".length())
50+
then:
51+
value == expected
52+
where:
53+
// spotless:off
54+
key | expected
55+
'garbage.One' | 1
56+
'garbage.com.Two' | 2
57+
'garbage.com.foo.Three' | 3
58+
'garbage.company.foo.Four' | 4
59+
'garbage.com.foobar.Five' | 5
60+
'garbage.company.foobar.Six' | 6
61+
'garbage.company.foobar.Sixty' | 60
62+
'garbage.com.f' | 7
63+
'garbage.com.foo.a' | 8
64+
'garbage.com.foobar.b' | 9
65+
'garbage.company.f' | 10
66+
'garbage.company.foo.a' | 11
67+
'garbage.company.foobar.S' | 12
68+
'garbage.com.Two$f' | 13
69+
'garbage.foobar.Two$b' | 14
70+
'garbage.' | -1
71+
'garbage.O' | -1
72+
'garbage._' | -1
73+
'garbage.On' | -1
74+
'garbage.O_' | -1
75+
'garbage.On_' | -1
76+
'garbage.OneNoMatch' | -1
77+
'garbage.com.Twos' | 7
78+
'garbage.com.foo.Threes' | 8
79+
'garbage.com.foobar.Fives' | 9
80+
'garbage.foobar.Thre' | -1
81+
'garbage.foobar.Three' | 15
82+
'garbage.foobar.ThreeMore' | 15
83+
// spotless:on
84+
}
85+
4786
def 'test internal name "#key" mapping'() {
4887
when:
4988
int value = TestClassNamesTrie.apply(key)

0 commit comments

Comments
 (0)