Skip to content

Commit f32dcef

Browse files
daniel-mohedanodaniel-mohedano
committed
fix: abstract repeated smoke test logic
1 parent a56d10f commit f32dcef

File tree

6 files changed

+111
-148
lines changed

6 files changed

+111
-148
lines changed

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,84 @@
11
package datadog.trace.civisibility
22

3+
import datadog.trace.api.Config
34
import datadog.trace.api.civisibility.config.TestFQN
5+
import datadog.trace.api.config.CiVisibilityConfig
6+
import datadog.trace.api.config.GeneralConfig
7+
import datadog.trace.util.Strings
48
import spock.lang.Specification
9+
import spock.util.environment.Jvm
510

611
abstract class CiVisibilitySmokeTest extends Specification {
712
static final List<String> SMOKE_IGNORED_TAGS = ["content.meta.['_dd.integration']"]
813

14+
protected static final String AGENT_JAR = System.getProperty("datadog.smoketest.agent.shadowJar.path")
15+
protected static final String TEST_ENVIRONMENT_NAME = "integration-test"
16+
protected static final String JAVAC_PLUGIN_VERSION = Config.get().ciVisibilityCompilerPluginVersion
17+
protected static final String JACOCO_PLUGIN_VERSION = Config.get().ciVisibilityJacocoPluginVersion
18+
19+
private static final Map<String,String> DEFAULT_TRACER_CONFIG = defaultJvmArguments()
20+
21+
protected static String buildJavaHome() {
22+
if (Jvm.current.isJava8()) {
23+
return System.getenv("JAVA_8_HOME")
24+
}
25+
return System.getenv("JAVA_" + Jvm.current.getJavaSpecificationVersion() + "_HOME")
26+
}
27+
28+
protected static String javaPath() {
29+
final String separator = System.getProperty("file.separator")
30+
return "${buildJavaHome()}${separator}bin${separator}java"
31+
}
32+
33+
protected static String javacPath() {
34+
final String separator = System.getProperty("file.separator")
35+
return "${buildJavaHome()}${separator}bin${separator}javac"
36+
}
37+
38+
private static Map<String, String> defaultJvmArguments() {
39+
Map<String, String> argMap = new HashMap<>()
40+
argMap.put(GeneralConfig.TRACE_DEBUG, "true")
41+
argMap.put(GeneralConfig.ENV, TEST_ENVIRONMENT_NAME)
42+
argMap.put(CiVisibilityConfig.CIVISIBILITY_ENABLED, "true")
43+
argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED, "true")
44+
argMap.put(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED, "false")
45+
argMap.put(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED, "false")
46+
argMap.put(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES, "true")
47+
argMap.put(CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION, JAVAC_PLUGIN_VERSION)
48+
return argMap
49+
}
50+
51+
private static Map<String, String> buildJvmArgMap(String mockBackendIntakeUrl, String serviceName, Map<String, String> additionalArgs) {
52+
Map<String, String> argMap = new HashMap<>(DEFAULT_TRACER_CONFIG)
53+
argMap.put(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL, mockBackendIntakeUrl)
54+
argMap.put(CiVisibilityConfig.CIVISIBILITY_INTAKE_AGENTLESS_URL, mockBackendIntakeUrl)
55+
argMap.putAll(additionalArgs)
56+
57+
if (serviceName != null) {
58+
argMap.put(GeneralConfig.SERVICE_NAME, serviceName)
59+
}
60+
61+
return argMap
62+
}
63+
64+
protected List<String> buildJvmArguments(String mockBackendIntakeUrl, String serviceName, Map<String, String> additionalArgs) {
65+
List<String> arguments = []
66+
Map<String, String> argMap = buildJvmArgMap(mockBackendIntakeUrl, serviceName, additionalArgs)
67+
68+
// for convenience when debugging locally
69+
if (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_PARENT") != null) {
70+
arguments += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
71+
}
72+
if (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null) {
73+
argMap.put(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT, "5055")
74+
}
75+
76+
String agentArgs = argMap.collect { k, v -> "${Strings.propertyNameToSystemPropertyName(k)}=${v}" }.join(",")
77+
arguments += "-javaagent:${AGENT_JAR}=${agentArgs}".toString()
78+
79+
return arguments
80+
}
81+
982
protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List<Map<String, Object>> events, List<Map<String, Object>> coverages, List<String> additionalDynamicTags = []) {
1083
def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"]
1184

‎dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy‎

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
package datadog.smoketest
22

33
import datadog.environment.JavaVirtualMachine
4-
import datadog.trace.api.Config
54
import datadog.trace.api.config.CiVisibilityConfig
65
import datadog.trace.api.config.GeneralConfig
76
import datadog.trace.api.config.TraceInstrumentationConfig
8-
import datadog.trace.util.Strings
97
import org.gradle.testkit.runner.BuildResult
108
import org.gradle.testkit.runner.GradleRunner
119
import org.gradle.testkit.runner.TaskOutcome
@@ -26,12 +24,9 @@ import java.nio.file.Path
2624
class GradleDaemonSmokeTest extends AbstractGradleTest {
2725

2826
private static final String TEST_SERVICE_NAME = "test-gradle-service"
29-
private static final String TEST_ENVIRONMENT_NAME = "integration-test"
3027

3128
private static final int GRADLE_DISTRIBUTION_NETWORK_TIMEOUT = 30_000 // Gradle's default timeout is 10s
3229

33-
private static final String JACOCO_PLUGIN_VERSION = Config.get().ciVisibilityJacocoPluginVersion
34-
3530
// TODO: Gradle daemons started by the TestKit have an idle period of 3 minutes
3631
// so by the time tests finish, at least some of the daemons are still alive.
3732
// Because of that the temporary TestKit folder cannot be fully deleted
@@ -156,40 +151,28 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
156151
}
157152

158153
private void givenGradleProperties() {
159-
String agentShadowJar = System.getProperty("datadog.smoketest.agent.shadowJar.path")
160-
assert new File(agentShadowJar).isFile()
154+
assert new File(AGENT_JAR).isFile()
161155

162156
def ddApiKeyPath = testKitFolder.resolve(".dd.api.key")
163157
Files.write(ddApiKeyPath, "dummy".getBytes())
164158

165-
def gradleProperties =
166-
"org.gradle.jvmargs=" +
167-
// for convenience when debugging locally
168-
(System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_PARENT") != null ? "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005 " : "") +
169-
"-javaagent:${agentShadowJar}=" +
170-
// for convenience when debugging locally
171-
(System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null ? "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT)}=5055," : "") +
172-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.TRACE_DEBUG)}=true," +
173-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.ENV)}=${TEST_ENVIRONMENT_NAME}," +
174-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.SERVICE_NAME)}=${TEST_SERVICE_NAME}," +
175-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.API_KEY_FILE)}=${ddApiKeyPath.toAbsolutePath().toString()}," +
176-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ENABLED)}=true," +
177-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED)}=true," +
178-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED)}=false," +
179-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED)}=false," +
159+
def additionalArgs = [
160+
(GeneralConfig.API_KEY_FILE): ddApiKeyPath.toAbsolutePath().toString(),
161+
(CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION): JACOCO_PLUGIN_VERSION,
180162
/*
181-
* Some of the smoke tests (in particular the one with the Gradle plugin), are using Gradle Test Kit for their tests.
182-
* Gradle Test Kit needs to do a "chmod" when starting a Gradle Daemon.
183-
* This "chmod" operation is traced by datadog.trace.instrumentation.java.lang.ProcessImplInstrumentation and is reported as a span.
184-
* The problem is that the "chmod" only happens when running in CI (could be due to differences in OS or FS permissions),
185-
* so when running the tests locally, the "chmod" span is not there.
186-
* This causes the tests to fail because the number of reported traces is different.
187-
* To avoid this discrepancy between local and CI runs, we disable tracing instrumentations.
188-
*/
189-
"${Strings.propertyNameToSystemPropertyName(TraceInstrumentationConfig.TRACE_ENABLED)}=false," +
190-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION)}=$JACOCO_PLUGIN_VERSION," +
191-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}"
192-
163+
* Some of the smoke tests (in particular the one with the Gradle plugin), are using Gradle Test Kit for their tests.
164+
* Gradle Test Kit needs to do a "chmod" when starting a Gradle Daemon.
165+
* This "chmod" operation is traced by datadog.trace.instrumentation.java.lang.ProcessImplInstrumentation and is reported as a span.
166+
* The problem is that the "chmod" only happens when running in CI (could be due to differences in OS or FS permissions),
167+
* so when running the tests locally, the "chmod" span is not there.
168+
* This causes the tests to fail because the number of reported traces is different.
169+
* To avoid this discrepancy between local and CI runs, we disable tracing instrumentations.
170+
*/
171+
(TraceInstrumentationConfig.TRACE_ENABLED): "false"
172+
]
173+
def arguments = buildJvmArguments(mockBackend.intakeUrl, TEST_SERVICE_NAME, additionalArgs)
174+
175+
def gradleProperties = "org.gradle.jvmargs=${arguments.join(" ")}".toString()
193176
Files.write(testKitFolder.resolve("gradle.properties"), gradleProperties.getBytes())
194177
}
195178

‎dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleLauncherSmokeTest.groovy‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package datadog.smoketest
33
import datadog.communication.util.IOUtils
44
import datadog.trace.civisibility.utils.ShellCommandExecutor
55
import org.opentest4j.AssertionFailedError
6-
import spock.util.environment.Jvm
76

87
/**
98
* This test runs Gradle Launcher with the Java Tracer injected
@@ -13,7 +12,6 @@ class GradleLauncherSmokeTest extends AbstractGradleTest {
1312

1413
private static final int GRADLE_BUILD_TIMEOUT_MILLIS = 90_000
1514

16-
private static final String AGENT_JAR = System.getProperty("datadog.smoketest.agent.shadowJar.path")
1715
private static final String JAVA_HOME = buildJavaHome()
1816

1917
def "test Gradle Launcher injects tracer into Gradle Daemon: v#gradleVersion, cmd line - #gradleDaemonCmdLineParams"() {
@@ -78,12 +76,4 @@ class GradleLauncherSmokeTest extends AbstractGradleTest {
7876
}
7977
return true
8078
}
81-
82-
private static String buildJavaHome() {
83-
if (Jvm.current.isJava8()) {
84-
return System.getenv("JAVA_8_HOME")
85-
}
86-
87-
return System.getenv("JAVA_" + Jvm.current.getJavaSpecificationVersion() + "_HOME")
88-
}
8979
}

‎dd-smoke-tests/junit-console/src/test/groovy/datadog/smoketest/JUnitConsoleSmokeTest.groovy‎

Lines changed: 7 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ package datadog.smoketest
44
import datadog.trace.api.config.CiVisibilityConfig
55
import datadog.trace.api.config.GeneralConfig
66
import datadog.trace.civisibility.CiVisibilitySmokeTest
7-
import datadog.trace.util.Strings
87
import java.nio.file.FileVisitResult
98
import java.nio.file.Files
109
import java.nio.file.Path
@@ -18,15 +17,13 @@ import org.slf4j.LoggerFactory
1817
import spock.lang.AutoCleanup
1918
import spock.lang.Shared
2019
import spock.lang.TempDir
21-
import spock.util.environment.Jvm
2220

2321
class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
2422
// CodeNarc incorrectly thinks ".class" is unnecessary in getLogger
2523
@SuppressWarnings('UnnecessaryDotClass')
2624
private static final Logger LOGGER = LoggerFactory.getLogger(JUnitConsoleSmokeTest.class)
2725

2826
private static final String TEST_SERVICE_NAME = "test-headless-service"
29-
private static final String TEST_ENVIRONMENT_NAME = "integration-test"
3027

3128
private static final int PROCESS_TIMEOUT_SECS = 60
3229
private static final String JUNIT_CONSOLE_JAR_PATH = System.getProperty("datadog.smoketest.junit.console.jar.path")
@@ -54,8 +51,8 @@ class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
5451
assert compileCode == 0
5552

5653
def exitCode = whenRunningJUnitConsole([
57-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT)}=3" as String,
58-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL)}=${mockBackend.intakeUrl}" as String
54+
(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_COUNT): "3",
55+
(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL): mockBackend.intakeUrl
5956
],
6057
[:])
6158
assert exitCode == 1
@@ -168,7 +165,7 @@ class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
168165
return javaFiles
169166
}
170167

171-
private int whenRunningJUnitConsole(List<String> additionalAgentArgs, Map<String, String> additionalEnvVars) {
168+
private int whenRunningJUnitConsole(Map<String, String> additionalAgentArgs, Map<String, String> additionalEnvVars) {
172169
def processBuilder = createConsoleProcessBuilder(["execute"], additionalAgentArgs, additionalEnvVars)
173170

174171
processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF")
@@ -190,7 +187,7 @@ class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
190187
return p.exitValue()
191188
}
192189

193-
ProcessBuilder createConsoleProcessBuilder(List<String> consoleCommand, List<String> additionalAgentArgs, Map<String, String> additionalEnvVars) {
190+
ProcessBuilder createConsoleProcessBuilder(List<String> consoleCommand, Map<String, String> additionalAgentArgs, Map<String, String> additionalEnvVars) {
194191
assert new File(JUNIT_CONSOLE_JAR_PATH).isFile()
195192

196193
List<String> command = new ArrayList<>()
@@ -220,50 +217,9 @@ class JUnitConsoleSmokeTest extends CiVisibilitySmokeTest {
220217
return processBuilder
221218
}
222219

223-
String javaPath() {
224-
final String separator = System.getProperty("file.separator")
225-
return JAVA_HOME + separator + "bin" + separator + "java"
226-
}
227-
228-
String javacPath() {
229-
final String separator = System.getProperty("file.separator")
230-
return JAVA_HOME + separator + "bin" + separator + "javac"
231-
}
232-
233-
String javaToolOptions(List<String> additionalAgentArgs) {
234-
def arguments = []
235-
236-
if (System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_PARENT") != null) {
237-
// for convenience when debugging locally
238-
arguments += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
239-
}
240-
241-
def agentShadowJar = System.getProperty("datadog.smoketest.agent.shadowJar.path")
242-
def agentArgument = "-javaagent:${agentShadowJar}=" +
243-
// for convenience when debugging locally
244-
(System.getenv("DD_CIVISIBILITY_SMOKETEST_DEBUG_CHILD") != null ? "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT)}=5055," : "") +
245-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.TRACE_DEBUG)}=true," +
246-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.ENV)}=${TEST_ENVIRONMENT_NAME}," +
247-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ENABLED)}=true," +
248-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED)}=true," +
249-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED)}=false," +
250-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED)}=false," +
251-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}," +
252-
"${Strings.propertyNameToSystemPropertyName(GeneralConfig.SERVICE_NAME)}=${TEST_SERVICE_NAME}," +
253-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED)}=false," +
254-
"${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES)}=true,"
255-
256-
agentArgument += additionalAgentArgs.join(",")
257-
258-
arguments += agentArgument.toString()
259-
return arguments.join("\\ ")
260-
}
261-
262-
private static String buildJavaHome() {
263-
if (Jvm.current.isJava8()) {
264-
return System.getenv("JAVA_8_HOME")
265-
}
266-
return System.getenv("JAVA_" + Jvm.current.getJavaSpecificationVersion() + "_HOME")
220+
String javaToolOptions(Map<String, String> additionalAgentArgs) {
221+
additionalAgentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false")
222+
return buildJvmArguments(mockBackend.intakeUrl, TEST_SERVICE_NAME, additionalAgentArgs).join("\\ ")
267223
}
268224

269225
private static class StreamConsumer extends Thread {

‎dd-smoke-tests/junit-console/src/test/resources/test_junit_console_failed_test_replay/events.ftl‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
"runtime.vendor" : ${content_meta_runtime_vendor},
139139
"runtime.version" : ${content_meta_runtime_version},
140140
"span.kind" : "test",
141+
"test.failure_suppressed" : "true",
141142
"test.framework" : "junit5",
142143
"test.framework_version" : "5.13.4",
143144
"test.module" : "test-headless-service",
@@ -208,6 +209,7 @@
208209
"runtime.vendor" : ${content_meta_runtime_vendor},
209210
"runtime.version" : ${content_meta_runtime_version},
210211
"span.kind" : "test",
212+
"test.failure_suppressed" : "true",
211213
"test.framework" : "junit5",
212214
"test.framework_version" : "5.13.4",
213215
"test.is_retry" : "true",
@@ -406,4 +408,4 @@
406408
},
407409
"type" : "test_module_end",
408410
"version" : 1
409-
} ]
411+
} ]

0 commit comments

Comments
 (0)