Skip to content

Add single parameter dd.data.jobs.enabled to enable DJM #6972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 7, 2024
Merged

Add single parameter dd.data.jobs.enabled to enable DJM #6972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0af3ab0
Add single parameter dd.data.jobs.enabled to enable DJM
paul-laffon-dd Apr 30, 2024
ab7079f
Merge branch 'master' into paul.laffon/data-jobs-enabled
paul-laffon-dd May 3, 2024
57d83ad
Keep other integrations enabled with DJM
paul-laffon-dd May 3, 2024
d703e90
Merge branch 'master' into paul.laffon/data-jobs-enabled
paul-laffon-dd May 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ private enum AgentFeature {
propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED), false),
USM(propertyNameToSystemPropertyName(UsmConfig.USM_ENABLED), false),
TELEMETRY(propertyNameToSystemPropertyName(GeneralConfig.TELEMETRY_ENABLED), true),
DEBUGGER(propertyNameToSystemPropertyName(DebuggerConfig.DEBUGGER_ENABLED), false);
DEBUGGER(propertyNameToSystemPropertyName(DebuggerConfig.DEBUGGER_ENABLED), false),
DATA_JOBS(propertyNameToSystemPropertyName(GeneralConfig.DATA_JOBS_ENABLED), false);

private final String systemProp;
private final boolean enabledByDefault;
Expand Down Expand Up @@ -197,6 +198,18 @@ public static void start(final Instrumentation inst, final URL agentJarURL, Stri
}
}

boolean dataJobsEnabled = isFeatureEnabled(AgentFeature.DATA_JOBS);
if (dataJobsEnabled) {
log.info("Data Jobs Monitoring enabled, enabling spark integrations");

setSystemPropertyDefault(
propertyNameToSystemPropertyName(TracerConfig.TRACE_LONG_RUNNING_ENABLED), "true");
setSystemPropertyDefault(
propertyNameToSystemPropertyName("integration.spark.enabled"), "true");
setSystemPropertyDefault(
propertyNameToSystemPropertyName("integration.spark-executor.enabled"), "true");
Comment on lines +207 to +210
Copy link

@gabe-ochoa gabe-ochoa May 2, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once the apm intake work is done for #6967, can we just turn on all integrations?

Copy link
Contributor

@kr-igor kr-igor May 2, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to disable sampling as well? It may be a big issue for some hight throughput systems with millions of operations per second.

Copy link
Contributor Author

@paul-laffon-dd paul-laffon-dd May 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed it so that dd.data.jobs.enabled=true does not change the default for all integrations (enabled by default), only enables spark/long running spans

}

if (!isSupportedAppSecArch()) {
log.debug(
"OS and architecture ({}/{}) not supported by AppSec, dd.appsec.enabled will default to false",
Expand Down
2 changes: 2 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ public final class ConfigDefaults {
static final boolean DEFAULT_CWS_ENABLED = false;
static final int DEFAULT_CWS_TLS_REFRESH = 5000;

static final boolean DEFAULT_DATA_JOBS_ENABLED = false;

static final boolean DEFAULT_DATA_STREAMS_ENABLED = false;
static final int DEFAULT_DATA_STREAMS_BUCKET_DURATION = 10; // seconds

Expand Down
2 changes: 2 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ public final class GeneralConfig {
public static final String AZURE_APP_SERVICES = "azure.app.services";
public static final String INTERNAL_EXIT_ON_FAILURE = "trace.internal.exit.on.failure";

public static final String DATA_JOBS_ENABLED = "data.jobs.enabled";

public static final String DATA_STREAMS_ENABLED = "data.streams.enabled";
public static final String DATA_STREAMS_BUCKET_DURATION_SECONDS =
"data.streams.bucket_duration.seconds";
Expand Down
10 changes: 9 additions & 1 deletion internal-api/src/main/java/datadog/trace/api/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import static datadog.trace.api.ConfigDefaults.DEFAULT_COUCHBASE_INTERNAL_SPANS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_TLS_REFRESH;
import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_JOBS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_BUCKET_DURATION;
import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_ENABLED;
import static datadog.trace.api.ConfigDefaults.DEFAULT_DB_CLIENT_HOST_SPLIT_BY_HOST;
Expand Down Expand Up @@ -233,6 +234,7 @@
import static datadog.trace.api.config.GeneralConfig.APPLICATION_KEY;
import static datadog.trace.api.config.GeneralConfig.APPLICATION_KEY_FILE;
import static datadog.trace.api.config.GeneralConfig.AZURE_APP_SERVICES;
import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED;
import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_BUCKET_DURATION_SECONDS;
import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_ENABLED;
import static datadog.trace.api.config.GeneralConfig.DOGSTATSD_ARGS;
Expand Down Expand Up @@ -900,6 +902,8 @@ static class HostNameHolder {
private final boolean cwsEnabled;
private final int cwsTlsRefresh;

private final boolean dataJobsEnabled;

private final boolean dataStreamsEnabled;
private final float dataStreamsBucketDurationSeconds;

Expand Down Expand Up @@ -1975,6 +1979,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
cwsEnabled = configProvider.getBoolean(CWS_ENABLED, DEFAULT_CWS_ENABLED);
cwsTlsRefresh = configProvider.getInteger(CWS_TLS_REFRESH, DEFAULT_CWS_TLS_REFRESH);

dataJobsEnabled = configProvider.getBoolean(DATA_JOBS_ENABLED, DEFAULT_DATA_JOBS_ENABLED);

dataStreamsEnabled =
configProvider.getBoolean(DATA_STREAMS_ENABLED, DEFAULT_DATA_STREAMS_ENABLED);
dataStreamsBucketDurationSeconds =
Expand Down Expand Up @@ -3452,7 +3458,7 @@ public boolean isAxisPromoteResourceName() {
}

public boolean isDataJobsEnabled() {
return instrumenterConfig.isDataJobsEnabled();
return dataJobsEnabled;
}

/** @return A map of tags to be applied only to the local application root span. */
Expand Down Expand Up @@ -4568,6 +4574,8 @@ public String toString() {
+ telemetryMetricsEnabled
+ ", appSecScaEnabled="
+ appSecScaEnabled
+ ", dataJobsEnabled="
+ dataJobsEnabled
+ '}';
}
}
7 changes: 0 additions & 7 deletions internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -271,12 +270,6 @@ public boolean isIntegrationShortcutMatchingEnabled(
integrationNames, "integration.", ".matching.shortcut.enabled", defaultEnabled);
}

public boolean isDataJobsEnabled() {
// there's no dedicated flag to enabled DJM, it's enough to just enable
// spark instrumentation
return isIntegrationEnabled(Collections.singletonList("spark"), false);
}

public boolean isTraceEnabled() {
return traceEnabled;
}
Expand Down