Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GOBBLIN-1969] Increase Visibility for Flow Compilation Failures #3840

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

[GOBBLIN-1969] Increase Visibility for Flow Compilation Failures #3840

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
99769c4
Add logging and metrics for flow compilation validation
Dec 1, 2023
37654ff
Change metric to count successful flow compilations
Dec 1, 2023
33ada42
Initialize FlowCatalog before DagManager started
Dec 2, 2023
028e21b
Revert "Initialize FlowCatalog before DagManager started"
Dec 2, 2023
ebbdad3
Rename metrics to be uniform
Dec 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Rename metrics to be uniform
  • Loading branch information
Urmi Mustafi committed Dec 2, 2023
commit ebbdad32f66bf88292f431c6e093df325664adb7
2 changes: 1 addition & 1 deletion ...ics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/ServiceMetricNames.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class ServiceMetricNames {
public static final String DAG_MANAGER_PREFIX = GOBBLIN_SERVICE_PREFIX_WITH_DELIMITER + "dagManager";
public static final String
DAG_MANAGER_FAILED_LAUNCH_EVENTS_ON_STARTUP_COUNT = DAG_MANAGER_PREFIX + ".failedLaunchEventsOnStartupCount";
public static final String DAG_MANAGER_SUCCESSFUL_LAUNCH_EVENTS_ON_STARTUP_COUNT = DAG_MANAGER_PREFIX + ".successfulLaunchEventsOnActivationCount";
public static final String DAG_MANAGER_SUCCESSFUL_LAUNCH_EVENTS_ON_STARTUP_COUNT = DAG_MANAGER_PREFIX + ".successfulLaunchEventsOnStartupCount";
public static final String FLOW_FAILED_FORWARD_TO_DAG_MANAGER_COUNT = DAG_MANAGER_PREFIX + ".flowFailedForwardToDagManagerCount";

//Job status poll timer
Expand Down
2 changes: 1 addition & 1 deletion ...in-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ public void handleLaunchFlowEvent(DagActionStore.DagAction launchAction) {
this.flowCompilationValidationHelper.createExecutionPlanIfValid(spec, Optional.absent());
if (optionalJobExecutionPlanDag.isPresent()) {
addDag(optionalJobExecutionPlanDag.get(), true, true);
this.dagManagerMetrics.incrementSuccessfulLaunchAttemptCount();
this.dagManagerMetrics.incrementSuccessfulLaunchCount();
} else {
log.warn("Failed flow compilation of spec causing launch flow event to be skipped on startup. Flow {}", flowId);
this.dagManagerMetrics.incrementFailedLaunchCount();
Expand Down
24 changes: 14 additions & 10 deletions ...ice/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagerMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ public class DagManagerMetrics {
private final Map<String, ContextAwareMeter> executorSlaExceededMeters = Maps.newConcurrentMap();
private final Map<String, ContextAwareMeter> executorJobSentMeters = Maps.newConcurrentMap();

// Metric for unexpected flow handling failures
private ContextAwareCounter failedLaunchEventsOnActivationCount;
private ContextAwareCounter successfulLaunchEventsOnActivationCount;
// Metric for unexpected flow handling outcomes
private ContextAwareCounter failedLaunchEventsOnStartupCount;
private ContextAwareCounter successfulLaunchEventsOnStartupCount;
MetricContext metricContext;

public DagManagerMetrics(MetricContext metricContext) {
Expand All @@ -105,10 +105,10 @@ public void activate() {
allRunningMeter = metricContext.contextAwareMeter(MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX,
ServiceMetricNames.JOBS_SENT_TO_SPEC_EXECUTOR));
// TODO: remove duplicate use of 'GOBBLIN_SERVICE_PREFIX' once startup functionality is stable
failedLaunchEventsOnActivationCount = metricContext.contextAwareCounter(
failedLaunchEventsOnStartupCount = metricContext.contextAwareCounter(
MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX,
ServiceMetricNames.DAG_MANAGER_FAILED_LAUNCH_EVENTS_ON_STARTUP_COUNT));
successfulLaunchEventsOnActivationCount = metricContext.contextAwareCounter(
successfulLaunchEventsOnStartupCount = metricContext.contextAwareCounter(
ServiceMetricNames.DAG_MANAGER_SUCCESSFUL_LAUNCH_EVENTS_ON_STARTUP_COUNT);
}
}
Expand Down Expand Up @@ -209,17 +209,21 @@ public void incrementCountsStartSlaExceeded(Dag.DagNode<JobExecutionPlan> node)
}
}

// Increment the count for num of failed launches during leader activation
/**
* Increment the count for num of failed launches during system startup
*/
public void incrementFailedLaunchCount() {
if (this.metricContext != null) {
this.failedLaunchEventsOnActivationCount.inc();
this.failedLaunchEventsOnStartupCount.inc();
}
}

// Increment the count for num of successful launches attempted during leader activation
public void incrementSuccessfulLaunchAttemptCount() {
/**
* Increment the count for num of successful launches during system startup
*/
public void incrementSuccessfulLaunchCount() {
if (this.metricContext != null) {
this.successfulLaunchEventsOnActivationCount.inc();
this.successfulLaunchEventsOnStartupCount.inc();
}
}

Expand Down