Add a class of scheduler-related metrics to expose more easily

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosModule.java

@@ -36,6 +36,7 @@ public void configure() {
     bind(SingularitySchedulerLock.class).in(Scopes.SINGLETON);
     bind(SingularityMesosSchedulerClient.class).in(Scopes.SINGLETON);
     bind(TaskLagGuardrail.class).in(Scopes.SINGLETON);
+    bind(SingularitySchedulerMetrics.class).in(Scopes.SINGLETON);
 
     Multibinder.newSetBinder(binder(), DeployAcceptanceHook.class);
   }

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java

@@ -82,6 +82,7 @@ public class SingularityMesosOfferScheduler {
   private final DisasterManager disasterManager;
   private final SingularityMesosSchedulerClient mesosSchedulerClient;
   private final OfferCache offerCache;
+  private final SingularitySchedulerMetrics metrics;
 
   private final double normalizedCpuWeight;
   private final double normalizedMemWeight;
@@ -108,7 +109,8 @@ public SingularityMesosOfferScheduler(
     SingularityManagedThreadPoolFactory threadPoolFactory,
     DisasterManager disasterManager,
     SingularityMesosSchedulerClient mesosSchedulerClient,
-    OfferCache offerCache
+    OfferCache offerCache,
+    SingularitySchedulerMetrics metrics
   ) {
     this.defaultResources =
       new Resources(
@@ -142,6 +144,7 @@ public SingularityMesosOfferScheduler(
     this.usageManager = usageManager;
     this.deployManager = deployManager;
     this.lock = lock;
+    this.metrics = metrics;
 
     double cpuWeight = mesosConfiguration.getCpuWeight();
     double memWeight = mesosConfiguration.getMemWeight();
@@ -331,6 +334,8 @@ public void resourceOffers(List<Offer> uncached) {
       throw t;
     }
 
+    metrics.getOfferLoopTime().update(System.currentTimeMillis() - start);
+
     LOG.info(
       "Finished handling {} new offer(s) {} from cache ({}), {} accepted, {} declined/cached",
       uncached.size(),
@@ -624,6 +629,7 @@ Collection<SingularityOfferHolder> checkOffers(
                               bestOffer.getSanitizedHost()
                             );
                             acceptTask(bestOffer, taskRequestHolder);
+                            metrics.getTasksScheduled().inc();
                             tasksScheduled.getAndIncrement();
                             updateAgentUsageScores(
                               taskRequestHolder,
@@ -651,6 +657,9 @@ Collection<SingularityOfferHolder> checkOffers(
       )
       .join();
 
+    metrics.getOfferLoopTasksRemaining().update(numDueTasks - tasksScheduled.get());
+    metrics.getOfferLoopOverLoadedHosts().update(overloadedHosts.size());
+    metrics.getOfferLoopNoMatches().update(noMatches.get());
     LOG.info(
       "{} tasks scheduled, {} tasks remaining after examining {} offers ({} overloaded hosts, {} had no offer matches)",
       tasksScheduled,

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySchedulerMetrics.java

@@ -0,0 +1,58 @@
+package com.hubspot.singularity.mesos;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.MetricRegistry;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class SingularitySchedulerMetrics {
+  private final Histogram offerLoopTime;
+  private final Counter tasksScheduled;
+  private final Histogram offerLoopTasksRemaining;
+  private final Histogram deployPollerTime;
+  private final Histogram lbUpdateTime;
+  private final Histogram offerLoopOverLoadedHosts;
+  private final Histogram offerLoopNoMatches;
+
+  @Inject
+  SingularitySchedulerMetrics(MetricRegistry metricRegistry) {
+    this.offerLoopTime = metricRegistry.histogram("offer-loop.time");
+    this.tasksScheduled = metricRegistry.counter("tasks-scheduled");
+    this.offerLoopTasksRemaining = metricRegistry.histogram("offer-loop.tasks-remaining");
+    this.deployPollerTime = metricRegistry.histogram("deploy-poller-time");
+    this.lbUpdateTime = metricRegistry.histogram("lb-update-time");
+    this.offerLoopOverLoadedHosts =
+      metricRegistry.histogram("offer-loop.overloaded-hosts");
+    this.offerLoopNoMatches = metricRegistry.histogram("offer-loop.no-matches");
+  }
+
+  public Histogram getOfferLoopTime() {
+    return offerLoopTime;
+  }
+
+  public Counter getTasksScheduled() {
+    return tasksScheduled;
+  }
+
+  public Histogram getOfferLoopTasksRemaining() {
+    return offerLoopTasksRemaining;
+  }
+
+  public Histogram getDeployPollerTime() {
+    return deployPollerTime;
+  }
+
+  public Histogram getLbUpdateTime() {
+    return lbUpdateTime;
+  }
+
+  public Histogram getOfferLoopOverLoadedHosts() {
+    return offerLoopOverLoadedHosts;
+  }
+
+  public Histogram getOfferLoopNoMatches() {
+    return offerLoopNoMatches;
+  }
+}

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java

@@ -44,6 +44,7 @@
 import com.hubspot.singularity.expiring.SingularityExpiringScale;
 import com.hubspot.singularity.hooks.LoadBalancerClient;
 import com.hubspot.singularity.mesos.SingularitySchedulerLock;
+import com.hubspot.singularity.mesos.SingularitySchedulerMetrics;
 import com.hubspot.singularity.scheduler.SingularityDeployHealthHelper.DeployHealth;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -77,6 +78,7 @@ public class SingularityDeployChecker {
   private final UsageManager usageManager;
   private final SingularityDeployAcceptanceManager deployAcceptanceManager;
   private final ExecutorService deployCheckExecutor;
+  private final SingularitySchedulerMetrics metrics;
 
   @Inject
   public SingularityDeployChecker(
@@ -90,7 +92,8 @@ public SingularityDeployChecker(
     SingularitySchedulerLock lock,
     UsageManager usageManager,
     SingularityManagedThreadPoolFactory threadPoolFactory,
-    SingularityDeployAcceptanceManager deployAcceptanceManager
+    SingularityDeployAcceptanceManager deployAcceptanceManager,
+    SingularitySchedulerMetrics metrics
   ) {
     this.configuration = configuration;
     this.lbClient = lbClient;
@@ -104,9 +107,11 @@ public SingularityDeployChecker(
     this.deployAcceptanceManager = deployAcceptanceManager;
     this.deployCheckExecutor =
       threadPoolFactory.get("deploy-checker", configuration.getCoreThreadpoolSize());
+    this.metrics = metrics;
   }
 
   public int checkDeploys() {
+    long start = System.currentTimeMillis();
     final List<SingularityPendingDeploy> pendingDeploys = deployManager.getPendingDeploys();
     final List<SingularityDeployMarker> cancelDeploys = deployManager.getCancelDeploys();
     final List<SingularityUpdatePendingDeployRequest> updateRequests = deployManager.getPendingDeployUpdates();
@@ -138,6 +143,7 @@ public int checkDeploys() {
     cancelDeploys.forEach(deployManager::deleteCancelDeployRequest);
     updateRequests.forEach(deployManager::deleteUpdatePendingDeployRequest);
 
+    metrics.getDeployPollerTime().update(System.currentTimeMillis() - start);
     return pendingDeploys.size();
   }
 
@@ -1589,6 +1595,12 @@ private SingularityDeployResult processLbState(
         lbUpdate,
         lbUpdateHolder
       );
+      metrics
+        .getLbUpdateTime()
+        .update(
+          System.currentTimeMillis() -
+          lbUpdateHolder.getLoadBalancerUpdate().getTimestamp()
+        );
       updatePendingDeploy(pendingDeploy, DeployState.WAITING, updatedProgress);
       // All tasks for current step are launched and in the LB if needed
       return markStepLaunchFinished(