Keep track of failed nodes when calculating block size and added some debugging statements for the PBS provider

Author: hategan

cogkit/modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/Block.java

@@ -65,7 +65,7 @@ private synchronized static BlockTaskSubmitter getSubmitter() {
         return submitter;
     }
 
-    private int workers, activeWorkers;
+    private int workers, activeWorkers, failedWorkers;
     private TimeInterval walltime, maxIdleTime;
     private Time endTime, startTime, deadline, creationTime, terminationTime, shutdownTime;
     private final SortedMap<Cpu, Time> scpus;
@@ -267,7 +267,7 @@ public double sizeLeft() {
         }
         else if (running) {
             return bqp.getMetric().size(
-                workers,
+                workers - failedWorkers,
                 (int) TimeInterval.max(getEndTime().subtract(Time.max(Time.now(), startTime)), NO_TIME).getSeconds());
         }
         else {
@@ -661,6 +661,8 @@ public void removeNode(Node node) {
         int left;
         synchronized(cpus) {
             nodes.remove(node);
+            this.activeWorkers -= node.getConcurrency();
+            this.failedWorkers += node.getConcurrency();
             for (Cpu cpu : node.getCpus()) {
                 scpus.remove(cpu);
                 cpus.remove(cpu);

cogkit/modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/common/AbstractJobSubmissionTaskHandler.java

@@ -104,6 +104,9 @@ public synchronized void cancel(String message)
 	}
 
 	public void processCompleted(int exitCode) {
+	    if (logger.isDebugEnabled()) {
+	        logger.debug("Process " + getTask().getIdentity() + " completed. Exit code: " + exitCode);
+	    }
 		if (getTask().getStatus().getStatusCode() != Status.FAILED) {
 			if (exitCode == 0) {
 				getTask().setStatus(Status.COMPLETED);
@@ -115,14 +118,23 @@ public void processCompleted(int exitCode) {
 	}
 
 	public void processFailed(String message) {
+	    if (logger.isDebugEnabled()) {
+            logger.debug("Process " + getTask().getIdentity() + " failed: " + message);
+        }
 		failTask(message, null);
 	}
 
 	public void processFailed(Exception e) {
+	    if (logger.isDebugEnabled()) {
+            logger.debug("Process " + getTask().getIdentity() + " failed: " + e);
+        }
 		failTask(null, e);
 	}
 
 	public void statusChanged(int status) {
+	    if (logger.isDebugEnabled()) {
+            logger.debug("Process " + getTask().getIdentity() + " status changed: " + status);
+        }
 		if (status == Job.STATE_RUNNING) {
 		    Job job = executor.getJob();
 		    String location = null;

cogkit/modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/common/Job.java

@@ -84,6 +84,9 @@ public boolean close(int tentativeState) {
 		}
 		File f = null;
 		if (exitcodeFileName != null) {
+		    if (logger.isDebugEnabled()) {
+		        logger.debug("Reading exit code file for " + jobID);
+		    }
 			f = new File(exitcodeFileName);	
 			if (f != null && !f.exists()) {
 				if (ticks == 5) {
@@ -100,6 +103,11 @@ else if (exitcode != NO_EXITCODE) {
 				f.delete();
 			}
 		}
+		else {
+		    if (logger.isDebugEnabled()) {
+                logger.debug("Job " + jobID + " has no exit code file");
+            }
+		}
 
 		processExitCode(tentativeState);
 		return true;