Incorporating review comments.

Author: Sundeep Narravula

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -1077,12 +1077,12 @@ class SparkContext(
   }
 
   /** Cancel a given job if it's scheduled or running */
-  def cancelJob(jobId: Int) {
+  private[spark] def cancelJob(jobId: Int) {
     dagScheduler.cancelJob(jobId)
   }
 
   /** Cancel a given stage and all jobs associated with it */
-  def cancelStage(stageId: Int) {
+  private[spark] def cancelStage(stageId: Int) {
     dagScheduler.cancelStage(stageId)
   }
 

core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala

@@ -494,7 +494,7 @@ class DAGScheduler(
   /**
    * Cancel a job that is running or waiting in the queue.
    */
-  private[spark] def cancelJob(jobId: Int) {
+  def cancelJob(jobId: Int) {
     logInfo("Asked to cancel job " + jobId)
     eventProcessActor ! JobCancelled(jobId)
   }
@@ -570,11 +570,13 @@ class DAGScheduler(
         val activeInGroup = activeJobs.filter(activeJob =>
           groupId == activeJob.properties.get(SparkContext.SPARK_JOB_GROUP_ID))
         val jobIds = activeInGroup.map(_.jobId)
-        jobIds.foreach(jobId => handleJobCancellation(jobId, "as part of cancelled job group %s".format(groupId)))
+        jobIds.foreach(jobId => handleJobCancellation(jobId,
+          "as part of cancelled job group %s".format(groupId)))
 
       case AllJobsCancelled =>
         // Cancel all running jobs.
-        runningStages.map(_.jobId).foreach(jobId => handleJobCancellation(jobId, "as part of cancellation of all jobs"))
+        runningStages.map(_.jobId).foreach(jobId => handleJobCancellation(jobId,
+          "as part of cancellation of all jobs"))
         activeJobs.clear()      // These should already be empty by this point,
         jobIdToActiveJob.clear()   // but just in case we lost track of some jobs...
 
@@ -1003,7 +1005,7 @@ class DAGScheduler(
 
   private def handleStageCancellation(stageId: Int) {
     if (stageIdToJobIds.contains(stageId)) {
-      val jobsThatUseStage: Array[Int] = stageIdToJobIds(stageId).toArray.sorted
+      val jobsThatUseStage: Array[Int] = stageIdToJobIds(stageId).toArray
       jobsThatUseStage.foreach(jobId => {
         handleJobCancellation(jobId, "because Stage %s was cancelled".format(stageId))
       })
@@ -1016,7 +1018,7 @@ class DAGScheduler(
     if (!jobIdToStageIds.contains(jobId)) {
       logDebug("Trying to cancel unregistered job " + jobId)
     } else {
-      failJobAndIndependentStages(jobIdToActiveJob(jobId), 
+      failJobAndIndependentStages(jobIdToActiveJob(jobId),
         "Job %d cancelled %s".format(jobId, reason), None)
     }
   }

core/src/main/scala/org/apache/spark/ui/SparkUI.scala

@@ -46,7 +46,7 @@ private[spark] class SparkUI(
   val live = sc != null
 
   val securityManager = if (live) sc.env.securityManager else new SecurityManager(conf)
-  val killEnabled = conf.get("spark.ui.killEnabled", "false").toBoolean
+  val killEnabled = conf.getBoolean("spark.ui.killEnabled", true)
 
   private val bindHost = Utils.localHostName()
   private val publicHost = Option(System.getenv("SPARK_PUBLIC_DNS")).getOrElse(bindHost)

core/src/main/scala/org/apache/spark/ui/jobs/IndexPage.scala

@@ -33,6 +33,7 @@ private[ui] class IndexPage(parent: JobProgressUI) {
   private val sc = parent.sc
   private lazy val listener = parent.listener
   private lazy val isFairScheduler = parent.isFairScheduler
+  private val killEnabled = parent.killEnabled
 
   def render(request: HttpServletRequest): Seq[Node] = {
     listener.synchronized {
@@ -41,6 +42,16 @@ private[ui] class IndexPage(parent: JobProgressUI) {
       val failedStages = listener.failedStages.reverse.toSeq
       val now = System.currentTimeMillis()
 
+      if (killEnabled) {
+        val killFlag = Option(request.getParameter("terminate")).getOrElse("false").toBoolean
+        val stageId = Option(request.getParameter("id")).getOrElse("-1").toInt
+
+        if (stageId >= 0 && killFlag && listener.activeStages.contains(stageId)) {
+          sc.cancelStage(stageId)
+        }
+      }
+
+
       val activeStagesTable =
         new StageTable(activeStages.sortBy(_.submissionTime).reverse, parent, parent.killEnabled)
       val completedStagesTable =

core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala

@@ -61,7 +61,6 @@ private[ui] class JobProgressListener(conf: SparkConf) extends SparkListener {
   val stageIdToPool = HashMap[Int, String]()
   val stageIdToDescription = HashMap[Int, String]()
   val poolToActiveStages = HashMap[String, HashMap[Int, StageInfo]]()
-  val jobIdToStageIds = HashMap[Int, Seq[Int]]()
 
   val executorIdToBlockManagerId = HashMap[String, BlockManagerId]()
 

core/src/main/scala/org/apache/spark/ui/jobs/JobProgressUI.scala

@@ -57,5 +57,5 @@ private[ui] class JobProgressUI(parent: SparkUI) {
       (request: HttpServletRequest) => poolPage.render(request), parent.securityManager, basePath),
     createServletHandler("/stages",
       (request: HttpServletRequest) => indexPage.render(request), parent.securityManager, basePath)
-    )
+  )
 }

core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala

@@ -32,20 +32,11 @@ private[ui] class StagePage(parent: JobProgressUI) {
   private val basePath = parent.basePath
   private lazy val listener = parent.listener
   private lazy val sc = parent.sc
-  private val killEnabled = parent.killEnabled
 
   def render(request: HttpServletRequest): Seq[Node] = {
     listener.synchronized {
       val stageId = request.getParameter("id").toInt
 
-      if (killEnabled) {
-        val killFlag = Option(request.getParameter("terminate")).getOrElse("false").toBoolean
-
-        if (killFlag && listener.activeStages.contains(stageId)) {
-          sc.cancelStage(stageId)
-        }
-      }
-
       if (!listener.stageIdToTaskData.contains(stageId)) {
         val content =
           <div>

core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala

@@ -27,7 +27,11 @@ import org.apache.spark.ui.{WebUI, UIUtils}
 import org.apache.spark.util.Utils
 
 /** Page showing list of all ongoing and recently finished stages */
-private[ui] class StageTable(stages: Seq[StageInfo], parent: JobProgressUI, killEnabled: Boolean = false) {
+private[ui] class StageTable(
+  stages: Seq[StageInfo],
+  parent: JobProgressUI,
+  killEnabled: Boolean = false) {
+
   private val basePath = parent.basePath
   private lazy val listener = parent.listener
   private lazy val isFairScheduler = parent.isFairScheduler
@@ -76,16 +80,15 @@ private[ui] class StageTable(stages: Seq[StageInfo], parent: JobProgressUI, kill
       <a href={"%s/stages/stage?id=%s".format(UIUtils.prependBaseUri(basePath), s.stageId)}>
         {s.name}
       </a>
-    val killButton =  if (killEnabled) {
-      <form action={"%s/stages/stage/".format(UIUtils.prependBaseUri(basePath))}>
-        <input type="hidden" value={"true"} name="terminate" />
-        <input type="hidden" value={"" + s.stageId} name="id" />
-        <input type="submit" value="Terminate Job"/>
-      </form>
+    val killLink = if (killEnabled) {
+      <div>[<a href={"%s/stages?id=%s&terminate=true".format(UIUtils.prependBaseUri(basePath), s.stageId)}>
+        Kill
+      </a>]</div>
+
     }
     val description = listener.stageIdToDescription.get(s.stageId)
-      .map(d => <div><em>{d}</em></div><div>{nameLink} {killButton}</div>)
-      .getOrElse(<div>{nameLink} {killButton}</div>)
+      .map(d => <div><em>{d}</em></div><div>{nameLink} {killLink}</div>)
+      .getOrElse(<div>{nameLink} {killLink}</div>)
 
     return description
   }

docs/configuration.md

@@ -190,6 +190,13 @@ Apart from these, the following properties are also available, and may be useful
     user that started the Spark job has view access.
   </td>
 </tr>
+<tr>
+  <td>spark.ui.killEnabled</td>
+  <td>true</td>
+  <td>
+    Allows stages and corresponding jobs to be killed from the web ui.
+  </td>
+</tr>
 <tr>
   <td>spark.shuffle.compress</td>
   <td>true</td>