Skip to content

Commit 190d8b0

Browse files
fjh100456srowen
authored andcommitted
[SPARK-20591][WEB UI] Succeeded tasks num not equal in all jobs page and job detail page on spark web ui when speculative task(s) exist.
## What changes were proposed in this pull request? Modified succeeded num in job detail page from "completed = stageData.completedIndices.size" to "completed = stageData.numCompleteTasks",which making succeeded tasks num in all jobs page and job detail page look more consistent, and more easily to find which stages the speculative task(s) were in. ## How was this patch tested? manual tests Author: fjh100456 <fu.jinhua6@zte.com.cn> Closes apache#17923 from fjh100456/master.
1 parent be846db commit 190d8b0

File tree

4 files changed

+5
-7
lines changed

4 files changed

+5
-7
lines changed

core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -631,7 +631,8 @@ private[ui] class JobPagedTable(
631631
{if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
632632
</td>
633633
<td class="progress-cell">
634-
{UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
634+
{UIUtils.makeProgressBar(started = job.numActiveTasks,
635+
completed = job.completedIndices.size,
635636
failed = job.numFailedTasks, skipped = job.numSkippedTasks,
636637
reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)}
637638
</td>

core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
423423
jobData.numActiveTasks -= 1
424424
taskEnd.reason match {
425425
case Success =>
426+
jobData.completedIndices.add((taskEnd.stageId, info.index))
426427
jobData.numCompletedTasks += 1
427428
case kill: TaskKilled =>
428429
jobData.reasonToNumKilled = jobData.reasonToNumKilled.updated(

core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ private[spark] object UIData {
6262
var numTasks: Int = 0,
6363
var numActiveTasks: Int = 0,
6464
var numCompletedTasks: Int = 0,
65+
var completedIndices: OpenHashSet[(Int, Int)] = new OpenHashSet[(Int, Int)](),
6566
var numSkippedTasks: Int = 0,
6667
var numFailedTasks: Int = 0,
6768
var reasonToNumKilled: Map[String, Int] = Map.empty,

core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -320,12 +320,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
320320
eventually(timeout(5 seconds), interval(50 milliseconds)) {
321321
goToUi(sc, "/jobs")
322322
find(cssSelector(".stage-progress-cell")).get.text should be ("2/2 (1 failed)")
323-
// Ideally, the following test would pass, but currently we overcount completed tasks
324-
// if task recomputations occur:
325-
// find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
326-
// Instead, we guarantee that the total number of tasks is always correct, while the number
327-
// of completed tasks may be higher:
328-
find(cssSelector(".progress-cell .progress")).get.text should be ("3/2 (1 failed)")
323+
find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
329324
}
330325
val jobJson = getJson(sc.ui.get, "jobs")
331326
(jobJson \ "numTasks").extract[Int]should be (2)

0 commit comments

Comments
 (0)