Skip to content

Commit 95cad7c

Browse files
jkbradleyjkbradley
committed
Merge remote-tracking branch 'upstream/master' into dt-opt3
Conflicts: mllib/src/main/scala/org/apache/spark/mllib/tree/DecisionTree.scala
2 parents 5f94342 + 115eeb3 commit 95cad7c

File tree

31 files changed

+176
-121
lines changed

31 files changed

+176
-121
lines changed

‎.travis.yml

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎core/src/main/scala/org/apache/spark/deploy/master/Master.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,7 @@ private[spark] class Master(
697697
appIdToUI(app.id) = ui
698698
webUi.attachSparkUI(ui)
699699
// Application UI is successfully rebuilt, so link the Master UI to it
700-
app.desc.appUiUrl = ui.basePath
700+
app.desc.appUiUrl = ui.getBasePath
701701
true
702702
} catch {
703703
case e: Exception =>

‎core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ private[spark] class EventLoggingListener(
5454
private val testing = sparkConf.getBoolean("spark.eventLog.testing", false)
5555
private val outputBufferSize = sparkConf.getInt("spark.eventLog.buffer.kb", 100) * 1024
5656
private val logBaseDir = sparkConf.get("spark.eventLog.dir", DEFAULT_LOG_DIR).stripSuffix("/")
57-
private val name = appName.replaceAll("[ :/]", "-").toLowerCase + "-" + System.currentTimeMillis
57+
private val name = appName.replaceAll("[ :/]", "-").replaceAll("[${}'\"]", "_")
58+
.toLowerCase + "-" + System.currentTimeMillis
5859
val logDir = Utils.resolveURI(logBaseDir) + "/" + name.stripSuffix("/")
5960

6061
protected val logger = new FileLogger(logDir, sparkConf, hadoopConf, outputBufferSize,

‎core/src/main/scala/org/apache/spark/ui/SparkUI.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ private[spark] class SparkUI(
7676
}
7777
}
7878

79+
def getAppName = appName
80+
7981
/** Set the app name for this UI. */
8082
def setAppName(name: String) {
8183
appName = name
@@ -100,6 +102,13 @@ private[spark] class SparkUI(
100102
private[spark] def appUIAddress = s"http://$appUIHostPort"
101103
}
102104

105+
private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String)
106+
extends WebUITab(parent, prefix) {
107+
108+
def appName: String = parent.getAppName
109+
110+
}
111+
103112
private[spark] object SparkUI {
104113
val DEFAULT_PORT = 4040
105114
val STATIC_RESOURCE_DIR = "org/apache/spark/ui/static"

‎core/src/main/scala/org/apache/spark/ui/UIUtils.scala

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -163,17 +163,15 @@ private[spark] object UIUtils extends Logging {
163163

164164
/** Returns a spark page with correctly formatted headers */
165165
def headerSparkPage(
166-
content: => Seq[Node],
167-
basePath: String,
168-
appName: String,
169166
title: String,
170-
tabs: Seq[WebUITab],
171-
activeTab: WebUITab,
167+
content: => Seq[Node],
168+
activeTab: SparkUITab,
172169
refreshInterval: Option[Int] = None): Seq[Node] = {
173170

174-
val header = tabs.map { tab =>
171+
val appName = activeTab.appName
172+
val header = activeTab.headerTabs.map { tab =>
175173
<li class={if (tab == activeTab) "active" else ""}>
176-
<a href={prependBaseUri(basePath, "/" + tab.prefix)}>{tab.name}</a>
174+
<a href={prependBaseUri(activeTab.basePath, "/" + tab.prefix)}>{tab.name}</a>
177175
</li>
178176
}
179177

‎core/src/main/scala/org/apache/spark/ui/WebUI.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ private[spark] abstract class WebUI(
5050
protected val publicHostName = Option(System.getenv("SPARK_PUBLIC_DNS")).getOrElse(localHostName)
5151
private val className = Utils.getFormattedClassName(this)
5252

53+
def getBasePath: String = basePath
5354
def getTabs: Seq[WebUITab] = tabs.toSeq
5455
def getHandlers: Seq[ServletContextHandler] = handlers.toSeq
5556
def getSecurityManager: SecurityManager = securityManager
@@ -135,6 +136,8 @@ private[spark] abstract class WebUITab(parent: WebUI, val prefix: String) {
135136

136137
/** Get a list of header tabs from the parent UI. */
137138
def headerTabs: Seq[WebUITab] = parent.getTabs
139+
140+
def basePath: String = parent.getBasePath
138141
}
139142

140143

‎core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ import scala.xml.Node
2424
import org.apache.spark.ui.{UIUtils, WebUIPage}
2525

2626
private[ui] class EnvironmentPage(parent: EnvironmentTab) extends WebUIPage("") {
27-
private val appName = parent.appName
28-
private val basePath = parent.basePath
2927
private val listener = parent.listener
3028

3129
def render(request: HttpServletRequest): Seq[Node] = {
@@ -45,7 +43,7 @@ private[ui] class EnvironmentPage(parent: EnvironmentTab) extends WebUIPage("")
4543
<h4>Classpath Entries</h4> {classpathEntriesTable}
4644
</span>
4745

48-
UIUtils.headerSparkPage(content, basePath, appName, "Environment", parent.headerTabs, parent)
46+
UIUtils.headerSparkPage("Environment", content, parent)
4947
}
5048

5149
private def propertyHeader = Seq("Name", "Value")

‎core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,7 @@ import org.apache.spark.annotation.DeveloperApi
2121
import org.apache.spark.scheduler._
2222
import org.apache.spark.ui._
2323

24-
private[ui] class EnvironmentTab(parent: SparkUI) extends WebUITab(parent, "environment") {
25-
val appName = parent.appName
26-
val basePath = parent.basePath
24+
private[ui] class EnvironmentTab(parent: SparkUI) extends SparkUITab(parent, "environment") {
2725
val listener = new EnvironmentListener
2826

2927
attachPage(new EnvironmentPage(this))

‎core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,6 @@ private case class ExecutorSummaryInfo(
4343
maxMemory: Long)
4444

4545
private[ui] class ExecutorsPage(parent: ExecutorsTab) extends WebUIPage("") {
46-
private val appName = parent.appName
47-
private val basePath = parent.basePath
4846
private val listener = parent.listener
4947

5048
def render(request: HttpServletRequest): Seq[Node] = {
@@ -101,8 +99,7 @@ private[ui] class ExecutorsPage(parent: ExecutorsTab) extends WebUIPage("") {
10199
</div>
102100
</div>;
103101

104-
UIUtils.headerSparkPage(content, basePath, appName, "Executors (" + execInfo.size + ")",
105-
parent.headerTabs, parent)
102+
UIUtils.headerSparkPage("Executors (" + execInfo.size + ")", content, parent)
106103
}
107104

108105
/** Render an HTML row representing an executor */

‎core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,9 @@ import org.apache.spark.ExceptionFailure
2323
import org.apache.spark.annotation.DeveloperApi
2424
import org.apache.spark.scheduler._
2525
import org.apache.spark.storage.StorageStatusListener
26-
import org.apache.spark.ui.{SparkUI, WebUITab}
26+
import org.apache.spark.ui.{SparkUI, SparkUITab}
2727

28-
private[ui] class ExecutorsTab(parent: SparkUI) extends WebUITab(parent, "executors") {
29-
val appName = parent.appName
30-
val basePath = parent.basePath
28+
private[ui] class ExecutorsTab(parent: SparkUI) extends SparkUITab(parent, "executors") {
3129
val listener = new ExecutorsListener(parent.storageStatusListener)
3230

3331
attachPage(new ExecutorsPage(this))

‎core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import org.apache.spark.ui.{WebUIPage, UIUtils}
2626

2727
/** Page showing list of all ongoing and recently finished stages and pools */
2828
private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("") {
29-
private val appName = parent.appName
30-
private val basePath = parent.basePath
3129
private val live = parent.live
3230
private val sc = parent.sc
3331
private val listener = parent.listener
@@ -94,7 +92,7 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("")
9492
<h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++
9593
failedStagesTable.toNodeSeq
9694

97-
UIUtils.headerSparkPage(content, basePath, appName, "Spark Stages", parent.headerTabs, parent)
95+
UIUtils.headerSparkPage("Spark Stages", content, parent)
9896
}
9997
}
10098
}

‎core/src/main/scala/org/apache/spark/ui/jobs/JobProgressTab.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,10 @@ import javax.servlet.http.HttpServletRequest
2121

2222
import org.apache.spark.SparkConf
2323
import org.apache.spark.scheduler.SchedulingMode
24-
import org.apache.spark.ui.{SparkUI, WebUITab}
24+
import org.apache.spark.ui.{SparkUI, SparkUITab}
2525

2626
/** Web UI showing progress status of all jobs in the given SparkContext. */
27-
private[ui] class JobProgressTab(parent: SparkUI) extends WebUITab(parent, "stages") {
28-
val appName = parent.appName
29-
val basePath = parent.basePath
27+
private[ui] class JobProgressTab(parent: SparkUI) extends SparkUITab(parent, "stages") {
3028
val live = parent.live
3129
val sc = parent.sc
3230
val conf = if (live) sc.conf else new SparkConf
@@ -53,4 +51,5 @@ private[ui] class JobProgressTab(parent: SparkUI) extends WebUITab(parent, "stag
5351
Thread.sleep(100)
5452
}
5553
}
54+
5655
}

‎core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import org.apache.spark.ui.{WebUIPage, UIUtils}
2626

2727
/** Page showing specific pool details */
2828
private[ui] class PoolPage(parent: JobProgressTab) extends WebUIPage("pool") {
29-
private val appName = parent.appName
30-
private val basePath = parent.basePath
3129
private val live = parent.live
3230
private val sc = parent.sc
3331
private val listener = parent.listener
@@ -51,8 +49,7 @@ private[ui] class PoolPage(parent: JobProgressTab) extends WebUIPage("pool") {
5149
<h4>Summary </h4> ++ poolTable.toNodeSeq ++
5250
<h4>{activeStages.size} Active Stages</h4> ++ activeStagesTable.toNodeSeq
5351

54-
UIUtils.headerSparkPage(content, basePath, appName, "Fair Scheduler Pool: " + poolName,
55-
parent.headerTabs, parent)
52+
UIUtils.headerSparkPage("Fair Scheduler Pool: " + poolName, content, parent)
5653
}
5754
}
5855
}

‎core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import org.apache.spark.ui.UIUtils
2525

2626
/** Table showing list of pools */
2727
private[ui] class PoolTable(pools: Seq[Schedulable], parent: JobProgressTab) {
28-
private val basePath = parent.basePath
2928
private val listener = parent.listener
3029

3130
def toNodeSeq: Seq[Node] = {
@@ -59,11 +58,11 @@ private[ui] class PoolTable(pools: Seq[Schedulable], parent: JobProgressTab) {
5958
case Some(stages) => stages.size
6059
case None => 0
6160
}
61+
val href = "%s/stages/pool?poolname=%s"
62+
.format(UIUtils.prependBaseUri(parent.basePath), p.name)
6263
<tr>
6364
<td>
64-
<a href={"%s/stages/pool?poolname=%s".format(UIUtils.prependBaseUri(basePath), p.name)}>
65-
{p.name}
66-
</a>
65+
<a href={href}>{p.name}</a>
6766
</td>
6867
<td>{p.minShare}</td>
6968
<td>{p.weight}</td>

‎core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ import org.apache.spark.scheduler.AccumulableInfo
2929

3030
/** Page showing statistics and task list for a given stage */
3131
private[ui] class StagePage(parent: JobProgressTab) extends WebUIPage("stage") {
32-
private val appName = parent.appName
33-
private val basePath = parent.basePath
3432
private val listener = parent.listener
3533

3634
def render(request: HttpServletRequest): Seq[Node] = {
@@ -44,8 +42,7 @@ private[ui] class StagePage(parent: JobProgressTab) extends WebUIPage("stage") {
4442
<h4>Summary Metrics</h4> No tasks have started yet
4543
<h4>Tasks</h4> No tasks have started yet
4644
</div>
47-
return UIUtils.headerSparkPage(content, basePath, appName,
48-
"Details for Stage %s".format(stageId), parent.headerTabs, parent)
45+
return UIUtils.headerSparkPage("Details for Stage %s".format(stageId), content, parent)
4946
}
5047

5148
val stageData = stageDataOption.get
@@ -227,8 +224,7 @@ private[ui] class StagePage(parent: JobProgressTab) extends WebUIPage("stage") {
227224
maybeAccumulableTable ++
228225
<h4>Tasks</h4> ++ taskTable
229226

230-
UIUtils.headerSparkPage(content, basePath, appName, "Details for Stage %d".format(stageId),
231-
parent.headerTabs, parent)
227+
UIUtils.headerSparkPage("Details for Stage %d".format(stageId), content, parent)
232228
}
233229
}
234230

‎core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ private[ui] class StageTableBase(
3232
parent: JobProgressTab,
3333
killEnabled: Boolean = false) {
3434

35-
private val basePath = parent.basePath
3635
private val listener = parent.listener
3736
protected def isFairScheduler = parent.isFairScheduler
3837

@@ -88,17 +87,19 @@ private[ui] class StageTableBase(
8887
private def makeDescription(s: StageInfo): Seq[Node] = {
8988
// scalastyle:off
9089
val killLink = if (killEnabled) {
90+
val killLinkUri = "%s/stages/stage/kill?id=%s&terminate=true"
91+
.format(UIUtils.prependBaseUri(parent.basePath), s.stageId)
92+
val confirm = "return window.confirm('Are you sure you want to kill stage %s ?');"
93+
.format(s.stageId)
9194
<span class="kill-link">
92-
(<a href={"%s/stages/stage/kill?id=%s&terminate=true".format(UIUtils.prependBaseUri(basePath), s.stageId)}
93-
onclick={"return window.confirm('Are you sure you want to kill stage %s ?');".format(s.stageId)}>kill</a>)
95+
(<a href={killLinkUri} onclick={confirm}>kill</a>)
9496
</span>
9597
}
9698
// scalastyle:on
9799

98-
val nameLink =
99-
<a href={"%s/stages/stage?id=%s".format(UIUtils.prependBaseUri(basePath), s.stageId)}>
100-
{s.name}
101-
</a>
100+
val nameLinkUri ="%s/stages/stage?id=%s"
101+
.format(UIUtils.prependBaseUri(parent.basePath), s.stageId)
102+
val nameLink = <a href={nameLinkUri}>{s.name}</a>
102103

103104
val cachedRddInfos = s.rddInfos.filter(_.numCachedPartitions > 0)
104105
val details = if (s.details.nonEmpty) {
@@ -111,7 +112,7 @@ private[ui] class StageTableBase(
111112
Text("RDD: ") ++
112113
// scalastyle:off
113114
cachedRddInfos.map { i =>
114-
<a href={"%s/storage/rdd?id=%d".format(UIUtils.prependBaseUri(basePath), i.id)}>{i.name}</a>
115+
<a href={"%s/storage/rdd?id=%d".format(UIUtils.prependBaseUri(parent.basePath), i.id)}>{i.name}</a>
115116
}
116117
// scalastyle:on
117118
}}
@@ -157,7 +158,7 @@ private[ui] class StageTableBase(
157158
{if (isFairScheduler) {
158159
<td>
159160
<a href={"%s/stages/pool?poolname=%s"
160-
.format(UIUtils.prependBaseUri(basePath), stageData.schedulingPool)}>
161+
.format(UIUtils.prependBaseUri(parent.basePath), stageData.schedulingPool)}>
161162
{stageData.schedulingPool}
162163
</a>
163164
</td>

‎core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,14 @@ import org.apache.spark.util.Utils
2727

2828
/** Page showing storage details for a given RDD */
2929
private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
30-
private val appName = parent.appName
31-
private val basePath = parent.basePath
3230
private val listener = parent.listener
3331

3432
def render(request: HttpServletRequest): Seq[Node] = {
3533
val rddId = request.getParameter("id").toInt
3634
val storageStatusList = listener.storageStatusList
3735
val rddInfo = listener.rddInfoList.find(_.id == rddId).getOrElse {
3836
// Rather than crashing, render an "RDD Not Found" page
39-
return UIUtils.headerSparkPage(Seq[Node](), basePath, appName, "RDD Not Found",
40-
parent.headerTabs, parent)
37+
return UIUtils.headerSparkPage("RDD Not Found", Seq[Node](), parent)
4138
}
4239

4340
// Worker table
@@ -96,8 +93,7 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
9693
</div>
9794
</div>;
9895

99-
UIUtils.headerSparkPage(content, basePath, appName, "RDD Storage Info for " + rddInfo.name,
100-
parent.headerTabs, parent)
96+
UIUtils.headerSparkPage("RDD Storage Info for " + rddInfo.name, content, parent)
10197
}
10298

10399
/** Header fields for the worker table */

‎core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,12 @@ import org.apache.spark.util.Utils
2727

2828
/** Page showing list of RDD's currently stored in the cluster */
2929
private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
30-
private val appName = parent.appName
31-
private val basePath = parent.basePath
3230
private val listener = parent.listener
3331

3432
def render(request: HttpServletRequest): Seq[Node] = {
3533
val rdds = listener.rddInfoList
3634
val content = UIUtils.listingTable(rddHeader, rddRow, rdds)
37-
UIUtils.headerSparkPage(content, basePath, appName, "Storage ", parent.headerTabs, parent)
35+
UIUtils.headerSparkPage("Storage", content, parent)
3836
}
3937

4038
/** Header fields for the RDD table */
@@ -52,7 +50,7 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
5250
// scalastyle:off
5351
<tr>
5452
<td>
55-
<a href={"%s/storage/rdd?id=%s".format(UIUtils.prependBaseUri(basePath), rdd.id)}>
53+
<a href={"%s/storage/rdd?id=%s".format(UIUtils.prependBaseUri(parent.basePath), rdd.id)}>
5654
{rdd.name}
5755
</a>
5856
</td>

0 commit comments

Comments
 (0)