Generalize SparkUI header to display tabs dynamically

andrewor14 · andrewor14 · commit ed25dfc704af · 2014-04-02T21:13:45.000-07:00
diff --git a/core/src/main/scala/org/apache/spark/ui/Page.scala b/core/src/main/scala/org/apache/spark/ui/Page.scala
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -55,8 +55,8 @@ private[spark] class SparkUI(
 
   /** Initialize all components of the server. Must be called before bind(). */
   def start() {
-    attachTab(new BlockManagerTab(this))
     attachTab(new JobProgressTab(this))
+    attachTab(new BlockManagerTab(this))
     attachTab(new EnvironmentTab(this))
     attachTab(new ExecutorsTab(this))
     attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -25,8 +25,6 @@ import scala.xml.Node
 /** Utility functions for generating XML pages with spark content. */
 private[spark] object UIUtils {
 
-  import Page._
-
   // SimpleDateFormat is not thread-safe. Don't expose it to avoid improper use.
   private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
     override def initialValue(): SimpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss")
@@ -62,26 +60,13 @@ private[spark] object UIUtils {
       basePath: String,
       appName: String,
       title: String,
-      page: Page.Value) : Seq[Node] = {
-    val jobs = page match {
-      case Stages =>
-        <li class="active"><a href={prependBaseUri(basePath, "/stages")}>Stages</a></li>
-      case _ => <li><a href={prependBaseUri(basePath, "/stages")}>Stages</a></li>
-    }
-    val storage = page match {
-      case Storage =>
-        <li class="active"><a href={prependBaseUri(basePath, "/storage")}>Storage</a></li>
-      case _ => <li><a href={prependBaseUri(basePath, "/storage")}>Storage</a></li>
-    }
-    val environment = page match {
-      case Environment => 
-        <li class="active"><a href={prependBaseUri(basePath, "/environment")}>Environment</a></li>
-      case _ => <li><a href={prependBaseUri(basePath, "/environment")}>Environment</a></li>
-    }
-    val executors = page match {
-      case Executors =>
-        <li class="active"><a href={prependBaseUri(basePath, "/executors")}>Executors</a></li>
-      case _ => <li><a href={prependBaseUri(basePath, "/executors")}>Executors</a></li>
+      tabs: Seq[UITab],
+      activeTab: UITab) : Seq[Node] = {
+
+    val header = tabs.map { tab =>
+      <li class={if (tab == activeTab) "active" else ""}>
+        <a href={prependBaseUri(basePath, "/" + tab.prefix)}>{tab.name}</a>
+      </li>
     }
 
     <html>
@@ -100,12 +85,7 @@ private[spark] object UIUtils {
             <a href={prependBaseUri(basePath, "/")} class="brand">
               <img src={prependBaseUri("/static/spark-logo-77x50px-hd.png")} />
             </a>
-            <ul class="nav">
-              {jobs}
-              {storage}
-              {environment}
-              {executors}
-            </ul>
+            <ul class="nav">{header}</ul>
             <p class="navbar-text pull-right"><strong>{appName}</strong> application UI</p>
           </div>
         </div>
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -42,6 +42,10 @@ private[spark] abstract class WebUI(securityManager: SecurityManager, basePath:
   protected val handlers = ArrayBuffer[ServletContextHandler]()
   protected var serverInfo: Option[ServerInfo] = None
 
+  def getTabs: Seq[UITab] = tabs.toSeq
+  def getHandlers: Seq[ServletContextHandler] = handlers.toSeq
+  def getListeners: Seq[SparkListener] = tabs.flatMap(_.listener)
+
   /** Attach a tab to this UI, along with all of its attached pages. Only valid before bind(). */
   def attachTab(tab: UITab) {
     tab.start()
@@ -65,12 +69,6 @@ private[spark] abstract class WebUI(securityManager: SecurityManager, basePath:
     handlers += handler
   }
 
-  /** Return a list of listeners attached to this UI. */
-  def getListeners = tabs.flatMap(_.listener)
-
-  /** Return a list of handlers attached to this UI. */
-  def getHandlers = handlers.toSeq
-
   /** Initialize all components of the server. Must be called before bind(). */
   def start()
 
@@ -98,6 +96,7 @@ private[spark] abstract class WebUI(securityManager: SecurityManager, basePath:
 private[spark] abstract class UITab(val prefix: String) {
   val pages = ArrayBuffer[UIPage]()
   var listener: Option[SparkListener] = None
+  var name = prefix.capitalize
 
   /** Attach a page to this tab. This prepends the page's prefix with the tab's own prefix. */
   def attachPage(page: UIPage) {
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentTab.scala
@@ -33,6 +33,8 @@ private[ui] class EnvironmentTab(parent: SparkUI) extends UITab("environment") {
     assert(listener.isDefined, "EnvironmentTab has not started yet!")
     listener.get.asInstanceOf[EnvironmentListener]
   }
+
+  def headerTabs: Seq[UITab] = parent.getTabs
 }
 
 /**
diff --git a/core/src/main/scala/org/apache/spark/ui/env/IndexPage.scala b/core/src/main/scala/org/apache/spark/ui/env/IndexPage.scala
@@ -22,7 +22,6 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.Node
 
 import org.apache.spark.ui.{UIUtils, UIPage}
-import org.apache.spark.ui.Page.Environment
 
 private[ui] class IndexPage(parent: EnvironmentTab) extends UIPage("") {
   private val appName = parent.appName
@@ -46,7 +45,7 @@ private[ui] class IndexPage(parent: EnvironmentTab) extends UIPage("") {
         <h4>Classpath Entries</h4> {classpathEntriesTable}
       </span>
 
-    UIUtils.headerSparkPage(content, basePath, appName, "Environment", Environment)
+    UIUtils.headerSparkPage(content, basePath, appName, "Environment", parent.headerTabs, parent)
   }
 
   private def propertyHeader = Seq("Name", "Value")
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
@@ -37,6 +37,8 @@ private[ui] class ExecutorsTab(parent: SparkUI) extends UITab("executors") {
     assert(listener.isDefined, "ExecutorsTab has not started yet!")
     listener.get.asInstanceOf[ExecutorsListener]
   }
+
+  def headerTabs: Seq[UITab] = parent.getTabs
 }
 
 /**
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/IndexPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/IndexPage.scala
@@ -22,7 +22,6 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.Node
 
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Executors
 import org.apache.spark.util.Utils
 
 private[ui] class IndexPage(parent: ExecutorsTab) extends UIPage("") {
@@ -56,8 +55,8 @@ private[ui] class IndexPage(parent: ExecutorsTab) extends UIPage("") {
         </div>
       </div>;
 
-    UIUtils.headerSparkPage(
-      content, basePath, appName, "Executors (" + execInfo.size + ")", Executors)
+    UIUtils.headerSparkPage(content, basePath, appName, "Executors (" + execInfo.size + ")",
+      parent.headerTabs, parent)
   }
 
   /** Header fields for the executors table */
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/IndexPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/IndexPage.scala
@@ -23,7 +23,6 @@ import scala.xml.{Node, NodeSeq}
 
 import org.apache.spark.scheduler.Schedulable
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Stages
 
 /** Page showing list of all ongoing and recently finished stages and pools */
 private[ui] class IndexPage(parent: JobProgressTab) extends UIPage("") {
@@ -92,7 +91,7 @@ private[ui] class IndexPage(parent: JobProgressTab) extends UIPage("") {
         <h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++
         failedStagesTable.toNodeSeq
 
-      UIUtils.headerSparkPage(content, basePath, appName, "Spark Stages", Stages)
+      UIUtils.headerSparkPage(content, basePath, appName, "Spark Stages", parent.headerTabs, parent)
     }
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressTab.scala
@@ -42,4 +42,6 @@ private[ui] class JobProgressTab(parent: SparkUI) extends UITab("stages") {
   }
 
   def isFairScheduler = jobProgressListener.schedulingMode.exists(_ == SchedulingMode.FAIR)
+
+  def headerTabs: Seq[UITab] = parent.getTabs
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
@@ -23,7 +23,6 @@ import scala.xml.Node
 
 import org.apache.spark.scheduler.{Schedulable, StageInfo}
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Stages
 
 /** Page showing specific pool details */
 private[ui] class PoolPage(parent: JobProgressTab) extends UIPage("pool") {
@@ -51,8 +50,8 @@ private[ui] class PoolPage(parent: JobProgressTab) extends UIPage("pool") {
         <h4>Summary </h4> ++ poolTable.toNodeSeq ++
         <h4>{activeStages.size} Active Stages</h4> ++ activeStagesTable.toNodeSeq
 
-      UIUtils.headerSparkPage(
-        content, basePath, appName, "Fair Scheduler Pool: " + poolName, Stages)
+      UIUtils.headerSparkPage(content, basePath, appName, "Fair Scheduler Pool: " + poolName,
+        parent.headerTabs, parent)
     }
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -23,7 +23,6 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.Node
 
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Stages
 import org.apache.spark.util.{Utils, Distribution}
 
 /** Page showing statistics and task list for a given stage */
@@ -42,8 +41,8 @@ private[ui] class StagePage(parent: JobProgressTab) extends UIPage("stage") {
             <h4>Summary Metrics</h4> No tasks have started yet
             <h4>Tasks</h4> No tasks have started yet
           </div>
-        return UIUtils.headerSparkPage(
-          content, basePath, appName, "Details for Stage %s".format(stageId), Stages)
+        return UIUtils.headerSparkPage(content, basePath, appName,
+          "Details for Stage %s".format(stageId), parent.headerTabs, parent)
       }
 
       val tasks = listener.stageIdToTaskData(stageId).values.toSeq.sortBy(_.taskInfo.launchTime)
@@ -204,8 +203,8 @@ private[ui] class StagePage(parent: JobProgressTab) extends UIPage("stage") {
         <h4>Aggregated Metrics by Executor</h4> ++ executorTable.toNodeSeq ++
         <h4>Tasks</h4> ++ taskTable
 
-      UIUtils.headerSparkPage(
-        content, basePath, appName, "Details for Stage %d".format(stageId), Stages)
+      UIUtils.headerSparkPage(content, basePath, appName, "Details for Stage %d".format(stageId),
+        parent.headerTabs, parent)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerTab.scala
@@ -38,6 +38,8 @@ private[ui] class BlockManagerTab(parent: SparkUI) extends UITab("storage") {
     assert(listener.isDefined, "BlockManagerTab has not started yet!")
     listener.get.asInstanceOf[BlockManagerListener]
   }
+
+  def headerTabs: Seq[UITab] = parent.getTabs
 }
 
 /**
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/IndexPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/IndexPage.scala
@@ -23,7 +23,6 @@ import scala.xml.Node
 
 import org.apache.spark.storage.RDDInfo
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Storage
 import org.apache.spark.util.Utils
 
 /** Page showing list of RDD's currently stored in the cluster */
@@ -35,7 +34,7 @@ private[ui] class IndexPage(parent: BlockManagerTab) extends UIPage("") {
   override def render(request: HttpServletRequest): Seq[Node] = {
     val rdds = listener.rddInfoList
     val content = UIUtils.listingTable(rddHeader, rddRow, rdds)
-    UIUtils.headerSparkPage(content, basePath, appName, "Storage ", Storage)
+    UIUtils.headerSparkPage(content, basePath, appName, "Storage ", parent.headerTabs, parent)
   }
 
   /** Header fields for the RDD table */
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
@@ -23,7 +23,6 @@ import scala.xml.Node
 
 import org.apache.spark.storage.{BlockId, BlockStatus, StorageStatus, StorageUtils}
 import org.apache.spark.ui.{UIPage, UIUtils}
-import org.apache.spark.ui.Page.Storage
 import org.apache.spark.util.Utils
 
 /** Page showing storage details for a given RDD */
@@ -37,7 +36,8 @@ private[ui] class RddPage(parent: BlockManagerTab) extends UIPage("rdd") {
     val storageStatusList = listener.storageStatusList
     val rddInfo = listener.rddInfoList.find(_.id == rddId).getOrElse {
       // Rather than crashing, render an "RDD Not Found" page
-      return UIUtils.headerSparkPage(Seq[Node](), basePath, appName, "RDD Not Found", Storage)
+      return UIUtils.headerSparkPage(Seq[Node](), basePath, appName, "RDD Not Found",
+        parent.headerTabs, parent)
     }
 
     // Worker table
@@ -95,8 +95,8 @@ private[ui] class RddPage(parent: BlockManagerTab) extends UIPage("rdd") {
         </div>
       </div>;
 
-    UIUtils.headerSparkPage(
-      content, basePath, appName, "RDD Storage Info for " + rddInfo.name, Storage)
+    UIUtils.headerSparkPage(content, basePath, appName, "RDD Storage Info for " + rddInfo.name,
+      parent.headerTabs, parent)
   }
 
   /** Header fields for the worker table */

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,8 @@ private[ui] class EnvironmentTab(parent: SparkUI) extends UITab("environment") {`
`33`	`33`	`assert(listener.isDefined, "EnvironmentTab has not started yet!")`
`34`	`34`	`listener.get.asInstanceOf[EnvironmentListener]`
`35`	`35`	`}`
	`36`	`+`
	`37`	`+ def headerTabs: Seq[UITab] = parent.getTabs`
`36`	`38`	`}`
`37`	`39`
`38`	`40`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,8 @@ private[ui] class ExecutorsTab(parent: SparkUI) extends UITab("executors") {`
`37`	`37`	`assert(listener.isDefined, "ExecutorsTab has not started yet!")`
`38`	`38`	`listener.get.asInstanceOf[ExecutorsListener]`
`39`	`39`	`}`
	`40`	`+`
	`41`	`+ def headerTabs: Seq[UITab] = parent.getTabs`
`40`	`42`	`}`
`41`	`43`
`42`	`44`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,6 @@ import scala.xml.{Node, NodeSeq}`
`23`	`23`
`24`	`24`	`import org.apache.spark.scheduler.Schedulable`
`25`	`25`	`import org.apache.spark.ui.{UIPage, UIUtils}`
`26`		`-import org.apache.spark.ui.Page.Stages`
`27`	`26`
`28`	`27`	`/** Page showing list of all ongoing and recently finished stages and pools */`
`29`	`28`	`private[ui] class IndexPage(parent: JobProgressTab) extends UIPage("") {`
`@@ -92,7 +91,7 @@ private[ui] class IndexPage(parent: JobProgressTab) extends UIPage("") {`
`92`	`91`	`<h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++`
`93`	`92`	`failedStagesTable.toNodeSeq`
`94`	`93`
`95`		`- UIUtils.headerSparkPage(content, basePath, appName, "Spark Stages", Stages)`
	`94`	`+ UIUtils.headerSparkPage(content, basePath, appName, "Spark Stages", parent.headerTabs, parent)`
`96`	`95`	`}`
`97`	`96`	`}`
`98`	`97`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,4 +42,6 @@ private[ui] class JobProgressTab(parent: SparkUI) extends UITab("stages") {`
`42`	`42`	`}`
`43`	`43`
`44`	`44`	`def isFairScheduler = jobProgressListener.schedulingMode.exists(_ == SchedulingMode.FAIR)`
	`45`	`+`
	`46`	`+ def headerTabs: Seq[UITab] = parent.getTabs`
`45`	`47`	`}`