Skip to content

Commit 74307cf

Browse files
zsxwingzsxwing
committed
Reuse the data for histogram graphs to reduce the page size
1 parent 2586916 commit 74307cf

File tree

2 files changed

+79
-94
lines changed

2 files changed

+79
-94
lines changed

‎streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala

Lines changed: 79 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ import org.apache.spark.ui._
3030
import org.apache.spark.ui.UIUtils._
3131

3232
/**
33-
* @param divId the `id` used in the html `div` tag
34-
* @param data the data for the timeline graph
33+
* @param timelineDivId the timeline `id` used in the html `div` tag
34+
* @param histogramDivId the timeline `id` used in the html `div` tag
35+
* @param data the data for the graph
3536
* @param minX the min value of X axis
3637
* @param maxX the max value of X axis
3738
* @param minY the min value of Y axis
@@ -40,52 +41,56 @@ import org.apache.spark.ui.UIUtils._
4041
* @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in
4142
* the graph
4243
*/
43-
private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long,
44-
minY: Double, maxY: Double, unitY: String, batchInterval: Option[Double] = None) {
44+
private[ui] class GraphUIData(
45+
timelineDivId: String,
46+
histogramDivId: String,
47+
data: Seq[(Long, Double)],
48+
minX: Long,
49+
maxX: Long,
50+
minY: Double,
51+
maxY: Double,
52+
unitY: String,
53+
batchInterval: Option[Double] = None) {
54+
55+
private var dataJavaScriptName: String = _
4556

46-
def toHtml(jsCollector: JsCollector): Seq[Node] = {
57+
def generateDataJs(jsCollector: JsCollector): Unit = {
4758
val jsForData = data.map { case (x, y) =>
4859
s"""{"x": $x, "y": $y}"""
4960
}.mkString("[", ",", "]")
61+
dataJavaScriptName = jsCollector.nextVariableName
62+
jsCollector.addPreparedStatement(s"var $dataJavaScriptName = $jsForData;")
63+
}
64+
65+
def generateTimelineHtml(jsCollector: JsCollector): Seq[Node] = {
5066
jsCollector.addPreparedStatement(s"registerTimeline($minY, $maxY);")
5167
if (batchInterval.isDefined) {
5268
jsCollector.addStatement(
5369
"drawTimeline(" +
54-
s"'#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY', ${batchInterval.get}" +
70+
s"'#$timelineDivId', $dataJavaScriptName, $minX, $maxX, $minY, $maxY, '$unitY'," +
71+
s" ${batchInterval.get}" +
5572
");")
5673
} else {
5774
jsCollector.addStatement(
58-
s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');")
75+
s"drawTimeline('#$timelineDivId', $dataJavaScriptName, $minX, $maxX, $minY, $maxY," +
76+
s" '$unitY');")
5977
}
60-
<div id={divId}></div>
78+
<div id={timelineDivId}></div>
6179
}
62-
}
6380

64-
/**
65-
* @param divId the `id` used in the html `div` tag
66-
* @param data the data for the histogram graph
67-
* @param minY the min value of Y axis
68-
* @param maxY the max value of Y axis
69-
* @param unitY the unit of Y axis
70-
* @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in
71-
* the graph
72-
*/
73-
private[ui] class HistogramUIData(
74-
divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String,
75-
batchInterval: Option[Double] = None) {
76-
77-
def toHtml(jsCollector: JsCollector): Seq[Node] = {
78-
val jsForData = data.mkString("[", ",", "]")
79-
jsCollector.addPreparedStatement(s"registerHistogram($jsForData, $minY, $maxY);")
81+
def generateHistogramHtml(jsCollector: JsCollector): Seq[Node] = {
82+
val histogramData = s"$dataJavaScriptName.map(function(d) { return d.y; })"
83+
jsCollector.addPreparedStatement(s"registerHistogram($histogramData, $minY, $maxY);")
8084
if (batchInterval.isDefined) {
8185
jsCollector.addStatement(
8286
"drawHistogram(" +
83-
s"'#$divId', $jsForData, $minY, $maxY, '$unitY', ${batchInterval.get}" +
87+
s"'#$histogramDivId', $histogramData, $minY, $maxY, '$unitY', ${batchInterval.get}" +
8488
");")
8589
} else {
86-
jsCollector.addStatement(s"drawHistogram('#$divId', $jsForData, $minY, $maxY, '$unitY');")
90+
jsCollector.addStatement(
91+
s"drawHistogram('#$histogramDivId', $histogramData, $minY, $maxY, '$unitY');")
8792
}
88-
<div id={divId}></div>
93+
<div id={histogramDivId}></div>
8994
}
9095
}
9196

@@ -246,77 +251,53 @@ private[ui] class StreamingPage(parent: StreamingTab)
246251

247252
val jsCollector = new JsCollector
248253

249-
val timelineDataForEventRateOfAllReceivers =
250-
new TimelineUIData(
254+
val graphUIDataForEventRateOfAllReceivers =
255+
new GraphUIData(
251256
"all-receiver-events-timeline",
257+
"all-receiver-events-histogram",
252258
eventRateForAllReceivers.data,
253259
minBatchTime,
254260
maxBatchTime,
255261
minEventRate,
256262
maxEventRate,
257-
"events/sec").toHtml(jsCollector)
263+
"events/sec")
264+
graphUIDataForEventRateOfAllReceivers.generateDataJs(jsCollector)
258265

259-
val histogramDataForEventRateOfAllReceivers =
260-
new HistogramUIData(
261-
"all-receiver-events-histogram",
262-
eventRateForAllReceivers.data.map(_._2),
263-
minEventRate,
264-
maxEventRate,
265-
"events/sec").toHtml(jsCollector)
266-
267-
val timelineDataForSchedulingDelay =
268-
new TimelineUIData(
266+
val graphUIDataForSchedulingDelay =
267+
new GraphUIData(
269268
"scheduling-delay-timeline",
269+
"scheduling-delay-histogram",
270270
schedulingDelay.timelineData(normalizedUnit),
271271
minBatchTime,
272272
maxBatchTime,
273273
minTime,
274274
maxTime,
275-
formattedUnit).toHtml(jsCollector)
276-
277-
val histogramDataForSchedulingDelay =
278-
new HistogramUIData(
279-
"scheduling-delay-histogram",
280-
schedulingDelay.histogramData(normalizedUnit),
281-
minTime,
282-
maxTime,
283-
formattedUnit).toHtml(jsCollector)
275+
formattedUnit)
276+
graphUIDataForSchedulingDelay.generateDataJs(jsCollector)
284277

285-
val timelineDataForProcessingTime =
286-
new TimelineUIData(
278+
val graphUIDataForProcessingTime =
279+
new GraphUIData(
287280
"processing-time-timeline",
281+
"processing-time-histogram",
288282
processingTime.timelineData(normalizedUnit),
289283
minBatchTime,
290284
maxBatchTime,
291285
minTime,
292286
maxTime,
293-
formattedUnit, Some(batchInterval)).toHtml(jsCollector)
294-
295-
val histogramDataForProcessingTime =
296-
new HistogramUIData(
297-
"processing-time-histogram",
298-
processingTime.histogramData(normalizedUnit),
299-
minTime,
300-
maxTime,
301-
formattedUnit, Some(batchInterval)).toHtml(jsCollector)
287+
formattedUnit, Some(batchInterval))
288+
graphUIDataForProcessingTime.generateDataJs(jsCollector)
302289

303-
val timelineDataForTotalDelay =
304-
new TimelineUIData(
290+
val graphUIDataForTotalDelay =
291+
new GraphUIData(
305292
"total-delay-timeline",
293+
"total-delay-histogram",
306294
totalDelay.timelineData(normalizedUnit),
307295
minBatchTime,
308296
maxBatchTime,
309297
minTime,
310298
maxTime,
311-
formattedUnit).toHtml(jsCollector)
312-
313-
val histogramDataForTotalDelay =
314-
new HistogramUIData(
315-
"total-delay-histogram",
316-
totalDelay.histogramData(normalizedUnit),
317-
minTime,
318-
maxTime,
319-
formattedUnit).toHtml(jsCollector)
299+
formattedUnit)
300+
graphUIDataForTotalDelay.generateDataJs(jsCollector)
320301

321302
val hasReceiver = listener.allReceivers.nonEmpty
322303

@@ -344,8 +325,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
344325
<div>Avg: {eventRateForAllReceivers.formattedAvg} events/sec</div>
345326
</div>
346327
</td>
347-
<td class="timeline">{timelineDataForEventRateOfAllReceivers}</td>
348-
<td class="histogram">{histogramDataForEventRateOfAllReceivers}</td>
328+
<td class="timeline">{graphUIDataForEventRateOfAllReceivers.generateTimelineHtml(jsCollector)}</td>
329+
<td class="histogram">{graphUIDataForEventRateOfAllReceivers.generateHistogramHtml(jsCollector)}</td>
349330
</tr>
350331
{if (hasReceiver) {
351332
<tr id="inputs-table" style="display: none;" >
@@ -361,8 +342,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
361342
<div>Avg: {schedulingDelay.formattedAvg}</div>
362343
</div>
363344
</td>
364-
<td class="timeline">{timelineDataForSchedulingDelay}</td>
365-
<td class="histogram">{histogramDataForSchedulingDelay}</td>
345+
<td class="timeline">{graphUIDataForSchedulingDelay.generateTimelineHtml(jsCollector)}</td>
346+
<td class="histogram">{graphUIDataForSchedulingDelay.generateHistogramHtml(jsCollector)}</td>
366347
</tr>
367348
<tr>
368349
<td style="vertical-align: middle;">
@@ -371,8 +352,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
371352
<div>Avg: {processingTime.formattedAvg}</div>
372353
</div>
373354
</td>
374-
<td class="timeline">{timelineDataForProcessingTime}</td>
375-
<td class="histogram">{histogramDataForProcessingTime}</td>
355+
<td class="timeline">{graphUIDataForProcessingTime.generateTimelineHtml(jsCollector)}</td>
356+
<td class="histogram">{graphUIDataForProcessingTime.generateHistogramHtml(jsCollector)}</td>
376357
</tr>
377358
<tr>
378359
<td style="vertical-align: middle;">
@@ -381,8 +362,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
381362
<div>Avg: {totalDelay.formattedAvg}</div>
382363
</div>
383364
</td>
384-
<td class="timeline">{timelineDataForTotalDelay}</td>
385-
<td class="histogram">{histogramDataForTotalDelay}</td>
365+
<td class="timeline">{graphUIDataForTotalDelay.generateTimelineHtml(jsCollector)}</td>
366+
<td class="histogram">{graphUIDataForTotalDelay.generateHistogramHtml(jsCollector)}</td>
386367
</tr>
387368
</tbody>
388369
</table>
@@ -442,23 +423,17 @@ private[ui] class StreamingPage(parent: StreamingTab)
442423
val receivedRecords =
443424
new EventRateUIData(listener.receivedEventRateWithBatchTime.get(receiverId).getOrElse(Seq()))
444425

445-
val timelineForEventRate =
446-
new TimelineUIData(
426+
val graphUIDataForEventRate =
427+
new GraphUIData(
447428
s"receiver-$receiverId-events-timeline",
429+
s"receiver-$receiverId-events-histogram",
448430
receivedRecords.data,
449431
minX,
450432
maxX,
451433
minY,
452434
maxY,
453-
"events/sec").toHtml(jsCollector)
454-
455-
val histogramForEventsRate =
456-
new HistogramUIData(
457-
s"receiver-$receiverId-events-histogram",
458-
receivedRecords.data.map(_._2),
459-
minY,
460-
maxY,
461-
"events/sec").toHtml(jsCollector)
435+
"events/sec")
436+
graphUIDataForEventRate.generateDataJs(jsCollector)
462437

463438
<tr>
464439
<td rowspan="2" style="vertical-align: middle; width: 151px;">
@@ -474,9 +449,9 @@ private[ui] class StreamingPage(parent: StreamingTab)
474449
</tr>
475450
<tr>
476451
<td colspan="3" class="timeline">
477-
{timelineForEventRate}
452+
{graphUIDataForEventRate.generateTimelineHtml(jsCollector)}
478453
</td>
479-
<td class="histogram">{histogramForEventsRate}</td>
454+
<td class="histogram">{graphUIDataForEventRate.generateHistogramHtml(jsCollector)}</td>
480455
</tr>
481456
}
482457

@@ -535,6 +510,17 @@ private[ui] object StreamingPage {
535510
* DOM has finished loading.
536511
*/
537512
private[ui] class JsCollector {
513+
514+
private var variableId = 0
515+
516+
/**
517+
* Return the next unused JavaScript variable name
518+
*/
519+
def nextVariableName: String = {
520+
variableId += 1
521+
"v" + variableId
522+
}
523+
538524
/**
539525
* JavaScript statements that will execute before `statements`
540526
*/

‎streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,6 @@ class UISeleniumSuite
105105
statTableHeaders should contain ("Histograms")
106106

107107
val statTableCells = findAll(cssSelector("#stat-table td")).map(_.text).toSeq
108-
println(statTableCells.toList)
109108
statTableCells.exists(_.contains("Input Rate")) should be (true)
110109
statTableCells.exists(_.contains("Streaming Scheduling Delay")) should be (true)
111110
statTableCells.exists(_.contains("Processing Time")) should be (true)

0 commit comments

Comments
 (0)