Skip to content

Commit 4c8b044

Browse files
sarutakkai-chi
sarutak
authored and
kai-chi
committed
[SPARK-28647][WEBUI][2.4] Recover additional metric feature
This PR is for backporting SPARK-28647(apache#25374) to branch-2.4. The original PR removed `additional-metrics.js` but branch-2.4 still uses it so I don't remove it and related things for branch-2.4. ### What changes were proposed in this pull request? Added checkboxes to enable users to select which optional metrics (`On Heap Memory`, `Off Heap Memory` and `Select All` in this case) to be shown in `ExecuorPage`. ### Why are the changes needed? By SPARK-17019, `On Heap Memory` and `Off Heap Memory` are introduced as optional metrics. But they are not displayed because they are made `display: none` in css and there are no way to appear them. ### Does this PR introduce any user-facing change? The previous `ExecutorPage` doesn't show optional metrics. This change adds checkboxes to `ExecutorPage` for optional metrics. We can choose which metrics should be shown by checking corresponding checkboxes. ![Screenshot from 2019-08-18 03-56-09](https://user-images.githubusercontent.com/4736016/63216148-2bfadb80-c16c-11e9-81e1-e1e66198dd6c.png) ### How was this patch tested? Manual test. Closes apache#25484 from sarutak/backport-SPARK-28647-branch-2.4. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent b38f1e3 commit 4c8b044

File tree

3 files changed

+110
-27
lines changed

3 files changed

+110
-27
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
-->
1717

1818
<script id="executors-summary-template" type="text/html">
19-
<h4 style="clear: left; display: inline-block;">Summary</h4>
19+
<div id="showAdditionalMetrics"></div>
20+
<h4 class="title-table">Summary</h4>
2021
<div class="container-fluid">
2122
<div class="container-fluid">
2223
<table id="summary-execs-table" class="table table-striped compact">
@@ -26,11 +27,11 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
2627
<th><span data-toggle="tooltip"
2728
title="Memory used / total available memory for storage of data like RDD partitions cached in memory.">Storage Memory</span>
2829
</th>
29-
<th class="on_heap_memory">
30+
<th>
3031
<span data-toggle="tooltip"
3132
title="Memory used / total available memory for on heap storage of data like RDD partitions cached in memory.">On Heap Storage Memory</span>
3233
</th>
33-
<th class="off_heap_memory">
34+
<th>
3435
<span data-toggle="tooltip"
3536
title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">Off Heap Storage Memory</span>
3637
</th>
@@ -81,11 +82,11 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
8182
<span data-toggle="tooltip" data-placement="top"
8283
title="Memory used / total available memory for storage of data like RDD partitions cached in memory.">
8384
Storage Memory</span></th>
84-
<th class="on_heap_memory">
85+
<th>
8586
<span data-toggle="tooltip" data-placement="top"
8687
title="Memory used / total available memory for on heap storage of data like RDD partitions cached in memory.">
8788
On Heap Storage Memory</span></th>
88-
<th class="off_heap_memory">
89+
<th>
8990
<span data-toggle="tooltip"
9091
title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">
9192
Off Heap Storage Memory</span></th>

‎core/src/main/resources/org/apache/spark/ui/static/executorspage.js‎

Lines changed: 90 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -177,14 +177,35 @@ function totalDurationColor(totalGCTime, totalDuration) {
177177
return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
178178
}
179179

180+
var sumOptionalColumns = [3, 4];
181+
var execOptionalColumns = [5, 6];
182+
var execDataTable;
183+
var sumDataTable;
184+
185+
function reselectCheckboxesBasedOnTaskTableState() {
186+
var allChecked = true;
187+
if (typeof execDataTable !== "undefined") {
188+
for (var k = 0; k < execOptionalColumns.length; k++) {
189+
if (execDataTable.column(execOptionalColumns[k]).visible()) {
190+
$("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
191+
} else {
192+
allChecked = false;
193+
}
194+
}
195+
}
196+
if (allChecked) {
197+
$("#select-all-box").prop("checked", true);
198+
}
199+
}
200+
180201
$(document).ready(function () {
181202
$.extend($.fn.dataTable.defaults, {
182203
stateSave: true,
183204
lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
184205
pageLength: 20
185206
});
186207

187-
executorsSummary = $("#active-executors");
208+
var executorsSummary = $("#active-executors");
188209

189210
getStandAloneppId(function (appId) {
190211

@@ -444,9 +465,6 @@ $(document).ready(function () {
444465
else
445466
return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
446467
formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
447-
},
448-
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
449-
$(nTd).addClass('on_heap_memory')
450468
}
451469
},
452470
{
@@ -456,9 +474,6 @@ $(document).ready(function () {
456474
else
457475
return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
458476
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
459-
},
460-
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
461-
$(nTd).addClass('off_heap_memory')
462477
}
463478
},
464479
{data: 'diskUsed', render: formatBytes},
@@ -505,12 +520,16 @@ $(document).ready(function () {
505520
}
506521
}
507522
],
508-
"order": [[0, "asc"]]
523+
"order": [[0, "asc"]],
524+
"columnDefs": [
525+
{"visible": false, "targets": 5},
526+
{"visible": false, "targets": 6}
527+
]
509528
};
510-
511-
var dt = $(selector).DataTable(conf);
512-
dt.column('executorLogsCol:name').visible(logsExist(response));
513-
dt.column('threadDumpCol:name').visible(getThreadDumpEnabled());
529+
530+
execDataTable = $(selector).DataTable(conf);
531+
execDataTable.column('executorLogsCol:name').visible(logsExist(response));
532+
execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
514533
$('#active-executors [data-toggle="tooltip"]').tooltip();
515534

516535
var sumSelector = "#summary-execs-table";
@@ -540,9 +559,6 @@ $(document).ready(function () {
540559
else
541560
return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
542561
formatBytes(row.allOnHeapMaxMemory, type));
543-
},
544-
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
545-
$(nTd).addClass('on_heap_memory')
546562
}
547563
},
548564
{
@@ -552,9 +568,6 @@ $(document).ready(function () {
552568
else
553569
return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
554570
formatBytes(row.allOffHeapMaxMemory, type));
555-
},
556-
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
557-
$(nTd).addClass('off_heap_memory')
558571
}
559572
},
560573
{data: 'allDiskUsed', render: formatBytes},
@@ -597,13 +610,69 @@ $(document).ready(function () {
597610
],
598611
"paging": false,
599612
"searching": false,
600-
"info": false
613+
"info": false,
614+
"columnDefs": [
615+
{"visible": false, "targets": 3},
616+
{"visible": false, "targets": 4}
617+
]
601618

602619
};
603620

604-
$(sumSelector).DataTable(sumConf);
621+
sumDataTable = $(sumSelector).DataTable(sumConf);
605622
$('#execSummary [data-toggle="tooltip"]').tooltip();
606-
623+
624+
$("#showAdditionalMetrics").append(
625+
"<div><a id='additionalMetrics'>" +
626+
"<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle-optional-metrics'></span>" +
627+
"Show Additional Metrics" +
628+
"</a></div>" +
629+
"<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>" +
630+
"<div><input type='checkbox' class='toggle-vis' id='select-all-box'>Select All</div>" +
631+
"<div id='on_heap_memory' class='on-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='3' data-exec-col-idx='5'>On Heap Memory</div>" +
632+
"<div id='off_heap_memory' class='off-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='4' data-exec-col-idx='6'>Off Heap Memory</div>" +
633+
"</div>");
634+
635+
reselectCheckboxesBasedOnTaskTableState();
636+
637+
$("#additionalMetrics").click(function() {
638+
$("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
639+
$("#toggle-metrics").toggle();
640+
if (window.localStorage) {
641+
window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
642+
}
643+
});
644+
645+
$(".toggle-vis").on("click", function() {
646+
var thisBox = $(this);
647+
if (thisBox.is("#select-all-box")) {
648+
var sumColumn = sumDataTable.columns(sumOptionalColumns);
649+
var execColumn = execDataTable.columns(execOptionalColumns);
650+
if (thisBox.is(":checked")) {
651+
$(".toggle-vis").prop("checked", true);
652+
sumColumn.visible(true);
653+
execColumn.visible(true);
654+
} else {
655+
$(".toggle-vis").prop("checked", false);
656+
sumColumn.visible(false);
657+
execColumn.visible(false);
658+
}
659+
} else {
660+
var execColIdx = thisBox.attr("data-exec-col-idx");
661+
var execCol = execDataTable.column(execColIdx);
662+
execCol.visible(!execCol.visible());
663+
var sumColIdx = thisBox.attr("data-sum-col-idx");
664+
var sumCol = sumDataTable.column(sumColIdx);
665+
sumCol.visible(!sumCol.visible());
666+
}
667+
});
668+
669+
if (window.localStorage) {
670+
if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
671+
window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
672+
$("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
673+
$("#toggle-metrics").toggle();
674+
}
675+
}
607676
});
608677
});
609678
});

‎core/src/main/resources/org/apache/spark/ui/static/webui.css‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,4 +260,17 @@ a.expandbutton {
260260
.paginate_button.active > a {
261261
color: #999999;
262262
text-decoration: underline;
263-
}
263+
}
264+
265+
.title-table {
266+
clear: left;
267+
display: inline-block;
268+
}
269+
270+
.table-dataTable {
271+
width: 100%;
272+
}
273+
274+
.container-fluid-div {
275+
width: 200px;
276+
}

0 commit comments

Comments
 (0)