Add GIL metric to dashboard (#7646)

* Add GIL metric to dashboard

Adds a GIL contention metric to the Event Loop
page when `distributed.admin.system-monitor.gil-contention`
is enabled and `gilknocker` is installed.

Otherwise, the chart will not show anything GIL
related, not even placeholders.

* Reduce horizonal white space between groups

* Update distributed/dashboard/components/scheduler.py

Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>

* Rename EventLoop -> Contention, and page name

---------

Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>

Author: milesgranger

distributed/dashboard/components/scheduler.py

@@ -3375,57 +3375,99 @@ def update(self):
         )
 
 
-class EventLoop(DashboardComponent):
-    """Event Loop Health"""
+class Contention(DashboardComponent):
+    """
+    Event Loop Health (and GIL Contention, if configured)
+    """
 
     @log_errors
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
-        self.source = ColumnDataSource(
-            {
-                "names": ["Scheduler", "Workers"],
-                "values": [0, 0],
-                "text": ["0", "0"],
-            }
+        self.data = dict(
+            names=[
+                ("Scheduler", "Event Loop"),
+                ("Scheduler", "GIL Contention"),
+                ("Workers", "Event Loop"),
+                ("Workers", "GIL Contention"),
+            ],
+            values=[0, 0, 0, 0],
+            text=["0s", "0%", "0s", "0%"],
         )
+        title = "Event Loop & GIL Contention"
+
+        # Remove GIL related names/values if not monitoring GIL
+        if not self.scheduler.monitor.monitor_gil_contention:
+            title = "Event Loop"
+            for key in self.data:
+                self.data[key] = self.data[key][::2]
 
+        self.source = ColumnDataSource(data=self.data)
         self.root = figure(
-            title="Event Loop Health",
-            x_range=["Scheduler", "Workers"],
-            y_range=[
-                0,
-                parse_timedelta(dask.config.get("distributed.admin.tick.interval"))
-                * 25,
-            ],
+            title=title,
+            x_range=FactorRange(*self.data["names"]),
+            y_range=(0, 1),
             tools="",
             toolbar_location="above",
             **kwargs,
         )
-        self.root.vbar(x="names", top="values", width=0.9, source=self.source)
+        self.root.vbar(
+            x="names",
+            top="values",
+            width=0.9,
+            line_color="white",
+            source=self.source,
+            fill_color=factor_cmap(
+                field_name="names",
+                palette=["#b8e0ce", "#81aae4"],
+                factors=["Event Loop", "GIL Contention"],
+                start=1,
+                end=2,
+            ),
+        )
 
+        self.root.x_range.group_padding = 0.25
         self.root.xaxis.minor_tick_line_alpha = 0
         self.root.ygrid.visible = True
         self.root.xgrid.visible = False
 
-        hover = HoverTool(tooltips=[("Interval", "@text s")], mode="vline")
+        hover = HoverTool(
+            tooltips=[("Name", "@names"), ("Value", "@text")], mode="vline"
+        )
         self.root.add_tools(hover)
 
     @without_property_validation
     @log_errors
     def update(self):
         s = self.scheduler
+        monitor_gil = s.monitor.monitor_gil_contention
+
+        self.data["values"] = [
+            s._tick_interval_observed,
+            self.gil_contention_scheduler,
+            sum(w.metrics["event_loop_interval"] for w in s.workers.values())
+            / (len(s.workers) or 1),
+            self.gil_contention_workers,
+        ][:: 1 if monitor_gil else 2]
+
+        # Format event loop as time and GIL (if configured) as %
+        self.data["text"] = [
+            f"{x * 100:.1f}%" if i % 2 and monitor_gil else format_time(x)
+            for i, x in enumerate(self.data["values"])
+        ]
+        update(self.source, self.data)
 
-        data = {
-            "names": ["Scheduler", "Workers"],
-            "values": [
-                s._tick_interval_observed,
-                sum(w.metrics["event_loop_interval"] for w in s.workers.values())
-                / (len(s.workers) or 1),
-            ],
-        }
-        data["text"] = [format_time(x) for x in data["values"]]
+    @property
+    def gil_contention_workers(self) -> float:
+        workers = self.scheduler.workers
+        if workers:
+            return sum(
+                w.metrics.get("gil_contention", 0) for w in workers.values()
+            ) / len(workers)
+        return float("NaN")
 
-        update(self.source, data)
+    @property
+    def gil_contention_scheduler(self) -> float:
+        return self.scheduler.monitor.recent().get("gil_contention", float("NaN"))
 
 
 class ExceptionsTable(DashboardComponent):
@@ -4071,7 +4113,7 @@ def shuffling_doc(scheduler, extra, doc):
     timeseries = SystemTimeseries(
         scheduler, width=1600, height=200, follow_interval=3000
     )
-    event_loop = EventLoop(scheduler, width=200, height=400)
+    event_loop = Contention(scheduler, width=200, height=400)
 
     add_periodic_callback(doc, shuffling, 200)
     add_periodic_callback(doc, workers_memory, 200)

distributed/dashboard/scheduler.py

@@ -17,8 +17,8 @@
     BandwidthWorkers,
     ClusterMemory,
     ComputePerKey,
+    Contention,
     CurrentLoad,
-    EventLoop,
     ExceptionsTable,
     MemoryByKey,
     Occupancy,
@@ -110,7 +110,7 @@
     "/individual-compute-time-per-key": individual_doc(ComputePerKey, 500),
     "/individual-aggregate-time-per-action": individual_doc(AggregateAction, 500),
     "/individual-scheduler-system": individual_doc(SystemMonitor, 500),
-    "/individual-event-loop": individual_doc(EventLoop, 500),
+    "/individual-contention": individual_doc(Contention, 500),
     "/individual-profile": individual_profile_doc,
     "/individual-profile-server": individual_profile_server_doc,
     "/individual-gpu-memory": gpu_memory_doc,

distributed/dashboard/tests/test_scheduler_bokeh.py

@@ -25,8 +25,8 @@
     AggregateAction,
     ClusterMemory,
     ComputePerKey,
+    Contention,
     CurrentLoad,
-    EventLoop,
     Events,
     Hardware,
     MemoryByKey,
@@ -108,7 +108,7 @@ async def test_basic(c, s, a, b):
         SystemMonitor,
         Occupancy,
         StealingTimeSeries,
-        EventLoop,
+        Contention,
     ]:
         ss = component(s)