OOMKill Dmesg enricher

helm/robusta/values.yaml

@@ -195,6 +195,7 @@ builtinPlaybooks:
       attach_logs: true
       container_memory_graph: true
       node_memory_graph: true
+  - pod_dmesg_enricher: {}
   stop: true
 
 # playbooks for prometheus alerts enrichment

playbooks/robusta_playbooks/node_enrichments.py

@@ -6,7 +6,6 @@
 from robusta.api import (
     BaseBlock,
     EnrichmentType,
-    FileBlock,
     Finding,
     FindingSeverity,
     FindingSource,
@@ -17,9 +16,9 @@
     NodeEvent,
     PodRunningParams,
     ResourceGraphEnricherParams,
-    RobustaPod,
     action,
     create_node_graph_enrichment,
+    dmesg_enricher,
     get_node_allocatable_resources_table_block,
     get_node_running_pods_table_block_or_none,
     get_node_status_table_block,
@@ -116,7 +115,7 @@ def node_status_enricher(event: NodeEvent):
 
     logging.info("node_status_enricher is depricated, use status_enricher instead")
 
-    event.add_enrichment(get_node_status_table_block(node))
+    event.add_enrichment([get_node_status_table_block(node)])
 
 
 @action
@@ -128,15 +127,7 @@ def node_dmesg_enricher(event: NodeEvent, params: PodRunningParams):
     if not node:
         logging.error(f"node_dmesg_enricher was called on event without node : {event}")
         return
-    exec_result = RobustaPod.exec_on_node(
-        pod_name="dmesg_pod", node_name=node.metadata.name, cmd="dmesg", custom_annotations=params.custom_annotations
-    )
-    if exec_result:
-        event.add_enrichment(
-            [FileBlock("dmesg.log", exec_result.encode())],
-            enrichment_type=EnrichmentType.text_file,
-            title="DMESG Info",
-        )
+    dmesg_enricher(event, node, params.custom_annotations)
 
 
 @action

playbooks/robusta_playbooks/pod_enrichments.py

@@ -5,13 +5,15 @@
     EnrichmentType,
     PodEvent,
     PodResourceGraphEnricherParams,
+    PodRunningParams,
     ResourceChartItemType,
     ResourceChartResourceType,
     ResourceGraphEnricherParams,
     XAxisLine,
     action,
     create_node_graph_enrichment,
     create_resource_enrichment,
+    dmesg_enricher,
     pod_limits,
 )
 from robusta.core.model.pods import pod_requests
@@ -83,3 +85,19 @@ def pod_node_graph_enricher(pod_event: PodEvent, params: ResourceGraphEnricherPa
         return
     graph_enrichment = create_node_graph_enrichment(params, node, metrics_legends_labels=["instance"])
     pod_event.add_enrichment([graph_enrichment], enrichment_type=EnrichmentType.graph, title="Pod Resources")
+
+
+@action
+def pod_dmesg_enricher(pod_event: PodEvent, params: PodRunningParams):
+    """
+    Gets the dmesg from a node
+    """
+    pod = pod_event.get_pod()
+    if not pod:
+        logging.error(f"cannot run pod_node_graph_enricher on event with no pod: {pod_event}")
+        return
+    node = pod.get_node()
+    if not node:
+        logging.warning(f"Node {pod.spec.nodeName} not found for pod {pod.metadata.name}")
+        return
+    dmesg_enricher(pod_event, node, params.custom_annotations)

src/robusta/api/__init__.py

@@ -117,7 +117,7 @@
     get_node_running_pods_table_block_or_none,
     get_node_status_table_block,
 )
-from robusta.core.playbooks.node_playbook_utils import create_node_graph_enrichment
+from robusta.core.playbooks.node_playbook_utils import create_node_graph_enrichment, dmesg_enricher
 from robusta.core.playbooks.pod_utils.crashloop_utils import get_crash_report_enrichments
 from robusta.core.playbooks.pod_utils.imagepull_utils import (
     get_image_pull_backoff_container_statuses,

src/robusta/core/playbooks/node_playbook_utils.py

@@ -1,15 +1,22 @@
+import logging
 from datetime import datetime
-from typing import Optional, List
+from typing import Dict, List, Optional
 
 from hikaru.model.rel_1_26 import Node
 
 from robusta.core.model.base_params import ResourceChartItemType, ResourceChartResourceType, ResourceGraphEnricherParams
+from robusta.core.model.events import ExecutionBaseEvent
 from robusta.core.playbooks.prometheus_enrichment_utils import create_resource_enrichment, get_node_internal_ip
-from robusta.core.reporting.blocks import GraphBlock
+from robusta.core.reporting.base import EnrichmentType
+from robusta.core.reporting.blocks import FileBlock, GraphBlock
+from robusta.integrations.kubernetes.custom_models import RobustaPod
 
 
-def create_node_graph_enrichment(params: ResourceGraphEnricherParams, node: Node,
-                                 metrics_legends_labels: Optional[List[str]] = None,) -> GraphBlock:
+def create_node_graph_enrichment(
+    params: ResourceGraphEnricherParams,
+    node: Node,
+    metrics_legends_labels: Optional[List[str]] = None,
+) -> GraphBlock:
     start_at = datetime.now()
     labels = {"node": node.metadata.name}
     internal_ip = get_node_internal_ip(node)
@@ -26,3 +33,18 @@ def create_node_graph_enrichment(params: ResourceGraphEnricherParams, node: Node
         metrics_legends_labels=metrics_legends_labels,
     )
     return graph_enrichment
+
+
+def dmesg_enricher(event: ExecutionBaseEvent, node: Node, custom_annotations: Optional[Dict[str, str]] = None):
+    if not node:
+        logging.error("_dmesg_enricher was called on event without node")
+        return
+    exec_result = RobustaPod.exec_on_node(
+        pod_name="dmesg_pod", node_name=node.metadata.name, cmd="dmesg", custom_annotations=custom_annotations
+    )
+    if exec_result:
+        event.add_enrichment(
+            [FileBlock("dmesg.log", exec_result.encode())],
+            enrichment_type=EnrichmentType.text_file,
+            title="DMESG Info",
+        )