Ai delayed health check

helm/robusta/Chart.yaml

@@ -15,6 +15,6 @@ dependencies:
   condition: enablePrometheusStack
   repository: "https://prometheus-community.github.io/helm-charts"
 - name: holmes
-  version: 0.2.6
+  version: 0.3.0
   condition: enableHolmesGPT
   repository: "https://robusta-charts.storage.googleapis.com"

src/robusta/core/model/base_params.py

@@ -196,7 +196,7 @@ class HolmesWorkloadHealthParams(HolmesParams):
     :var alert_history_since_hours: Timespan of historic data to use in hours. 24 by default.
     :var stored_instrucitons: Use remote instructions specified for the workload.
     :var instructions: List of extra instructions to supply.
-
+    :var silent_healthy: Does not create findings in the case of healthy workload.
 
     :example ask: What are all the issues in my cluster right now?
     """
@@ -209,6 +209,7 @@ class HolmesWorkloadHealthParams(HolmesParams):
     instructions: List[str] = []
     include_tool_calls: bool = True
     include_tool_call_results: bool = True
+    silent_healthy: bool = False
 
 
 class NamespacedResourcesParams(ActionParams):

src/robusta/core/playbooks/internal/ai_integration.py

@@ -3,7 +3,12 @@
 
 import requests
 
-from robusta.core.model.base_params import AIInvestigateParams, HolmesConversationParams, HolmesWorkloadHealthParams
+from robusta.core.model.base_params import (
+    AIInvestigateParams,
+    HolmesConversationParams,
+    HolmesWorkloadHealthParams,
+    ResourceInfo,
+)
 from robusta.core.model.events import ExecutionBaseEvent
 from robusta.core.playbooks.actions_registry import action
 from robusta.core.reporting import Finding, FindingSubject
@@ -16,6 +21,8 @@
     HolmesResult,
     HolmesResultsBlock,
 )
+from robusta.core.schedule.model import FixedDelayRepeat
+from robusta.integrations.kubernetes.autogenerated.events import KubernetesAnyChangeEvent
 from robusta.integrations.prometheus.utils import HolmesDiscovery
 from robusta.utils.error_codes import ActionException, ErrorCodes
 
@@ -102,9 +109,20 @@ def holmes_workload_health(event: ExecutionBaseEvent, params: HolmesWorkloadHeal
 
         holmes_result = HolmesResult(**json.loads(result.text))
 
+        healthy = True
+        try:
+            analysis = json.loads(holmes_result.analysis)
+            healthy = analysis.get("workload_healthy")
+        except Exception:
+            logging.exception("Error in holmes response format, analysis did not return the expecrted json format.")
+            pass
+
+        if params.silent_healthy and healthy:
+            return
+
         finding = Finding(
-            title=f"AI Analysis of {params.resource}",
-            aggregation_key="HolmesInvestigationResult",
+            title=f"AI Health check of {params.resource}",
+            aggregation_key="HolmesHealthCheck",
             subject=FindingSubject(
                 name=params.resource.name if params.resource else "",
                 namespace=params.resource.namespace if params.resource else "",
@@ -197,3 +215,31 @@ def holmes_conversation(event: ExecutionBaseEvent, params: HolmesConversationPar
             raise ActionException(ErrorCodes.HOLMES_REQUEST_ERROR, "Holmes internal configuration error.")
         else:
             raise ActionException(ErrorCodes.HOLMES_UNEXPECTED_ERROR, "An unexpected error occured.")
+
+
+class DelayedHealthCheckParams(HolmesWorkloadHealthParams):
+    delay_seconds: int = 120
+
+
+@action
+def delayed_health_check(event: KubernetesAnyChangeEvent, action_params: DelayedHealthCheckParams):
+    """
+    runs a holmes workload health action with a delay
+    """
+    metadata = event.obj and event.obj.metadata
+
+    if not action_params.ask:
+        action_params.ask = f"help me diagnose an issue with a workload {metadata.namespace}/{event.obj.kind}/{metadata.name} running in my Kubernetes cluster. Can you assist with identifying potential issues and pinpoint the root cause."
+
+    action_params.resource = ResourceInfo(name=metadata.name, namespace=metadata.namespace, kind=event.obj.kind)
+
+    logging.info(f"Scheduling health check. {metadata.name} delays: {action_params.delay_seconds}")
+    event.get_scheduler().schedule_action(
+        action_func=holmes_workload_health,
+        task_id=f"health_check_{metadata.name}_{metadata.namespace}",
+        scheduling_params=FixedDelayRepeat(repeat=1, seconds_delay=action_params.delay_seconds),
+        named_sinks=event.named_sinks,
+        action_params=action_params,
+        replace_existing=True,
+        standalone_task=True,
+    )