fix(fleet): slm-agent 404 heartbeat backoff + vnc target_path detection + optional-role health classification (#9965)

autobot-slm-agent/agent.py

@@ -132,6 +132,11 @@ def __init__(
         self.role_detector = RoleDetector()
         self._role_definitions_loaded = False
 
+        # Registration-pending backoff state (#9965).
+        # 404 responses mean the node row doesn't exist yet — keep retrying
+        # with bounded exponential backoff instead of treating it as fatal.
+        self._reg_backoff_seconds: float = 0  # 0 = not in backoff
+
     def _init_buffer_db(self):
         """Initialize SQLite buffer database."""
         Path(self.buffer_db).parent.mkdir(parents=True, exist_ok=True)
@@ -263,6 +268,9 @@ def _build_heartbeat_payload(self, health: dict, os_info: str, code_version: Opt
             },
         }
 
+    # Maximum registration-pending backoff in seconds (#9965).
+    _REG_BACKOFF_CAP: float = 300
+
     async def _send_heartbeat_request(self, payload: dict) -> bool:
         """
         Send heartbeat HTTP request to admin server.
@@ -272,7 +280,7 @@ async def _send_heartbeat_request(self, payload: dict) -> bool:
 
         Returns:
             True if heartbeat was accepted, False otherwise.
-        Issue #620.
+        Issue #620, #9965.
         """
         try:
             async with aiohttp.ClientSession() as session:
@@ -287,15 +295,32 @@ async def _send_heartbeat_request(self, payload: dict) -> bool:
                         # Issue #741: Process heartbeat response
                         response_data = await response.json()
                         self._process_heartbeat_response(response_data)
+                        if self._reg_backoff_seconds:
+                            logger.info("Heartbeat accepted — node registration confirmed")
+                            self._reg_backoff_seconds = 0
                         logger.debug("Heartbeat sent successfully")
                         return True
-                    else:
-                        logger.warning(
-                            "Heartbeat rejected: %s %s",
-                            response.status,
-                            await response.text(),
+                    if response.status == 404:
+                        # Node row not yet created — registration still in progress.
+                        # Back off and retry; do NOT buffer or treat as fatal (#9965).
+                        new_backoff = min(
+                            max(self._reg_backoff_seconds * 2, 5),
+                            self._REG_BACKOFF_CAP,
+                        )
+                        self._reg_backoff_seconds = new_backoff
+                        logger.debug(
+                            "Heartbeat 404: node %s not yet registered — "
+                            "retrying in %.0fs (registration pending, #9965)",
+                            self.node_id,
+                            new_backoff,
                         )
                         return False
+                    logger.warning(
+                        "Heartbeat rejected: %s %s",
+                        response.status,
+                        await response.text(),
+                    )
+                    return False
         except aiohttp.ClientError as e:
             logger.warning("Failed to send heartbeat: %s", e)
             self.buffer_event("heartbeat", payload)
@@ -380,15 +405,25 @@ async def run(self, enable_notify_server: bool = False, notify_port: int = DEFAU
             # Send systemd watchdog notification (prevents timeout restart)
             sd_notify("WATCHDOG=1")
 
-            # Send heartbeat
-            success = await self.send_heartbeat()
-
-            # If connected, try to sync buffered events
-            if success:
-                await self.sync_buffered_events()
-
-            # Wait for next heartbeat
-            await asyncio.sleep(self.heartbeat_interval)
+            try:
+                # Send heartbeat
+                success = await self.send_heartbeat()
+
+                # If connected, try to sync buffered events
+                if success:
+                    await self.sync_buffered_events()
+            except Exception as exc:
+                # Catch-all: an unhandled error in any heartbeat step must
+                # NOT kill the loop (#9965).  Log and continue.
+                logger.exception("Unhandled error in heartbeat loop: %s", exc)
+
+            # When in registration-pending backoff, sleep the longer of
+            # the normal interval or the computed backoff delay (#9965).
+            sleep_secs = max(
+                float(self.heartbeat_interval),
+                self._reg_backoff_seconds,
+            )
+            await asyncio.sleep(sleep_secs)
 
         # Notify systemd we're stopping
         sd_notify("STOPPING=1")

autobot-slm-agent/role_detector.py

@@ -46,13 +46,18 @@ class RoleStatus:
 
     @property
     def status(self) -> str:
-        """Determine overall status."""
-        if self.path_exists and self.service_running:
-            return "active"
-        elif self.path_exists:
-            return "inactive"
-        else:
+        """Determine overall status.
+
+        - active: path exists and service running (or no service required)
+        - inactive: path exists but service is down
+        - not_installed: path absent
+        """
+        if not self.path_exists:
             return "not_installed"
+        # No service name means it's a library/passive role — path = active
+        if not self.service_name:
+            return "active"
+        return "active" if self.service_running else "inactive"
 
 
 class RoleDetector:
@@ -74,12 +79,16 @@ def load_definitions(self, definitions: List[Dict]) -> None:
         """Load role definitions from SLM server response."""
         self.roles = {}
         for defn in definitions:
-            if not defn.get("target_path"):
-                continue  # Skip roles without target path (e.g., code-source)
+            # Skip roles that have neither a target path nor a systemd service —
+            # there is nothing to detect on the node (#9965; was: skip if no
+            # target_path, which caused service-only roles like vnc to be silently
+            # omitted even when tigervncserver was running).
+            if not defn.get("target_path") and not defn.get("systemd_service"):
+                continue
 
             self.roles[defn["name"]] = RoleDefinition(
                 name=defn["name"],
-                target_path=defn["target_path"],
+                target_path=defn.get("target_path", ""),
                 systemd_service=defn.get("systemd_service"),
                 health_check_port=defn.get("health_check_port"),
             )
@@ -100,10 +109,13 @@ def _detect_role(self, role: RoleDefinition, listening_ports: set) -> RoleStatus
         """Detect a single role."""
         status = RoleStatus()
 
-        # Check path
-        target_path = Path(role.target_path)
-        status.path_exists = target_path.exists()
-        status.path = str(target_path) if status.path_exists else None
+        # Check path (service-only roles have no target_path — always pass, #9965)
+        if role.target_path:
+            target_path = Path(role.target_path)
+            status.path_exists = target_path.exists()
+            status.path = str(target_path) if status.path_exists else None
+        else:
+            status.path_exists = True  # no path requirement
 
         # Check service
         if role.systemd_service:
@@ -115,9 +127,9 @@ def _detect_role(self, role: RoleDefinition, listening_ports: set) -> RoleStatus
             if role.health_check_port in listening_ports:
                 status.ports.append(role.health_check_port)
 
-        # Read version if path exists
-        if status.path_exists:
-            status.version = self._read_version(target_path)
+        # Read version if path exists and path was specified
+        if status.path_exists and role.target_path:
+            status.version = self._read_version(Path(role.target_path))
 
         return status
 

autobot-slm-backend/ansible/roles/slm_agent/files/slm/agent/agent.py

@@ -136,6 +136,11 @@ def __init__(
         # connection pool + SSL context allocation (#9086)
         self._session: aiohttp.ClientSession | None = None
 
+        # Registration-pending backoff state (#9965).
+        # 404 responses mean the node row doesn't exist yet — keep retrying
+        # with bounded exponential backoff instead of treating it as fatal.
+        self._reg_backoff_seconds: float = 0  # 0 = not in backoff
+
     def _init_buffer_db(self):
         """Initialize SQLite buffer database."""
         Path(self.buffer_db).parent.mkdir(parents=True, exist_ok=True)
@@ -267,6 +272,9 @@ def _build_heartbeat_payload(self, health: dict, os_info: str, code_version: str
             },
         }
 
+    # Maximum registration-pending backoff in seconds (#9965).
+    _REG_BACKOFF_CAP: float = 300
+
     async def _send_heartbeat_request(self, payload: dict) -> bool:
         """
         Send heartbeat HTTP request to admin server.
@@ -276,7 +284,7 @@ async def _send_heartbeat_request(self, payload: dict) -> bool:
 
         Returns:
             True if heartbeat was accepted, False otherwise.
-        Issue #620.
+        Issue #620, #9965.
         """
         assert self._session is not None
         try:
@@ -291,15 +299,32 @@ async def _send_heartbeat_request(self, payload: dict) -> bool:
                     # Issue #741: Process heartbeat response
                     response_data = await response.json()
                     self._process_heartbeat_response(response_data)
+                    if self._reg_backoff_seconds:
+                        logger.info("Heartbeat accepted — node registration confirmed")
+                        self._reg_backoff_seconds = 0
                     logger.debug("Heartbeat sent successfully")
                     return True
-                else:
-                    logger.warning(
-                        "Heartbeat rejected: %s %s",
-                        response.status,
-                        await response.text(),
+                if response.status == 404:
+                    # Node row not yet created — registration still in progress.
+                    # Back off and retry; do NOT buffer or treat as fatal (#9965).
+                    new_backoff = min(
+                        max(self._reg_backoff_seconds * 2, 5),
+                        self._REG_BACKOFF_CAP,
+                    )
+                    self._reg_backoff_seconds = new_backoff
+                    logger.debug(
+                        "Heartbeat 404: node %s not yet registered — "
+                        "retrying in %.0fs (registration pending, #9965)",
+                        self.node_id,
+                        new_backoff,
                     )
                     return False
+                logger.warning(
+                    "Heartbeat rejected: %s %s",
+                    response.status,
+                    await response.text(),
+                )
+                return False
         except aiohttp.ClientError as e:
             logger.warning("Failed to send heartbeat: %s", e)
             self.buffer_event("heartbeat", payload)
@@ -418,15 +443,25 @@ async def run(self, enable_notify_server: bool = False, notify_port: int = DEFAU
                 # Send systemd watchdog notification (prevents timeout restart)
                 sd_notify("WATCHDOG=1")
 
-                # Send heartbeat
-                success = await self.send_heartbeat()
-
-                # If connected, try to sync buffered events
-                if success:
-                    await self.sync_buffered_events()
-
-                # Wait for next heartbeat
-                await asyncio.sleep(self.heartbeat_interval)
+                try:
+                    # Send heartbeat
+                    success = await self.send_heartbeat()
+
+                    # If connected, try to sync buffered events
+                    if success:
+                        await self.sync_buffered_events()
+                except Exception as exc:
+                    # Catch-all: an unhandled error in any heartbeat step must
+                    # NOT kill the loop (#9965).  Log and continue.
+                    logger.exception("Unhandled error in heartbeat loop: %s", exc)
+
+                # When in registration-pending backoff, sleep the longer of
+                # the normal interval or the computed backoff delay (#9965).
+                sleep_secs = max(
+                    float(self.heartbeat_interval),
+                    self._reg_backoff_seconds,
+                )
+                await asyncio.sleep(sleep_secs)
 
             self._session = None
 

autobot-slm-backend/api/setup_wizard.py

@@ -1146,7 +1146,21 @@ async def validate_fleet(
         # Only check roles that are actually assigned to nodes (#2747)
         # Roles not yet assigned via wizard are not "missing"
         assigned_roles = (await session.execute(select(NodeRole.role_name).distinct())).scalars().all()
-        missing_roles = []
+
+        # Look up which of the assigned roles are required (#9965).
+        # Optional roles (required=False) that are assigned but not yet active
+        # should not degrade the fleet health score — report them separately.
+        from models.database import Role as RoleModel
+
+        role_required_map: dict[str, bool] = {}
+        for role_name in assigned_roles:
+            role_row = await session.execute(select(RoleModel).where(RoleModel.name == role_name))
+            role_obj = role_row.scalar_one_or_none()
+            # Default required=True when the DB row is missing (fail-safe)
+            role_required_map[role_name] = role_obj.required if role_obj is not None else True
+
+        missing_required_roles: list[str] = []
+        inactive_optional_roles: list[str] = []
         for role_name in assigned_roles:
             active = await session.execute(
                 select(func.count(NodeRole.id)).where(
@@ -1155,10 +1169,13 @@ async def validate_fleet(
                 )
             )
             if (active.scalar() or 0) == 0:
-                missing_roles.append(role_name)
+                if role_required_map.get(role_name, True):
+                    missing_required_roles.append(role_name)
+                else:
+                    inactive_optional_roles.append(role_name)
 
     health = "healthy"
-    if missing_roles:
+    if missing_required_roles:
         health = "degraded"
     elif online_nodes < total_nodes:
         health = "degraded"
@@ -1167,6 +1184,8 @@ async def validate_fleet(
         "health": health,
         "total_nodes": total_nodes,
         "online_nodes": online_nodes,
-        "missing_required_roles": missing_roles,
+        "missing_required_roles": missing_required_roles,
+        # Informational: optional roles assigned but not yet active — not health-degrading
+        "inactive_optional_roles": inactive_optional_roles,
         "ready": total_nodes > 0,
     }

autobot-slm-backend/services/role_registry.py

@@ -310,7 +310,11 @@
         "display_name": "VNC Server",
         "sync_type": None,
         "source_paths": [],
-        "target_path": "",
+        # /usr/bin/Xtigervnc is the TigerVNC binary installed by tigervnc-standalone-server.
+        # Providing a non-empty target_path lets role_detector.py skip its early-exit
+        # guard (which checks only target_path) and activate path-existence + systemd
+        # checks — mirrors the postgres fix in #9853 (#9965).
+        "target_path": "/usr/bin/Xtigervnc",
         "systemd_service": "tigervncserver",
         "auto_restart": False,
         "required": False,