feat(tui): implement delete operations and update kill shortcut

- Add delete_job_safely to StateProvider with proper running job check
- Add delete_experiment to WorkspaceStateProvider with cache clearing
- Add DELETE_JOB_SAFELY and DELETE_EXPERIMENT RPC methods for SSH monitoring
- Emit JobStateChangedEvent(state=unscheduled) on job deletion for TUI refresh
- Change kill shortcut from 'k' to 'Ctrl+K' for consistency with Ctrl+D delete

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: bpiwowar

docs/source/interfaces.md

@@ -97,7 +97,7 @@ Press `?` in the TUI to show the help screen with all shortcuts.
 | `Enter` | Select experiment |
 | `d` | Show experiment runs |
 | `Ctrl+d` | Delete experiment |
-| `k` | Kill all running jobs |
+| `Ctrl+k` | Kill all running jobs |
 | `S` | Sort by status |
 | `D` | Sort by date |
 
@@ -107,7 +107,7 @@ Press `?` in the TUI to show the help screen with all shortcuts.
 |-----|--------|
 | `l` | View job logs |
 | `Ctrl+d` | Delete job |
-| `k` | Kill running job |
+| `Ctrl+k` | Kill running job |
 | `/` | Open search filter |
 | `c` | Clear search filter |
 | `S` | Sort by status |

src/experimaestro/scheduler/remote/client.py

@@ -702,6 +702,35 @@ def clean_job(self, job: BaseJob, perform: bool = False) -> bool:
         result = self._call_sync(RPCMethod.CLEAN_JOB, params)
         return result.get("success", False)
 
+    def delete_job_safely(self, job: BaseJob, perform: bool = True) -> tuple[bool, str]:
+        """Safely delete a job and its data via remote server"""
+        if not perform:
+            # Dry run - check if job can be deleted (not running)
+            if job.state and job.state.running():
+                return False, f"Cannot delete running job {job.identifier}"
+            return True, f"Job {job.identifier} can be deleted"
+
+        params = {
+            "job_id": job.identifier,
+            "experiment_id": getattr(job, "experiment_id", ""),
+            "run_id": getattr(job, "run_id", ""),
+            "perform": True,
+        }
+        result = self._call_sync(RPCMethod.DELETE_JOB_SAFELY, params)
+        return result.get("success", False), result.get("message", "")
+
+    def delete_experiment(
+        self, experiment_id: str, delete_jobs: bool = False, perform: bool = True
+    ) -> tuple[bool, str]:
+        """Delete an experiment and optionally its job data via remote server"""
+        params = {
+            "experiment_id": experiment_id,
+            "delete_jobs": delete_jobs,
+            "perform": perform,
+        }
+        result = self._call_sync(RPCMethod.DELETE_EXPERIMENT, params)
+        return result.get("success", False), result.get("message", "")
+
     def get_process_info(self, job: BaseJob):
         """Get process information for a job
 

src/experimaestro/scheduler/remote/protocol.py

@@ -58,6 +58,8 @@ class RPCMethod(str, Enum):
     GET_DEPENDENCIES_MAP = "get_dependencies_map"
     KILL_JOB = "kill_job"
     CLEAN_JOB = "clean_job"
+    DELETE_JOB_SAFELY = "delete_job_safely"
+    DELETE_EXPERIMENT = "delete_experiment"
     GET_SYNC_INFO = "get_sync_info"
     GET_PROCESS_INFO = "get_process_info"
 

src/experimaestro/scheduler/remote/server.py

@@ -82,6 +82,8 @@ def __init__(
             RPCMethod.GET_DEPENDENCIES_MAP.value: self._handle_get_dependencies_map,
             RPCMethod.KILL_JOB.value: self._handle_kill_job,
             RPCMethod.CLEAN_JOB.value: self._handle_clean_job,
+            RPCMethod.DELETE_JOB_SAFELY.value: self._handle_delete_job_safely,
+            RPCMethod.DELETE_EXPERIMENT.value: self._handle_delete_experiment,
             RPCMethod.GET_SYNC_INFO.value: self._handle_get_sync_info,
             RPCMethod.GET_PROCESS_INFO.value: self._handle_get_process_info,
         }
@@ -445,3 +447,45 @@ def _handle_get_process_info(self, params: Dict) -> Optional[Dict]:
             "type": pinfo.type,
             "running": pinfo.running,
         }
+
+    def _handle_delete_job_safely(self, params: Dict) -> Dict:
+        """Handle delete_job_safely request"""
+        job_id = params.get("job_id")
+        experiment_id = params.get("experiment_id")
+        run_id = params.get("run_id")
+        perform = params.get("perform", True)
+
+        if not job_id or not experiment_id:
+            raise TypeError("job_id and experiment_id are required")
+
+        # Get the job first
+        job = self._state_provider.get_job(job_id, experiment_id, run_id)
+        if job is None:
+            return {"success": False, "message": "Job not found"}
+
+        # Delete the job safely
+        try:
+            success, message = self._state_provider.delete_job_safely(
+                job, perform=perform
+            )
+            return {"success": success, "message": message}
+        except Exception as e:
+            return {"success": False, "message": str(e)}
+
+    def _handle_delete_experiment(self, params: Dict) -> Dict:
+        """Handle delete_experiment request"""
+        experiment_id = params.get("experiment_id")
+        delete_jobs = params.get("delete_jobs", False)
+        perform = params.get("perform", True)
+
+        if not experiment_id:
+            raise TypeError("experiment_id is required")
+
+        # Delete the experiment
+        try:
+            success, message = self._state_provider.delete_experiment(
+                experiment_id, delete_jobs=delete_jobs, perform=perform
+            )
+            return {"success": success, "message": message}
+        except Exception as e:
+            return {"success": False, "message": str(e)}

src/experimaestro/scheduler/state_provider.py

@@ -312,13 +312,44 @@ def get_stray_jobs(self) -> List[BaseJob]:
         return [j for j in self.get_orphan_jobs() if j.state and j.state.running()]
 
     def delete_job_safely(self, job: BaseJob, perform: bool = True) -> Tuple[bool, str]:
-        """Safely delete a job and its data"""
-        return False, "Not implemented"
+        """Safely delete a job and its data
+
+        Only deletes jobs that are finished (not running). Uses clean_job
+        for the actual deletion.
+
+        Args:
+            job: The job to delete
+            perform: If True, actually perform deletion; if False, just check
+
+        Returns:
+            Tuple of (success, message)
+        """
+        # Check if job is running - cannot delete running jobs
+        if job.state and job.state.running():
+            return False, f"Cannot delete running job {job.identifier}"
+
+        # Use clean_job for the actual deletion
+        if self.clean_job(job, perform=perform):
+            if perform:
+                return True, f"Deleted job {job.identifier}"
+            else:
+                return True, f"Job {job.identifier} can be deleted"
+        else:
+            return False, f"Failed to delete job {job.identifier}"
 
     def delete_experiment(
-        self, experiment_id: str, perform: bool = True
+        self, experiment_id: str, delete_jobs: bool = False, perform: bool = True
     ) -> Tuple[bool, str]:
-        """Delete an experiment and all its data"""
+        """Delete an experiment and optionally its job data
+
+        Args:
+            experiment_id: Experiment identifier to delete
+            delete_jobs: If True, also delete job directories (default: False)
+            perform: If True, actually perform deletion; if False, just check
+
+        Returns:
+            Tuple of (success, message)
+        """
         return False, "Not implemented"
 
     def cleanup_orphan_partials(self, perform: bool = False) -> List[str]:

src/experimaestro/scheduler/workspace_state_provider.py

@@ -1187,6 +1187,21 @@ def clean_job(self, job: MockJob, perform: bool = False) -> bool:
 
             if job.path.exists():
                 shutil.rmtree(job.path)
+
+            # Clear job from cache
+            with self._job_cache_lock:
+                self._job_cache.pop(job.identifier, None)
+
+            # Emit state change event (unscheduled = job removed)
+            from experimaestro.scheduler.state_status import JobStateChangedEvent
+
+            self._notify_state_listeners(
+                JobStateChangedEvent(
+                    job_id=job.identifier,
+                    state="unscheduled",
+                )
+            )
+
             return True
         except Exception as e:
             logger.warning("Failed to clean job %s: %s", job.identifier, e)
@@ -1547,6 +1562,88 @@ def get_process_info(self, job: MockJob) -> Optional[ProcessInfo]:
         except (json.JSONDecodeError, OSError):
             return None
 
+    # =========================================================================
+    # Experiment deletion
+    # =========================================================================
+
+    def delete_experiment(
+        self, experiment_id: str, delete_jobs: bool = False, perform: bool = True
+    ) -> tuple[bool, str]:
+        """Delete an experiment and optionally its job data
+
+        Args:
+            experiment_id: Experiment identifier to delete
+            delete_jobs: If True, also delete job directories (default: False)
+            perform: If True, actually perform deletion; if False, just check
+
+        Returns:
+            Tuple of (success, message)
+        """
+        import shutil
+
+        # Check for running jobs first
+        jobs = self.get_jobs(experiment_id=experiment_id)
+        running_jobs = [j for j in jobs if j.state and j.state.running()]
+        if running_jobs:
+            return False, f"Cannot delete: {len(running_jobs)} jobs are still running"
+
+        # Find experiment directories (v2 layout)
+        exp_dir = self.workspace_path / "experiments" / experiment_id
+        events_dir = self._experiments_dir / experiment_id
+
+        # Check for v1 layout
+        v1_exp_dir = self.workspace_path / "xp" / experiment_id
+
+        if not exp_dir.exists() and not v1_exp_dir.exists():
+            return False, f"Experiment {experiment_id} not found"
+
+        if not perform:
+            return True, f"Experiment {experiment_id} can be deleted"
+
+        errors = []
+
+        # Delete job directories if requested
+        if delete_jobs:
+            for job in jobs:
+                if job.path and job.path.exists():
+                    try:
+                        shutil.rmtree(job.path)
+                    except OSError as e:
+                        errors.append(f"Failed to delete job {job.identifier}: {e}")
+
+        # Delete v2 experiment directory
+        if exp_dir.exists():
+            try:
+                shutil.rmtree(exp_dir)
+            except OSError as e:
+                errors.append(f"Failed to delete experiment dir: {e}")
+
+        # Delete events directory
+        if events_dir.exists():
+            try:
+                shutil.rmtree(events_dir)
+            except OSError as e:
+                errors.append(f"Failed to delete events dir: {e}")
+
+        # Delete v1 experiment directory
+        if v1_exp_dir.exists():
+            try:
+                shutil.rmtree(v1_exp_dir)
+            except OSError as e:
+                errors.append(f"Failed to delete v1 experiment dir: {e}")
+
+        # Clear caches
+        self._clear_experiment_cache(experiment_id)
+        with self._job_cache_lock:
+            job_ids_to_remove = [j.identifier for j in jobs]
+            for job_id in job_ids_to_remove:
+                self._job_cache.pop(job_id, None)
+
+        if errors:
+            return False, f"Partial deletion: {'; '.join(errors)}"
+
+        return True, f"Deleted experiment {experiment_id}"
+
     # =========================================================================
     # Lifecycle
     # =========================================================================

src/experimaestro/tui/dialogs.py

@@ -130,14 +130,14 @@ def compose(self) -> ComposeResult:
   Enter     Select experiment
   d         Show experiment runs
   Ctrl+d    Delete experiment
-  k         Kill all running jobs
+  Ctrl+k    Kill all running jobs
   S         Sort by status
   D         Sort by date
 
 [bold cyan]Jobs[/bold cyan]
   l         View job logs
   Ctrl+d    Delete job
-  k         Kill running job
+  Ctrl+k    Kill running job
   /         Open search filter
   c         Clear search filter
   S         Sort by status

src/experimaestro/tui/widgets/experiments.py

@@ -27,7 +27,7 @@ class ExperimentsList(Widget):
     BINDINGS = [
         Binding("d", "show_runs", "Runs"),
         Binding("ctrl+d", "delete_experiment", "Delete", show=False),
-        Binding("k", "kill_experiment", "Kill", show=False),
+        Binding("ctrl+k", "kill_experiment", "Kill", show=False),
         Binding("S", "sort_by_status", "Sort ⚑", show=False),
         Binding("D", "sort_by_date", "Sort Date", show=False),
     ]

src/experimaestro/tui/widgets/jobs.py

@@ -456,7 +456,7 @@ class JobsTable(Vertical):
 
     BINDINGS = [
         Binding("ctrl+d", "delete_job", "Delete", show=False),
-        Binding("k", "kill_job", "Kill", show=False),
+        Binding("ctrl+k", "kill_job", "Kill", show=False),
         Binding("l", "view_logs", "Logs", key_display="l"),
         Binding("f", "copy_path", "Copy Path", show=False),
         Binding("/", "toggle_search", "Search"),