CI twin for PR #23

crates/subc-client-rs/Cargo.toml

@@ -11,7 +11,7 @@ repository = "https://github.com/cortexkit/subconscious"
 async-trait = "0.1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-subc-control = { path = "../subc-control", version = "0.1" }
+subc-control = { path = "../subc-control", version = "0.2" }
 subc-protocol = { path = "../subc-protocol", version = "0.11" }
 subc-transport = { path = "../subc-transport", version = "0.5" }
 tokio = { version = "1", features = ["io-util", "macros", "net", "rt", "sync", "time"] }

crates/subc-control/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "subc-control"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 publish = true
 description = "Client-facing subc control-plane wire shapes."

crates/subc-control/src/lib.rs

@@ -46,6 +46,7 @@ pub mod ops {
     pub const SUPERVISOR_HEALTH_PROBE: &str = "supervisor.health_probe";
     pub const SUPERVISOR_HEALTH: &str = "supervisor.health";
     pub const SUPERVISOR_STDERR_TAIL: &str = "supervisor.stderr_tail";
+    pub const SUPERVISOR_TERMINALS: &str = "supervisor.terminals";
 }
 
 /// Client-originated channel-0 control RPC body.
@@ -150,6 +151,20 @@ pub enum ClientControlRequest {
         #[serde(default, skip_serializing_if = "Option::is_none")]
         max_bytes: Option<u32>,
     },
+    /// Retained terminal exits for one module.
+    ///
+    /// This stays separate from `supervisor.list`: a history grows with every
+    /// incident, while the list is a current-state read most callers issue often.
+    ///
+    /// The read MUST stay off the supervisor command channel — it reads the
+    /// module's shared ring directly. This is a requirement, not an
+    /// optimisation: when the supervision task itself dies, every
+    /// command-channel op returns `CommandClosed`, and that is precisely the
+    /// moment an operator needs the exit history most. A history reachable only
+    /// through the machinery whose death you are diagnosing is unreachable when
+    /// it matters. Proven failure mode, not a hypothetical.
+    #[serde(rename = "supervisor.terminals")]
+    SupervisorTerminals { module_id: String },
 }
 
 /// subc's channel-0 response body for client control RPCs.
@@ -230,6 +245,12 @@ pub enum ClientControlResponse {
         #[serde(flatten)]
         tail: StderrTail,
     },
+    #[serde(rename = "supervisor.terminals")]
+    SupervisorTerminals {
+        module_id: String,
+        #[serde(flatten)]
+        terminals: TerminalHistory,
+    },
 }
 
 /// A module's retained stderr, oldest entry first.
@@ -290,6 +311,39 @@ fn is_zero_u64(value: &u64) -> bool {
     *value == 0
 }
 
+/// Bounded terminal history for one module, oldest retained record first.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TerminalHistory {
+    /// Unix milliseconds at daemon start. The history is in-memory and resets with
+    /// the daemon, so an empty list only means "no exits during this incarnation".
+    pub daemon_started_at_ms: u64,
+    pub entries: Vec<TerminalEntry>,
+    /// Earlier terminal exits evicted by the bounded ring.
+    #[serde(default, skip_serializing_if = "is_zero_u64")]
+    pub dropped: u64,
+}
+
+/// One terminal child exit and the supervisor action it selected.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TerminalEntry {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub exit_code: Option<i32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub exit_signal: Option<i32>,
+    pub at_ms: u64,
+    pub disposition: TerminalDisposition,
+}
+
+/// The supervisor disposition selected after observing a terminal exit.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum TerminalDisposition {
+    Stopped,
+    Disabled,
+    Failed,
+    Restarting,
+}
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum PollKind {
@@ -400,6 +454,11 @@ pub struct SupervisorEntry {
     /// (often a panic-abort). Survives respawn.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub last_exit_signal: Option<i32>,
+    /// Unix milliseconds when the most recent child exit was observed. Present
+    /// even when the terminal ring is not queried, so existing list readers can
+    /// order their latest observed exit against events they already received.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub last_exit_ms: Option<u64>,
     /// Replacement processes spawned for this module so far, against the budget
     /// that disables it.
     ///

crates/subc-control/tests/golden/client_control_response_catalog_list.json

@@ -91,6 +91,7 @@
     "supervisor.set_enabled",
     "supervisor.health_probe",
     "supervisor.health",
-    "supervisor.stderr_tail"
+    "supervisor.stderr_tail",
+    "supervisor.terminals"
   ]
 }

crates/subc-control/tests/golden/client_control_response_server_describe.json

@@ -17,6 +17,7 @@
     "supervisor.set_enabled",
     "supervisor.health_probe",
     "supervisor.health",
-    "supervisor.stderr_tail"
+    "supervisor.stderr_tail",
+    "supervisor.terminals"
   ]
 }

crates/subc-control/tests/golden/client_control_response_server_describe_with_counters.json

@@ -28,6 +28,7 @@
     "supervisor.set_enabled",
     "supervisor.health_probe",
     "supervisor.health",
-    "supervisor.stderr_tail"
+    "supervisor.stderr_tail",
+    "supervisor.terminals"
   ]
 }

crates/subc-control/tests/golden/client_control_response_supervisor_list.json

@@ -4,6 +4,7 @@
     {
       "enabled": true,
       "health": "degraded",
+      "last_exit_ms": 1700000000123,
       "last_probe_ms": 1700000000000,
       "live": true,
       "max_restarts": 3,

crates/subc-control/tests/golden/client_control_response_supervisor_terminals.json

@@ -0,0 +1,17 @@
+{
+  "daemon_started_at_ms": 1700000000000,
+  "entries": [
+    {
+      "at_ms": 1700000000001,
+      "disposition": "restarting",
+      "exit_code": 23
+    },
+    {
+      "at_ms": 1700000000002,
+      "disposition": "failed",
+      "exit_code": 23
+    }
+  ],
+  "module_id": "terminal-golden",
+  "op": "supervisor.terminals"
+}

crates/subc-control/tests/golden/supervisor_entry.json

@@ -1,6 +1,7 @@
 {
   "enabled": true,
   "health": "degraded",
+  "last_exit_ms": 1700000000123,
   "last_probe_ms": 1700000000000,
   "live": true,
   "max_restarts": 3,

crates/subc-control/tests/golden_json.rs

@@ -346,6 +346,7 @@ fn thin_core_ops() -> Vec<String> {
         "supervisor.health_probe".to_string(),
         "supervisor.health".to_string(),
         "supervisor.stderr_tail".to_string(),
+        "supervisor.terminals".to_string(),
     ]
 }
 
@@ -376,6 +377,7 @@ fn supervisor_entry() -> SupervisorEntry {
         last_probe_ms: Some(1_700_000_000_000),
         last_exit_code: None,
         last_exit_signal: None,
+        last_exit_ms: Some(1_700_000_000_123),
         // Non-equal and non-zero so the pin would catch the two fields being
         // swapped, which equal values or a zeroed count could not.
         restart_count: Some(2),

crates/subc-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "subc-core"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 publish = false
 description = "subc daemon core: loopback TCP transport and opaque-byte splice router."

crates/subc-core/src/bin/ck.rs

@@ -41,7 +41,7 @@ const PROD_CONNECTION_RELATIVE_PATH: &[&str] =
 const QUOTA_MODULE_ID: &str = "insula";
 const CK_HARNESS: &str = "ck";
 
-const TOP_HELP_BASE: &str = "ck — CortexKit operator CLI\n\nusage:\n  ck [--subc <connection-file>] [--json] <domain> [<verb>] [<args>]\n\ndomains:\n  module    supervised modules: list, status, stderr, restart, stop, start, rescan\n  health    one-line health for every supervised module\n  quota     AI-provider quota and usage windows\n  daemon    daemon version, uptime, and connection info";
+const TOP_HELP_BASE: &str = "ck — CortexKit operator CLI\n\nusage:\n  ck [--subc <connection-file>] [--json] <domain> [<verb>] [<args>]\n\ndomains:\n  module    supervised modules: list, status, stderr, terminals, restart, stop, start, rescan\n  health    one-line health for every supervised module\n  quota     AI-provider quota and usage windows\n  daemon    daemon version, uptime, and connection info";
 
 const TOP_HELP_TAIL: &str = "flags:\n  --subc <file>   use a specific connection file (default: auto-discover)\n  --json          raw JSON output instead of tables\n\nrun 'ck <domain>' with no verb to see that domain's commands";
 
@@ -106,7 +106,7 @@ fn discover_external_domains() -> Vec<String> {
 }
 
 const MODULE_HELP: &str = "ck module — inspect and control supervised modules\n\nusage: ck [--json] module <verb> [<args>]\n\nverbs:\n  ck module list            all modules with state and health\n  ck module status <id>     one module in detail
-  ck module stderr <id>     retained stderr for a module (-n <count> to limit)\n  ck module restart <id>    drain-restart a module\n  ck module stop <id>       disable and stop a module (persists until start)\n  ck module start <id>      enable and spawn a module\n  ck module rescan          re-read subc.jsonc and reconcile the module set\n  ck module rescan --dry-run  show what a rescan would change, without changing it";
+  ck module stderr <id>     retained stderr for a module (-n <count> to limit)\n  ck module terminals <id>  retained terminal exits for a module\n  ck module restart <id>    drain-restart a module\n  ck module stop <id>       disable and stop a module (persists until start)\n  ck module start <id>      enable and spawn a module\n  ck module rescan          re-read subc.jsonc and reconcile the module set\n  ck module rescan --dry-run  show what a rescan would change, without changing it";
 
 const QUOTA_HELP: &str = "ck quota - AI-provider quota and usage windows\n\nusage: ck [--json] quota [--verbose] [<provider-id>]\n\n  ck quota              connected providers and their usage windows\n  ck quota --verbose    all tracked providers, including unavailable ones\n  ck quota claude       one provider's windows and status in detail";
 
@@ -151,6 +151,9 @@ async fn run(argv: impl IntoIterator<Item = OsString>) -> Result<(), CkError> {
             module_id,
             max_lines,
         }) => module_stderr_tail(&mut client, &module_id, max_lines, args.json).await,
+        Command::Module(ModuleCommand::Terminals { module_id }) => {
+            module_terminals(&mut client, &module_id, args.json).await
+        }
         Command::Module(ModuleCommand::Restart { module_id }) => {
             module_restart(&mut client, &module_id, args.json).await
         }
@@ -241,6 +244,9 @@ enum ModuleCommand {
         module_id: String,
         max_lines: Option<u32>,
     },
+    Terminals {
+        module_id: String,
+    },
 }
 
 struct ResolvedConnection {
@@ -598,6 +604,56 @@ async fn module_stderr_tail(
     Ok(())
 }
 
+async fn module_terminals(
+    client: &mut CkClient,
+    module_id: &str,
+    json_output: bool,
+) -> Result<(), CkError> {
+    let response = client
+        .rpc_value(ClientControlRequest::SupervisorTerminals {
+            module_id: module_id.to_string(),
+        })
+        .await?;
+
+    if json_output {
+        print_json(&response)?;
+        return Ok(());
+    }
+
+    let daemon_started_at_ms = response
+        .get("daemon_started_at_ms")
+        .and_then(Value::as_u64)
+        .map(|value| value.to_string())
+        .unwrap_or_else(|| "unknown".to_string());
+    let dropped = response.get("dropped").and_then(Value::as_u64).unwrap_or(0);
+    let entries = response
+        .get("entries")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    println!(
+        "daemon_started_at_ms={daemon_started_at_ms} · {} terminal record(s) · {dropped} dropped",
+        entries.len()
+    );
+    for entry in entries {
+        let at_ms = entry.get("at_ms").and_then(Value::as_u64).unwrap_or(0);
+        let disposition = entry
+            .get("disposition")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        let exit = match (
+            entry.get("exit_signal").and_then(Value::as_i64),
+            entry.get("exit_code").and_then(Value::as_i64),
+        ) {
+            (Some(signal), _) => format!("signal {signal}"),
+            (None, Some(code)) => format!("code {code}"),
+            (None, None) => "unknown exit".to_string(),
+        };
+        println!("{at_ms} {exit} → {disposition}");
+    }
+    Ok(())
+}
+
 async fn module_restart(
     client: &mut CkClient,
     module_id: &str,
@@ -2735,6 +2791,7 @@ fn parse_command(domain: &str, tail: &[OsString]) -> Result<Command, CkError> {
                     module_id: id(1)?,
                     max_lines: parse_tail_count(tail)?,
                 },
+                "terminals" => ModuleCommand::Terminals { module_id: id(1)? },
                 "restart" => ModuleCommand::Restart { module_id: id(1)? },
                 "stop" => ModuleCommand::Stop { module_id: id(1)? },
                 "start" => ModuleCommand::Start { module_id: id(1)? },
@@ -3573,4 +3630,18 @@ mod tests {
         ));
         assert!(QUOTA_HELP.contains("--verbose"));
     }
+
+    #[test]
+    fn module_terminals_is_parsed_and_documented() {
+        let command = parse_command(
+            "module",
+            &[OsString::from("terminals"), OsString::from("aft")],
+        )
+        .unwrap();
+        assert!(matches!(
+            command,
+            Command::Module(ModuleCommand::Terminals { module_id }) if module_id == "aft"
+        ));
+        assert!(MODULE_HELP.contains("ck module terminals <id>"));
+    }
 }