oh boy, backtraces.

Author: raulk

fvm/src/call_manager/backtrace.rs

@@ -7,7 +7,9 @@ use fvm_shared::{ActorID, MethodNum};
 
 use crate::kernel::SyscallError;
 
-/// A call backtrace records _why_ an actor exited with a specific error code.
+/// A call backtrace records the actors an error was propagated through, from
+/// the moment it was emitted. The original error is the _cause_. Backtraces are
+/// useful for identifying the root cause of an error.
 #[derive(Debug, Default, Clone)]
 pub struct Backtrace {
     /// The actors through which this error was propagated from bottom (source) to top.
@@ -34,22 +36,26 @@ impl Backtrace {
         self.frames.is_empty() && self.cause.is_none()
     }
 
-    /// Clear the backtrace. This should be called:
-    ///
-    /// 1. Before all syscalls except "abort"
-    /// 2. After an actor returns with a 0 exit code.
+    /// Clear the backtrace.
     pub fn clear(&mut self) {
         self.cause = None;
         self.frames.clear();
     }
 
-    /// Set the backtrace cause. If there is an existing backtrace, this will clear it.
-    pub fn set_cause(&mut self, cause: Cause) {
+    /// Begins a new backtrace. If there is an existing backtrace, this will clear it.
+    ///
+    /// Note: Backtraces are populated _backwards_. That is, a frame is inserted
+    /// every time an actor returns. That's why `begin()` resets any currently
+    /// accumulated state, as once an error occurs, we want to track its
+    /// propagation all the way up.
+    pub fn begin(&mut self, cause: Cause) {
         self.cause = Some(cause);
         self.frames.clear();
     }
 
     /// Push a "frame" (actor exit) onto the backtrace.
+    ///
+    /// This should be called every time an actor exits.
     pub fn push_frame(&mut self, frame: Frame) {
         self.frames.push(frame)
     }
@@ -83,9 +89,7 @@ impl Display for Frame {
     }
 }
 
-/// The ultimate "cause" of a failed message.
-#[derive(Clone, Debug)]
-pub struct Cause {
+struct SyscallCause {
     /// The syscall "module".
     pub module: &'static str,
     /// The syscall function name.
@@ -96,23 +100,45 @@ pub struct Cause {
     pub message: String,
 }
 
+/// The ultimate "cause" of a failed message.
+#[derive(Clone, Debug)]
+pub enum Cause {
+    /// The original cause was a syscall error.
+    Syscall(SyscallCause),
+    /// The original cause was a fatal error.
+    Fatal(String),
+}
+
 impl Cause {
-    pub fn new(module: &'static str, function: &'static str, err: SyscallError) -> Self {
-        Self {
+    /// Records a failing syscall as the cause of a backtrace.
+    pub fn from_syscall(module: &'static str, function: &'static str, err: SyscallError) -> Self {
+        Self::Syscall(SyscallCause {
             module,
             function,
             error: err.1,
             message: err.0,
-        }
+        })
+    }
+
+    /// Records a fatal error as the cause of a backtrace.
+    pub fn from_fatal(err: anyhow::Error) -> Self {
+        Self::Fatal(err.into())
     }
 }
 
 impl Display for Cause {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}::{} -- {} ({}: {})",
-            self.module, self.function, &self.message, self.error as u32, self.error,
-        )
+        match self {
+            Cause::Syscall(cause) => {
+                write!(
+                    f,
+                    "{}::{} -- {} ({}: {})",
+                    cause.module, cause.function, &cause.message, cause.error as u32, cause.error,
+                )
+            }
+            Cause::Fatal(msg) => {
+                write!(f, "[FATAL] {}", msg)
+            }
+        }
     }
 }

fvm/src/call_manager/default.rs

@@ -392,6 +392,11 @@ where
             let ret = match result {
                 Ok(value) => Ok(InvocationResult::Return(value)),
                 Err(abort) => {
+                    let cause = match abort {
+                        Abort::Exit(_, _) => {}
+                        Abort::OutOfGas => {}
+                        Abort::Fatal(_) => {}
+                    };
                     if let Some(err) = last_error {
                         cm.backtrace.set_cause(err);
                     }

fvm/src/executor/default.rs

@@ -130,7 +130,7 @@ where
                     ErrorNumber::Forbidden => ExitCode::SYS_ASSERTION_FAILED,
                 };
 
-                backtrace.set_cause(backtrace::Cause::new("send", "send", err));
+                backtrace.begin(backtrace::Cause::from_syscall("send", "send", err));
                 Receipt {
                     exit_code,
                     return_data: Default::default(),