transmute fatal errors into SYS_ASSERTION_FAILED exit code.

fvm/src/call_manager/backtrace.rs

@@ -7,7 +7,9 @@ use fvm_shared::{ActorID, MethodNum};
 
 use crate::kernel::SyscallError;
 
-/// A call backtrace records _why_ an actor exited with a specific error code.
+/// A call backtrace records the actors an error was propagated through, from
+/// the moment it was emitted. The original error is the _cause_. Backtraces are
+/// useful for identifying the root cause of an error.
 #[derive(Debug, Default, Clone)]
 pub struct Backtrace {
     /// The actors through which this error was propagated from bottom (source) to top.
@@ -34,22 +36,35 @@ impl Backtrace {
         self.frames.is_empty() && self.cause.is_none()
     }
 
-    /// Clear the backtrace. This should be called:
-    ///
-    /// 1. Before all syscalls except "abort"
-    /// 2. After an actor returns with a 0 exit code.
+    /// Clear the backtrace.
     pub fn clear(&mut self) {
         self.cause = None;
         self.frames.clear();
     }
 
-    /// Set the backtrace cause. If there is an existing backtrace, this will clear it.
-    pub fn set_cause(&mut self, cause: Cause) {
+    /// Begins a new backtrace. If there is an existing backtrace, this will clear it.
+    ///
+    /// Note: Backtraces are populated _backwards_. That is, a frame is inserted
+    /// every time an actor returns. That's why `begin()` resets any currently
+    /// accumulated state, as once an error occurs, we want to track its
+    /// propagation all the way up.
+    pub fn begin(&mut self, cause: Cause) {
         self.cause = Some(cause);
         self.frames.clear();
     }
 
+    /// Sets the cause of a backtrace.
+    ///
+    /// This is useful to stamp a backtrace with its cause after the frames
+    /// have been collected, such as when we ultimately handle a fatal error at
+    /// the top of its propagation chain.
+    pub fn set_cause(&mut self, cause: Cause) {
+        self.cause = Some(cause);
+    }
+
     /// Push a "frame" (actor exit) onto the backtrace.
+    ///
+    /// This should be called every time an actor exits.
     pub fn push_frame(&mut self, frame: Frame) {
         self.frames.push(frame)
     }
@@ -85,34 +100,69 @@ impl Display for Frame {
 
 /// The ultimate "cause" of a failed message.
 #[derive(Clone, Debug)]
-pub struct Cause {
-    /// The syscall "module".
-    pub module: &'static str,
-    /// The syscall function name.
-    pub function: &'static str,
-    /// The exact syscall error.
-    pub error: ErrorNumber,
-    /// The informational syscall message.
-    pub message: String,
+pub enum Cause {
+    /// The original cause was a syscall error.
+    Syscall {
+        /// The syscall "module".
+        module: &'static str,
+        /// The syscall function name.
+        function: &'static str,
+        /// The exact syscall error.
+        error: ErrorNumber,
+        /// The informational syscall message.
+        message: String,
+    },
+    /// The original cause was a fatal error.
+    Fatal {
+        /// The alternate-formatted message from the anyhow error.
+        error_msg: String,
+        /// The backtrace, captured if the relevant
+        /// [environment variables](https://doc.rust-lang.org/std/backtrace/index.html#environment-variables) are enabled.
+        backtrace: String,
+    },
 }
 
 impl Cause {
-    pub fn new(module: &'static str, function: &'static str, err: SyscallError) -> Self {
-        Self {
+    /// Records a failing syscall as the cause of a backtrace.
+    pub fn from_syscall(module: &'static str, function: &'static str, err: SyscallError) -> Self {
+        Self::Syscall {
             module,
             function,
             error: err.1,
             message: err.0,
         }
     }
+
+    /// Records a fatal error as the cause of a backtrace.
+    pub fn from_fatal(err: anyhow::Error) -> Self {
+        Self::Fatal {
+            error_msg: format!("{:#}", err),
+            backtrace: err.backtrace().to_string(),
+        }
+    }
 }
 
 impl Display for Cause {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}::{} -- {} ({}: {})",
-            self.module, self.function, &self.message, self.error as u32, self.error,
-        )
+        match self {
+            Cause::Syscall {
+                module,
+                function,
+                error,
+                message,
+            } => {
+                write!(
+                    f,
+                    "{}::{} -- {} ({}: {})",
+                    module, function, &message, *error as u32, error,
+                )
+            }
+            Cause::Fatal {
+                error_msg,
+                backtrace,
+            } => {
+                write!(f, "[FATAL] Error: {}, Backtrace:\n{}", error_msg, backtrace)
+            }
+        }
     }
 }

fvm/src/call_manager/default.rs

@@ -393,7 +393,7 @@ where
                 Ok(value) => Ok(InvocationResult::Return(value)),
                 Err(abort) => {
                     if let Some(err) = last_error {
-                        cm.backtrace.set_cause(err);
+                        cm.backtrace.begin(err);
                     }
 
                     let (code, message, res) = match abort {

fvm/src/executor/default.rs

@@ -130,22 +130,36 @@ where
                     ErrorNumber::Forbidden => ExitCode::SYS_ASSERTION_FAILED,
                 };
 
-                backtrace.set_cause(backtrace::Cause::new("send", "send", err));
+                backtrace.begin(backtrace::Cause::from_syscall("send", "send", err));
                 Receipt {
                     exit_code,
                     return_data: Default::default(),
                     gas_used,
                 }
             }
-            Err(ExecutionError::Fatal(e)) => {
-                return Err(e.context(format!(
-                    "[from={}, to={}, seq={}, m={}, h={}] fatal error",
+            Err(ExecutionError::Fatal(err)) => {
+                // We produce a receipt with SYS_ASSERTION_FAILED exit code, and
+                // we consume the full gas amount so that, in case of a network-
+                // wide fatal errors, all nodes behave deterministically.
+                //
+                // We set the backtrace from the fatal error to aid diagnosis.
+                // Note that we use backtrace#set_cause instead of backtrace#begin
+                // because we want to retain the propagation chain that we've
+                // accumulated on the way out.
+                let err = err.context(format!(
+                    "[from={}, to={}, seq={}, m={}, h={}]",
                     msg.from,
                     msg.to,
                     msg.sequence,
                     msg.method_num,
-                    self.context().epoch
-                )));
+                    self.context().epoch,
+                ));
+                backtrace.set_cause(backtrace::Cause::from_fatal(err));
+                Receipt {
+                    exit_code: ExitCode::SYS_ASSERTION_FAILED,
+                    return_data: Default::default(),
+                    gas_used: msg.gas_limit,
+                }
             }
         };
 

fvm/src/machine/default.rs

@@ -113,6 +113,9 @@ where
         // This interface works for now because we know all actor CIDs
         // ahead of time, but with user-supplied code, we won't have that
         // guarantee.
+        // Skip preloading all builtin actors when testing. This results in JIT
+        // bytecode to machine code compilation, and leads to faster tests.
+        #[cfg(not(feature = "testing"))]
         engine.preload(state_tree.store(), builtin_actors.left_values())?;
 
         Ok(DefaultMachine {

fvm/src/syscalls/bind.rs

@@ -141,7 +141,7 @@ macro_rules! impl_bind_syscalls {
                             Ok(Err(err)) => {
                                 let code = err.1;
                                 log::trace!("syscall {}::{}: fail ({})", module, name, code as u32);
-                                data.last_error = Some(backtrace::Cause::new(module, name, err));
+                                data.last_error = Some(backtrace::Cause::from_syscall(module, name, err));
                                 Ok(code as u32)
                             },
                             Err(e) => Err(e.into()),
@@ -163,7 +163,7 @@ macro_rules! impl_bind_syscalls {
                         if (ret as u64) > (memory.len() as u64)
                             || memory.len() - (ret as usize) < mem::size_of::<Ret::Value>() {
                             let code = ErrorNumber::IllegalArgument;
-                            data.last_error = Some(backtrace::Cause::new(module, name, SyscallError(format!("no space for return value"), code)));
+                            data.last_error = Some(backtrace::Cause::from_syscall(module, name, SyscallError(format!("no space for return value"), code)));
                             return Ok(code as u32);
                         }
 
@@ -178,7 +178,7 @@ macro_rules! impl_bind_syscalls {
                             Ok(Err(err)) => {
                                 let code = err.1;
                                 log::trace!("syscall {}::{}: fail ({})", module, name, code as u32);
-                                data.last_error = Some(backtrace::Cause::new(module, name, err));
+                                data.last_error = Some(backtrace::Cause::from_syscall(module, name, err));
                                 Ok(code as u32)
                             },
                             Err(e) => Err(e.into()),

testing/integration/examples/integration.rs

@@ -1,5 +1,6 @@
 use fvm::executor::{ApplyKind, Executor};
 use fvm_integration_tests::tester::{Account, Tester};
+use fvm_ipld_blockstore::MemoryBlockstore;
 use fvm_ipld_encoding::tuple::*;
 use fvm_shared::address::Address;
 use fvm_shared::bigint::BigInt;
@@ -25,7 +26,12 @@ struct State {
 
 pub fn main() {
     // Instantiate tester
-    let mut tester = Tester::new(NetworkVersion::V15, StateTreeVersion::V4).unwrap();
+    let mut tester = Tester::new(
+        NetworkVersion::V15,
+        StateTreeVersion::V4,
+        MemoryBlockstore::default(),
+    )
+    .unwrap();
 
     let sender: [Account; 1] = tester.create_accounts().unwrap();
 

testing/integration/src/builtin.rs

@@ -5,7 +5,7 @@ use cid::Cid;
 use futures::executor::block_on;
 use fvm::state_tree::{ActorState, StateTree};
 use fvm::{init_actor, system_actor};
-use fvm_ipld_blockstore::{Blockstore, MemoryBlockstore};
+use fvm_ipld_blockstore::Blockstore;
 use fvm_ipld_car::load_car;
 use fvm_ipld_encoding::CborStore;
 use fvm_shared::actor::builtin::{load_manifest, Type};
@@ -24,7 +24,7 @@ const BUNDLES: [(NetworkVersion, &[u8]); 3] = [
 
 // Import built-in actors
 pub fn import_builtin_actors(
-    blockstore: &MemoryBlockstore,
+    blockstore: &impl Blockstore,
 ) -> Result<BTreeMap<NetworkVersion, Cid>> {
     BUNDLES
         .into_iter()
@@ -40,7 +40,7 @@ pub fn import_builtin_actors(
 
 // Retrieve system, init and accounts actors code CID
 pub fn fetch_builtin_code_cid(
-    blockstore: &MemoryBlockstore,
+    blockstore: &impl Blockstore,
     builtin_actors: &Cid,
     ver: u32,
 ) -> Result<(Cid, Cid, Cid)> {