Catch panics in encoding NIF results (#656)

Also adds tests for panicking in parameters, return values and the NIF
function itself.

Fixes #655

Author: filmor

CHANGELOG.md

@@ -24,6 +24,7 @@ versions.
   now (#638)
 - API functions for Windows are correctly assigned now for NIF version 2.15 and
   above (#635)
+- Panics in encoding the result of NIF function are caught (#656)
 
 ### Changed
 

rustler/src/codegen_runtime.rs

@@ -3,6 +3,7 @@
 use std::ffi::CString;
 use std::fmt;
 
+use crate::types::atom;
 use crate::{Encoder, Env, OwnedBinary, Term};
 
 // Re-export of inventory
@@ -26,10 +27,16 @@ pub unsafe trait NifReturnable {
 
 unsafe impl<T> NifReturnable for T
 where
-    T: crate::Encoder,
+    T: crate::Encoder + std::panic::RefUnwindSafe,
 {
     unsafe fn into_returned(self, env: Env) -> NifReturned {
-        NifReturned::Term(self.encode(env).as_c_arg())
+        if let Ok(res) = std::panic::catch_unwind(|| NifReturned::Term(self.encode(env).as_c_arg()))
+        {
+            res
+        } else {
+            let term = atom::nif_panicked().as_c_arg();
+            NifReturned::Raise(term)
+        }
     }
 }
 
@@ -126,7 +133,7 @@ where
             Err(err) => match err.downcast::<NifReturned>() {
                 Ok(ty) => NifReturned::Term(ty.apply(env)),
                 Err(_) => {
-                    let term = crate::types::atom::nif_panicked().as_c_arg();
+                    let term = atom::nif_panicked().as_c_arg();
                     NifReturned::Raise(term)
                 }
             },

rustler_tests/lib/rustler_test.ex

@@ -148,6 +148,10 @@ defmodule RustlerTest do
 
   def append_to_path(_path, _to_append), do: err()
 
+  def panic_in_nif(), do: err()
+  def panic_in_encode(), do: err()
+  def panic_in_decode(_), do: err()
+
   if Helper.has_nif_version("2.16") do
     def perform_dyncall(_res, _a, _b, _c), do: err()
   end

rustler_tests/native/rustler_test/src/lib.rs

@@ -10,6 +10,7 @@ mod test_list;
 mod test_local_pid;
 mod test_map;
 mod test_nif_attrs;
+mod test_panic;
 mod test_path;
 mod test_primitives;
 mod test_range;

rustler_tests/native/rustler_test/src/test_panic.rs

@@ -0,0 +1,30 @@
+use rustler::{Decoder, Encoder, Env, Term};
+
+#[rustler::nif]
+pub fn panic_in_nif() -> i32 {
+    panic!("panic!")
+}
+
+struct Panicking;
+
+impl Encoder for Panicking {
+    fn encode<'a>(&self, _env: Env<'a>) -> Term<'a> {
+        panic!("panic in encode!")
+    }
+}
+
+impl<'a> Decoder<'a> for Panicking {
+    fn decode(_term: Term<'a>) -> rustler::NifResult<Self> {
+        panic!("panic in decode!")
+    }
+}
+
+#[rustler::nif]
+pub fn panic_in_encode() -> Panicking {
+    Panicking
+}
+
+#[rustler::nif]
+pub fn panic_in_decode(_p: Panicking) -> i32 {
+    0
+}

rustler_tests/test/panic_test.exs

@@ -0,0 +1,12 @@
+defmodule PanicTest do
+  use ExUnit.Case
+
+  test "panics in NIFs are caught" do
+    assert_raise ErlangError, &RustlerTest.panic_in_nif/0
+    assert_raise ErlangError, &RustlerTest.panic_in_encode/0
+
+    assert_raise ErlangError, fn ->
+      RustlerTest.panic_in_decode(:anything)
+    end
+  end
+end