Cranelift: initial try_call / try_call_indirect (exception) support.

This PR adds `try_call` and `try_call_indirect` instructions, and
lowerings on four of five ISAs (x86-64, aarch64, riscv64, pulley; s390x
has its own non-shared ABI code that will need separate work).

It extends CLIF to support these instructions as new kinds of branches,
and extends block-calls to accept `retN` and `exnN` block-call args that
carry the normal return values or exception payloads (respectively) into
the appropriate successor blocks.

It wires up the "normal return path" so that it continues to work.
It updates the ABI so that unwinding is possible without an initial
register state at throw: specifically, as per our RFC, all registers are
clobbered. It also includes metadata in the `MachBuffer` that describes
exception-catch destinations. However, no unwinder exists to interpret
these catch-destinations yet, so they are untested.

Author: cfallin

cranelift/codegen/meta/src/gen_inst.rs

@@ -336,7 +336,7 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
                             fmtln!(fmt, "::core::hash::Hash::hash(&{len}, state);");
                             fmt.add_block(&format!("for &block in {blocks}"), |fmt| {
                                 fmtln!(fmt, "::core::hash::Hash::hash(&block.block(pool), state);");
-                                fmt.add_block("for &arg in block.args_slice(pool)", |fmt| {
+                                fmt.add_block("for arg in block.args(pool)", |fmt| {
                                     fmtln!(fmt, "::core::hash::Hash::hash(&arg, state);");
                                 });
                             });
@@ -964,6 +964,7 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
     let mut tmpl_types = Vec::new();
     let mut into_args = Vec::new();
     let mut block_args = Vec::new();
+    let mut lifetime_param = None;
     for op in &inst.operands_in {
         if op.kind.is_block() {
             args.push(format!("{}_label: {}", op.name, "ir::Block"));
@@ -972,7 +973,14 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
                 op.name, "Destination basic block"
             ));
 
-            args.push(format!("{}_args: {}", op.name, "&[Value]"));
+            let lifetime = *lifetime_param.get_or_insert_with(|| {
+                tmpl_types.insert(0, "'a".to_string());
+                "'a"
+            });
+            args.push(format!(
+                "{}_args: impl IntoIterator<Item = &{} BlockArg>",
+                op.name, lifetime,
+            ));
             args_doc.push(format!("- {}_args: {}", op.name, "Block arguments"));
 
             block_args.push(op);

cranelift/codegen/meta/src/shared/entities.rs

@@ -43,6 +43,9 @@ pub(crate) struct EntityRefs {
     /// A reference to a jump table declared in the function preamble.
     pub(crate) jump_table: OperandKind,
 
+    /// A reference to an exception table declared in the function preamble.
+    pub(crate) exception_table: OperandKind,
+
     /// A variable-sized list of value operands. Use for Block and function call arguments.
     pub(crate) varargs: OperandKind,
 }
@@ -84,6 +87,8 @@ impl EntityRefs {
 
             jump_table: new("table", "ir::JumpTable", "A jump table."),
 
+            exception_table: new("exception", "ir::ExceptionTable", "An exception table."),
+
             varargs: OperandKind::new(
                 "",
                 "&[Value]",

cranelift/codegen/meta/src/shared/formats.rs

@@ -12,6 +12,8 @@ pub(crate) struct Formats {
     pub(crate) brif: Rc<InstructionFormat>,
     pub(crate) call: Rc<InstructionFormat>,
     pub(crate) call_indirect: Rc<InstructionFormat>,
+    pub(crate) try_call: Rc<InstructionFormat>,
+    pub(crate) try_call_indirect: Rc<InstructionFormat>,
     pub(crate) cond_trap: Rc<InstructionFormat>,
     pub(crate) float_compare: Rc<InstructionFormat>,
     pub(crate) func_addr: Rc<InstructionFormat>,
@@ -133,6 +135,18 @@ impl Formats {
                 .varargs()
                 .build(),
 
+            try_call: Builder::new("TryCall")
+                .imm(&entities.func_ref)
+                .varargs()
+                .imm(&&entities.exception_table)
+                .build(),
+
+            try_call_indirect: Builder::new("TryCallIndirect")
+                .value()
+                .varargs()
+                .imm(&&entities.exception_table)
+                .build(),
+
             func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(),
 
             atomic_rmw: Builder::new("AtomicRmw")

cranelift/codegen/meta/src/shared/instructions.rs

@@ -307,6 +307,75 @@ fn define_control_flow(
             .with_doc("function to call, declared by `function`")])
         .operands_out(vec![Operand::new("addr", iAddr)]),
     );
+
+    ig.push(
+        Inst::new(
+            "try_call",
+            r#"
+        Call a function, catching the specified exceptions.
+
+        Call the function pointed to by `callee` with the given arguments. On
+        normal return, branch to the first target, with function returns
+        available as `retN` block arguments. On exceptional return,
+        look up the thrown exception tag in the provided exception table;
+        if the tag matches one of the targets, branch to the matching
+        target with the exception payloads available as `exnN` block arguments.
+        If no tag matches, then propagate the exception up the stack.
+
+        It is the Cranelift embedder's responsibility to define the meaning
+        of tags: they are accepted by this instruction and passed through
+        to unwind metadata tables in Cranelift's output. Actual unwinding is
+        outside the purview of the core Cranelift compiler.
+
+        Payload values on exception are passed in fixed register(s) that are
+        defined by the platform and ABI. See the documentation on `CallConv`
+        for details.
+        "#,
+            &formats.try_call,
+        )
+        .operands_in(vec![
+            Operand::new("callee", &entities.func_ref)
+                .with_doc("function to call, declared by `function`"),
+            Operand::new("args", &entities.varargs).with_doc("call arguments"),
+            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
+        ])
+        .call()
+        .branches(),
+    );
+
+    ig.push(
+        Inst::new(
+            "try_call_indirect",
+            r#"
+        Call a function, catching the specified exceptions.
+
+        Call the function pointed to by `callee` with the given arguments. On
+        normal return, branch to the first target, with function returns
+        available as `retN` block arguments. On exceptional return,
+        look up the thrown exception tag in the provided exception table;
+        if the tag matches one of the targets, branch to the matching
+        target with the exception payloads available as `exnN` block arguments.
+        If no tag matches, then propagate the exception up the stack.
+
+        It is the Cranelift embedder's responsibility to define the meaning
+        of tags: they are accepted by this instruction and passed through
+        to unwind metadata tables in Cranelift's output. Actual unwinding is
+        outside the purview of the core Cranelift compiler.
+
+        Payload values on exception are passed in fixed register(s) that are
+        defined by the platform and ABI. See the documentation on `CallConv`
+        for details.
+        "#,
+            &formats.try_call_indirect,
+        )
+        .operands_in(vec![
+            Operand::new("callee", iAddr).with_doc("address of function to call"),
+            Operand::new("args", &entities.varargs).with_doc("call arguments"),
+            Operand::new("ET", &entities.exception_table).with_doc("exception table"),
+        ])
+        .call()
+        .branches(),
+    );
 }
 
 #[inline(never)]

cranelift/codegen/src/dominator_tree.rs

@@ -644,7 +644,7 @@ mod tests {
         cur.insert_block(block1);
         let v1 = cur.ins().iconst(I32, 1);
         let v2 = cur.ins().iadd(v0, v1);
-        cur.ins().jump(block0, &[v2]);
+        cur.ins().jump(block0, &[v2.into()]);
 
         cur.insert_block(block2);
         cur.ins().return_(&[v0]);

cranelift/codegen/src/dominator_tree/simple.rs

@@ -365,7 +365,7 @@ mod tests {
         cur.insert_block(block1);
         let v1 = cur.ins().iconst(I32, 1);
         let v2 = cur.ins().iadd(v0, v1);
-        cur.ins().jump(block0, &[v2]);
+        cur.ins().jump(block0, &[v2.into()]);
 
         cur.insert_block(block2);
         cur.ins().return_(&[v0]);

cranelift/codegen/src/inst_predicates.rs

@@ -200,6 +200,16 @@ pub(crate) fn visit_block_succs<F: FnMut(Inst, Block, bool)>(
                 }
             }
 
+            ir::InstructionData::TryCall { exception, .. }
+            | ir::InstructionData::TryCallIndirect { exception, .. } => {
+                let pool = &f.dfg.value_lists;
+                let exdata = &f.stencil.dfg.exception_tables[*exception];
+
+                for dest in exdata.all_branches() {
+                    visit(inst, dest.block(pool), false);
+                }
+            }
+
             inst => debug_assert!(!inst.opcode().is_branch()),
         }
     }

cranelift/codegen/src/ir/builder.rs

@@ -6,8 +6,8 @@
 use crate::ir;
 use crate::ir::instructions::InstructionFormat;
 use crate::ir::types;
+use crate::ir::{BlockArg, Inst, Opcode, Type, Value};
 use crate::ir::{DataFlowGraph, InstructionData};
-use crate::ir::{Inst, Opcode, Type, Value};
 
 /// Base trait for instruction builders.
 ///

cranelift/codegen/src/ir/dfg.rs

@@ -8,9 +8,9 @@ use crate::ir::instructions::{CallInfo, InstructionData};
 use crate::ir::pcc::Fact;
 use crate::ir::user_stack_maps::{UserStackMapEntry, UserStackMapEntryVec};
 use crate::ir::{
-    types, Block, BlockCall, ConstantData, ConstantPool, DynamicType, ExtFuncData, FuncRef,
-    Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type, Value,
-    ValueLabelAssignments, ValueList, ValueListPool,
+    types, Block, BlockArg, BlockCall, ConstantData, ConstantPool, DynamicType, ExceptionTables,
+    ExtFuncData, FuncRef, Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type,
+    Value, ValueLabelAssignments, ValueList, ValueListPool,
 };
 use crate::packed_option::ReservedValue;
 use crate::write::write_operands;
@@ -158,6 +158,9 @@ pub struct DataFlowGraph {
 
     /// Jump tables used in this function.
     pub jump_tables: JumpTables,
+
+    /// Exception tables used in this function.
+    pub exception_tables: ExceptionTables,
 }
 
 impl DataFlowGraph {
@@ -178,6 +181,7 @@ impl DataFlowGraph {
             constants: ConstantPool::new(),
             immediates: PrimaryMap::new(),
             jump_tables: JumpTables::new(),
+            exception_tables: ExceptionTables::new(),
         }
     }
 
@@ -226,8 +230,12 @@ impl DataFlowGraph {
     }
 
     /// Make a BlockCall, bundling together the block and its arguments.
-    pub fn block_call(&mut self, block: Block, args: &[Value]) -> BlockCall {
-        BlockCall::new(block, args, &mut self.value_lists)
+    pub fn block_call<'a>(
+        &mut self,
+        block: Block,
+        args: impl IntoIterator<Item = &'a BlockArg>,
+    ) -> BlockCall {
+        BlockCall::new(block, args.into_iter().copied(), &mut self.value_lists)
     }
 
     /// Get the total number of values.
@@ -437,13 +445,18 @@ impl DataFlowGraph {
 
         // Rewrite InstructionData in `self.insts`.
         for inst in self.insts.0.values_mut() {
-            inst.map_values(&mut self.value_lists, &mut self.jump_tables, |arg| {
-                if let ValueData::Alias { original, .. } = self.values[arg].into() {
-                    original
-                } else {
-                    arg
-                }
-            });
+            inst.map_values(
+                &mut self.value_lists,
+                &mut self.jump_tables,
+                &mut self.exception_tables,
+                |arg| {
+                    if let ValueData::Alias { original, .. } = self.values[arg].into() {
+                        original
+                    } else {
+                        arg
+                    }
+                },
+            );
         }
 
         // - `results` and block-params in `blocks` are not aliases, by
@@ -843,23 +856,29 @@ impl DataFlowGraph {
         &'dfg self,
         inst: Inst,
     ) -> impl DoubleEndedIterator<Item = Value> + 'dfg {
-        self.inst_args(inst)
-            .iter()
-            .chain(
-                self.insts[inst]
-                    .branch_destination(&self.jump_tables)
-                    .into_iter()
-                    .flat_map(|branch| branch.args_slice(&self.value_lists).iter()),
-            )
-            .copied()
+        self.inst_args(inst).iter().copied().chain(
+            self.insts[inst]
+                .branch_destination(&self.jump_tables, &self.exception_tables)
+                .into_iter()
+                .flat_map(|branch| {
+                    branch
+                        .args(&self.value_lists)
+                        .filter_map(|arg| arg.as_value())
+                }),
+        )
     }
 
     /// Map a function over the values of the instruction.
     pub fn map_inst_values<F>(&mut self, inst: Inst, body: F)
     where
         F: FnMut(Value) -> Value,
     {
-        self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, body);
+        self.insts[inst].map_values(
+            &mut self.value_lists,
+            &mut self.jump_tables,
+            &mut self.exception_tables,
+            body,
+        );
     }
 
     /// Overwrite the instruction's value references with values from the iterator.
@@ -869,9 +888,12 @@ impl DataFlowGraph {
     where
         I: Iterator<Item = Value>,
     {
-        self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, |_| {
-            values.next().unwrap()
-        });
+        self.insts[inst].map_values(
+            &mut self.value_lists,
+            &mut self.jump_tables,
+            &mut self.exception_tables,
+            |_| values.next().unwrap(),
+        );
     }
 
     /// Get all value arguments on `inst` as a slice.
@@ -1078,26 +1100,30 @@ impl DataFlowGraph {
     /// Get the call signature of a direct or indirect call instruction.
     /// Returns `None` if `inst` is not a call instruction.
     pub fn call_signature(&self, inst: Inst) -> Option<SigRef> {
-        match self.insts[inst].analyze_call(&self.value_lists) {
+        match self.insts[inst].analyze_call(&self.value_lists, &self.exception_tables) {
             CallInfo::NotACall => None,
             CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature),
+            CallInfo::DirectWithSig(_, s, _) => Some(s),
             CallInfo::Indirect(s, _) => Some(s),
         }
     }
 
-    /// Like `call_signature` but returns none for tail call instructions.
-    fn non_tail_call_signature(&self, inst: Inst) -> Option<SigRef> {
+    /// Like `call_signature` but returns none for tail call
+    /// instructions and try-call (exception-handling invoke)
+    /// instructions.
+    fn non_tail_call_or_try_call_signature(&self, inst: Inst) -> Option<SigRef> {
         let sig = self.call_signature(inst)?;
         match self.insts[inst].opcode() {
             ir::Opcode::ReturnCall | ir::Opcode::ReturnCallIndirect => None,
+            ir::Opcode::TryCall | ir::Opcode::TryCallIndirect => None,
             _ => Some(sig),
         }
     }
 
     // Only for use by the verifier. Everyone else should just use
     // `dfg.inst_results(inst).len()`.
     pub(crate) fn num_expected_results_for_verifier(&self, inst: Inst) -> usize {
-        match self.non_tail_call_signature(inst) {
+        match self.non_tail_call_or_try_call_signature(inst) {
             Some(sig) => self.signatures[sig].returns.len(),
             None => {
                 let constraints = self.insts[inst].opcode().constraints();
@@ -1112,7 +1138,7 @@ impl DataFlowGraph {
         inst: Inst,
         ctrl_typevar: Type,
     ) -> impl iter::ExactSizeIterator<Item = Type> + 'a {
-        return match self.non_tail_call_signature(inst) {
+        return match self.non_tail_call_or_try_call_signature(inst) {
             Some(sig) => InstResultTypes::Signature(self, sig, 0),
             None => {
                 let constraints = self.insts[inst].opcode().constraints();