Cranelift: add patchable call instructions.

cranelift/codegen/meta/src/shared/instructions.rs

@@ -3918,4 +3918,31 @@ pub(crate) fn define(
         )
         .other_side_effects(),
     );
+
+    ig.push(
+        Inst::new(
+            "patchable_call",
+            r#"
+         A call instruction that can be turned on/off by patching machine code.
+
+         This call is different than an ordinary call in a few ways:
+         - It can only call functions with the `patchable` ABI.
+         - As a result of that, the called function cannot have any
+           return values.
+         - The call emits metadata into the MachBuffer that allows the consumer
+           of the machine code to edit the code to either make the call or not.
+
+         Note that the lack of return values is necessary: because the call
+         may or may not occur, dynamically, we cannot rely on any values
+         actually being defined.
+         "#,
+            &formats.call,
+        )
+        .operands_in(vec![
+            Operand::new("FN", &entities.func_ref)
+                .with_doc("function to call, declared by `function`"),
+            Operand::new("args", &entities.varargs).with_doc("call arguments"),
+        ])
+        .call(),
+    );
 }

cranelift/codegen/src/inst_predicates.rs

@@ -149,7 +149,11 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool {
         | Opcode::Fence
         | Opcode::Debugtrap
         | Opcode::SequencePoint => true,
-        Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true,
+        Opcode::Call
+        | Opcode::CallIndirect
+        | Opcode::TryCall
+        | Opcode::TryCallIndirect
+        | Opcode::PatchableCall => true,
         op if op.can_trap() => true,
         _ => false,
     }

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -817,6 +817,9 @@
        ;; An indirect return-call macro instruction.
        (ReturnCallInd (info BoxReturnCallIndInfo))
 
+       ;; A patchable call instruction.
+       (PatchableCall (info BoxCallInfo))
+
        ;; A pseudo-instruction that captures register arguments in vregs.
        (Args
         (args VecArgPair))
@@ -4510,6 +4513,11 @@
 (rule (return_call_ind_impl info)
       (SideEffectNoResult.Inst (MInst.ReturnCallInd info)))
 
+;; Helper for creating `MInst.PatchableCall` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+      (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
 ;; Helpers for pinned register manipulation.
 
 (decl write_pinned_reg (Reg) SideEffectNoResult)

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -2941,7 +2941,8 @@ impl MachInstEmit for Inst {
                     sink.put4(0xd65f0bff | (op2 << 9)); // reta{key}
                 }
             }
-            &Inst::Call { ref info } => {
+            &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {
+                let is_patchable = matches!(self, Inst::PatchableCall { .. });
                 let user_stack_map = state.take_stack_map();
                 sink.add_reloc(Reloc::Arm64Call, &info.dest, 0);
                 sink.put4(enc_jump26(0b100101, 0));
@@ -2955,6 +2956,8 @@ impl MachInstEmit for Inst {
                         Some(state.frame_layout.sp_to_fp()),
                         try_call.exception_handlers(&state.frame_layout),
                     );
+                } else if is_patchable {
+                    sink.add_patchable_call_site(4);
                 } else {
                     sink.add_call_site();
                 }

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -801,7 +801,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
         }
         Inst::Ret { .. } | Inst::AuthenticatedRet { .. } => {}
         Inst::Jump { .. } => {}
-        Inst::Call { info, .. } => {
+        Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {
             let CallInfo { uses, defs, .. } = &mut **info;
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
@@ -1008,6 +1008,7 @@ impl MachInst for Inst {
         match self {
             Inst::Call { .. }
             | Inst::CallInd { .. }
+            | Inst::PatchableCall { .. }
             | Inst::ElfTlsGetAddr { .. }
             | Inst::MachOTlsGetAddr { .. } => CallType::Regular,
 
@@ -1092,7 +1093,7 @@ impl MachInst for Inst {
 
     fn is_safepoint(&self) -> bool {
         match self {
-            Inst::Call { .. } | Inst::CallInd { .. } => true,
+            Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,
             _ => false,
         }
     }
@@ -1110,6 +1111,10 @@ impl MachInst for Inst {
         Inst::Nop4
     }
 
+    fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+        smallvec![0x1f, 0x20, 0x03, 0xd5]
+    }
+
     fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
             I8 => Ok((&[RegClass::Int], &[I8])),
@@ -2594,6 +2599,9 @@ impl Inst {
                     .unwrap_or_default();
                 format!("blr {rn}{try_call}")
             }
+            &Inst::PatchableCall { .. } => {
+                format!("bl 0 ; patchable")
+            }
             &Inst::ReturnCall { ref info } => {
                 let mut s = format!(
                     "return_call {:?} new_stack_arg_size:{}",

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2573,6 +2573,17 @@
             (_ Unit (emit_side_effect (call_ind_impl info))))
         output))
 
+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+      (let ((abi Sig (abi_sig sig_ref))
+            (uses CallArgList (gen_call_args abi args))
+            (defs CallRetList (gen_patchable_call_rets))
+            (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+            (_ Unit (emit_side_effect (patchable_call_impl info))))
+        (output_none)))
+
 ;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to an in-range function.

cranelift/codegen/src/isa/pulley_shared/inst.isle

@@ -51,6 +51,9 @@
     ;; An indirect call to an unknown callee.
     (IndirectCall (info BoxCallIndInfo))
 
+    ;; A patchable direct call to a known callee.
+    (PatchableCall (info BoxCallInfo))
+
     ;; A direct return-call macro instruction.
     (ReturnCall (info BoxReturnCallInfo))
 
@@ -749,6 +752,11 @@
 (rule (indirect_call_host_impl info)
       (SideEffectNoResult.Inst (MInst.IndirectCallHost info)))
 
+;; Helper for creating `MInst.Call` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+      (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
 ;; Helper for creating `MInst.ReturnCall` instructions.
 (decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
 (rule (return_call_impl info)

cranelift/codegen/src/isa/pulley_shared/inst/emit.rs

@@ -166,7 +166,10 @@ fn pulley_emit<P>(
             sink.add_reloc_at_offset(end - size, Reloc::Abs8, &**name, *offset);
         }
 
-        Inst::Call { info } => {
+        Inst::Call { info } | Inst::PatchableCall { info } => {
+            let is_patchable = matches!(inst, Inst::PatchableCall { .. });
+            let start = sink.cur_offset();
+
             // If arguments happen to already be in the right register for the
             // ABI then remove them from this list. Otherwise emit the
             // appropriate `Call` instruction depending on how many arguments we
@@ -196,6 +199,8 @@ fn pulley_emit<P>(
                     Some(state.frame_layout.sp_to_fp()),
                     try_call.exception_handlers(&state.frame_layout),
                 );
+            } else if is_patchable {
+                sink.add_patchable_call_site(sink.cur_offset() - start);
             } else {
                 sink.add_call_site();
             }

cranelift/codegen/src/isa/pulley_shared/inst/mod.rs

@@ -148,7 +148,7 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_def(dst);
         }
 
-        Inst::Call { info } => {
+        Inst::Call { info } | Inst::PatchableCall { info } => {
             let CallInfo {
                 uses,
                 defs,
@@ -435,7 +435,8 @@ where
             }
             | Inst::Call { .. }
             | Inst::IndirectCall { .. }
-            | Inst::IndirectCallHost { .. } => true,
+            | Inst::IndirectCallHost { .. }
+            | Inst::PatchableCall { .. } => true,
             _ => false,
         }
     }
@@ -498,9 +499,10 @@ where
 
     fn call_type(&self) -> CallType {
         match &self.inst {
-            Inst::Call { .. } | Inst::IndirectCall { .. } | Inst::IndirectCallHost { .. } => {
-                CallType::Regular
-            }
+            Inst::Call { .. }
+            | Inst::IndirectCall { .. }
+            | Inst::IndirectCallHost { .. }
+            | Inst::PatchableCall { .. } => CallType::Regular,
 
             Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => CallType::TailCall,
 
@@ -533,6 +535,16 @@ where
         todo!()
     }
 
+    fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+        let mut bytes = smallvec::smallvec![];
+        let nop = pulley_interpreter::op::Nop {};
+        nop.encode(&mut bytes);
+        // NOP needs to be a 1-byte opcode so it can be used to
+        // overwrite a callsite of any length.
+        assert_eq!(bytes.len(), 1);
+        bytes
+    }
+
     fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
             I8 => Ok((&[RegClass::Int], &[I8])),
@@ -712,6 +724,10 @@ impl Inst {
                 format!("indirect_call {callee}, {info:?}{try_call}")
             }
 
+            Inst::PatchableCall { info } => {
+                format!("patchable_call {info:?}")
+            }
+
             Inst::ReturnCall { info } => {
                 format!("return_call {info:?}")
             }

cranelift/codegen/src/isa/pulley_shared/lower.isle

@@ -171,6 +171,17 @@
             (_ Unit (emit_side_effect (indirect_call_impl info))))
         output))
 
+;;;; Rules for `patchable_call` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Direct call to an in-range function.
+(rule (lower (patchable_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
+      (let ((abi Sig (abi_sig sig_ref))
+            (uses CallArgList (gen_call_args abi args))
+            (defs CallRetList (gen_patchable_call_rets))
+            (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none)))
+            (_ Unit (emit_side_effect (patchable_call_impl info))))
+        (output_none)))
+
 ;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to a pulley function.

cranelift/codegen/src/isa/riscv64/inst.isle

@@ -111,11 +111,15 @@
       (from_bits u8)
       (to_bits u8))
 
+    ;; A machine direct-call instruction.
     (Call (info BoxCallInfo))
 
-      ;; A machine indirect-call instruction.
+    ;; A machine indirect-call instruction.
     (CallInd (info BoxCallIndInfo))
 
+    ;; A patchable machine direct-call instruction.
+    (PatchableCall (info BoxCallInfo))
+
     ;; A direct return-call macro instruction.
     (ReturnCall (info BoxReturnCallInfo))
 
@@ -3114,6 +3118,11 @@
 (rule (call_ind_impl info)
       (SideEffectNoResult.Inst (MInst.CallInd info)))
 
+;; Helper for creating `MInst.PatchableCall` instructions.
+(decl patchable_call_impl (BoxCallInfo) SideEffectNoResult)
+(rule (patchable_call_impl info)
+      (SideEffectNoResult.Inst (MInst.PatchableCall info)))
+
 ;; Helper for creating `MInst.ReturnCall` instructions.
 (decl return_call_impl (BoxReturnCallInfo) SideEffectNoResult)
 (rule (return_call_impl info)

cranelift/codegen/src/isa/riscv64/inst/emit.rs

@@ -167,6 +167,7 @@ impl Inst {
             | Inst::Extend { .. }
             | Inst::Call { .. }
             | Inst::CallInd { .. }
+            | Inst::PatchableCall { .. }
             | Inst::ReturnCall { .. }
             | Inst::ReturnCallInd { .. }
             | Inst::Jal { .. }
@@ -1193,9 +1194,12 @@ impl Inst {
                     .for_each(|i| i.emit(sink, emit_info, state));
             }
 
-            &Inst::Call { ref info } => {
+            &Inst::Call { ref info } | &Inst::PatchableCall { ref info } => {
+                let is_patchable = matches!(self, Inst::PatchableCall { .. });
+
                 sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0);
 
+                let start = sink.cur_offset();
                 Inst::construct_auipc_and_jalr(Some(writable_link_reg()), writable_link_reg(), 0)
                     .into_iter()
                     .for_each(|i| i.emit_uncompressed(sink, emit_info, state, start_off));
@@ -1210,6 +1214,9 @@ impl Inst {
                         Some(state.frame_layout.sp_to_fp()),
                         try_call.exception_handlers(&state.frame_layout),
                     );
+                } else if is_patchable {
+                    let len = sink.cur_offset() - start;
+                    sink.add_patchable_call_site(len);
                 } else {
                     sink.add_call_site();
                 }

cranelift/codegen/src/isa/riscv64/inst/mod.rs

@@ -330,7 +330,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_use(rn);
             collector.reg_def(rd);
         }
-        Inst::Call { info, .. } => {
+        Inst::Call { info, .. } | Inst::PatchableCall { info, .. } => {
             let CallInfo { uses, defs, .. } = &mut **info;
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
@@ -725,7 +725,7 @@ impl MachInst for Inst {
 
     fn is_safepoint(&self) -> bool {
         match self {
-            Inst::Call { .. } | Inst::CallInd { .. } => true,
+            Inst::Call { .. } | Inst::CallInd { .. } | Inst::PatchableCall { .. } => true,
             _ => false,
         }
     }
@@ -764,9 +764,10 @@ impl MachInst for Inst {
 
     fn call_type(&self) -> CallType {
         match self {
-            Inst::Call { .. } | Inst::CallInd { .. } | Inst::ElfTlsGetAddr { .. } => {
-                CallType::Regular
-            }
+            Inst::Call { .. }
+            | Inst::CallInd { .. }
+            | Inst::PatchableCall { .. }
+            | Inst::ElfTlsGetAddr { .. } => CallType::Regular,
 
             Inst::ReturnCall { .. } | Inst::ReturnCallInd { .. } => CallType::TailCall,
 
@@ -810,6 +811,10 @@ impl MachInst for Inst {
         Inst::Nop4
     }
 
+    fn gen_nop_unit() -> SmallVec<[u8; 8]> {
+        smallvec![0x13, 0x00, 0x00, 0x00]
+    }
+
     fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
             I8 => Ok((&[RegClass::Int], &[I8])),
@@ -1352,6 +1357,9 @@ impl Inst {
                     .unwrap_or_default();
                 format!("callind {rd}{try_call}")
             }
+            &MInst::PatchableCall { ref info } => {
+                format!("call {} ; patchable", info.dest.display(None))
+            }
             &MInst::ReturnCall { ref info } => {
                 let mut s = format!(
                     "return_call {:?} new_stack_arg_size:{}",