Use relative call instructions between wasm functions

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -110,7 +110,13 @@ fn machreg_to_gpr_or_vec(m: Reg) -> u32 {
     u32::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
 }
 
-fn enc_arith_rrr(bits_31_21: u32, bits_15_10: u32, rd: Writable<Reg>, rn: Reg, rm: Reg) -> u32 {
+pub(crate) fn enc_arith_rrr(
+    bits_31_21: u32,
+    bits_15_10: u32,
+    rd: Writable<Reg>,
+    rn: Reg,
+    rm: Reg,
+) -> u32 {
     (bits_31_21 << 21)
         | (bits_15_10 << 10)
         | machreg_to_gpr(rd.to_reg())
@@ -243,7 +249,7 @@ fn enc_ldst_reg(
         | machreg_to_gpr_or_vec(rd)
 }
 
-fn enc_ldst_imm19(op_31_24: u32, imm19: u32, rd: Reg) -> u32 {
+pub(crate) fn enc_ldst_imm19(op_31_24: u32, imm19: u32, rd: Reg) -> u32 {
     (op_31_24 << 24) | (imm19 << 5) | machreg_to_gpr_or_vec(rd)
 }
 
@@ -320,11 +326,11 @@ fn enc_bit_rr(size: u32, opcode2: u32, opcode1: u32, rn: Reg, rd: Writable<Reg>)
         | machreg_to_gpr(rd.to_reg())
 }
 
-fn enc_br(rn: Reg) -> u32 {
+pub(crate) fn enc_br(rn: Reg) -> u32 {
     0b1101011_0000_11111_000000_00000_00000 | (machreg_to_gpr(rn) << 5)
 }
 
-fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
+pub(crate) fn enc_adr(off: i32, rd: Writable<Reg>) -> u32 {
     let off = u32::try_from(off).unwrap();
     let immlo = off & 3;
     let immhi = (off >> 2) & ((1 << 19) - 1);
@@ -2694,7 +2700,7 @@ impl MachInstEmit for Inst {
                         dest: BranchTarget::Label(jump_around_label),
                     };
                     jmp.emit(sink, emit_info, state);
-                    sink.emit_island();
+                    sink.emit_island(needed_space + 4);
                     sink.bind_label(jump_around_label);
                 }
             }

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -3,7 +3,7 @@
 // Some variants are not constructed, but we still want them as options in the future.
 #![allow(dead_code)]
 
-use crate::binemit::CodeOffset;
+use crate::binemit::{Addend, CodeOffset, Reloc};
 use crate::ir::types::{
     B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64,
 };
@@ -4786,13 +4786,18 @@ impl MachInstLabelUse for LabelUse {
     fn supports_veneer(self) -> bool {
         match self {
             LabelUse::Branch19 => true, // veneer is a Branch26
+            LabelUse::Branch26 => true, // veneer is a PCRel32
             _ => false,
         }
     }
 
     /// How large is the veneer, if supported?
     fn veneer_size(self) -> CodeOffset {
-        4
+        match self {
+            LabelUse::Branch19 => 4,
+            LabelUse::Branch26 => 20,
+            _ => unreachable!(),
+        }
     }
 
     /// Generate a veneer into the buffer, given that this veneer is at `veneer_offset`, and return
@@ -4810,7 +4815,47 @@ impl MachInstLabelUse for LabelUse {
                 buffer[0..4].clone_from_slice(&u32::to_le_bytes(insn_word));
                 (veneer_offset, LabelUse::Branch26)
             }
+
+            // This is promoting a 26-bit call/jump to a 32-bit call/jump to
+            // get a further range. This jump translates to a jump to a
+            // relative location based on the address of the constant loaded
+            // from here.
+            //
+            // If this path is taken from a call instruction then caller-saved
+            // registers are available (minus arguments), so x16/x17 are
+            // available. Otherwise for intra-function jumps we also reserve
+            // x16/x17 as spill-style registers. In both cases these are
+            // available for us to use.
+            LabelUse::Branch26 => {
+                let tmp1 = regs::spilltmp_reg();
+                let tmp1_w = regs::writable_spilltmp_reg();
+                let tmp2 = regs::tmp2_reg();
+                let tmp2_w = regs::writable_tmp2_reg();
+                // ldrsw x16, 16
+                let ldr = emit::enc_ldst_imm19(0b1001_1000, 16 / 4, tmp1);
+                // adr x17, 12
+                let adr = emit::enc_adr(12, tmp2_w);
+                // add x16, x16, x17
+                let add = emit::enc_arith_rrr(0b10001011_000, 0, tmp1_w, tmp1, tmp2);
+                // br x16
+                let br = emit::enc_br(tmp1);
+                buffer[0..4].clone_from_slice(&u32::to_le_bytes(ldr));
+                buffer[4..8].clone_from_slice(&u32::to_le_bytes(adr));
+                buffer[8..12].clone_from_slice(&u32::to_le_bytes(add));
+                buffer[12..16].clone_from_slice(&u32::to_le_bytes(br));
+                // the 4-byte signed immediate we'll load is after these
+                // instructions, 16-bytes in.
+                (veneer_offset + 16, LabelUse::PCRel32)
+            }
+
             _ => panic!("Unsupported label-reference type for veneer generation!"),
         }
     }
+
+    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<LabelUse> {
+        match (reloc, addend) {
+            (Reloc::Arm64Call, 0) => Some(LabelUse::Branch26),
+            _ => None,
+        }
+    }
 }

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -4,7 +4,10 @@ use crate::ir::condcodes::IntCC;
 use crate::ir::Function;
 use crate::isa::aarch64::settings as aarch64_settings;
 use crate::isa::Builder as IsaBuilder;
-use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
+use crate::machinst::{
+    compile, MachBackend, MachCompileResult, MachTextSectionBuilder, TargetIsaAdapter,
+    TextSectionBuilder, VCode,
+};
 use crate::result::CodegenResult;
 use crate::settings as shared_settings;
 use alloc::{boxed::Box, vec::Vec};
@@ -161,6 +164,10 @@ impl MachBackend for AArch64Backend {
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         Some(inst::unwind::systemv::create_cie())
     }
+
+    fn text_section_builder(&self, num_funcs: u32) -> Box<dyn TextSectionBuilder> {
+        Box::new(MachTextSectionBuilder::<inst::Inst>::new(num_funcs))
+    }
 }
 
 /// Create a new `isa::Builder`.

cranelift/codegen/src/isa/arm32/inst/mod.rs

@@ -2,7 +2,7 @@
 
 #![allow(dead_code)]
 
-use crate::binemit::CodeOffset;
+use crate::binemit::{Addend, CodeOffset, Reloc};
 use crate::ir::types::{B1, B16, B32, B8, I16, I32, I8, IFLAGS};
 use crate::ir::{ExternalName, Opcode, TrapCode, Type};
 use crate::machinst::*;
@@ -1317,6 +1317,10 @@ impl MachInstLabelUse for LabelUse {
     ) -> (CodeOffset, LabelUse) {
         panic!("Veneer not supported yet.")
     }
+
+    fn from_reloc(_reloc: Reloc, _addend: Addend) -> Option<LabelUse> {
+        None
+    }
 }
 
 #[cfg(test)]

cranelift/codegen/src/isa/arm32/mod.rs

@@ -3,7 +3,10 @@
 use crate::ir::condcodes::IntCC;
 use crate::ir::Function;
 use crate::isa::Builder as IsaBuilder;
-use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
+use crate::machinst::{
+    compile, MachBackend, MachCompileResult, MachTextSectionBuilder, TargetIsaAdapter,
+    TextSectionBuilder, VCode,
+};
 use crate::result::CodegenResult;
 use crate::settings;
 
@@ -115,6 +118,10 @@ impl MachBackend for Arm32Backend {
         // Carry flag clear.
         IntCC::UnsignedLessThan
     }
+
+    fn text_section_builder(&self, num_funcs: u32) -> Box<dyn TextSectionBuilder> {
+        Box::new(MachTextSectionBuilder::<inst::Inst>::new(num_funcs))
+    }
 }
 
 /// Create a new `isa::Builder`.

cranelift/codegen/src/isa/s390x/inst/mod.rs

@@ -3,7 +3,7 @@
 // Some variants are not constructed, but we still want them as options in the future.
 #![allow(dead_code)]
 
-use crate::binemit::CodeOffset;
+use crate::binemit::{Addend, CodeOffset, Reloc};
 use crate::ir::{types, ExternalName, Opcode, TrapCode, Type, ValueLabel};
 use crate::isa::unwind::UnwindInst;
 use crate::machinst::*;
@@ -3686,4 +3686,8 @@ impl MachInstLabelUse for LabelUse {
     ) -> (CodeOffset, LabelUse) {
         unreachable!();
     }
+
+    fn from_reloc(_reloc: Reloc, _addend: Addend) -> Option<Self> {
+        None
+    }
 }

cranelift/codegen/src/isa/s390x/mod.rs

@@ -6,7 +6,10 @@ use crate::isa::s390x::settings as s390x_settings;
 #[cfg(feature = "unwind")]
 use crate::isa::unwind::systemv::RegisterMappingError;
 use crate::isa::Builder as IsaBuilder;
-use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
+use crate::machinst::{
+    compile, MachBackend, MachCompileResult, MachTextSectionBuilder, TargetIsaAdapter,
+    TextSectionBuilder, VCode,
+};
 use crate::result::CodegenResult;
 use crate::settings as shared_settings;
 
@@ -165,6 +168,10 @@ impl MachBackend for S390xBackend {
     fn map_reg_to_dwarf(&self, reg: Reg) -> Result<u16, RegisterMappingError> {
         inst::unwind::systemv::map_reg(reg).map(|reg| reg.0)
     }
+
+    fn text_section_builder(&self, num_funcs: u32) -> Box<dyn TextSectionBuilder> {
+        Box::new(MachTextSectionBuilder::<inst::Inst>::new(num_funcs))
+    }
 }
 
 /// Create a new `isa::Builder`.

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -1,6 +1,6 @@
 //! This module defines x86_64-specific machine instruction types.
 
-use crate::binemit::{CodeOffset, StackMap};
+use crate::binemit::{Addend, CodeOffset, Reloc, StackMap};
 use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
 use crate::isa::unwind::UnwindInst;
 use crate::isa::x64::abi::X64ABIMachineSpec;
@@ -3005,4 +3005,11 @@ impl MachInstLabelUse for LabelUse {
             }
         }
     }
+
+    fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
+        match (reloc, addend) {
+            (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
+            _ => None,
+        }
+    }
 }

cranelift/codegen/src/isa/x64/mod.rs

@@ -8,7 +8,10 @@ use crate::ir::{condcodes::IntCC, Function};
 use crate::isa::unwind::systemv;
 use crate::isa::x64::{inst::regs::create_reg_universe_systemv, settings as x64_settings};
 use crate::isa::Builder as IsaBuilder;
-use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
+use crate::machinst::{
+    compile, MachBackend, MachCompileResult, MachTextSectionBuilder, TargetIsaAdapter,
+    TextSectionBuilder, VCode,
+};
 use crate::result::CodegenResult;
 use crate::settings::{self as shared_settings, Flags};
 use alloc::{boxed::Box, vec::Vec};
@@ -158,6 +161,10 @@ impl MachBackend for X64Backend {
     fn map_reg_to_dwarf(&self, reg: Reg) -> Result<u16, systemv::RegisterMappingError> {
         inst::unwind::systemv::map_reg(reg).map(|reg| reg.0)
     }
+
+    fn text_section_builder(&self, num_funcs: u32) -> Box<dyn TextSectionBuilder> {
+        Box::new(MachTextSectionBuilder::<inst::Inst>::new(num_funcs))
+    }
 }
 
 /// Create a new `isa::Builder`.

cranelift/codegen/src/lib.rs

@@ -85,6 +85,7 @@ pub mod write;
 
 pub use crate::entity::packed_option;
 pub use crate::machinst::buffer::MachSrcLoc;
+pub use crate::machinst::TextSectionBuilder;
 
 mod abi;
 mod bitset;