inline_asm.rs

//! Codegen of `asm!` invocations.

use std::fmt::Write;

use cranelift_codegen::isa::CallConv;
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_span::sym;
use rustc_target::asm::*;
use target_lexicon::BinaryFormat;

use crate::prelude::*;

pub(crate) enum CInlineAsmOperand<'tcx> {
    In {
        reg: InlineAsmRegOrRegClass,
        value: Value,
    },
    Out {
        reg: InlineAsmRegOrRegClass,
        late: bool,
        place: Option<CPlace<'tcx>>,
    },
    InOut {
        reg: InlineAsmRegOrRegClass,
        _late: bool,
        in_value: Value,
        out_place: Option<CPlace<'tcx>>,
    },
    Const {
        value: String,
    },
    Symbol {
        symbol: String,
    },
}

pub(crate) fn codegen_inline_asm_terminator<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    span: Span,
    template: &[InlineAsmTemplatePiece],
    operands: &[InlineAsmOperand<'tcx>],
    options: InlineAsmOptions,
    destination: Option<mir::BasicBlock>,
) {
    // Used by panic_abort on Windows, but uses a syntax which only happens to work with
    // asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
    // the LLVM backend.
    if template.len() == 1
        && template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string())
    {
        fx.bcx.ins().trap(TrapCode::User(1));
        return;
    }

    let operands = operands
        .iter()
        .map(|operand| match *operand {
            InlineAsmOperand::In { reg, ref value } => CInlineAsmOperand::In {
                reg,
                value: crate::base::codegen_operand(fx, value).load_scalar(fx),
            },
            InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
                reg,
                late,
                place: place.map(|place| crate::base::codegen_place(fx, place)),
            },
            InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
                CInlineAsmOperand::InOut {
                    reg,
                    _late: late,
                    in_value: crate::base::codegen_operand(fx, in_value).load_scalar(fx),
                    out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
                }
            }
            InlineAsmOperand::Const { ref value } => {
                let (const_value, ty) = crate::constant::eval_mir_constant(fx, value);
                let value = rustc_codegen_ssa::common::asm_const_to_str(
                    fx.tcx,
                    span,
                    const_value,
                    fx.layout_of(ty),
                );
                CInlineAsmOperand::Const { value }
            }
            InlineAsmOperand::SymFn { ref value } => {
                if cfg!(not(feature = "inline_asm_sym")) {
                    fx.tcx
                        .dcx()
                        .span_err(span, "asm! and global_asm! sym operands are not yet supported");
                }

                let const_ = fx.monomorphize(value.const_);
                if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                    let instance = ty::Instance::resolve_for_fn_ptr(
                        fx.tcx,
                        ty::ParamEnv::reveal_all(),
                        def_id,
                        args,
                    )
                    .unwrap();
                    let symbol = fx.tcx.symbol_name(instance);

                    // Pass a wrapper rather than the function itself as the function itself may not
                    // be exported from the main codegen unit and may thus be unreachable from the
                    // object file created by an external assembler.
                    let inline_asm_index = fx.cx.inline_asm_index.get();
                    fx.cx.inline_asm_index.set(inline_asm_index + 1);
                    let wrapper_name = format!(
                        "__inline_asm_{}_wrapper_n{}",
                        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
                        inline_asm_index
                    );
                    let sig =
                        get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
                    create_wrapper_function(
                        fx.module,
                        &mut fx.cx.unwind_context,
                        sig,
                        &wrapper_name,
                        symbol.name,
                    );

                    CInlineAsmOperand::Symbol { symbol: wrapper_name }
                } else {
                    span_bug!(span, "invalid type for asm sym (fn)");
                }
            }
            InlineAsmOperand::SymStatic { def_id } => {
                assert!(fx.tcx.is_static(def_id));
                let instance = Instance::mono(fx.tcx, def_id);
                CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
            }
            InlineAsmOperand::Label { .. } => {
                span_bug!(span, "asm! label operands are not yet supported");
            }
        })
        .collect::<Vec<_>>();

    codegen_inline_asm_inner(fx, template, &operands, options);

    match destination {
        Some(destination) => {
            let destination_block = fx.get_block(destination);
            fx.bcx.ins().jump(destination_block, &[]);
        }
        None => {
            fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
        }
    }
}

pub(crate) fn codegen_inline_asm_inner<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    template: &[InlineAsmTemplatePiece],
    operands: &[CInlineAsmOperand<'tcx>],
    options: InlineAsmOptions,
) {
    // FIXME add .eh_frame unwind info directives

    let mut asm_gen = InlineAssemblyGenerator {
        tcx: fx.tcx,
        arch: fx.tcx.sess.asm_arch.unwrap(),
        enclosing_def_id: fx.instance.def_id(),
        template,
        operands,
        options,
        registers: Vec::new(),
        stack_slots_clobber: Vec::new(),
        stack_slots_input: Vec::new(),
        stack_slots_output: Vec::new(),
        stack_slot_size: Size::from_bytes(0),
        is_naked: false,
    };
    asm_gen.allocate_registers();
    asm_gen.allocate_stack_slots();

    let inline_asm_index = fx.cx.inline_asm_index.get();
    fx.cx.inline_asm_index.set(inline_asm_index + 1);
    let asm_name = format!(
        "__inline_asm_{}_n{}",
        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
        inline_asm_index
    );

    let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
    fx.cx.global_asm.push_str(&generated_asm);

    let mut inputs = Vec::new();
    let mut outputs = Vec::new();
    for (i, operand) in operands.iter().enumerate() {
        match operand {
            CInlineAsmOperand::In { reg: _, value } => {
                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
            }
            CInlineAsmOperand::Out { reg: _, late: _, place } => {
                if let Some(place) = place {
                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
                }
            }
            CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *in_value));
                if let Some(out_place) = out_place {
                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place));
                }
            }
            CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
        }
    }

    call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
}

pub(crate) fn codegen_naked_asm<'tcx>(
    tcx: TyCtxt<'tcx>,
    cx: &mut crate::CodegenCx,
    module: &mut dyn Module,
    instance: Instance<'tcx>,
    span: Span,
    symbol_name: &str,
    template: &[InlineAsmTemplatePiece],
    operands: &[InlineAsmOperand<'tcx>],
    options: InlineAsmOptions,
) {
    // FIXME add .eh_frame unwind info directives

    let operands = operands
        .iter()
        .map(|operand| match *operand {
            InlineAsmOperand::In { .. }
            | InlineAsmOperand::Out { .. }
            | InlineAsmOperand::InOut { .. } => {
                span_bug!(span, "invalid operand type for naked asm")
            }
            InlineAsmOperand::Const { ref value } => {
                let cv = instance.instantiate_mir_and_normalize_erasing_regions(
                    tcx,
                    ty::ParamEnv::reveal_all(),
                    ty::EarlyBinder::bind(value.const_),
                );
                let const_value = cv
                    .eval(tcx, ty::ParamEnv::reveal_all(), value.span)
                    .expect("erroneous constant missed by mono item collection");

                let value = rustc_codegen_ssa::common::asm_const_to_str(
                    tcx,
                    span,
                    const_value,
                    RevealAllLayoutCx(tcx).layout_of(cv.ty()),
                );
                CInlineAsmOperand::Const { value }
            }
            InlineAsmOperand::SymFn { ref value } => {
                if cfg!(not(feature = "inline_asm_sym")) {
                    tcx.dcx()
                        .span_err(span, "asm! and global_asm! sym operands are not yet supported");
                }

                let const_ = instance.instantiate_mir_and_normalize_erasing_regions(
                    tcx,
                    ty::ParamEnv::reveal_all(),
                    ty::EarlyBinder::bind(value.const_),
                );
                if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                    let instance = ty::Instance::resolve_for_fn_ptr(
                        tcx,
                        ty::ParamEnv::reveal_all(),
                        def_id,
                        args,
                    )
                    .unwrap();
                    let symbol = tcx.symbol_name(instance);

                    // Pass a wrapper rather than the function itself as the function itself may not
                    // be exported from the main codegen unit and may thus be unreachable from the
                    // object file created by an external assembler.
                    let inline_asm_index = cx.inline_asm_index.get();
                    cx.inline_asm_index.set(inline_asm_index + 1);
                    let wrapper_name = format!(
                        "__inline_asm_{}_wrapper_n{}",
                        cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
                        inline_asm_index
                    );
                    let sig =
                        get_function_sig(tcx, module.target_config().default_call_conv, instance);
                    create_wrapper_function(
                        module,
                        &mut cx.unwind_context,
                        sig,
                        &wrapper_name,
                        symbol.name,
                    );

                    CInlineAsmOperand::Symbol { symbol: wrapper_name }
                } else {
                    span_bug!(span, "invalid type for asm sym (fn)");
                }
            }
            InlineAsmOperand::SymStatic { def_id } => {
                assert!(tcx.is_static(def_id));
                let instance = Instance::mono(tcx, def_id);
                CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() }
            }
            InlineAsmOperand::Label { .. } => {
                span_bug!(span, "asm! label operands are not yet supported");
            }
        })
        .collect::<Vec<_>>();

    let asm_gen = InlineAssemblyGenerator {
        tcx,
        arch: tcx.sess.asm_arch.unwrap(),
        enclosing_def_id: instance.def_id(),
        template,
        operands: &operands,
        options,
        registers: Vec::new(),
        stack_slots_clobber: Vec::new(),
        stack_slots_input: Vec::new(),
        stack_slots_output: Vec::new(),
        stack_slot_size: Size::from_bytes(0),
        is_naked: true,
    };

    let generated_asm = asm_gen.generate_asm_wrapper(symbol_name);
    cx.global_asm.push_str(&generated_asm);
}

struct InlineAssemblyGenerator<'a, 'tcx> {
    tcx: TyCtxt<'tcx>,
    arch: InlineAsmArch,
    enclosing_def_id: DefId,
    template: &'a [InlineAsmTemplatePiece],
    operands: &'a [CInlineAsmOperand<'tcx>],
    options: InlineAsmOptions,
    registers: Vec<Option<InlineAsmReg>>,
    stack_slots_clobber: Vec<Option<Size>>,
    stack_slots_input: Vec<Option<Size>>,
    stack_slots_output: Vec<Option<Size>>,
    stack_slot_size: Size,
    is_naked: bool,
}

impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
    fn allocate_registers(&mut self) {
        assert!(!self.is_naked);

        let sess = self.tcx.sess;
        let map = allocatable_registers(
            self.arch,
            sess.relocation_model(),
            self.tcx.asm_target_features(self.enclosing_def_id),
            &sess.target,
        );
        let mut allocated = FxHashMap::<_, (bool, bool)>::default();
        let mut regs = vec![None; self.operands.len()];

        // Add explicit registers to the allocated set.
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                    regs[i] = Some(reg);
                    allocated.entry(reg).or_default().0 = true;
                }
                CInlineAsmOperand::Out {
                    reg: InlineAsmRegOrRegClass::Reg(reg),
                    late: true,
                    ..
                } => {
                    regs[i] = Some(reg);
                    allocated.entry(reg).or_default().1 = true;
                }
                CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
                | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                    regs[i] = Some(reg);
                    allocated.insert(reg, (true, true));
                }
                _ => (),
            }
        }

        // Allocate out/inout/inlateout registers first because they are more constrained.
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::Out {
                    reg: InlineAsmRegOrRegClass::RegClass(class),
                    late: false,
                    ..
                }
                | CInlineAsmOperand::InOut {
                    reg: InlineAsmRegOrRegClass::RegClass(class), ..
                } => {
                    let mut alloc_reg = None;
                    for &reg in &map[&class] {
                        let mut used = false;
                        reg.overlapping_regs(|r| {
                            if allocated.contains_key(&r) {
                                used = true;
                            }
                        });

                        if !used {
                            alloc_reg = Some(reg);
                            break;
                        }
                    }

                    let reg = alloc_reg.expect("cannot allocate registers");
                    regs[i] = Some(reg);
                    allocated.insert(reg, (true, true));
                }
                _ => (),
            }
        }

        // Allocate in/lateout.
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
                    let mut alloc_reg = None;
                    for &reg in &map[&class] {
                        let mut used = false;
                        reg.overlapping_regs(|r| {
                            if allocated.get(&r).copied().unwrap_or_default().0 {
                                used = true;
                            }
                        });

                        if !used {
                            alloc_reg = Some(reg);
                            break;
                        }
                    }

                    let reg = alloc_reg.expect("cannot allocate registers");
                    regs[i] = Some(reg);
                    allocated.entry(reg).or_default().0 = true;
                }
                CInlineAsmOperand::Out {
                    reg: InlineAsmRegOrRegClass::RegClass(class),
                    late: true,
                    ..
                } => {
                    let mut alloc_reg = None;
                    for &reg in &map[&class] {
                        let mut used = false;
                        reg.overlapping_regs(|r| {
                            if allocated.get(&r).copied().unwrap_or_default().1 {
                                used = true;
                            }
                        });

                        if !used {
                            alloc_reg = Some(reg);
                            break;
                        }
                    }

                    let reg = alloc_reg.expect("cannot allocate registers");
                    regs[i] = Some(reg);
                    allocated.entry(reg).or_default().1 = true;
                }
                _ => (),
            }
        }

        self.registers = regs;
    }

    fn allocate_stack_slots(&mut self) {
        assert!(!self.is_naked);

        let mut slot_size = Size::from_bytes(0);
        let mut slots_clobber = vec![None; self.operands.len()];
        let mut slots_input = vec![None; self.operands.len()];
        let mut slots_output = vec![None; self.operands.len()];

        let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
            let reg_size =
                reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
            let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
            let offset = slot_size.align_to(align);
            *slot_size = offset + reg_size;
            offset
        };
        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);

        // Allocate stack slots for saving clobbered registers
        let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C)
            .unwrap()
            .clobbered_regs();
        for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
            let mut need_save = true;
            // If the register overlaps with a register clobbered by function call, then
            // we don't need to save it.
            for r in abi_clobber {
                r.overlapping_regs(|r| {
                    if r == reg {
                        need_save = false;
                    }
                });

                if !need_save {
                    break;
                }
            }

            if need_save {
                slots_clobber[i] = Some(new_slot(reg.reg_class()));
            }
        }

        // Allocate stack slots for inout
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
                    let slot = new_slot(reg.reg_class());
                    slots_input[i] = Some(slot);
                    slots_output[i] = Some(slot);
                }
                _ => (),
            }
        }

        let slot_size_before_input = slot_size;
        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);

        // Allocate stack slots for input
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::In { reg, .. }
                | CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
                    slots_input[i] = Some(new_slot(reg.reg_class()));
                }
                _ => (),
            }
        }

        // Reset slot size to before input so that input and output operands can overlap
        // and save some memory.
        let slot_size_after_input = slot_size;
        slot_size = slot_size_before_input;
        let mut new_slot = |x| new_slot_fn(&mut slot_size, x);

        // Allocate stack slots for output
        for (i, operand) in self.operands.iter().enumerate() {
            match *operand {
                CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
                    slots_output[i] = Some(new_slot(reg.reg_class()));
                }
                _ => (),
            }
        }

        slot_size = slot_size.max(slot_size_after_input);

        self.stack_slots_clobber = slots_clobber;
        self.stack_slots_input = slots_input;
        self.stack_slots_output = slots_output;
        self.stack_slot_size = slot_size;
    }

    fn generate_asm_wrapper(&self, asm_name: &str) -> String {
        let binary_format = crate::target_triple(self.tcx.sess).binary_format;

        let mut generated_asm = String::new();
        match binary_format {
            BinaryFormat::Elf => {
                writeln!(generated_asm, ".globl {}", asm_name).unwrap();
                writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
                writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
                writeln!(generated_asm, "{}:", asm_name).unwrap();
            }
            BinaryFormat::Macho => {
                writeln!(generated_asm, ".globl _{}", asm_name).unwrap();
                writeln!(generated_asm, "_{}:", asm_name).unwrap();
            }
            BinaryFormat::Coff => {
                writeln!(generated_asm, ".globl {}", asm_name).unwrap();
                writeln!(generated_asm, "{}:", asm_name).unwrap();
            }
            _ => self
                .tcx
                .dcx()
                .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
        }

        let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);

        if is_x86 {
            generated_asm.push_str(".intel_syntax noprefix\n");
        }
        if !self.is_naked {
            Self::prologue(&mut generated_asm, self.arch);

            // Save clobbered registers
            if !self.options.contains(InlineAsmOptions::NORETURN) {
                for (reg, slot) in self
                    .registers
                    .iter()
                    .zip(self.stack_slots_clobber.iter().copied())
                    .filter_map(|(r, s)| r.zip(s))
                {
                    Self::save_register(&mut generated_asm, self.arch, reg, slot);
                }
            }

            // Write input registers
            for (reg, slot) in self
                .registers
                .iter()
                .zip(self.stack_slots_input.iter().copied())
                .filter_map(|(r, s)| r.zip(s))
            {
                Self::restore_register(&mut generated_asm, self.arch, reg, slot);
            }
        }

        if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
            generated_asm.push_str(".att_syntax\n");
        }

        // The actual inline asm
        for piece in self.template {
            match piece {
                InlineAsmTemplatePiece::String(s) => {
                    generated_asm.push_str(s);
                }
                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
                    match self.operands[*operand_idx] {
                        CInlineAsmOperand::In { .. }
                        | CInlineAsmOperand::Out { .. }
                        | CInlineAsmOperand::InOut { .. } => {
                            if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
                                generated_asm.push('%');
                            }

                            let reg = self.registers[*operand_idx].unwrap();
                            match self.arch {
                                InlineAsmArch::X86_64 => match reg {
                                    InlineAsmReg::X86(reg)
                                        if reg as u32 >= X86InlineAsmReg::xmm0 as u32
                                            && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
                                    {
                                        // rustc emits x0 rather than xmm0
                                        let class = match *modifier {
                                            None | Some('x') => "xmm",
                                            Some('y') => "ymm",
                                            Some('z') => "zmm",
                                            _ => unreachable!(),
                                        };
                                        write!(
                                            generated_asm,
                                            "{class}{}",
                                            reg as u32 - X86InlineAsmReg::xmm0 as u32
                                        )
                                        .unwrap();
                                    }
                                    _ => reg
                                        .emit(&mut generated_asm, InlineAsmArch::X86_64, *modifier)
                                        .unwrap(),
                                },
                                _ => reg.emit(&mut generated_asm, self.arch, *modifier).unwrap(),
                            }
                        }
                        CInlineAsmOperand::Const { ref value } => {
                            generated_asm.push_str(value);
                        }
                        CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
                    }
                }
            }
        }
        generated_asm.push('\n');

        if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
            generated_asm.push_str(".intel_syntax noprefix\n");
        }

        if !self.is_naked {
            if !self.options.contains(InlineAsmOptions::NORETURN) {
                // Read output registers
                for (reg, slot) in self
                    .registers
                    .iter()
                    .zip(self.stack_slots_output.iter().copied())
                    .filter_map(|(r, s)| r.zip(s))
                {
                    Self::save_register(&mut generated_asm, self.arch, reg, slot);
                }

                // Restore clobbered registers
                for (reg, slot) in self
                    .registers
                    .iter()
                    .zip(self.stack_slots_clobber.iter().copied())
                    .filter_map(|(r, s)| r.zip(s))
                {
                    Self::restore_register(&mut generated_asm, self.arch, reg, slot);
                }

                Self::epilogue(&mut generated_asm, self.arch);
            } else {
                Self::epilogue_noreturn(&mut generated_asm, self.arch);
            }
        }

        if is_x86 {
            generated_asm.push_str(".att_syntax\n");
        }

        match binary_format {
            BinaryFormat::Elf => {
                writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
                generated_asm.push_str(".text\n");
            }
            BinaryFormat::Macho | BinaryFormat::Coff => {}
            _ => self
                .tcx
                .dcx()
                .fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
        }

        generated_asm.push_str("\n\n");

        generated_asm
    }

    fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
        match arch {
            InlineAsmArch::X86_64 => {
                generated_asm.push_str("    push rbp\n");
                generated_asm.push_str("    mov rbp,rsp\n");
                generated_asm.push_str("    push rbx\n"); // rbx is callee saved
                // rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
                generated_asm.push_str("    mov rbx,rdi\n");
            }
            InlineAsmArch::AArch64 => {
                generated_asm.push_str("    stp fp, lr, [sp, #-32]!\n");
                generated_asm.push_str("    mov fp, sp\n");
                generated_asm.push_str("    str x19, [sp, #24]\n"); // x19 is callee saved
                // x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
                generated_asm.push_str("    mov x19, x0\n");
            }
            InlineAsmArch::RiscV64 => {
                generated_asm.push_str("    addi sp, sp, -16\n");
                generated_asm.push_str("    sd ra, 8(sp)\n");
                generated_asm.push_str("    sd s1, 0(sp)\n"); // s1 is callee saved
                // s1/x9 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
                generated_asm.push_str("    mv s1, a0\n");
            }
            _ => unimplemented!("prologue for {:?}", arch),
        }
    }

    fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
        match arch {
            InlineAsmArch::X86_64 => {
                generated_asm.push_str("    pop rbx\n");
                generated_asm.push_str("    pop rbp\n");
                generated_asm.push_str("    ret\n");
            }
            InlineAsmArch::AArch64 => {
                generated_asm.push_str("    ldr x19, [sp, #24]\n");
                generated_asm.push_str("    ldp fp, lr, [sp], #32\n");
                generated_asm.push_str("    ret\n");
            }
            InlineAsmArch::RiscV64 => {
                generated_asm.push_str("    ld s1, 0(sp)\n");
                generated_asm.push_str("    ld ra, 8(sp)\n");
                generated_asm.push_str("    addi sp, sp, 16\n");
                generated_asm.push_str("    ret\n");
            }
            _ => unimplemented!("epilogue for {:?}", arch),
        }
    }

    fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
        match arch {
            InlineAsmArch::X86_64 => {
                generated_asm.push_str("    ud2\n");
            }
            InlineAsmArch::AArch64 => {
                generated_asm.push_str("    brk     #0x1\n");
            }
            InlineAsmArch::RiscV64 => {
                generated_asm.push_str("    ebreak\n");
            }
            _ => unimplemented!("epilogue_noreturn for {:?}", arch),
        }
    }

    fn save_register(
        generated_asm: &mut String,
        arch: InlineAsmArch,
        reg: InlineAsmReg,
        offset: Size,
    ) {
        match arch {
            InlineAsmArch::X86_64 => {
                match reg {
                    InlineAsmReg::X86(reg)
                        if reg as u32 >= X86InlineAsmReg::xmm0 as u32
                            && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
                    {
                        // rustc emits x0 rather than xmm0
                        write!(generated_asm, "    movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
                        write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
                            .unwrap();
                    }
                    _ => {
                        write!(generated_asm, "    mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
                        reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
                    }
                }
                generated_asm.push('\n');
            }
            InlineAsmArch::AArch64 => {
                generated_asm.push_str("    str ");
                reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
                writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
            }
            InlineAsmArch::RiscV64 => {
                generated_asm.push_str("    sd ");
                reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
                writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
            }
            _ => unimplemented!("save_register for {:?}", arch),
        }
    }

    fn restore_register(
        generated_asm: &mut String,
        arch: InlineAsmArch,
        reg: InlineAsmReg,
        offset: Size,
    ) {
        match arch {
            InlineAsmArch::X86_64 => {
                match reg {
                    InlineAsmReg::X86(reg)
                        if reg as u32 >= X86InlineAsmReg::xmm0 as u32
                            && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
                    {
                        // rustc emits x0 rather than xmm0
                        write!(
                            generated_asm,
                            "    movups xmm{}",
                            reg as u32 - X86InlineAsmReg::xmm0 as u32
                        )
                        .unwrap();
                    }
                    _ => {
                        generated_asm.push_str("    mov ");
                        reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
                    }
                }
                writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
            }
            InlineAsmArch::AArch64 => {
                generated_asm.push_str("    ldr ");
                reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
                writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
            }
            InlineAsmArch::RiscV64 => {
                generated_asm.push_str("    ld ");
                reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
                writeln!(generated_asm, ", 0x{:x}(s1)", offset.bytes()).unwrap();
            }
            _ => unimplemented!("restore_register for {:?}", arch),
        }
    }
}

fn call_inline_asm<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    asm_name: &str,
    slot_size: Size,
    inputs: Vec<(Size, Value)>,
    outputs: Vec<(Size, CPlace<'tcx>)>,
) {
    let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);

    let inline_asm_func = fx
        .module
        .declare_function(
            asm_name,
            Linkage::Import,
            &Signature {
                call_conv: CallConv::SystemV,
                params: vec![AbiParam::new(fx.pointer_type)],
                returns: vec![],
            },
        )
        .unwrap();
    let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, fx.bcx.func);
    if fx.clif_comments.enabled() {
        fx.add_comment(inline_asm_func, asm_name);
    }

    for (offset, value) in inputs {
        stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
            fx,
            value,
            MemFlags::trusted(),
        );
    }

    let stack_slot_addr = stack_slot.get_addr(fx);
    fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);

    for (offset, place) in outputs {
        let ty = if place.layout().ty.is_simd() {
            let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
            asm_clif_type(fx, lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
        } else {
            asm_clif_type(fx, place.layout().ty).unwrap()
        };
        let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
            fx,
            ty,
            MemFlags::trusted(),
        );
        place.write_cvalue(fx, CValue::by_val(value, place.layout()));
    }
}

fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
    match ty.kind() {
        // Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
        ty::Adt(adt, args) if Some(adt.did()) == fx.tcx.lang_items().maybe_uninit() => {
            let fields = &adt.non_enum_variant().fields;
            let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args);
            let ty::Adt(ty, args) = ty.kind() else {
                unreachable!("expected first field of `MaybeUninit` to be an ADT")
            };
            assert!(
                ty.is_manually_drop(),
                "expected first field of `MaybeUninit` to be `ManuallyDrop`"
            );
            let fields = &ty.non_enum_variant().fields;
            let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args);
            fx.clif_type(ty)
        }
        _ => fx.clif_type(ty),
    }
}