Skip to content

Improve intrinsic handling in cg_ssa #141404

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 28, 2025
Merged

Improve intrinsic handling in cg_ssa #141404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1e9e177
Move some code around in codegen_call_terminator
bjorn3 May 21, 2025
e4700e7
Always use fn_span in codegen_call_terminator
bjorn3 May 22, 2025
c83358b
Move caller_location handling into codegen_intrinsic_call
bjorn3 May 22, 2025
6016f84
Pass PlaceRef rather than Bx::Value to codegen_intrinsic_call
bjorn3 May 22, 2025
0a14e1b
Remove usage of FnAbi in codegen_intrinsic_call
bjorn3 May 22, 2025
7122648
Don't depend on FnAbi for intrinsics
bjorn3 May 22, 2025
165fb98
Reduce indentation in codegen_panic_intrinsic
bjorn3 May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 20 additions & 33 deletions compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ use rustc_codegen_ssa::traits::{
};
use rustc_middle::bug;
#[cfg(feature = "master")]
use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
use rustc_middle::ty::{self, Instance, Ty};
use rustc_span::{Span, Symbol, sym};
use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
use rustc_target::callconv::{ArgAbi, PassMode};
use rustc_target::spec::PanicStrategy;

#[cfg(feature = "master")]
Expand Down Expand Up @@ -200,9 +200,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
fn codegen_intrinsic_call(
&mut self,
instance: Instance<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[OperandRef<'tcx, RValue<'gcc>>],
llresult: RValue<'gcc>,
result: PlaceRef<'tcx, RValue<'gcc>>,
span: Span,
) -> Result<(), Instance<'tcx>> {
let tcx = self.tcx;
Expand All @@ -221,7 +220,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
let name_str = name.as_str();

let llret_ty = self.layout_of(ret_ty).gcc_type(self);
let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);

let simple = get_simple_intrinsic(self, name);
let simple_func = get_simple_function(self, name);
Expand Down Expand Up @@ -271,7 +269,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
args[0].immediate(),
args[1].immediate(),
args[2].immediate(),
llresult,
result,
);
return Ok(());
}
Expand All @@ -286,17 +284,10 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
}

sym::volatile_load | sym::unaligned_volatile_load => {
let tp_ty = fn_args.type_at(0);
let ptr = args[0].immediate();
let layout = self.layout_of(tp_ty);
let load = if let PassMode::Cast { cast: ref ty, pad_i32: _ } = fn_abi.ret.mode {
let gcc_ty = ty.gcc_type(self);
self.volatile_load(gcc_ty, ptr)
} else {
self.volatile_load(layout.gcc_type(self), ptr)
};
let load = self.volatile_load(result.layout.gcc_type(self), ptr);
// TODO(antoyo): set alignment.
if let BackendRepr::Scalar(scalar) = layout.backend_repr {
if let BackendRepr::Scalar(scalar) = result.layout.backend_repr {
self.to_immediate_scalar(load, scalar)
} else {
load
Expand Down Expand Up @@ -511,16 +502,14 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
_ => return Err(Instance::new_raw(instance.def_id(), instance.args)),
};

if !fn_abi.ret.is_ignore() {
if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
let ptr = self.pointercast(result.val.llval, ptr_llty);
self.store(value, ptr, result.val.align);
} else {
OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
.val
.store(self, result);
}
if result.layout.ty.is_bool() {
OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
.val
.store(self, result);
} else if !result.layout.ty.is_unit() {
let ptr_llty = self.type_ptr_to(result.layout.gcc_type(self));
let ptr = self.pointercast(result.val.llval, ptr_llty);
self.store(value, ptr, result.val.align);
}
Ok(())
}
Expand Down Expand Up @@ -1230,14 +1219,13 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(
try_func: RValue<'gcc>,
data: RValue<'gcc>,
_catch_func: RValue<'gcc>,
dest: RValue<'gcc>,
dest: PlaceRef<'tcx, RValue<'gcc>>,
) {
if bx.sess().panic_strategy() == PanicStrategy::Abort {
bx.call(bx.type_void(), None, None, try_func, &[data], None, None);
// Return 0 unconditionally from the intrinsic call;
// we can never unwind.
let ret_align = bx.tcx.data_layout.i32_align.abi;
bx.store(bx.const_i32(0), dest, ret_align);
OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
} else {
if wants_msvc_seh(bx.sess()) {
unimplemented!();
Expand All @@ -1261,12 +1249,12 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(
// functions in play. By calling a shim we're guaranteed that our shim will have
// the right personality function.
#[cfg(feature = "master")]
fn codegen_gnu_try<'gcc>(
bx: &mut Builder<'_, 'gcc, '_>,
fn codegen_gnu_try<'gcc, 'tcx>(
bx: &mut Builder<'_, 'gcc, 'tcx>,
try_func: RValue<'gcc>,
data: RValue<'gcc>,
catch_func: RValue<'gcc>,
dest: RValue<'gcc>,
dest: PlaceRef<'tcx, RValue<'gcc>>,
) {
let cx: &CodegenCx<'gcc, '_> = bx.cx;
let (llty, func) = get_rust_try_fn(cx, &mut |mut bx| {
Expand Down Expand Up @@ -1322,8 +1310,7 @@ fn codegen_gnu_try<'gcc>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
OperandValue::Immediate(ret).store(bx, dest);
}

// Helper function used to get a handle to the `__rust_try` function used to
Expand Down
80 changes: 32 additions & 48 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,10 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty};
use rustc_middle::{bug, span_bug};
use rustc_span::{Span, Symbol, sym};
use rustc_symbol_mangling::mangle_internal_symbol;
use rustc_target::callconv::{FnAbi, PassMode};
use rustc_target::spec::{HasTargetSpec, PanicStrategy};
use tracing::debug;

use crate::abi::{FnAbiLlvmExt, LlvmType};
use crate::abi::FnAbiLlvmExt;
use crate::builder::Builder;
use crate::context::CodegenCx;
use crate::llvm::{self, Metadata};
Expand Down Expand Up @@ -165,9 +164,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
fn codegen_intrinsic_call(
&mut self,
instance: ty::Instance<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[OperandRef<'tcx, &'ll Value>],
llresult: &'ll Value,
result: PlaceRef<'tcx, &'ll Value>,
span: Span,
) -> Result<(), ty::Instance<'tcx>> {
let tcx = self.tcx;
Expand All @@ -184,7 +182,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let name = tcx.item_name(def_id);

let llret_ty = self.layout_of(ret_ty).llvm_type(self);
let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);

let simple = get_simple_intrinsic(self, name);
let llval = match name {
Expand Down Expand Up @@ -255,7 +252,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
args[0].immediate(),
args[1].immediate(),
args[2].immediate(),
llresult,
result,
);
return Ok(());
}
Expand All @@ -264,7 +261,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
}
sym::va_arg => {
match fn_abi.ret.layout.backend_repr {
match result.layout.backend_repr {
BackendRepr::Scalar(scalar) => {
match scalar.primitive() {
Primitive::Int(..) => {
Expand Down Expand Up @@ -299,18 +296,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}

sym::volatile_load | sym::unaligned_volatile_load => {
let tp_ty = fn_args.type_at(0);
let ptr = args[0].immediate();
let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode {
let llty = ty.llvm_type(self);
self.volatile_load(llty, ptr)
} else {
self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr)
};
let load = self.volatile_load(result.layout.llvm_type(self), ptr);
let align = if name == sym::unaligned_volatile_load {
1
} else {
self.align_of(tp_ty).bytes() as u32
result.layout.align.abi.bytes() as u32
};
unsafe {
llvm::LLVMSetAlignment(load, align);
Expand Down Expand Up @@ -629,14 +620,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
};

if !fn_abi.ret.is_ignore() {
if let PassMode::Cast { .. } = &fn_abi.ret.mode {
self.store(llval, result.val.llval, result.val.align);
} else {
OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
.val
.store(self, result);
}
if result.layout.ty.is_bool() {
OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
.val
.store(self, result);
} else if !result.layout.ty.is_unit() {
self.store_to_place(llval, result.val);
}
Ok(())
}
Expand Down Expand Up @@ -688,20 +677,19 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
}

fn catch_unwind_intrinsic<'ll>(
bx: &mut Builder<'_, 'll, '_>,
fn catch_unwind_intrinsic<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
try_func: &'ll Value,
data: &'ll Value,
catch_func: &'ll Value,
dest: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
if bx.sess().panic_strategy() == PanicStrategy::Abort {
let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
bx.call(try_func_ty, None, None, try_func, &[data], None, None);
// Return 0 unconditionally from the intrinsic call;
// we can never unwind.
let ret_align = bx.tcx().data_layout.i32_align.abi;
bx.store(bx.const_i32(0), dest, ret_align);
OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
} else if wants_msvc_seh(bx.sess()) {
codegen_msvc_try(bx, try_func, data, catch_func, dest);
} else if wants_wasm_eh(bx.sess()) {
Expand All @@ -720,12 +708,12 @@ fn catch_unwind_intrinsic<'ll>(
// instructions are meant to work for all targets, as of the time of this
// writing, however, LLVM does not recommend the usage of these new instructions
// as the old ones are still more optimized.
fn codegen_msvc_try<'ll>(
bx: &mut Builder<'_, 'll, '_>,
fn codegen_msvc_try<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
try_func: &'ll Value,
data: &'ll Value,
catch_func: &'ll Value,
dest: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
bx.set_personality_fn(bx.eh_personality());
Expand Down Expand Up @@ -865,17 +853,16 @@ fn codegen_msvc_try<'ll>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
OperandValue::Immediate(ret).store(bx, dest);
}

// WASM's definition of the `rust_try` function.
fn codegen_wasm_try<'ll>(
bx: &mut Builder<'_, 'll, '_>,
fn codegen_wasm_try<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
try_func: &'ll Value,
data: &'ll Value,
catch_func: &'ll Value,
dest: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
bx.set_personality_fn(bx.eh_personality());
Expand Down Expand Up @@ -939,8 +926,7 @@ fn codegen_wasm_try<'ll>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
OperandValue::Immediate(ret).store(bx, dest);
}

// Definition of the standard `try` function for Rust using the GNU-like model
Expand All @@ -954,12 +940,12 @@ fn codegen_wasm_try<'ll>(
// function calling it, and that function may already have other personality
// functions in play. By calling a shim we're guaranteed that our shim will have
// the right personality function.
fn codegen_gnu_try<'ll>(
bx: &mut Builder<'_, 'll, '_>,
fn codegen_gnu_try<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
try_func: &'ll Value,
data: &'ll Value,
catch_func: &'ll Value,
dest: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
// Codegens the shims described above:
Expand Down Expand Up @@ -1006,19 +992,18 @@ fn codegen_gnu_try<'ll>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
OperandValue::Immediate(ret).store(bx, dest);
}

// Variant of codegen_gnu_try used for emscripten where Rust panics are
// implemented using C++ exceptions. Here we use exceptions of a specific type
// (`struct rust_panic`) to represent Rust panics.
fn codegen_emcc_try<'ll>(
bx: &mut Builder<'_, 'll, '_>,
fn codegen_emcc_try<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
try_func: &'ll Value,
data: &'ll Value,
catch_func: &'ll Value,
dest: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
// Codegens the shims described above:
Expand Down Expand Up @@ -1089,8 +1074,7 @@ fn codegen_emcc_try<'ll>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
OperandValue::Immediate(ret).store(bx, dest);
}

// Helper function to give a Block to a closure to codegen a shim function.
Expand Down
Loading
Loading