Skip to content

Commit 740338c

Browse files
saethlinsaethlin
committed
Add "algebraic" versions of the fast-math intrinsics
1 parent b656f51 commit 740338c

File tree

11 files changed

+193
-15
lines changed

11 files changed

+193
-15
lines changed

‎compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,17 +1156,26 @@ fn codegen_regular_intrinsic_call<'tcx>(
11561156
ret.write_cvalue(fx, ret_val);
11571157
}
11581158

1159-
sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
1159+
sym::fadd_fast
1160+
| sym::fsub_fast
1161+
| sym::fmul_fast
1162+
| sym::fdiv_fast
1163+
| sym::frem_fast
1164+
| sym::fadd_algebraic
1165+
| sym::fsub_algebraic
1166+
| sym::fmul_algebraic
1167+
| sym::fdiv_algebraic
1168+
| sym::frem_algebraic => {
11601169
intrinsic_args!(fx, args => (x, y); intrinsic);
11611170

11621171
let res = crate::num::codegen_float_binop(
11631172
fx,
11641173
match intrinsic {
1165-
sym::fadd_fast => BinOp::Add,
1166-
sym::fsub_fast => BinOp::Sub,
1167-
sym::fmul_fast => BinOp::Mul,
1168-
sym::fdiv_fast => BinOp::Div,
1169-
sym::frem_fast => BinOp::Rem,
1174+
sym::fadd_fast | sym::fadd_algebraic => BinOp::Add,
1175+
sym::fsub_fast | sym::fsub_algebraic => BinOp::Sub,
1176+
sym::fmul_fast | sym::fmul_algebraic => BinOp::Mul,
1177+
sym::fdiv_fast | sym::fdiv_algebraic => BinOp::Div,
1178+
sym::frem_fast | sym::frem_algebraic => BinOp::Rem,
11701179
_ => unreachable!(),
11711180
},
11721181
x,

‎compiler/rustc_codegen_gcc/src/builder.rs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -705,6 +705,31 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
705705
self.frem(lhs, rhs)
706706
}
707707

708+
fn fadd_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
709+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
710+
lhs + rhs
711+
}
712+
713+
fn fsub_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
714+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
715+
lhs - rhs
716+
}
717+
718+
fn fmul_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
719+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
720+
lhs * rhs
721+
}
722+
723+
fn fdiv_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
724+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
725+
lhs / rhs
726+
}
727+
728+
fn frem_algebraic(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
729+
// NOTE: it seems like we cannot enable fast-mode for a single operation in GCC.
730+
self.frem(lhs, rhs)
731+
}
732+
708733
fn checked_binop(&mut self, oop: OverflowOp, typ: Ty<'_>, lhs: Self::Value, rhs: Self::Value) -> (Self::Value, Self::Value) {
709734
self.gcc_checked_binop(oop, typ, lhs, rhs)
710735
}

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
340340
}
341341
}
342342

343+
fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
344+
unsafe {
345+
let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
346+
llvm::LLVMRustSetAlgebraicMath(instr);
347+
instr
348+
}
349+
}
350+
351+
fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
352+
unsafe {
353+
let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
354+
llvm::LLVMRustSetAlgebraicMath(instr);
355+
instr
356+
}
357+
}
358+
359+
fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
360+
unsafe {
361+
let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
362+
llvm::LLVMRustSetAlgebraicMath(instr);
363+
instr
364+
}
365+
}
366+
367+
fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
368+
unsafe {
369+
let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
370+
llvm::LLVMRustSetAlgebraicMath(instr);
371+
instr
372+
}
373+
}
374+
375+
fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
376+
unsafe {
377+
let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
378+
llvm::LLVMRustSetAlgebraicMath(instr);
379+
instr
380+
}
381+
}
382+
343383
fn checked_binop(
344384
&mut self,
345385
oop: OverflowOp,
@@ -1327,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
13271367
pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13281368
unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
13291369
}
1330-
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1370+
pub fn vector_reduce_fadd_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13311371
unsafe {
13321372
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1333-
llvm::LLVMRustSetFastMath(instr);
1373+
llvm::LLVMRustSetAlgebraicMath(instr);
13341374
instr
13351375
}
13361376
}
1337-
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1377+
pub fn vector_reduce_fmul_algebraic(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
13381378
unsafe {
13391379
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1340-
llvm::LLVMRustSetFastMath(instr);
1380+
llvm::LLVMRustSetAlgebraicMath(instr);
13411381
instr
13421382
}
13431383
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1875,14 +1875,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18751875
arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
18761876
arith_red!(
18771877
simd_reduce_add_unordered: vector_reduce_add,
1878-
vector_reduce_fadd_fast,
1878+
vector_reduce_fadd_algebraic,
18791879
false,
18801880
add,
18811881
0.0
18821882
);
18831883
arith_red!(
18841884
simd_reduce_mul_unordered: vector_reduce_mul,
1885-
vector_reduce_fmul_fast,
1885+
vector_reduce_fmul_algebraic,
18861886
false,
18871887
mul,
18881888
1.0

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1618,6 +1618,7 @@ extern "C" {
16181618
) -> &'a Value;
16191619

16201620
pub fn LLVMRustSetFastMath(Instr: &Value);
1621+
pub fn LLVMRustSetAlgebraicMath(Instr: &Value);
16211622

16221623
// Miscellaneous instructions
16231624
pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,38 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
249249
}
250250
}
251251
}
252+
sym::fadd_algebraic
253+
| sym::fsub_algebraic
254+
| sym::fmul_algebraic
255+
| sym::fdiv_algebraic
256+
| sym::frem_algebraic => match float_type_width(arg_tys[0]) {
257+
Some(_width) => match name {
258+
sym::fadd_algebraic => {
259+
bx.fadd_algebraic(args[0].immediate(), args[1].immediate())
260+
}
261+
sym::fsub_algebraic => {
262+
bx.fsub_algebraic(args[0].immediate(), args[1].immediate())
263+
}
264+
sym::fmul_algebraic => {
265+
bx.fmul_algebraic(args[0].immediate(), args[1].immediate())
266+
}
267+
sym::fdiv_algebraic => {
268+
bx.fdiv_algebraic(args[0].immediate(), args[1].immediate())
269+
}
270+
sym::frem_algebraic => {
271+
bx.frem_algebraic(args[0].immediate(), args[1].immediate())
272+
}
273+
_ => bug!(),
274+
},
275+
None => {
276+
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
277+
span,
278+
name,
279+
ty: arg_tys[0],
280+
});
281+
return;
282+
}
283+
},
252284

253285
sym::float_to_int_unchecked => {
254286
if float_type_width(arg_tys[0]).is_none() {

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,22 +86,27 @@ pub trait BuilderMethods<'a, 'tcx>:
8686
fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8787
fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8888
fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
89+
fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
8990
fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9091
fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9192
fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
93+
fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9294
fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9395
fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9496
fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
97+
fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9598
fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
9699
fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
97100
fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
98101
fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
99102
fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
100103
fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
104+
fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
101105
fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
102106
fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
103107
fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
104108
fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
109+
fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
105110
fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
106111
fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
107112
fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,12 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
113113
| sym::black_box
114114
| sym::variant_count
115115
| sym::ptr_mask
116-
| sym::debug_assertions => hir::Unsafety::Normal,
116+
| sym::debug_assertions
117+
| sym::fadd_algebraic
118+
| sym::fsub_algebraic
119+
| sym::fmul_algebraic
120+
| sym::fdiv_algebraic
121+
| sym::frem_algebraic => hir::Unsafety::Normal,
117122
_ => hir::Unsafety::Unsafe,
118123
};
119124

@@ -381,6 +386,11 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
381386
sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
382387
(1, vec![param(0), param(0)], param(0))
383388
}
389+
sym::fadd_algebraic
390+
| sym::fsub_algebraic
391+
| sym::fmul_algebraic
392+
| sym::fdiv_algebraic
393+
| sym::frem_algebraic => (1, vec![param(0), param(0)], param(0)),
384394
sym::float_to_int_unchecked => (2, vec![param(0)], param(1)),
385395

386396
sym::assume => (0, vec![tcx.types.bool], Ty::new_unit(tcx)),

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,8 +417,7 @@ extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
417417
report_fatal_error("bad MemoryEffects.");
418418
}
419419
}
420-
421-
// Enable a fast-math flag
420+
// Enable all fast-math flags
422421
//
423422
// https://llvm.org/docs/LangRef.html#fast-math-flags
424423
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
@@ -427,6 +426,18 @@ extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
427426
}
428427
}
429428

429+
// Enable fast-math flags which do not let operations return poison.
430+
//
431+
// https://llvm.org/docs/LangRef.html#fast-math-flags
432+
extern "C" void LLVMRustSetAlgebraicMath(LLVMValueRef V) {
433+
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
434+
I->setHasAllowReassoc(true);
435+
I->setHasAllowContract(true);
436+
I->setHasAllowReciprocal(true);
437+
I->setHasNoSignedZeros(true);
438+
}
439+
}
440+
430441
extern "C" LLVMValueRef
431442
LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source,
432443
const char *Name, LLVMAtomicOrdering Order) {

‎compiler/rustc_span/src/symbol.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,8 +764,10 @@ symbols! {
764764
f64_nan,
765765
fabsf32,
766766
fabsf64,
767+
fadd_algebraic,
767768
fadd_fast,
768769
fake_variadic,
770+
fdiv_algebraic,
769771
fdiv_fast,
770772
feature,
771773
fence,
@@ -785,6 +787,7 @@ symbols! {
785787
fmaf32,
786788
fmaf64,
787789
fmt,
790+
fmul_algebraic,
788791
fmul_fast,
789792
fn_align,
790793
fn_delegation,
@@ -810,6 +813,7 @@ symbols! {
810813
format_unsafe_arg,
811814
freeze,
812815
freg,
816+
frem_algebraic,
813817
frem_fast,
814818
from,
815819
from_desugaring,
@@ -823,6 +827,7 @@ symbols! {
823827
from_usize,
824828
from_yeet,
825829
fs_create_dir,
830+
fsub_algebraic,
826831
fsub_fast,
827832
fundamental,
828833
future,

‎library/core/src/intrinsics.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1882,6 +1882,46 @@ extern "rust-intrinsic" {
18821882
#[rustc_nounwind]
18831883
pub fn frem_fast<T: Copy>(a: T, b: T) -> T;
18841884

1885+
/// Float addition that allows optimizations based on algebraic rules.
1886+
///
1887+
/// This intrinsic does not have a stable counterpart.
1888+
#[rustc_nounwind]
1889+
#[rustc_safe_intrinsic]
1890+
#[cfg(not(bootstrap))]
1891+
pub fn fadd_algebraic<T: Copy>(a: T, b: T) -> T;
1892+
1893+
/// Float subtraction that allows optimizations based on algebraic rules.
1894+
///
1895+
/// This intrinsic does not have a stable counterpart.
1896+
#[rustc_nounwind]
1897+
#[rustc_safe_intrinsic]
1898+
#[cfg(not(bootstrap))]
1899+
pub fn fsub_algebraic<T: Copy>(a: T, b: T) -> T;
1900+
1901+
/// Float multiplication that allows optimizations based on algebraic rules.
1902+
///
1903+
/// This intrinsic does not have a stable counterpart.
1904+
#[rustc_nounwind]
1905+
#[rustc_safe_intrinsic]
1906+
#[cfg(not(bootstrap))]
1907+
pub fn fmul_algebraic<T: Copy>(a: T, b: T) -> T;
1908+
1909+
/// Float division that allows optimizations based on algebraic rules.
1910+
///
1911+
/// This intrinsic does not have a stable counterpart.
1912+
#[rustc_nounwind]
1913+
#[rustc_safe_intrinsic]
1914+
#[cfg(not(bootstrap))]
1915+
pub fn fdiv_algebraic<T: Copy>(a: T, b: T) -> T;
1916+
1917+
/// Float remainder that allows optimizations based on algebraic rules.
1918+
///
1919+
/// This intrinsic does not have a stable counterpart.
1920+
#[rustc_nounwind]
1921+
#[rustc_safe_intrinsic]
1922+
#[cfg(not(bootstrap))]
1923+
pub fn frem_algebraic<T: Copy>(a: T, b: T) -> T;
1924+
18851925
/// Convert with LLVM’s fptoui/fptosi, which may return undef for values out of range
18861926
/// (<https://github.com/rust-lang/rust/issues/10184>)
18871927
///

0 commit comments

Comments
 (0)