Skip to content

Commit 346ae2a

Browse files
antoyoantoyo
authored
Merge pull request rust-lang#183 from sadlerap/simd-reduction-intrinsics
simd: Implement missing reduction intrinsics
2 parents b4626b3 + 4df874f commit 346ae2a

File tree

5 files changed

+105
-17
lines changed

5 files changed

+105
-17
lines changed

‎failing-ui-tests.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ src/test/ui/simd/intrinsic/generic-as.rs
4040
src/test/ui/simd/intrinsic/generic-bitmask-pass.rs
4141
src/test/ui/simd/intrinsic/generic-comparison-pass.rs
4242
src/test/ui/simd/intrinsic/generic-gather-pass.rs
43-
src/test/ui/simd/intrinsic/generic-reduction-pass.rs
4443
src/test/ui/simd/intrinsic/generic-select-pass.rs
4544
src/test/ui/simd/issue-17170.rs
4645
src/test/ui/simd/issue-39720.rs

‎failing-ui-tests12.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ src/test/ui/simd/intrinsic/generic-arithmetic-saturating-pass.rs
1111
src/test/ui/simd/intrinsic/generic-cast-pass.rs
1212
src/test/ui/simd/intrinsic/generic-cast-pointer-width.rs
1313
src/test/ui/simd/intrinsic/generic-elements-pass.rs
14+
src/test/ui/simd/intrinsic/generic-reduction-pass.rs
1415
src/test/ui/simd/intrinsic/inlining-issue67557-ice.rs
1516
src/test/ui/simd/intrinsic/inlining-issue67557.rs
1617
src/test/ui/simd/monomorphize-shuffle-index.rs

‎src/builder.rs

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,26 +1460,99 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
14601460
unimplemented!();
14611461
}
14621462

1463+
#[cfg(feature="master")]
1464+
pub fn vector_reduce_fadd(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
1465+
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1466+
let element_count = vector_type.get_num_units();
1467+
(0..element_count).into_iter()
1468+
.map(|i| self.context
1469+
.new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
1470+
.to_rvalue())
1471+
.fold(acc, |x, i| x + i)
1472+
}
1473+
1474+
#[cfg(not(feature="master"))]
1475+
pub fn vector_reduce_fadd(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1476+
unimplemented!();
1477+
}
1478+
14631479
pub fn vector_reduce_fmul_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
14641480
unimplemented!();
14651481
}
14661482

1483+
#[cfg(feature="master")]
1484+
pub fn vector_reduce_fmul(&mut self, acc: RValue<'gcc>, src: RValue<'gcc>) -> RValue<'gcc> {
1485+
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1486+
let element_count = vector_type.get_num_units();
1487+
(0..element_count).into_iter()
1488+
.map(|i| self.context
1489+
.new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
1490+
.to_rvalue())
1491+
.fold(acc, |x, i| x * i)
1492+
}
1493+
1494+
#[cfg(not(feature="master"))]
1495+
pub fn vector_reduce_fmul(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1496+
unimplemented!()
1497+
}
1498+
14671499
// Inspired by Hacker's Delight min implementation.
14681500
pub fn vector_reduce_min(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
14691501
self.vector_reduce(src, |a, b, context| {
14701502
let differences_or_zeros = difference_or_zero(a, b, context);
1471-
context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
1503+
context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
14721504
})
14731505
}
14741506

14751507
// Inspired by Hacker's Delight max implementation.
14761508
pub fn vector_reduce_max(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
14771509
self.vector_reduce(src, |a, b, context| {
14781510
let differences_or_zeros = difference_or_zero(a, b, context);
1479-
context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
1511+
context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
14801512
})
14811513
}
14821514

1515+
#[cfg(feature="master")]
1516+
pub fn vector_reduce_fmin(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1517+
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1518+
let element_count = vector_type.get_num_units();
1519+
let mut acc = self.context.new_vector_access(None, src, self.context.new_rvalue_zero(self.int_type)).to_rvalue();
1520+
for i in 1..element_count {
1521+
let elem = self.context
1522+
.new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
1523+
.to_rvalue();
1524+
let cmp = self.context.new_comparison(None, ComparisonOp::LessThan, acc, elem);
1525+
acc = self.select(cmp, acc, elem);
1526+
}
1527+
acc
1528+
}
1529+
1530+
#[cfg(not(feature="master"))]
1531+
pub fn vector_reduce_fmin(&mut self, _src: RValue<'gcc>) -> RValue<'gcc> {
1532+
unimplemented!();
1533+
}
1534+
1535+
#[cfg(feature="master")]
1536+
pub fn vector_reduce_fmax(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1537+
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1538+
let element_count = vector_type.get_num_units();
1539+
let mut acc = self.context.new_vector_access(None, src, self.context.new_rvalue_zero(self.int_type)).to_rvalue();
1540+
for i in 1..element_count {
1541+
let elem = self.context
1542+
.new_vector_access(None, src, self.context.new_rvalue_from_int(self.int_type, i as _))
1543+
.to_rvalue();
1544+
let cmp = self.context.new_comparison(None, ComparisonOp::GreaterThan, acc, elem);
1545+
acc = self.select(cmp, acc, elem);
1546+
}
1547+
acc
1548+
}
1549+
1550+
#[cfg(not(feature="master"))]
1551+
pub fn vector_reduce_fmax(&mut self, _src: RValue<'gcc>) -> RValue<'gcc> {
1552+
unimplemented!();
1553+
}
1554+
1555+
14831556
pub fn vector_select(&mut self, cond: RValue<'gcc>, then_val: RValue<'gcc>, else_val: RValue<'gcc>) -> RValue<'gcc> {
14841557
// cond is a vector of integers, not of bools.
14851558
let cond_type = cond.get_type();

‎src/intrinsic/simd.rs

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
use gccjit::{ToRValue, ComparisonOp, UnaryOp};
33
use gccjit::{BinaryOp, RValue, Type};
44
use rustc_codegen_ssa::base::compare_simd_types;
5-
use rustc_codegen_ssa::common::{TypeKind, span_invalid_monomorphization_error};
5+
use rustc_codegen_ssa::common::{IntPredicate, TypeKind, span_invalid_monomorphization_error};
66
use rustc_codegen_ssa::mir::operand::OperandRef;
77
use rustc_codegen_ssa::mir::place::PlaceRef;
88
use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
@@ -667,9 +667,24 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
667667
mul,
668668
1.0
669669
);
670+
arith_red!(
671+
simd_reduce_add_ordered: BinaryOp::Plus,
672+
vector_reduce_fadd,
673+
true,
674+
add,
675+
0.0
676+
);
677+
arith_red!(
678+
simd_reduce_mul_ordered: BinaryOp::Mult,
679+
vector_reduce_fmul,
680+
true,
681+
mul,
682+
1.0
683+
);
684+
670685

671686
macro_rules! minmax_red {
672-
($name:ident: $reduction:ident) => {
687+
($name:ident: $int_red:ident, $float_red:ident) => {
673688
if name == sym::$name {
674689
require!(
675690
ret_ty == in_elem,
@@ -679,7 +694,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
679694
ret_ty
680695
);
681696
return match in_elem.kind() {
682-
ty::Int(_) | ty::Uint(_) | ty::Float(_) => Ok(bx.$reduction(args[0].immediate())),
697+
ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())),
698+
ty::Float(_) => Ok(bx.$float_red(args[0].immediate())),
683699
_ => return_error!(
684700
"unsupported {} from `{}` with element `{}` to `{}`",
685701
sym::$name,
@@ -692,8 +708,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
692708
};
693709
}
694710

695-
minmax_red!(simd_reduce_min: vector_reduce_min);
696-
minmax_red!(simd_reduce_max: vector_reduce_max);
711+
minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
712+
minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
713+
// TODO(sadlerap): revisit these intrinsics to generate more optimal reductions
714+
minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin);
715+
minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax);
697716

698717
macro_rules! bitwise_red {
699718
($name:ident : $op:expr, $boolean:expr) => {
@@ -719,15 +738,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
719738
),
720739
}
721740

722-
// boolean reductions operate on vectors of i1s:
723-
let i1 = bx.type_i1();
724-
let i1xn = bx.type_vector(i1, in_len as u64);
725-
bx.trunc(args[0].immediate(), i1xn)
741+
args[0].immediate()
726742
};
727743
return match in_elem.kind() {
728744
ty::Int(_) | ty::Uint(_) => {
729745
let r = bx.vector_reduce_op(input, $op);
730-
Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
746+
Ok(if !$boolean { r } else { bx.icmp(IntPredicate::IntNE, r, bx.context.new_rvalue_zero(r.get_type())) })
731747
}
732748
_ => return_error!(
733749
"unsupported {} from `{}` with element `{}` to `{}`",
@@ -743,6 +759,9 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
743759

744760
bitwise_red!(simd_reduce_and: BinaryOp::BitwiseAnd, false);
745761
bitwise_red!(simd_reduce_or: BinaryOp::BitwiseOr, false);
762+
bitwise_red!(simd_reduce_xor: BinaryOp::BitwiseXor, false);
763+
bitwise_red!(simd_reduce_all: BinaryOp::BitwiseAnd, true);
764+
bitwise_red!(simd_reduce_any: BinaryOp::BitwiseOr, true);
746765

747766
unimplemented!("simd {}", name);
748767
}

‎src/type_.rs

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,10 +247,6 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
247247

248248
self.context.new_array_type(None, ty, len)
249249
}
250-
251-
pub fn type_bool(&self) -> Type<'gcc> {
252-
self.context.new_type::<bool>()
253-
}
254250
}
255251

256252
pub fn struct_fields<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout<'tcx>) -> (Vec<Type<'gcc>>, bool) {

0 commit comments

Comments
 (0)