Skip to content

Commit 11571a0

Browse files
beetreesbeetrees
authored
Fix legalizing FNEG and FABS with TypeSoftPromoteHalf (#156343)
Based on top of #157211. `FNEG` and `FABS` must preserve signalling NaNs, meaning they should not convert to f32 to perform the operation. Instead legalize to `XOR` and `AND`. Fixes almost all of #104915
1 parent c18de24 commit 11571a0

File tree

12 files changed

+565
-467
lines changed

12 files changed

+565
-467
lines changed

‎llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3313,7 +3313,6 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
33133313
case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
33143314

33153315
// Unary FP Operations
3316-
case ISD::FABS:
33173316
case ISD::FACOS:
33183317
case ISD::FASIN:
33193318
case ISD::FATAN:
@@ -3329,7 +3328,6 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
33293328
case ISD::FLOG2:
33303329
case ISD::FLOG10:
33313330
case ISD::FNEARBYINT:
3332-
case ISD::FNEG:
33333331
case ISD::FREEZE:
33343332
case ISD::FRINT:
33353333
case ISD::FROUND:
@@ -3341,6 +3339,12 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
33413339
case ISD::FTAN:
33423340
case ISD::FTANH:
33433341
case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3342+
case ISD::FABS:
3343+
R = SoftPromoteHalfRes_FABS(N);
3344+
break;
3345+
case ISD::FNEG:
3346+
R = SoftPromoteHalfRes_FNEG(N);
3347+
break;
33443348
case ISD::AssertNoFPClass:
33453349
R = SoftPromoteHalfRes_AssertNoFPClass(N);
33463350
break;
@@ -3670,6 +3674,24 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
36703674
return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
36713675
}
36723676

3677+
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
3678+
SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3679+
SDLoc dl(N);
3680+
3681+
// Clear the sign bit.
3682+
return DAG.getNode(ISD::AND, dl, MVT::i16, Op,
3683+
DAG.getConstant(0x7fff, dl, MVT::i16));
3684+
}
3685+
3686+
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
3687+
SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3688+
SDLoc dl(N);
3689+
3690+
// Invert the sign bit.
3691+
return DAG.getNode(ISD::XOR, dl, MVT::i16, Op,
3692+
DAG.getConstant(0x8000, dl, MVT::i16));
3693+
}
3694+
36733695
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
36743696
return GetSoftPromotedHalf(N->getOperand(0));
36753697
}

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
832832
SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
833833
SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
834834
SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
835+
SDValue SoftPromoteHalfRes_FABS(SDNode *N);
836+
SDValue SoftPromoteHalfRes_FNEG(SDNode *N);
835837
SDValue SoftPromoteHalfRes_AssertNoFPClass(SDNode *N);
836838
SDValue SoftPromoteHalfRes_XINT_TO_FP(SDNode *N);
837839
SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);

‎llvm/test/CodeGen/AMDGPU/bf16.ll‎

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -21204,18 +21204,14 @@ define bfloat @v_fabs_bf16(bfloat %a) {
2120421204
; GCN: ; %bb.0:
2120521205
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2120621206
; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
21207-
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21208-
; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
21209-
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21207+
; GCN-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
2121021208
; GCN-NEXT: s_setpc_b64 s[30:31]
2121121209
;
2121221210
; GFX7-LABEL: v_fabs_bf16:
2121321211
; GFX7: ; %bb.0:
2121421212
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2121521213
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
21216-
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21217-
; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
21218-
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21214+
; GFX7-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
2121921215
; GFX7-NEXT: s_setpc_b64 s[30:31]
2122021216
;
2122121217
; GFX8-LABEL: v_fabs_bf16:
@@ -21440,21 +21436,15 @@ define bfloat @v_fneg_fabs_bf16(bfloat %a) {
2144021436
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2144121437
; GCN-NEXT: v_mul_f32_e32 v0, 1.0, v0
2144221438
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21443-
; GCN-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
21444-
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21445-
; GCN-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
21446-
; GCN-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21439+
; GCN-NEXT: v_or_b32_e32 v0, 0x80000000, v0
2144721440
; GCN-NEXT: s_setpc_b64 s[30:31]
2144821441
;
2144921442
; GFX7-LABEL: v_fneg_fabs_bf16:
2145021443
; GFX7: ; %bb.0:
2145121444
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2145221445
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
2145321446
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21454-
; GFX7-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
21455-
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21456-
; GFX7-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
21457-
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
21447+
; GFX7-NEXT: v_or_b32_e32 v0, 0x80000000, v0
2145821448
; GFX7-NEXT: s_setpc_b64 s[30:31]
2145921449
;
2146021450
; GFX8-LABEL: v_fneg_fabs_bf16:
@@ -21510,23 +21500,17 @@ define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) {
2151021500
; GCN-LABEL: s_fneg_fabs_bf16:
2151121501
; GCN: ; %bb.0:
2151221502
; GCN-NEXT: v_mul_f32_e64 v0, 1.0, s0
21503+
; GCN-NEXT: v_lshrrev_b32_e32 v0, 16, v0
21504+
; GCN-NEXT: v_or_b32_e32 v0, 0x8000, v0
2151321505
; GCN-NEXT: v_readfirstlane_b32 s0, v0
21514-
; GCN-NEXT: s_and_b32 s0, s0, 0xffff0000
21515-
; GCN-NEXT: s_bitset0_b32 s0, 31
21516-
; GCN-NEXT: s_and_b32 s0, s0, 0xffff0000
21517-
; GCN-NEXT: s_xor_b32 s0, s0, 0x80000000
21518-
; GCN-NEXT: s_lshr_b32 s0, s0, 16
2151921506
; GCN-NEXT: ; return to shader part epilog
2152021507
;
2152121508
; GFX7-LABEL: s_fneg_fabs_bf16:
2152221509
; GFX7: ; %bb.0:
2152321510
; GFX7-NEXT: v_mul_f32_e64 v0, 1.0, s0
21511+
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
21512+
; GFX7-NEXT: v_or_b32_e32 v0, 0x8000, v0
2152421513
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
21525-
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff0000
21526-
; GFX7-NEXT: s_bitset0_b32 s0, 31
21527-
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff0000
21528-
; GFX7-NEXT: s_xor_b32 s0, s0, 0x80000000
21529-
; GFX7-NEXT: s_lshr_b32 s0, s0, 16
2153021514
; GFX7-NEXT: ; return to shader part epilog
2153121515
;
2153221516
; GFX8-LABEL: s_fneg_fabs_bf16:

‎llvm/test/CodeGen/AMDGPU/fabs.bf16.ll‎

Lines changed: 18 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -218,19 +218,11 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
218218
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
219219
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
220220
; CI-NEXT: s_waitcnt lgkmcnt(0)
221-
; CI-NEXT: s_and_b32 s4, s3, 0xffff0000
222-
; CI-NEXT: s_lshl_b32 s3, s3, 16
223-
; CI-NEXT: s_and_b32 s5, s2, 0xffff0000
224-
; CI-NEXT: v_mul_f32_e64 v0, 1.0, |s4|
225-
; CI-NEXT: v_mul_f32_e64 v1, 1.0, |s3|
226-
; CI-NEXT: v_mul_f32_e64 v2, 1.0, |s5|
227-
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
228-
; CI-NEXT: s_lshl_b32 s2, s2, 16
229-
; CI-NEXT: v_alignbit_b32 v1, v0, v1, 16
230-
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v2
231-
; CI-NEXT: v_mul_f32_e64 v2, 1.0, |s2|
232-
; CI-NEXT: v_alignbit_b32 v0, v0, v2, 16
221+
; CI-NEXT: s_and_b32 s3, s3, 0x7fff7fff
222+
; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
233223
; CI-NEXT: v_mov_b32_e32 v3, s1
224+
; CI-NEXT: v_mov_b32_e32 v0, s2
225+
; CI-NEXT: v_mov_b32_e32 v1, s3
234226
; CI-NEXT: v_mov_b32_e32 v2, s0
235227
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
236228
; CI-NEXT: s_endpgm
@@ -537,16 +529,15 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad
537529
; CI-NEXT: v_mov_b32_e32 v0, s0
538530
; CI-NEXT: v_mov_b32_e32 v1, s1
539531
; CI-NEXT: s_waitcnt vmcnt(0)
540-
; CI-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
541-
; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
542-
; CI-NEXT: v_mul_f32_e64 v4, 1.0, |v3|
543-
; CI-NEXT: v_mul_f32_e64 v5, 1.0, |v2|
544-
; CI-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
545-
; CI-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
546-
; CI-NEXT: v_mul_f32_e32 v3, v4, v3
547-
; CI-NEXT: v_mul_f32_e32 v2, v5, v2
548-
; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v3
549-
; CI-NEXT: v_alignbit_b32 v2, v3, v2, 16
532+
; CI-NEXT: v_and_b32_e32 v3, 0x7fff, v2
533+
; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v2
534+
; CI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
535+
; CI-NEXT: v_and_b32_e32 v2, 0x7fff0000, v2
536+
; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
537+
; CI-NEXT: v_mul_f32_e32 v2, v2, v5
538+
; CI-NEXT: v_mul_f32_e32 v3, v3, v4
539+
; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
540+
; CI-NEXT: v_alignbit_b32 v2, v2, v3, 16
550541
; CI-NEXT: flat_store_dword v[0:1], v2
551542
; CI-NEXT: s_endpgm
552543
;
@@ -898,16 +889,13 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {
898889
; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
899890
; CI-NEXT: flat_load_dword v0, v[0:1]
900891
; CI-NEXT: s_waitcnt vmcnt(0)
901-
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v0
902-
; CI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
903-
; CI-NEXT: v_mul_f32_e64 v1, 1.0, |v1|
904-
; CI-NEXT: v_mul_f32_e64 v0, 1.0, |v0|
905-
; CI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
906-
; CI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
907-
; CI-NEXT: v_mul_f32_e32 v1, 4.0, v1
892+
; CI-NEXT: v_and_b32_e32 v1, 0x7fff, v0
893+
; CI-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
894+
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
908895
; CI-NEXT: v_add_f32_e32 v0, 2.0, v0
909-
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
896+
; CI-NEXT: v_mul_f32_e32 v1, 4.0, v1
910897
; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
898+
; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
911899
; CI-NEXT: flat_store_short v[0:1], v1
912900
; CI-NEXT: s_waitcnt vmcnt(0)
913901
; CI-NEXT: flat_store_short v[0:1], v0

0 commit comments

Comments
 (0)