Skip to content

Commit ca4a329

Browse files
nhaehnlenhaehnle
committed
AMDGPU: Generate VALU ThreeOp Integer instructions
Summary: Original patch by: Fabian Wahlster <razor@singul4rity.com> Change-Id: I148f692a88432541fad468963f58da9ddf79fac5 Reviewers: arsenm, rampitec Subscribers: kzhuravl, jvesely, wdng, yaxunl, dstuttard, tpr, t-tye, b-sumner, llvm-commits Differential Revision: https://reviews.llvm.org/D51995 llvm-svn: 348488
1 parent f479fbb commit ca4a329

File tree

9 files changed

+873
-2
lines changed

9 files changed

+873
-2
lines changed

‎llvm/lib/Target/AMDGPU/VOP3Instructions.td

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,37 @@ defm: Ternary_i16_Pats<mul, add, V_MAD_I16, sext>;
491491

492492
} // End Predicates = [Has16BitInsts]
493493

494+
class ThreeOpFrag<SDPatternOperator op1, SDPatternOperator op2> : PatFrag<
495+
(ops node:$x, node:$y, node:$z),
496+
// When the inner operation is used multiple times, selecting 3-op
497+
// instructions may still be beneficial -- if the other users can be
498+
// combined similarly. Let's be conservative for now.
499+
(op2 (HasOneUseBinOp<op1> node:$x, node:$y), node:$z),
500+
[{
501+
// Only use VALU ops when the result is divergent.
502+
if (!N->isDivergent())
503+
return false;
504+
505+
// Check constant bus limitations.
506+
//
507+
// Note: Use !isDivergent as a conservative proxy for whether the value
508+
// is in an SGPR (uniform values can end up in VGPRs as well).
509+
unsigned ConstantBusUses = 0;
510+
for (unsigned i = 0; i < 3; ++i) {
511+
if (!Operands[i]->isDivergent() &&
512+
!isInlineImmediate(Operands[i].getNode())) {
513+
ConstantBusUses++;
514+
if (ConstantBusUses >= 2)
515+
return false;
516+
}
517+
}
518+
519+
return true;
520+
}]
521+
> {
522+
let PredicateCodeUsesOperands = 1;
523+
}
524+
494525
let SubtargetPredicate = isGFX9 in {
495526
def V_PACK_B32_F16 : VOP3Inst <"v_pack_b32_f16", VOP3_Profile<VOP_B32_F16_F16, VOP3_OPSEL>>;
496527
def V_LSHL_ADD_U32 : VOP3Inst <"v_lshl_add_u32", VOP3_Profile<VOP_I32_I32_I32_I32>>;
@@ -525,6 +556,22 @@ def V_CVT_PKNORM_U16_F16 : VOP3Inst <"v_cvt_pknorm_u16_f16", VOP3_Profile<VOP_B3
525556

526557
def V_ADD_I32_gfx9 : VOP3Inst <"v_add_i32_gfx9", VOP3_Profile<VOP_I32_I32_I32>>;
527558
def V_SUB_I32_gfx9 : VOP3Inst <"v_sub_i32_gfx9", VOP3_Profile<VOP_I32_I32_I32>>;
559+
560+
561+
class ThreeOp_i32_Pats <SDPatternOperator op1, SDPatternOperator op2, Instruction inst> : GCNPat <
562+
// This matches (op2 (op1 i32:$src0, i32:$src1), i32:$src2) with conditions.
563+
(ThreeOpFrag<op1, op2> i32:$src0, i32:$src1, i32:$src2),
564+
(inst i32:$src0, i32:$src1, i32:$src2)
565+
>;
566+
567+
def : ThreeOp_i32_Pats<shl, add, V_LSHL_ADD_U32>;
568+
def : ThreeOp_i32_Pats<add, shl, V_ADD_LSHL_U32>;
569+
def : ThreeOp_i32_Pats<add, add, V_ADD3_U32>;
570+
def : ThreeOp_i32_Pats<shl, or, V_LSHL_OR_B32>;
571+
def : ThreeOp_i32_Pats<and, or, V_AND_OR_B32>;
572+
def : ThreeOp_i32_Pats<or, or, V_OR3_B32>;
573+
def : ThreeOp_i32_Pats<xor, add, V_XAD_U32>;
574+
528575
} // End SubtargetPredicate = isGFX9
529576

530577
//===----------------------------------------------------------------------===//

‎llvm/test/CodeGen/AMDGPU/add3.ll

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2+
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=fiji -verify-machineinstrs | FileCheck -check-prefix=VI %s
3+
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 -verify-machineinstrs | FileCheck -check-prefix=GFX9 %s
4+
5+
; ===================================================================================
6+
; V_ADD3_U32
7+
; ===================================================================================
8+
9+
define amdgpu_ps float @add3(i32 %a, i32 %b, i32 %c) {
10+
; VI-LABEL: add3:
11+
; VI: ; %bb.0:
12+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
13+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
14+
; VI-NEXT: ; return to shader part epilog
15+
;
16+
; GFX9-LABEL: add3:
17+
; GFX9: ; %bb.0:
18+
; GFX9-NEXT: v_add3_u32 v0, v0, v1, v2
19+
; GFX9-NEXT: ; return to shader part epilog
20+
%x = add i32 %a, %b
21+
%result = add i32 %x, %c
22+
%bc = bitcast i32 %result to float
23+
ret float %bc
24+
}
25+
26+
; ThreeOp instruction variant not used due to Constant Bus Limitations
27+
; TODO: with reassociation it is possible to replace a v_add_u32_e32 with a s_add_i32
28+
define amdgpu_ps float @add3_vgpr_b(i32 inreg %a, i32 %b, i32 inreg %c) {
29+
; VI-LABEL: add3_vgpr_b:
30+
; VI: ; %bb.0:
31+
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
32+
; VI-NEXT: v_add_u32_e32 v0, vcc, s3, v0
33+
; VI-NEXT: ; return to shader part epilog
34+
;
35+
; GFX9-LABEL: add3_vgpr_b:
36+
; GFX9: ; %bb.0:
37+
; GFX9-NEXT: v_add_u32_e32 v0, s2, v0
38+
; GFX9-NEXT: v_add_u32_e32 v0, s3, v0
39+
; GFX9-NEXT: ; return to shader part epilog
40+
%x = add i32 %a, %b
41+
%result = add i32 %x, %c
42+
%bc = bitcast i32 %result to float
43+
ret float %bc
44+
}
45+
46+
define amdgpu_ps float @add3_vgpr_all2(i32 %a, i32 %b, i32 %c) {
47+
; VI-LABEL: add3_vgpr_all2:
48+
; VI: ; %bb.0:
49+
; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v2
50+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
51+
; VI-NEXT: ; return to shader part epilog
52+
;
53+
; GFX9-LABEL: add3_vgpr_all2:
54+
; GFX9: ; %bb.0:
55+
; GFX9-NEXT: v_add3_u32 v0, v1, v2, v0
56+
; GFX9-NEXT: ; return to shader part epilog
57+
%x = add i32 %b, %c
58+
%result = add i32 %a, %x
59+
%bc = bitcast i32 %result to float
60+
ret float %bc
61+
}
62+
63+
define amdgpu_ps float @add3_vgpr_bc(i32 inreg %a, i32 %b, i32 %c) {
64+
; VI-LABEL: add3_vgpr_bc:
65+
; VI: ; %bb.0:
66+
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
67+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
68+
; VI-NEXT: ; return to shader part epilog
69+
;
70+
; GFX9-LABEL: add3_vgpr_bc:
71+
; GFX9: ; %bb.0:
72+
; GFX9-NEXT: v_add3_u32 v0, s2, v0, v1
73+
; GFX9-NEXT: ; return to shader part epilog
74+
%x = add i32 %a, %b
75+
%result = add i32 %x, %c
76+
%bc = bitcast i32 %result to float
77+
ret float %bc
78+
}
79+
80+
define amdgpu_ps float @add3_vgpr_const(i32 %a, i32 %b) {
81+
; VI-LABEL: add3_vgpr_const:
82+
; VI: ; %bb.0:
83+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
84+
; VI-NEXT: v_add_u32_e32 v0, vcc, 16, v0
85+
; VI-NEXT: ; return to shader part epilog
86+
;
87+
; GFX9-LABEL: add3_vgpr_const:
88+
; GFX9: ; %bb.0:
89+
; GFX9-NEXT: v_add3_u32 v0, v0, v1, 16
90+
; GFX9-NEXT: ; return to shader part epilog
91+
%x = add i32 %a, %b
92+
%result = add i32 %x, 16
93+
%bc = bitcast i32 %result to float
94+
ret float %bc
95+
}
96+
97+
define amdgpu_ps <2 x float> @add3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x) {
98+
; VI-LABEL: add3_multiuse_outer:
99+
; VI: ; %bb.0:
100+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
101+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
102+
; VI-NEXT: v_mul_lo_i32 v1, v0, v3
103+
; VI-NEXT: ; return to shader part epilog
104+
;
105+
; GFX9-LABEL: add3_multiuse_outer:
106+
; GFX9: ; %bb.0:
107+
; GFX9-NEXT: v_add3_u32 v0, v0, v1, v2
108+
; GFX9-NEXT: v_mul_lo_i32 v1, v0, v3
109+
; GFX9-NEXT: ; return to shader part epilog
110+
%inner = add i32 %a, %b
111+
%outer = add i32 %inner, %c
112+
%x1 = mul i32 %outer, %x
113+
%r1 = insertelement <2 x i32> undef, i32 %outer, i32 0
114+
%r0 = insertelement <2 x i32> %r1, i32 %x1, i32 1
115+
%bc = bitcast <2 x i32> %r0 to <2 x float>
116+
ret <2 x float> %bc
117+
}
118+
119+
define amdgpu_ps <2 x float> @add3_multiuse_inner(i32 %a, i32 %b, i32 %c) {
120+
; VI-LABEL: add3_multiuse_inner:
121+
; VI: ; %bb.0:
122+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
123+
; VI-NEXT: v_add_u32_e32 v1, vcc, v0, v2
124+
; VI-NEXT: ; return to shader part epilog
125+
;
126+
; GFX9-LABEL: add3_multiuse_inner:
127+
; GFX9: ; %bb.0:
128+
; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
129+
; GFX9-NEXT: v_add_u32_e32 v1, v0, v2
130+
; GFX9-NEXT: ; return to shader part epilog
131+
%inner = add i32 %a, %b
132+
%outer = add i32 %inner, %c
133+
%r1 = insertelement <2 x i32> undef, i32 %inner, i32 0
134+
%r0 = insertelement <2 x i32> %r1, i32 %outer, i32 1
135+
%bc = bitcast <2 x i32> %r0 to <2 x float>
136+
ret <2 x float> %bc
137+
}
138+
139+
; A case where uniform values end up in VGPRs -- we could use v_add3_u32 here,
140+
; but we don't.
141+
define amdgpu_ps float @add3_uniform_vgpr(float inreg %a, float inreg %b, float inreg %c) {
142+
; VI-LABEL: add3_uniform_vgpr:
143+
; VI: ; %bb.0:
144+
; VI-NEXT: v_mov_b32_e32 v2, 0x40400000
145+
; VI-NEXT: v_add_f32_e64 v0, s2, 1.0
146+
; VI-NEXT: v_add_f32_e64 v1, s3, 2.0
147+
; VI-NEXT: v_add_f32_e32 v2, s4, v2
148+
; VI-NEXT: v_add_u32_e32 v0, vcc, v1, v0
149+
; VI-NEXT: v_add_u32_e32 v0, vcc, v2, v0
150+
; VI-NEXT: ; return to shader part epilog
151+
;
152+
; GFX9-LABEL: add3_uniform_vgpr:
153+
; GFX9: ; %bb.0:
154+
; GFX9-NEXT: v_mov_b32_e32 v2, 0x40400000
155+
; GFX9-NEXT: v_add_f32_e64 v0, s2, 1.0
156+
; GFX9-NEXT: v_add_f32_e64 v1, s3, 2.0
157+
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
158+
; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
159+
; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
160+
; GFX9-NEXT: ; return to shader part epilog
161+
%a1 = fadd float %a, 1.0
162+
%b2 = fadd float %b, 2.0
163+
%c3 = fadd float %c, 3.0
164+
%bc.a = bitcast float %a1 to i32
165+
%bc.b = bitcast float %b2 to i32
166+
%bc.c = bitcast float %c3 to i32
167+
%x = add i32 %bc.a, %bc.b
168+
%result = add i32 %x, %bc.c
169+
%bc = bitcast i32 %result to float
170+
ret float %bc
171+
}

‎llvm/test/CodeGen/AMDGPU/add_shl.ll

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2+
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=fiji -verify-machineinstrs | FileCheck -check-prefix=VI %s
3+
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 -verify-machineinstrs | FileCheck -check-prefix=GFX9 %s
4+
5+
; ===================================================================================
6+
; V_ADD_LSHL_U32
7+
; ===================================================================================
8+
9+
define amdgpu_ps float @add_shl(i32 %a, i32 %b, i32 %c) {
10+
; VI-LABEL: add_shl:
11+
; VI: ; %bb.0:
12+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
13+
; VI-NEXT: v_lshlrev_b32_e32 v0, v2, v0
14+
; VI-NEXT: ; return to shader part epilog
15+
;
16+
; GFX9-LABEL: add_shl:
17+
; GFX9: ; %bb.0:
18+
; GFX9-NEXT: v_add_lshl_u32 v0, v0, v1, v2
19+
; GFX9-NEXT: ; return to shader part epilog
20+
%x = add i32 %a, %b
21+
%result = shl i32 %x, %c
22+
%bc = bitcast i32 %result to float
23+
ret float %bc
24+
}
25+
26+
define amdgpu_ps float @add_shl_vgpr_c(i32 inreg %a, i32 inreg %b, i32 %c) {
27+
; VI-LABEL: add_shl_vgpr_c:
28+
; VI: ; %bb.0:
29+
; VI-NEXT: s_add_i32 s2, s2, s3
30+
; VI-NEXT: v_lshlrev_b32_e64 v0, v0, s2
31+
; VI-NEXT: ; return to shader part epilog
32+
;
33+
; GFX9-LABEL: add_shl_vgpr_c:
34+
; GFX9: ; %bb.0:
35+
; GFX9-NEXT: s_add_i32 s2, s2, s3
36+
; GFX9-NEXT: v_lshlrev_b32_e64 v0, v0, s2
37+
; GFX9-NEXT: ; return to shader part epilog
38+
%x = add i32 %a, %b
39+
%result = shl i32 %x, %c
40+
%bc = bitcast i32 %result to float
41+
ret float %bc
42+
}
43+
44+
define amdgpu_ps float @add_shl_vgpr_ac(i32 %a, i32 inreg %b, i32 %c) {
45+
; VI-LABEL: add_shl_vgpr_ac:
46+
; VI: ; %bb.0:
47+
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
48+
; VI-NEXT: v_lshlrev_b32_e32 v0, v1, v0
49+
; VI-NEXT: ; return to shader part epilog
50+
;
51+
; GFX9-LABEL: add_shl_vgpr_ac:
52+
; GFX9: ; %bb.0:
53+
; GFX9-NEXT: v_add_lshl_u32 v0, v0, s2, v1
54+
; GFX9-NEXT: ; return to shader part epilog
55+
%x = add i32 %a, %b
56+
%result = shl i32 %x, %c
57+
%bc = bitcast i32 %result to float
58+
ret float %bc
59+
}
60+
61+
define amdgpu_ps float @add_shl_vgpr_const(i32 %a, i32 %b) {
62+
; VI-LABEL: add_shl_vgpr_const:
63+
; VI: ; %bb.0:
64+
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
65+
; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0
66+
; VI-NEXT: ; return to shader part epilog
67+
;
68+
; GFX9-LABEL: add_shl_vgpr_const:
69+
; GFX9: ; %bb.0:
70+
; GFX9-NEXT: v_add_lshl_u32 v0, v0, v1, 9
71+
; GFX9-NEXT: ; return to shader part epilog
72+
%x = add i32 %a, %b
73+
%result = shl i32 %x, 9
74+
%bc = bitcast i32 %result to float
75+
ret float %bc
76+
}
77+
78+
define amdgpu_ps float @add_shl_vgpr_const_inline_const(i32 %a) {
79+
; VI-LABEL: add_shl_vgpr_const_inline_const:
80+
; VI: ; %bb.0:
81+
; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0
82+
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x7e800, v0
83+
; VI-NEXT: ; return to shader part epilog
84+
;
85+
; GFX9-LABEL: add_shl_vgpr_const_inline_const:
86+
; GFX9: ; %bb.0:
87+
; GFX9-NEXT: v_mov_b32_e32 v1, 0x7e800
88+
; GFX9-NEXT: v_lshl_add_u32 v0, v0, 9, v1
89+
; GFX9-NEXT: ; return to shader part epilog
90+
%x = add i32 %a, 1012
91+
%result = shl i32 %x, 9
92+
%bc = bitcast i32 %result to float
93+
ret float %bc
94+
}
95+
96+
; TODO: Non-optimal code generation because SelectionDAG combines
97+
; (shl (add x, CONST), y) ---> (add (shl x, y), CONST').
98+
;
99+
define amdgpu_ps float @add_shl_vgpr_inline_const_x2(i32 %a) {
100+
; VI-LABEL: add_shl_vgpr_inline_const_x2:
101+
; VI: ; %bb.0:
102+
; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0
103+
; VI-NEXT: v_add_u32_e32 v0, vcc, 0x600, v0
104+
; VI-NEXT: ; return to shader part epilog
105+
;
106+
; GFX9-LABEL: add_shl_vgpr_inline_const_x2:
107+
; GFX9: ; %bb.0:
108+
; GFX9-NEXT: v_mov_b32_e32 v1, 0x600
109+
; GFX9-NEXT: v_lshl_add_u32 v0, v0, 9, v1
110+
; GFX9-NEXT: ; return to shader part epilog
111+
%x = add i32 %a, 3
112+
%result = shl i32 %x, 9
113+
%bc = bitcast i32 %result to float
114+
ret float %bc
115+
}

0 commit comments

Comments
 (0)