Skip to content

Commit 68a93cb

Browse files
rj-jesusrj-jesus
committed
Restrict patterns to INCB/DECB, ALL, {1, 2, 4}
1 parent 8a67870 commit 68a93cb

File tree

7 files changed

+44
-36
lines changed

7 files changed

+44
-36
lines changed

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,11 @@ def AArch64st1q_scatter : SDNode<"AArch64ISD::SST1Q_PRED", SDT_AArch64_SCATTER_V
142142

143143
// SVE CNT/INC/RDVL
144144
def sve_rdvl_imm : ComplexPattern<i64, 1, "SelectRDVLImm<-32, 31, 16>">;
145-
def sve_cntb_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 16>">;
146145
def sve_cnth_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 8>">;
147146
def sve_cntw_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 4>">;
148147
def sve_cntd_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 2>">;
149148

150149
// SVE DEC
151-
def sve_cntb_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -16>">;
152150
def sve_cnth_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -8>">;
153151
def sve_cntw_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -4>">;
154152
def sve_cntd_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -2>">;
@@ -2680,31 +2678,50 @@ let Predicates = [HasSVE_or_SME] in {
26802678
}
26812679

26822680
let Predicates = [HasSVE_or_SME, UseScalarIncVL], AddedComplexity = 5 in {
2683-
def : Pat<(add GPR64:$op, (vscale (sve_cntb_imm i32:$imm))),
2684-
(INCB_XPiI GPR64:$op, 31, $imm)>;
2681+
// Some INCB/DECB forms have better latency and throughput than ADDVL on
2682+
// microarchitectures such as the Neoverse V2 and Neoverse V3, so we prefer
2683+
// using them here.
2684+
foreach imm = [ 1, 2, 4 ] in
2685+
let AddedComplexity = 6 in {
2686+
def : Pat<(add GPR64:$op, (vscale !mul(imm, 16))),
2687+
(INCB_XPiI GPR64:$op, 31, imm)>;
2688+
2689+
def : Pat<(add GPR32:$op, (i32 (trunc (vscale !mul(imm, 16))))),
2690+
(EXTRACT_SUBREG (INCB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
2691+
GPR32:$op, sub_32), 31, imm),
2692+
sub_32)>;
2693+
2694+
def : Pat<(add GPR64:$op, (vscale !mul(imm, -16))),
2695+
(DECB_XPiI GPR64:$op, 31, imm)>;
2696+
2697+
def : Pat<(add GPR32:$op, (i32 (trunc (vscale !mul(imm, -16))))),
2698+
(EXTRACT_SUBREG (DECB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
2699+
GPR32:$op, sub_32), 31, imm),
2700+
sub_32)>;
2701+
}
2702+
2703+
def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))),
2704+
(ADDVL_XXI GPR64:$op, $imm)>;
2705+
2706+
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_rdvl_imm i32:$imm))))),
2707+
(EXTRACT_SUBREG (ADDVL_XXI (INSERT_SUBREG (IMPLICIT_DEF),
2708+
GPR32:$op, sub_32), $imm),
2709+
sub_32)>;
2710+
26852711
def : Pat<(add GPR64:$op, (vscale (sve_cnth_imm i32:$imm))),
26862712
(INCH_XPiI GPR64:$op, 31, $imm)>;
26872713
def : Pat<(add GPR64:$op, (vscale (sve_cntw_imm i32:$imm))),
26882714
(INCW_XPiI GPR64:$op, 31, $imm)>;
26892715
def : Pat<(add GPR64:$op, (vscale (sve_cntd_imm i32:$imm))),
26902716
(INCD_XPiI GPR64:$op, 31, $imm)>;
26912717

2692-
def : Pat<(add GPR64:$op, (vscale (sve_cntb_imm_neg i32:$imm))),
2693-
(DECB_XPiI GPR64:$op, 31, $imm)>;
26942718
def : Pat<(add GPR64:$op, (vscale (sve_cnth_imm_neg i32:$imm))),
26952719
(DECH_XPiI GPR64:$op, 31, $imm)>;
26962720
def : Pat<(add GPR64:$op, (vscale (sve_cntw_imm_neg i32:$imm))),
26972721
(DECW_XPiI GPR64:$op, 31, $imm)>;
26982722
def : Pat<(add GPR64:$op, (vscale (sve_cntd_imm_neg i32:$imm))),
26992723
(DECD_XPiI GPR64:$op, 31, $imm)>;
27002724

2701-
def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))),
2702-
(ADDVL_XXI GPR64:$op, $imm)>;
2703-
2704-
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cntb_imm i32:$imm))))),
2705-
(EXTRACT_SUBREG (INCB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
2706-
GPR32:$op, sub_32), 31, $imm),
2707-
sub_32)>;
27082725
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cnth_imm i32:$imm))))),
27092726
(EXTRACT_SUBREG (INCH_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
27102727
GPR32:$op, sub_32), 31, $imm),
@@ -2718,10 +2735,6 @@ let Predicates = [HasSVE_or_SME] in {
27182735
GPR32:$op, sub_32), 31, $imm),
27192736
sub_32)>;
27202737

2721-
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cntb_imm_neg i32:$imm))))),
2722-
(EXTRACT_SUBREG (DECB_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
2723-
GPR32:$op, sub_32), 31, $imm),
2724-
sub_32)>;
27252738
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cnth_imm_neg i32:$imm))))),
27262739
(EXTRACT_SUBREG (DECH_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
27272740
GPR32:$op, sub_32), 31, $imm),
@@ -2734,11 +2747,6 @@ let Predicates = [HasSVE_or_SME] in {
27342747
(EXTRACT_SUBREG (DECD_XPiI (INSERT_SUBREG (IMPLICIT_DEF),
27352748
GPR32:$op, sub_32), 31, $imm),
27362749
sub_32)>;
2737-
2738-
def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_rdvl_imm i32:$imm))))),
2739-
(EXTRACT_SUBREG (ADDVL_XXI (INSERT_SUBREG (IMPLICIT_DEF),
2740-
GPR32:$op, sub_32), $imm),
2741-
sub_32)>;
27422750
}
27432751

27442752
// For big endian, only BITCASTs involving same sized vector types with same

‎llvm/test/CodeGen/AArch64/sme-intrinsics-loads.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -271,9 +271,9 @@ define void @ldr_with_off_15(ptr %ptr) {
271271
define void @ldr_with_off_15mulvl(ptr %ptr) {
272272
; CHECK-LABEL: ldr_with_off_15mulvl:
273273
; CHECK: // %bb.0:
274-
; CHECK-NEXT: incb x0, all, mul #15
275274
; CHECK-NEXT: mov w12, #15 // =0xf
276-
; CHECK-NEXT: ldr za[w12, 0], [x0]
275+
; CHECK-NEXT: addvl x8, x0, #15
276+
; CHECK-NEXT: ldr za[w12, 0], [x8]
277277
; CHECK-NEXT: ret
278278
%vscale = call i64 @llvm.vscale.i64()
279279
%mulvl = mul i64 %vscale, 240
@@ -285,9 +285,9 @@ define void @ldr_with_off_15mulvl(ptr %ptr) {
285285
define void @ldr_with_off_16mulvl(ptr %ptr) {
286286
; CHECK-LABEL: ldr_with_off_16mulvl:
287287
; CHECK: // %bb.0:
288-
; CHECK-NEXT: incb x0, all, mul #16
289288
; CHECK-NEXT: mov w12, #16 // =0x10
290-
; CHECK-NEXT: ldr za[w12, 0], [x0]
289+
; CHECK-NEXT: addvl x8, x0, #16
290+
; CHECK-NEXT: ldr za[w12, 0], [x8]
291291
; CHECK-NEXT: ret
292292
%vscale = call i64 @llvm.vscale.i64()
293293
%mulvl = mul i64 %vscale, 256

‎llvm/test/CodeGen/AArch64/sme-intrinsics-stores.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -271,9 +271,9 @@ define void @str_with_off_15(ptr %ptr) {
271271
define void @str_with_off_15mulvl(ptr %ptr) {
272272
; CHECK-LABEL: str_with_off_15mulvl:
273273
; CHECK: // %bb.0:
274-
; CHECK-NEXT: incb x0, all, mul #15
275274
; CHECK-NEXT: mov w12, #15 // =0xf
276-
; CHECK-NEXT: str za[w12, 0], [x0]
275+
; CHECK-NEXT: addvl x8, x0, #15
276+
; CHECK-NEXT: str za[w12, 0], [x8]
277277
; CHECK-NEXT: ret
278278
%vscale = call i64 @llvm.vscale.i64()
279279
%mulvl = mul i64 %vscale, 240
@@ -285,9 +285,9 @@ define void @str_with_off_15mulvl(ptr %ptr) {
285285
define void @str_with_off_16mulvl(ptr %ptr) {
286286
; CHECK-LABEL: str_with_off_16mulvl:
287287
; CHECK: // %bb.0:
288-
; CHECK-NEXT: incb x0, all, mul #16
289288
; CHECK-NEXT: mov w12, #16 // =0x10
290-
; CHECK-NEXT: str za[w12, 0], [x0]
289+
; CHECK-NEXT: addvl x8, x0, #16
290+
; CHECK-NEXT: str za[w12, 0], [x8]
291291
; CHECK-NEXT: ret
292292
%vscale = call i64 @llvm.vscale.i64()
293293
%mulvl = mul i64 %vscale, 256

‎llvm/test/CodeGen/AArch64/sve-vl-arith.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ define i32 @incb_scalar_i32(i32 %a) {
264264
; CHECK-LABEL: incb_scalar_i32:
265265
; CHECK: // %bb.0:
266266
; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
267-
; CHECK-NEXT: incb x0, all, mul #3
267+
; CHECK-NEXT: addvl x0, x0, #3
268268
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
269269
; CHECK-NEXT: ret
270270

‎llvm/test/CodeGen/AArch64/sve2p1-intrinsics-ld1-single.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,8 @@ define <vscale x 2 x i64> @test_svld1udq_i64_si(<vscale x 1 x i1> %pred, ptr %ba
101101
define <vscale x 2 x i64> @test_svld1udq_i64_out_of_bound(<vscale x 1 x i1> %pred, ptr %base) {
102102
; CHECK-LABEL: test_svld1udq_i64_out_of_bound:
103103
; CHECK: // %bb.0:
104-
; CHECK-NEXT: decb x0, all, mul #5
105-
; CHECK-NEXT: ld1d { z0.q }, p0/z, [x0]
104+
; CHECK-NEXT: addvl x8, x0, #-5
105+
; CHECK-NEXT: ld1d { z0.q }, p0/z, [x8]
106106
; CHECK-NEXT: ret
107107
%gep = getelementptr inbounds <vscale x 1 x i64>, ptr %base, i64 -10
108108
%res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1udq.nxv2i64(<vscale x 1 x i1> %pred, ptr %gep)

‎llvm/test/CodeGen/AArch64/sve2p1-intrinsics-st1-single.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,8 @@ define void @test_svst1dq_i64_si(<vscale x 2 x i64> %zt, <vscale x 1 x i1> %pred
9191
define void @test_svst1dq_i64_out_of_bound(<vscale x 2 x i64> %zt, <vscale x 1 x i1> %pred, ptr %base) {
9292
; CHECK-LABEL: test_svst1dq_i64_out_of_bound:
9393
; CHECK: // %bb.0:
94-
; CHECK-NEXT: decb x0, all, mul #5
95-
; CHECK-NEXT: st1d { z0.q }, p0, [x0]
94+
; CHECK-NEXT: addvl x8, x0, #-5
95+
; CHECK-NEXT: st1d { z0.q }, p0, [x8]
9696
; CHECK-NEXT: ret
9797
%gep = getelementptr inbounds <vscale x 1 x i64>, ptr %base, i64 -10
9898
call void @llvm.aarch64.sve.st1dq.nxv2i64(<vscale x 2 x i64> %zt, <vscale x 1 x i1> %pred, ptr %gep)

‎llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ define void @mulvl123_addressing(ptr %src, ptr %dst, i64 %count) #0 {
1717
; COMMON-NEXT: ldr z1, [x0, #1, mul vl]
1818
; COMMON-NEXT: ldr z2, [x0, #2, mul vl]
1919
; COMMON-NEXT: ldr z3, [x0, #3, mul vl]
20-
; COMMON-NEXT: incb x0, all, mul #5
20+
; COMMON-NEXT: addvl x0, x0, #5
2121
; COMMON-NEXT: umax z0.b, p0/m, z0.b, z1.b
2222
; COMMON-NEXT: movprfx z1, z2
2323
; COMMON-NEXT: umax z1.b, p0/m, z1.b, z3.b

0 commit comments

Comments
 (0)