Skip to content

Commit 35f865b

Browse files
fhahngithub-actions[bot]
fhahn
authored and
github-actions[bot]
committed
Automerge: [VPlan] Support multiple F(Max|Min)Num reductions. (#161735)
Generalize handleMaxMinNumReductions to handle any number of F(Max|Min)Num reductions by collecting a vector of reductions to convert. We then add NaN checks for all of them, followed by adjusting the branch controlling the vector loop region, and updating the resume phis. Addresses a TODO from llvm/llvm-project#148239 PR: llvm/llvm-project#161735
2 parents 94c3902 + 3ee2f07 commit 35f865b

File tree

6 files changed

+259
-96
lines changed

6 files changed

+259
-96
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp‎

Lines changed: 65 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -823,7 +823,8 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
823823
};
824824

825825
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
826-
VPReductionPHIRecipe *RedPhiR = nullptr;
826+
SmallVector<std::pair<VPReductionPHIRecipe *, VPValue *>>
827+
MinMaxNumReductionsToHandle;
827828
bool HasUnsupportedPhi = false;
828829
for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
829830
if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
@@ -834,19 +835,20 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
834835
HasUnsupportedPhi = true;
835836
continue;
836837
}
837-
// For now, only a single reduction is supported.
838-
// TODO: Support multiple MaxNum/MinNum reductions and other reductions.
839-
if (RedPhiR)
840-
return false;
841838
if (!RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(
842839
Cur->getRecurrenceKind())) {
843840
HasUnsupportedPhi = true;
844841
continue;
845842
}
846-
RedPhiR = Cur;
843+
844+
VPValue *MinMaxOp = GetMinMaxCompareValue(Cur);
845+
if (!MinMaxOp)
846+
return false;
847+
848+
MinMaxNumReductionsToHandle.emplace_back(Cur, MinMaxOp);
847849
}
848850

849-
if (!RedPhiR)
851+
if (MinMaxNumReductionsToHandle.empty())
850852
return true;
851853

852854
// We won't be able to resume execution in the scalar tail, if there are
@@ -855,14 +857,6 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
855857
if (HasUnsupportedPhi || !Plan.hasScalarTail())
856858
return false;
857859

858-
VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
859-
if (!MinMaxOp)
860-
return false;
861-
862-
assert(RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(
863-
RedPhiR->getRecurrenceKind()) &&
864-
"unsupported reduction");
865-
866860
/// Check if the vector loop of \p Plan can early exit and restart
867861
/// execution of last vector iteration in the scalar loop. This requires all
868862
/// recipes up to early exit point be side-effect free as they are
@@ -879,52 +873,68 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
879873
}
880874

881875
VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
882-
VPBuilder Builder(LatchVPBB->getTerminator());
883-
auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
884-
assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
876+
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
877+
VPValue *AllNaNLanes = nullptr;
878+
SmallPtrSet<VPValue *, 2> RdxResults;
879+
for (const auto &[_, MinMaxOp] : MinMaxNumReductionsToHandle) {
880+
VPValue *RedNaNLanes =
881+
LatchBuilder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
882+
AllNaNLanes = AllNaNLanes ? LatchBuilder.createOr(AllNaNLanes, RedNaNLanes)
883+
: RedNaNLanes;
884+
}
885+
886+
VPValue *AnyNaNLane =
887+
LatchBuilder.createNaryOp(VPInstruction::AnyOf, {AllNaNLanes});
888+
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
889+
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->begin());
890+
for (const auto &[RedPhiR, _] : MinMaxNumReductionsToHandle) {
891+
assert(RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(
892+
RedPhiR->getRecurrenceKind()) &&
893+
"unsupported reduction");
894+
895+
// If we exit early due to NaNs, compute the final reduction result based on
896+
// the reduction phi at the beginning of the last vector iteration.
897+
auto *RdxResult = find_singleton<VPSingleDefRecipe>(
898+
RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
899+
auto *VPI = dyn_cast<VPInstruction>(U);
900+
if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
901+
return VPI;
902+
return nullptr;
903+
});
904+
905+
auto *NewSel = MiddleBuilder.createSelect(AnyNaNLane, RedPhiR,
906+
RdxResult->getOperand(1));
907+
RdxResult->setOperand(1, NewSel);
908+
assert(!RdxResults.contains(RdxResult) && "RdxResult already used");
909+
RdxResults.insert(RdxResult);
910+
}
911+
912+
auto *LatchExitingBranch = LatchVPBB->getTerminator();
913+
assert(match(LatchExitingBranch, m_BranchOnCount(m_VPValue(), m_VPValue())) &&
885914
"Unexpected terminator");
886-
auto *IsLatchExitTaken =
887-
Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
888-
LatchExitingBranch->getOperand(1));
889-
890-
VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
891-
VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
892-
auto *AnyExitTaken =
893-
Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
894-
Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
915+
auto *IsLatchExitTaken = LatchBuilder.createICmp(
916+
CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
917+
LatchExitingBranch->getOperand(1));
918+
auto *AnyExitTaken = LatchBuilder.createNaryOp(
919+
Instruction::Or, {AnyNaNLane, IsLatchExitTaken});
920+
LatchBuilder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
895921
LatchExitingBranch->eraseFromParent();
896922

897-
// If we exit early due to NaNs, compute the final reduction result based on
898-
// the reduction phi at the beginning of the last vector iteration.
899-
auto *RdxResult = find_singleton<VPSingleDefRecipe>(
900-
RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
901-
auto *VPI = dyn_cast<VPInstruction>(U);
902-
if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
903-
return VPI;
904-
return nullptr;
905-
});
906-
907-
auto *MiddleVPBB = Plan.getMiddleBlock();
908-
Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
909-
auto *NewSel =
910-
Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
911-
RdxResult->setOperand(1, NewSel);
912-
913-
auto *ScalarPH = Plan.getScalarPreheader();
914-
// Update resume phis for inductions in the scalar preheader. If AnyNaN is
923+
// Update resume phis for inductions in the scalar preheader. If AnyNaNLane is
915924
// true, the resume from the start of the last vector iteration via the
916925
// canonical IV, otherwise from the original value.
917-
for (auto &R : ScalarPH->phis()) {
926+
for (auto &R : Plan.getScalarPreheader()->phis()) {
918927
auto *ResumeR = cast<VPPhi>(&R);
919928
VPValue *VecV = ResumeR->getOperand(0);
920-
if (VecV == RdxResult)
929+
if (RdxResults.contains(VecV))
921930
continue;
922931
if (auto *DerivedIV = dyn_cast<VPDerivedIVRecipe>(VecV)) {
923932
if (DerivedIV->getNumUsers() == 1 &&
924933
DerivedIV->getOperand(1) == &Plan.getVectorTripCount()) {
925-
auto *NewSel = Builder.createSelect(
926-
AnyNaN, LoopRegion->getCanonicalIV(), &Plan.getVectorTripCount());
927-
DerivedIV->moveAfter(&*Builder.getInsertPoint());
934+
auto *NewSel =
935+
MiddleBuilder.createSelect(AnyNaNLane, LoopRegion->getCanonicalIV(),
936+
&Plan.getVectorTripCount());
937+
DerivedIV->moveAfter(&*MiddleBuilder.getInsertPoint());
928938
DerivedIV->setOperand(1, NewSel);
929939
continue;
930940
}
@@ -936,15 +946,16 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
936946
"FMaxNum/FMinNum reduction.\n");
937947
return false;
938948
}
939-
auto *NewSel =
940-
Builder.createSelect(AnyNaN, LoopRegion->getCanonicalIV(), VecV);
949+
auto *NewSel = MiddleBuilder.createSelect(
950+
AnyNaNLane, LoopRegion->getCanonicalIV(), VecV);
941951
ResumeR->setOperand(0, NewSel);
942952
}
943953

944954
auto *MiddleTerm = MiddleVPBB->getTerminator();
945-
Builder.setInsertPoint(MiddleTerm);
955+
MiddleBuilder.setInsertPoint(MiddleTerm);
946956
VPValue *MiddleCond = MiddleTerm->getOperand(0);
947-
VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN));
957+
VPValue *NewCond =
958+
MiddleBuilder.createAnd(MiddleCond, MiddleBuilder.createNot(AnyNaNLane));
948959
MiddleTerm->setOperand(0, NewCond);
949960
return true;
950961
}

‎llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll‎

Lines changed: 69 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@ define float @fmaxnum(ptr %src, i64 %n) {
5959
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
6060
; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
6161
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
62-
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
6362
; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
6463
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
6564
; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP3]]
6665
; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP4]]
6766
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP18]], [[TMP15]]
6867
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
68+
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
6969
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
7070
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7171
; CHECK: [[MIDDLE_BLOCK]]:
@@ -112,27 +112,84 @@ exit:
112112
ret float %max.next
113113
}
114114

115+
; TODO: Could fold pairs of `fcmp uno` together.
115116
define float @test_fmax_and_fmin(ptr %src.0, ptr %src.1, i64 %n) {
116117
; CHECK-LABEL: define float @test_fmax_and_fmin(
117118
; CHECK-SAME: ptr [[SRC_0:%.*]], ptr [[SRC_1:%.*]], i64 [[N:%.*]]) {
118119
; CHECK-NEXT: [[ENTRY:.*]]:
119-
; CHECK-NEXT: br label %[[LOOP:.*]]
120-
; CHECK: [[LOOP]]:
121-
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
122-
; CHECK-NEXT: [[MIN:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MIN_NEXT:%.*]], %[[LOOP]] ]
123-
; CHECK-NEXT: [[MAX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
120+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
121+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
122+
; CHECK: [[VECTOR_PH]]:
123+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
124+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
125+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
126+
; CHECK: [[VECTOR_BODY]]:
127+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
128+
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
129+
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
130+
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
131+
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
124132
; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_0]], i64 [[IV]]
125133
; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV]]
126-
; CHECK-NEXT: [[L_0:%.*]] = load float, ptr [[GEP_SRC_0]], align 4
127-
; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
134+
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_0]], i32 4
135+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[GEP_SRC_0]], align 4
136+
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
137+
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC_1]], i32 4
138+
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[GEP_SRC_1]], align 4
139+
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
140+
; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI2]], <4 x float> [[WIDE_LOAD]])
141+
; CHECK-NEXT: [[TMP5]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI3]], <4 x float> [[WIDE_LOAD4]])
142+
; CHECK-NEXT: [[TMP6]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD5]])
143+
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD6]])
144+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
145+
; CHECK-NEXT: [[TMP8:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD5]], [[WIDE_LOAD5]]
146+
; CHECK-NEXT: [[TMP9:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD6]]
147+
; CHECK-NEXT: [[TMP14:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
148+
; CHECK-NEXT: [[TMP15:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
149+
; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP8]], [[TMP14]]
150+
; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP9]], [[TMP15]]
151+
; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP12]]
152+
; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP13]]
153+
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
154+
; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
155+
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
156+
; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP21]]
157+
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
158+
; CHECK: [[MIDDLE_BLOCK]]:
159+
; CHECK-NEXT: [[TMP23:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
160+
; CHECK-NEXT: [[TMP24:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP7]]
161+
; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI2]], <4 x float> [[TMP4]]
162+
; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP19]], <4 x float> [[VEC_PHI3]], <4 x float> [[TMP5]]
163+
; CHECK-NEXT: [[TMP27:%.*]] = select i1 [[TMP19]], i64 [[IV]], i64 [[N_VEC]]
164+
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP24]])
165+
; CHECK-NEXT: [[TMP28:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX]])
166+
; CHECK-NEXT: [[RDX_MINMAX9:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP25]], <4 x float> [[TMP26]])
167+
; CHECK-NEXT: [[TMP29:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX9]])
168+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
169+
; CHECK-NEXT: [[TMP30:%.*]] = xor i1 [[TMP19]], true
170+
; CHECK-NEXT: [[TMP31:%.*]] = and i1 [[CMP_N]], [[TMP30]]
171+
; CHECK-NEXT: br i1 [[TMP31]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
172+
; CHECK: [[SCALAR_PH]]:
173+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
174+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
175+
; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi float [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
176+
; CHECK-NEXT: br label %[[LOOP:.*]]
177+
; CHECK: [[LOOP]]:
178+
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
179+
; CHECK-NEXT: [[MIN:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MIN_NEXT:%.*]], %[[LOOP]] ]
180+
; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX8]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
181+
; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_0]], i64 [[IV1]]
182+
; CHECK-NEXT: [[GEP_SRC_3:%.*]] = getelementptr inbounds nuw float, ptr [[SRC_1]], i64 [[IV1]]
183+
; CHECK-NEXT: [[L_0:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
184+
; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_SRC_3]], align 4
128185
; CHECK-NEXT: [[MAX_NEXT]] = tail call noundef float @llvm.maxnum.f32(float [[MAX]], float [[L_0]])
129186
; CHECK-NEXT: [[MIN_NEXT]] = tail call noundef float @llvm.minnum.f32(float [[MIN]], float [[L_1]])
130-
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
187+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
131188
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
132-
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
189+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
133190
; CHECK: [[EXIT]]:
134-
; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
135-
; CHECK-NEXT: [[MIN_NEXT_LCSSA:%.*]] = phi float [ [[MIN_NEXT]], %[[LOOP]] ]
191+
; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP29]], %[[MIDDLE_BLOCK]] ]
192+
; CHECK-NEXT: [[MIN_NEXT_LCSSA:%.*]] = phi float [ [[MIN_NEXT]], %[[LOOP]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
136193
; CHECK-NEXT: [[SUB:%.*]] = fsub float [[MAX_NEXT_LCSSA]], [[MIN_NEXT_LCSSA]]
137194
; CHECK-NEXT: ret float [[SUB]]
138195
;

‎llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@ define float @fminnum(ptr %src, i64 %n) {
5959
; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
6060
; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
6161
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
62-
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
6362
; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
6463
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
6564
; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP3]]
6665
; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP4]]
6766
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP15]], [[TMP18]]
6867
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
68+
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
6969
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
7070
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7171
; CHECK: [[MIDDLE_BLOCK]]:

0 commit comments

Comments
 (0)