From b2a744682e24361cc417793e8877aa48b9d63fc9 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Fri, 11 Oct 2024 16:39:07 +0100 Subject: [PATCH] ValueTracking: introduce llvm::isLanewiseOperation Factor out and unify common code from InstSimplify and InstCombine that partially guard against cross-lane vector operations into llvm::isLanewiseOperation in ValueTracking. Alive2 proofs for changed tests: https://alive2.llvm.org/ce/z/r9jCpP --- llvm/include/llvm/Analysis/ValueTracking.h | 4 ++++ llvm/lib/Analysis/InstructionSimplify.cpp | 8 +++---- llvm/lib/Analysis/ValueTracking.cpp | 24 +++++++++++++++++++ .../InstCombine/InstCombineCalls.cpp | 17 +------------ llvm/test/Transforms/InstCombine/ispow2.ll | 12 ++-------- .../InstCombine/select-ctlz-to-cttz.ll | 4 +--- llvm/test/Transforms/InstSimplify/select.ll | 7 ++---- 7 files changed, 37 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 5749a34d511dd7..780b88107fd67b 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -791,6 +791,10 @@ bool onlyUsedByLifetimeMarkers(const Value *V); /// droppable instructions. bool onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V); +/// Return true if the instruction is known to be a vector lane-wise operation +/// i.e. if it doesn't potentially cross vector lanes. +bool isLanewiseOperation(const Instruction *I); + /// Return true if the instruction does not have any effects besides /// calculating the result and does not have undefined behavior. /// diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 90f05d43a2b147..c84292fb30cd4d 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4343,13 +4343,11 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp, if (isa(I)) return nullptr; - if (Op->getType()->isVectorTy()) { + if (Op->getType()->isVectorTy() && + (!I->getType()->isVectorTy() || !isLanewiseOperation(I))) // For vector types, the simplification must hold per-lane, so forbid // potentially cross-lane operations like shufflevector. - if (!I->getType()->isVectorTy() || isa(I) || - isa(I) || isa(I)) - return nullptr; - } + return nullptr; // Don't fold away llvm.is.constant checks based on assumptions. if (match(I, m_Intrinsic())) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 446ff42f3e243d..284123777a5350 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -6947,6 +6947,30 @@ bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) { V, /* AllowLifetime */ true, /* AllowDroppable */ true); } +bool llvm::isLanewiseOperation(const Instruction *I) { + if (auto *II = dyn_cast(I)) { + switch (II->getIntrinsicID()) { + case Intrinsic::ctlz: + case Intrinsic::cttz: + case Intrinsic::ctpop: + case Intrinsic::umin: + case Intrinsic::umax: + case Intrinsic::smin: + case Intrinsic::smax: + case Intrinsic::usub_sat: + case Intrinsic::uadd_sat: + case Intrinsic::ssub_sat: + case Intrinsic::sadd_sat: + return true; + default: + return false; + } + } + auto *Shuffle = dyn_cast(I); + return (!Shuffle || Shuffle->isIdentity() || Shuffle->isSelect()) && + !isa(I) && !isa(I); +} + bool llvm::isSafeToSpeculativelyExecute(const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC, diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index f7a9406791801c..be9d3e3abd2df5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3629,26 +3629,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // * The intrinsic is speculatable. // * The select condition is not a vector, or the intrinsic does not // perform cross-lane operations. - switch (IID) { - case Intrinsic::ctlz: - case Intrinsic::cttz: - case Intrinsic::ctpop: - case Intrinsic::umin: - case Intrinsic::umax: - case Intrinsic::smin: - case Intrinsic::smax: - case Intrinsic::usub_sat: - case Intrinsic::uadd_sat: - case Intrinsic::ssub_sat: - case Intrinsic::sadd_sat: + if (isLanewiseOperation(II)) for (Value *Op : II->args()) if (auto *Sel = dyn_cast(Op)) if (Instruction *R = FoldOpIntoSelect(*II, Sel)) return R; - [[fallthrough]]; - default: - break; - } if (Instruction *Shuf = foldShuffledIntrinsicOperands(II, Builder)) return Shuf; diff --git a/llvm/test/Transforms/InstCombine/ispow2.ll b/llvm/test/Transforms/InstCombine/ispow2.ll index c21ad95f83a1c4..2bc629a5cddf24 100644 --- a/llvm/test/Transforms/InstCombine/ispow2.ll +++ b/llvm/test/Transforms/InstCombine/ispow2.ll @@ -977,11 +977,7 @@ define i1 @is_pow2or0_ctpop_wrong_pred2_logical(i32 %x) { define <2 x i1> @is_pow2or0_ctpop_commute_vec_wrong_pred3(<2 x i8> %x) { ; CHECK-LABEL: @is_pow2or0_ctpop_commute_vec_wrong_pred3( -; CHECK-NEXT: [[T0:%.*]] = tail call range(i8 0, 9) <2 x i8> @llvm.ctpop.v2i8(<2 x i8> [[X:%.*]]) -; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[T0]], -; CHECK-NEXT: [[ISZERO:%.*]] = icmp eq <2 x i8> [[X]], zeroinitializer -; CHECK-NEXT: [[R:%.*]] = and <2 x i1> [[CMP]], [[ISZERO]] -; CHECK-NEXT: ret <2 x i1> [[R]] +; CHECK-NEXT: ret <2 x i1> zeroinitializer ; %t0 = tail call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> %x) %cmp = icmp eq <2 x i8> %t0, @@ -1174,11 +1170,7 @@ define i1 @isnot_pow2nor0_ctpop_wrong_pred2_logical(i32 %x) { define <2 x i1> @isnot_pow2nor0_wrong_pred3_ctpop_commute_vec(<2 x i8> %x) { ; CHECK-LABEL: @isnot_pow2nor0_wrong_pred3_ctpop_commute_vec( -; CHECK-NEXT: [[T0:%.*]] = tail call range(i8 0, 9) <2 x i8> @llvm.ctpop.v2i8(<2 x i8> [[X:%.*]]) -; CHECK-NEXT: [[CMP:%.*]] = icmp ne <2 x i8> [[T0]], -; CHECK-NEXT: [[NOTZERO:%.*]] = icmp ne <2 x i8> [[X]], zeroinitializer -; CHECK-NEXT: [[R:%.*]] = or <2 x i1> [[CMP]], [[NOTZERO]] -; CHECK-NEXT: ret <2 x i1> [[R]] +; CHECK-NEXT: ret <2 x i1> ; %t0 = tail call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> %x) %cmp = icmp ne <2 x i8> %t0, diff --git a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll index cc8f5d53fddddd..2e6fe26c03051b 100644 --- a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll +++ b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll @@ -208,10 +208,8 @@ define <2 x i32> @select_clz_to_ctz_vec_with_undef(<2 x i32> %a) { ; CHECK-NEXT: [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[A:%.*]] ; CHECK-NEXT: [[AND:%.*]] = and <2 x i32> [[A]], [[SUB]] ; CHECK-NEXT: [[LZ:%.*]] = tail call range(i32 0, 33) <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[AND]], i1 true) -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq <2 x i32> [[A]], zeroinitializer ; CHECK-NEXT: [[SUB1:%.*]] = xor <2 x i32> [[LZ]], -; CHECK-NEXT: [[COND:%.*]] = select <2 x i1> [[TOBOOL]], <2 x i32> [[LZ]], <2 x i32> [[SUB1]] -; CHECK-NEXT: ret <2 x i32> [[COND]] +; CHECK-NEXT: ret <2 x i32> [[SUB1]] ; %sub = sub <2 x i32> zeroinitializer, %a %and = and <2 x i32> %sub, %a diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll index 1e503afae1a69d..4618e1143dc8f9 100644 --- a/llvm/test/Transforms/InstSimplify/select.ll +++ b/llvm/test/Transforms/InstSimplify/select.ll @@ -1087,13 +1087,10 @@ define i32 @select_ctpop_zero(i32 %x) { ret i32 %sel } -; FIXME: This is safe to fold. define <2 x i32> @select_ctpop_zero_vec(<2 x i32> %x) { ; CHECK-LABEL: @select_ctpop_zero_vec( -; CHECK-NEXT: [[T0:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer -; CHECK-NEXT: [[T1:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X]]) -; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[T0]], <2 x i32> zeroinitializer, <2 x i32> [[T1]] -; CHECK-NEXT: ret <2 x i32> [[SEL]] +; CHECK-NEXT: [[T1:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[X:%.*]]) +; CHECK-NEXT: ret <2 x i32> [[T1]] ; %t0 = icmp eq <2 x i32> %x, zeroinitializer %t1 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %x)