Skip to content

Commit 0476371

Browse files
bjopebjope
committed
[SelectionDAG] Improve isGuaranteedNotToBeUndefOrPoison
Add special handling of EXTRACT_SUBVECTOR, INSERT_SUBVECTOR, EXTRACT_VECTOR_ELT, INSERT_VECTOR_ELT and SCALAR_TO_VECTOR in isGuaranteedNotToBeUndefOrPoison. Make use of DemandedElts to improve the analysis and only check relevant elements for each operand. Also start using DemandedElts in the recursive calls that check isGuaranteedNotToBeUndefOrPoison for all operands for operations that do not create undef/poison. We can do that for a number of elementwise operations for which the DemandedElts can be applied to every operand (e.g. ADD, OR, BITREVERSE, TRUNCATE).
1 parent eda3161 commit 0476371

File tree

4 files changed

+171
-22
lines changed

4 files changed

+171
-22
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAGNodes.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1889,6 +1889,12 @@ LLVM_ABI SDValue peekThroughExtractSubvectors(SDValue V);
18891889
/// If \p V is not a truncation, it is returned as-is.
18901890
LLVM_ABI SDValue peekThroughTruncates(SDValue V);
18911891

1892+
/// Recursively peek through INSERT_VECTOR_ELT nodes, returning the source
1893+
/// vector operand of \p V, as long as \p V is an INSERT_VECTOR_ELT operation
1894+
/// that do not insert into any of the demanded vector elts.
1895+
LLVM_ABI SDValue peekThroughInsertVectorElt(SDValue V,
1896+
const APInt &DemandedElts);
1897+
18921898
/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
18931899
/// constant is canonicalized to be operand 1.
18941900
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5456,6 +5456,85 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
54565456
}
54575457
return true;
54585458

5459+
case ISD::EXTRACT_SUBVECTOR: {
5460+
SDValue Src = Op.getOperand(0);
5461+
if (Src.getValueType().isScalableVector())
5462+
break;
5463+
uint64_t Idx = Op.getConstantOperandVal(1);
5464+
unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5465+
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
5466+
return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,
5467+
Depth + 1);
5468+
}
5469+
5470+
case ISD::INSERT_SUBVECTOR: {
5471+
if (Op.getValueType().isScalableVector())
5472+
break;
5473+
SDValue Src = Op.getOperand(0);
5474+
SDValue Sub = Op.getOperand(1);
5475+
uint64_t Idx = Op.getConstantOperandVal(2);
5476+
unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5477+
APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
5478+
APInt DemandedSrcElts = DemandedElts;
5479+
DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
5480+
5481+
if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5482+
Sub, DemandedSubElts, PoisonOnly, Depth + 1))
5483+
return false;
5484+
if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5485+
Src, DemandedSrcElts, PoisonOnly, Depth + 1))
5486+
return false;
5487+
return true;
5488+
}
5489+
5490+
case ISD::EXTRACT_VECTOR_ELT: {
5491+
SDValue Src = Op.getOperand(0);
5492+
auto *IndexC = dyn_cast<ConstantSDNode>(Op.getOperand(1));
5493+
EVT SrcVT = Src.getValueType();
5494+
if (SrcVT.isFixedLengthVector() && IndexC &&
5495+
IndexC->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
5496+
APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
5497+
IndexC->getZExtValue());
5498+
return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,
5499+
Depth + 1);
5500+
}
5501+
break;
5502+
}
5503+
5504+
case ISD::INSERT_VECTOR_ELT: {
5505+
SDValue InVec = Op.getOperand(0);
5506+
SDValue InVal = Op.getOperand(1);
5507+
SDValue EltNo = Op.getOperand(2);
5508+
EVT VT = InVec.getValueType();
5509+
auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
5510+
if (IndexC && VT.isFixedLengthVector() &&
5511+
IndexC->getZExtValue() < VT.getVectorNumElements()) {
5512+
if (DemandedElts[IndexC->getZExtValue()] &&
5513+
!isGuaranteedNotToBeUndefOrPoison(InVal, PoisonOnly, Depth + 1))
5514+
return false;
5515+
APInt InVecDemandedElts = DemandedElts;
5516+
InVecDemandedElts.clearBit(IndexC->getZExtValue());
5517+
if (!!InVecDemandedElts &&
5518+
!isGuaranteedNotToBeUndefOrPoison(
5519+
peekThroughInsertVectorElt(InVec, InVecDemandedElts),
5520+
InVecDemandedElts, PoisonOnly, Depth + 1))
5521+
return false;
5522+
return true;
5523+
}
5524+
break;
5525+
}
5526+
5527+
case ISD::SCALAR_TO_VECTOR:
5528+
// Check upper (known undef) elements.
5529+
if (DemandedElts.ugt(1) && !PoisonOnly)
5530+
return false;
5531+
// Check element zero.
5532+
if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0),
5533+
PoisonOnly,
5534+
Depth + 1))
5535+
return false;
5536+
return true;
5537+
54595538
case ISD::SPLAT_VECTOR:
54605539
return isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), PoisonOnly,
54615540
Depth + 1);
@@ -5478,6 +5557,52 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
54785557
return true;
54795558
}
54805559

5560+
case ISD::SHL:
5561+
case ISD::SRL:
5562+
case ISD::SRA:
5563+
// Shift amount operand is checked by canCreateUndefOrPoison. So it is
5564+
// enough to check operand 0 if Op can't create undef/poison.
5565+
return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5566+
/*ConsiderFlags*/ true, Depth) &&
5567+
isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,
5568+
PoisonOnly, Depth + 1);
5569+
5570+
case ISD::BSWAP:
5571+
case ISD::CTPOP:
5572+
case ISD::BITREVERSE:
5573+
case ISD::AND:
5574+
case ISD::OR:
5575+
case ISD::XOR:
5576+
case ISD::ADD:
5577+
case ISD::SUB:
5578+
case ISD::MUL:
5579+
case ISD::SADDSAT:
5580+
case ISD::UADDSAT:
5581+
case ISD::SSUBSAT:
5582+
case ISD::USUBSAT:
5583+
case ISD::SSHLSAT:
5584+
case ISD::USHLSAT:
5585+
case ISD::SMIN:
5586+
case ISD::SMAX:
5587+
case ISD::UMIN:
5588+
case ISD::UMAX:
5589+
case ISD::ZERO_EXTEND:
5590+
case ISD::SIGN_EXTEND:
5591+
case ISD::ANY_EXTEND:
5592+
case ISD::TRUNCATE:
5593+
case ISD::VSELECT: {
5594+
// If Op can't create undef/poison and none of its operands are undef/poison
5595+
// then Op is never undef/poison. A difference from the more common check
5596+
// below, outside the switch, is that we handle elementwise operations for
5597+
// which the DemandedElts mask is valid for all operands here.
5598+
return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5599+
/*ConsiderFlags*/ true, Depth) &&
5600+
all_of(Op->ops(), [&](SDValue V) {
5601+
return isGuaranteedNotToBeUndefOrPoison(V, DemandedElts,
5602+
PoisonOnly, Depth + 1);
5603+
});
5604+
}
5605+
54815606
// TODO: Search for noundef attributes from library functions.
54825607

54835608
// TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
@@ -12463,6 +12588,23 @@ SDValue llvm::peekThroughTruncates(SDValue V) {
1246312588
return V;
1246412589
}
1246512590

12591+
SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
12592+
while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
12593+
SDValue InVec = V.getOperand(0);
12594+
SDValue EltNo = V.getOperand(2);
12595+
EVT VT = InVec.getValueType();
12596+
auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
12597+
if (IndexC && VT.isFixedLengthVector() &&
12598+
IndexC->getZExtValue() < VT.getVectorNumElements() &&
12599+
!DemandedElts[IndexC->getZExtValue()]) {
12600+
V = InVec;
12601+
continue;
12602+
}
12603+
break;
12604+
}
12605+
return V;
12606+
}
12607+
1246612608
bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
1246712609
if (V.getOpcode() != ISD::XOR)
1246812610
return false;

‎llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
172172
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
173173
; X86-NEXT: sbbl %eax, %edi
174174
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
175-
; X86-NEXT: movl 40(%ebp), %ecx
176-
; X86-NEXT: movl %ecx, %edx
175+
; X86-NEXT: movl 40(%ebp), %esi
176+
; X86-NEXT: movl %esi, %edx
177177
; X86-NEXT: sarl $31, %edx
178-
; X86-NEXT: movl %ecx, %esi
179178
; X86-NEXT: xorl %edx, %esi
180179
; X86-NEXT: movl 36(%ebp), %ecx
181180
; X86-NEXT: xorl %edx, %ecx
@@ -204,14 +203,14 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
204203
; X86-NEXT: sete %al
205204
; X86-NEXT: orb %cl, %al
206205
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
207-
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
208-
; X86-NEXT: bsrl %eax, %edx
206+
; X86-NEXT: bsrl %esi, %edx
209207
; X86-NEXT: xorl $31, %edx
210-
; X86-NEXT: addl $32, %edx
211-
; X86-NEXT: bsrl %esi, %ecx
208+
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
209+
; X86-NEXT: bsrl %eax, %ecx
212210
; X86-NEXT: xorl $31, %ecx
211+
; X86-NEXT: addl $32, %ecx
213212
; X86-NEXT: testl %esi, %esi
214-
; X86-NEXT: cmovel %edx, %ecx
213+
; X86-NEXT: cmovnel %edx, %ecx
215214
; X86-NEXT: bsrl %ebx, %edx
216215
; X86-NEXT: xorl $31, %edx
217216
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -226,14 +225,14 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
226225
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
227226
; X86-NEXT: orl %esi, %edx
228227
; X86-NEXT: cmovnel %ecx, %edi
229-
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
230-
; X86-NEXT: xorl $31, %edx
231-
; X86-NEXT: addl $32, %edx
232228
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
233-
; X86-NEXT: bsrl %eax, %ecx
229+
; X86-NEXT: bsrl %eax, %edx
230+
; X86-NEXT: xorl $31, %edx
231+
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
234232
; X86-NEXT: xorl $31, %ecx
233+
; X86-NEXT: addl $32, %ecx
235234
; X86-NEXT: testl %eax, %eax
236-
; X86-NEXT: cmovel %edx, %ecx
235+
; X86-NEXT: cmovnel %edx, %ecx
237236
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
238237
; X86-NEXT: bsrl %ebx, %esi
239238
; X86-NEXT: xorl $31, %esi
@@ -380,9 +379,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
380379
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
381380
; X86-NEXT: adcl $-1, %eax
382381
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
383-
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
384-
; X86-NEXT: adcl $-1, %eax
385-
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
382+
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
383+
; X86-NEXT: adcl $-1, %ecx
384+
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
386385
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
387386
; X86-NEXT: adcl $-1, %ecx
388387
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill

‎llvm/test/CodeGen/X86/pr62286.ll‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ define i64 @PR62286(i32 %a) {
2828
; AVX1-NEXT: vmovd %edi, %xmm0
2929
; AVX1-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
3030
; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm0
31+
; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
32+
; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
3133
; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
32-
; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
3334
; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0
3435
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
3536
; AVX1-NEXT: vpmovsxdq %xmm1, %xmm1
@@ -58,12 +59,13 @@ define i64 @PR62286(i32 %a) {
5859
; AVX512-LABEL: PR62286:
5960
; AVX512: # %bb.0:
6061
; AVX512-NEXT: vmovd %edi, %xmm0
61-
; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,1,0]
62-
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
63-
; AVX512-NEXT: movw $4369, %ax # imm = 0x1111
62+
; AVX512-NEXT: movb $8, %al
6463
; AVX512-NEXT: kmovd %eax, %k1
65-
; AVX512-NEXT: vpaddd %zmm0, %zmm0, %zmm1 {%k1}
66-
; AVX512-NEXT: vpmovsxdq %ymm1, %zmm0
64+
; AVX512-NEXT: vpexpandd %ymm0, %ymm1 {%k1} {z}
65+
; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
66+
; AVX512-NEXT: vpaddd %ymm0, %ymm0, %ymm0
67+
; AVX512-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
68+
; AVX512-NEXT: vpmovsxdq %ymm0, %zmm0
6769
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
6870
; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
6971
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1

0 commit comments

Comments
 (0)