Skip to content

Commit 67d4c99

Browse files
sdesmalen-armsdesmalen-arm
committed
Add support for (expressing) vscale.
In LLVM IR, vscale can be represented with an intrinsic. For some targets, this is equivalent to the constexpr: getelementptr <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1 This can be used to propagate the value in CodeGenPrepare. In ISel we add a node that can be legalized to one or more instructions to materialize the runtime vector length. This patch also adds SVE CodeGen support for VSCALE, which maps this node to RDVL instructions (for scaled multiples of 16bytes) or CNT[HSD] instructions (scaled multiples of 2, 4, or 8 bytes, respectively). Reviewers: rengolin, cameron.mcinally, hfinkel, sebpop, SjoerdMeijer, efriedma, lattner Reviewed by: efriedma Tags: #llvm Differential Revision: https://reviews.llvm.org/D68203
1 parent e57a9ab commit 67d4c99

18 files changed

+402
-2
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17889,6 +17889,34 @@ information on the *based on* terminology see
1788917889
mask argument does not match the pointer size of the target, the mask is
1789017890
zero-extended or truncated accordingly.
1789117891

17892+
.. _int_vscale:
17893+
17894+
'``llvm.vscale``' Intrinsic
17895+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
17896+
17897+
Syntax:
17898+
"""""""
17899+
17900+
::
17901+
17902+
declare i32 llvm.vscale.i32()
17903+
declare i64 llvm.vscale.i64()
17904+
17905+
Overview:
17906+
"""""""""
17907+
17908+
The ``llvm.vscale`` intrinsic returns the value for ``vscale`` in scalable
17909+
vectors such as ``<vscale x 16 x i8>``.
17910+
17911+
Semantics:
17912+
""""""""""
17913+
17914+
``vscale`` is a positive value that is constant throughout program
17915+
execution, but is unknown at compile time.
17916+
If the result value does not fit in the result type, then the result is
17917+
a :ref:`poison value <poisonvalues>`.
17918+
17919+
1789217920
Stack Map Intrinsics
1789317921
--------------------
1789417922

‎llvm/include/llvm/CodeGen/ISDOpcodes.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,11 @@ namespace ISD {
921921
/// known nonzero constant. The only operand here is the chain.
922922
GET_DYNAMIC_AREA_OFFSET,
923923

924+
/// VSCALE(IMM) - Returns the runtime scaling factor used to calculate the
925+
/// number of elements within a scalable vector. IMM is a constant integer
926+
/// multiplier that is applied to the runtime value.
927+
VSCALE,
928+
924929
/// Generic reduction nodes. These nodes represent horizontal vector
925930
/// reduction operations, producing a scalar result.
926931
/// The STRICT variants perform reductions in sequential order. The first

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -914,6 +914,13 @@ class SelectionDAG {
914914
return getNode(ISD::UNDEF, SDLoc(), VT);
915915
}
916916

917+
/// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
918+
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
919+
assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&
920+
"Immediate does not fit VT");
921+
return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
922+
}
923+
917924
/// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
918925
SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
919926
return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,11 @@ def int_preserve_struct_access_index : Intrinsic<[llvm_anyptr_ty],
13421342
[IntrNoMem, ImmArg<1>,
13431343
ImmArg<2>]>;
13441344

1345+
//===---------- Intrinsics to query properties of scalable vectors --------===//
1346+
def int_vscale : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>;
1347+
1348+
//===----------------------------------------------------------------------===//
1349+
13451350
//===----------------------------------------------------------------------===//
13461351
// Target-specific intrinsics
13471352
//===----------------------------------------------------------------------===//

‎llvm/include/llvm/IR/PatternMatch.h

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "llvm/ADT/APInt.h"
3333
#include "llvm/IR/Constant.h"
3434
#include "llvm/IR/Constants.h"
35+
#include "llvm/IR/DataLayout.h"
3536
#include "llvm/IR/InstrTypes.h"
3637
#include "llvm/IR/Instruction.h"
3738
#include "llvm/IR/Instructions.h"
@@ -2002,6 +2003,42 @@ inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) {
20022003
return ExtractValue_match<Ind, Val_t>(V);
20032004
}
20042005

2006+
/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
2007+
/// the constant expression
2008+
/// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
2009+
/// under the right conditions determined by DataLayout.
2010+
struct VScaleVal_match {
2011+
private:
2012+
template <typename Base, typename Offset>
2013+
inline BinaryOp_match<Base, Offset, Instruction::GetElementPtr>
2014+
m_OffsetGep(const Base &B, const Offset &O) {
2015+
return BinaryOp_match<Base, Offset, Instruction::GetElementPtr>(B, O);
2016+
}
2017+
2018+
public:
2019+
const DataLayout &DL;
2020+
VScaleVal_match(const DataLayout &DL) : DL(DL) {}
2021+
2022+
template <typename ITy> bool match(ITy *V) {
2023+
if (m_Intrinsic<Intrinsic::vscale>().match(V))
2024+
return true;
2025+
2026+
if (m_PtrToInt(m_OffsetGep(m_Zero(), m_SpecificInt(1))).match(V)) {
2027+
Type *PtrTy = cast<Operator>(V)->getOperand(0)->getType();
2028+
Type *DerefTy = PtrTy->getPointerElementType();
2029+
if (DerefTy->isVectorTy() && DerefTy->getVectorIsScalable() &&
2030+
DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8)
2031+
return true;
2032+
}
2033+
2034+
return false;
2035+
}
2036+
};
2037+
2038+
inline VScaleVal_match m_VScale(const DataLayout &DL) {
2039+
return VScaleVal_match(DL);
2040+
}
2041+
20052042
} // end namespace PatternMatch
20062043
} // end namespace llvm
20072044

‎llvm/include/llvm/Target/TargetSelectionDAG.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,7 @@ def vt : SDNode<"ISD::VALUETYPE" , SDTOther , [], "VTSDNode">;
316316
def bb : SDNode<"ISD::BasicBlock", SDTOther , [], "BasicBlockSDNode">;
317317
def cond : SDNode<"ISD::CONDCODE" , SDTOther , [], "CondCodeSDNode">;
318318
def undef : SDNode<"ISD::UNDEF" , SDTUNDEF , []>;
319+
def vscale : SDNode<"ISD::VSCALE" , SDTIntUnaryOp, []>;
319320
def globaladdr : SDNode<"ISD::GlobalAddress", SDTPtrLeaf, [],
320321
"GlobalAddressSDNode">;
321322
def tglobaladdr : SDNode<"ISD::TargetGlobalAddress", SDTPtrLeaf, [],

‎llvm/lib/Analysis/ConstantFolding.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -828,7 +828,8 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
828828
Type *SrcElemTy = GEP->getSourceElementType();
829829
Type *ResElemTy = GEP->getResultElementType();
830830
Type *ResTy = GEP->getType();
831-
if (!SrcElemTy->isSized())
831+
if (!SrcElemTy->isSized() ||
832+
(SrcElemTy->isVectorTy() && SrcElemTy->getVectorIsScalable()))
832833
return nullptr;
833834

834835
if (Constant *C = CastGEPIndices(SrcElemTy, Ops, ResTy,

‎llvm/lib/CodeGen/CodeGenPrepare.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2010,6 +2010,22 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
20102010
return despeculateCountZeros(II, TLI, DL, ModifiedDT);
20112011
case Intrinsic::dbg_value:
20122012
return fixupDbgValue(II);
2013+
case Intrinsic::vscale: {
2014+
// If datalayout has no special restrictions on vector data layout,
2015+
// replace `llvm.vscale` by an equivalent constant expression
2016+
// to benefit from cheap constant propagation.
2017+
Type *ScalableVectorTy =
2018+
VectorType::get(Type::getInt8Ty(II->getContext()), 1, true);
2019+
if (DL->getTypeAllocSize(ScalableVectorTy).getKnownMinSize() == 8) {
2020+
auto Null = Constant::getNullValue(ScalableVectorTy->getPointerTo());
2021+
auto One = ConstantInt::getSigned(II->getType(), 1);
2022+
auto *CGep =
2023+
ConstantExpr::getGetElementPtr(ScalableVectorTy, Null, One);
2024+
II->replaceAllUsesWith(ConstantExpr::getPtrToInt(CGep, II->getType()));
2025+
II->eraseFromParent();
2026+
return true;
2027+
}
2028+
}
20132029
}
20142030

20152031
if (TLI) {

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
9191
case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
9292
case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
9393
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
94+
case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
9495

9596
case ISD::EXTRACT_SUBVECTOR:
9697
Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
@@ -1179,6 +1180,13 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
11791180
N->getValueType(0)));
11801181
}
11811182

1183+
SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1184+
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1185+
1186+
APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1187+
return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1188+
}
1189+
11821190
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
11831191
SDValue Chain = N->getOperand(0); // Get the chain.
11841192
SDValue Ptr = N->getOperand(1); // Get the pointer.

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
326326
SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
327327
SDValue PromoteIntRes_UNDEF(SDNode *N);
328328
SDValue PromoteIntRes_VAARG(SDNode *N);
329+
SDValue PromoteIntRes_VSCALE(SDNode *N);
329330
SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
330331
SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
331332
SDValue PromoteIntRes_MULFIX(SDNode *N);

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5185,11 +5185,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
51855185
if (N2C && N2C->isNullValue())
51865186
return N1;
51875187
break;
5188+
case ISD::MUL:
5189+
assert(VT.isInteger() && "This operator does not apply to FP types!");
5190+
assert(N1.getValueType() == N2.getValueType() &&
5191+
N1.getValueType() == VT && "Binary operator types must match!");
5192+
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5193+
APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
5194+
APInt N2CImm = N2C->getAPIntValue();
5195+
return getVScale(DL, VT, MulImm * N2CImm);
5196+
}
5197+
break;
51885198
case ISD::UDIV:
51895199
case ISD::UREM:
51905200
case ISD::MULHU:
51915201
case ISD::MULHS:
5192-
case ISD::MUL:
51935202
case ISD::SDIV:
51945203
case ISD::SREM:
51955204
case ISD::SMIN:
@@ -5222,6 +5231,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
52225231
"Invalid FCOPYSIGN!");
52235232
break;
52245233
case ISD::SHL:
5234+
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5235+
APInt MulImm = cast<ConstantSDNode>(N1->getOperand(0))->getAPIntValue();
5236+
APInt ShiftImm = N2C->getAPIntValue();
5237+
return getVScale(DL, VT, MulImm << ShiftImm);
5238+
}
5239+
LLVM_FALLTHROUGH;
52255240
case ISD::SRA:
52265241
case ISD::SRL:
52275242
if (SDValue V = simplifyShift(N1, N2))

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1482,6 +1482,9 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
14821482
TLI.getPointerTy(DAG.getDataLayout(), AS));
14831483
}
14841484

1485+
if (match(C, m_VScale(DAG.getDataLayout())))
1486+
return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1487+
14851488
if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
14861489
return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
14871490

@@ -5772,6 +5775,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
57725775
// By default, turn this into a target intrinsic node.
57735776
visitTargetIntrinsic(I, Intrinsic);
57745777
return;
5778+
case Intrinsic::vscale: {
5779+
match(&I, m_VScale(DAG.getDataLayout()));
5780+
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5781+
setValue(&I,
5782+
DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)));
5783+
return;
5784+
}
57755785
case Intrinsic::vastart: visitVAStart(I); return;
57765786
case Intrinsic::vaend: visitVAEnd(I); return;
57775787
case Intrinsic::vacopy: visitVACopy(I); return;

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
170170
case ISD::CopyToReg: return "CopyToReg";
171171
case ISD::CopyFromReg: return "CopyFromReg";
172172
case ISD::UNDEF: return "undef";
173+
case ISD::VSCALE: return "vscale";
173174
case ISD::MERGE_VALUES: return "merge_values";
174175
case ISD::INLINEASM: return "inlineasm";
175176
case ISD::INLINEASM_BR: return "inlineasm_br";

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
6262
unsigned ConstraintID,
6363
std::vector<SDValue> &OutOps) override;
6464

65+
template <signed Low, signed High, signed Scale>
66+
bool SelectRDVLImm(SDValue N, SDValue &Imm);
67+
6568
bool tryMLAV64LaneV128(SDNode *N);
6669
bool tryMULLV64LaneV128(unsigned IntNo, SDNode *N);
6770
bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
@@ -679,6 +682,23 @@ static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
679682
return SDValue(Node, 0);
680683
}
681684

685+
// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
686+
template<signed Low, signed High, signed Scale>
687+
bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
688+
if (!isa<ConstantSDNode>(N))
689+
return false;
690+
691+
int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
692+
if ((MulImm % std::abs(Scale)) == 0) {
693+
int64_t RDVLImm = MulImm / Scale;
694+
if ((RDVLImm >= Low) && (RDVLImm <= High)) {
695+
Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
696+
return true;
697+
}
698+
}
699+
700+
return false;
701+
}
682702

683703
/// SelectArithExtendedRegister - Select a "extended register" operand. This
684704
/// operand folds in an extend followed by an optional left shift.

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
836836
}
837837
}
838838

839+
if (Subtarget->hasSVE())
840+
setOperationAction(ISD::VSCALE, MVT::i32, Custom);
841+
839842
setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
840843
}
841844

@@ -3254,6 +3257,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
32543257
return LowerATOMIC_LOAD_AND(Op, DAG);
32553258
case ISD::DYNAMIC_STACKALLOC:
32563259
return LowerDYNAMIC_STACKALLOC(Op, DAG);
3260+
case ISD::VSCALE:
3261+
return LowerVSCALE(Op, DAG);
32573262
}
32583263
}
32593264

@@ -8641,6 +8646,17 @@ AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
86418646
return DAG.getMergeValues(Ops, dl);
86428647
}
86438648

8649+
SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
8650+
SelectionDAG &DAG) const {
8651+
EVT VT = Op.getValueType();
8652+
assert(VT != MVT::i64 && "Expected illegal VSCALE node");
8653+
8654+
SDLoc DL(Op);
8655+
APInt MulImm = cast<ConstantSDNode>(Op.getOperand(0))->getAPIntValue();
8656+
return DAG.getZExtOrTrunc(DAG.getVScale(DL, MVT::i64, MulImm.sextOrSelf(64)),
8657+
DL, VT);
8658+
}
8659+
86448660
/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
86458661
/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
86468662
/// specified in the intrinsic calls.

‎llvm/lib/Target/AArch64/AArch64ISelLowering.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -748,6 +748,7 @@ class AArch64TargetLowering : public TargetLowering {
748748
SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
749749
SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
750750
SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
751+
SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
751752
SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
752753
SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
753754
SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@ def AArch64ld1_gather_uxtw_scaled : SDNode<"AArch64ISD::GLD1_UXTW_SCALED",
4646
def AArch64ld1_gather_sxtw_scaled : SDNode<"AArch64ISD::GLD1_SXTW_SCALED", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
4747
def AArch64ld1_gather_imm : SDNode<"AArch64ISD::GLD1_IMM", SDT_AArch64_GLD1_IMM, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
4848

49+
// SVE CNT/INC/RDVL
50+
def sve_rdvl_imm : ComplexPattern<i32, 1, "SelectRDVLImm<-32, 31, 16>">;
51+
def sve_cnth_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 8>">;
52+
def sve_cntw_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 4>">;
53+
def sve_cntd_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 2>">;
54+
55+
// SVE DEC
56+
def sve_cnth_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -8>">;
57+
def sve_cntw_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -4>">;
58+
def sve_cntd_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -2>">;
59+
4960
def AArch64ld1s_gather : SDNode<"AArch64ISD::GLD1S", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
5061
def AArch64ld1s_gather_scaled : SDNode<"AArch64ISD::GLD1S_SCALED", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
5162
def AArch64ld1s_gather_uxtw : SDNode<"AArch64ISD::GLD1S_UXTW", SDT_AArch64_GLD1, [SDNPHasChain, SDNPMayLoad, SDNPOptInGlue]>;
@@ -1105,6 +1116,23 @@ let Predicates = [HasSVE] in {
11051116
def : Pat<(sext_inreg (nxv4i32 ZPR:$Zs), nxv4i8), (SXTB_ZPmZ_S (IMPLICIT_DEF), (PTRUE_S 31), ZPR:$Zs)>;
11061117
def : Pat<(sext_inreg (nxv8i16 ZPR:$Zs), nxv8i8), (SXTB_ZPmZ_H (IMPLICIT_DEF), (PTRUE_H 31), ZPR:$Zs)>;
11071118

1119+
// General case that we ideally never want to match.
1120+
def : Pat<(vscale GPR64:$scale), (MADDXrrr (UBFMXri (RDVLI_XI 1), 4, 63), $scale, XZR)>;
1121+
1122+
let AddedComplexity = 5 in {
1123+
def : Pat<(vscale (i64 1)), (UBFMXri (RDVLI_XI 1), 4, 63)>;
1124+
def : Pat<(vscale (i64 -1)), (SBFMXri (RDVLI_XI -1), 4, 63)>;
1125+
1126+
def : Pat<(vscale (sve_rdvl_imm i32:$imm)), (RDVLI_XI $imm)>;
1127+
def : Pat<(vscale (sve_cnth_imm i32:$imm)), (CNTH_XPiI 31, $imm)>;
1128+
def : Pat<(vscale (sve_cntw_imm i32:$imm)), (CNTW_XPiI 31, $imm)>;
1129+
def : Pat<(vscale (sve_cntd_imm i32:$imm)), (CNTD_XPiI 31, $imm)>;
1130+
1131+
def : Pat<(vscale (sve_cnth_imm_neg i32:$imm)), (SUBXrs XZR, (CNTH_XPiI 31, $imm), 0)>;
1132+
def : Pat<(vscale (sve_cntw_imm_neg i32:$imm)), (SUBXrs XZR, (CNTW_XPiI 31, $imm), 0)>;
1133+
def : Pat<(vscale (sve_cntd_imm_neg i32:$imm)), (SUBXrs XZR, (CNTD_XPiI 31, $imm), 0)>;
1134+
}
1135+
11081136
def : Pat<(nxv16i8 (bitconvert (nxv8i16 ZPR:$src))), (nxv16i8 ZPR:$src)>;
11091137
def : Pat<(nxv16i8 (bitconvert (nxv4i32 ZPR:$src))), (nxv16i8 ZPR:$src)>;
11101138
def : Pat<(nxv16i8 (bitconvert (nxv2i64 ZPR:$src))), (nxv16i8 ZPR:$src)>;

0 commit comments

Comments
 (0)