Skip to content

Commit d1279c2

Browse files
LiqinWengLiqinWeng
committed
[LV][VPlan] Extract the implementation of transform Recipe to EVLRecipe into a small function
1 parent 7f27bc1 commit d1279c2

File tree

1 file changed

+89
-97
lines changed

1 file changed

+89
-97
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 89 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,13 +1442,93 @@ void VPlanTransforms::addActiveLaneMask(
14421442
HeaderMask->replaceAllUsesWith(LaneMask);
14431443
}
14441444

1445+
// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
1446+
// nullptr if no EVL-based recipe could be created.
1447+
// \p HeaderMask Header Mask.
1448+
// \p CurRecipe Recipe to be transform.
1449+
// \p TypeInfo VPlan-based type analysis.
1450+
// \p AllOneMask The vector mask parameter of vector-predication intrinsics.
1451+
// \p EVL The explicit vector length parameter of vector-predication
1452+
// intrinsics.
1453+
static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
1454+
VPRecipeBase &CurRecipe,
1455+
VPTypeAnalysis &TypeInfo,
1456+
VPValue &AllOneMask, VPValue &EVL) {
1457+
using namespace llvm::VPlanPatternMatch;
1458+
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1459+
assert(OrigMask && "Unmasked recipe when folding tail");
1460+
return HeaderMask == OrigMask ? nullptr : OrigMask;
1461+
};
1462+
1463+
return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(&CurRecipe)
1464+
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1465+
VPValue *NewMask = GetNewMask(L->getMask());
1466+
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1467+
})
1468+
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1469+
VPValue *NewMask = GetNewMask(S->getMask());
1470+
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1471+
})
1472+
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1473+
unsigned Opcode = W->getOpcode();
1474+
if (!Instruction::isBinaryOp(Opcode) && !Instruction::isUnaryOp(Opcode))
1475+
return nullptr;
1476+
return new VPWidenEVLRecipe(*W, EVL);
1477+
})
1478+
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1479+
VPValue *NewMask = GetNewMask(Red->getCondOp());
1480+
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1481+
})
1482+
.Case<VPWidenIntrinsicRecipe, VPWidenCastRecipe>(
1483+
[&](auto *CR) -> VPRecipeBase * {
1484+
Intrinsic::ID VPID;
1485+
if (auto *CallR = dyn_cast<VPWidenIntrinsicRecipe>(CR))
1486+
VPID =
1487+
VPIntrinsic::getForIntrinsic(CallR->getVectorIntrinsicID());
1488+
else if (auto *CastR = dyn_cast<VPWidenCastRecipe>(CR))
1489+
VPID = VPIntrinsic::getForOpcode(CastR->getOpcode());
1490+
assert(VPID != Intrinsic::not_intrinsic && "Expected VP intrinsic");
1491+
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1492+
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1493+
"Expected VP intrinsic");
1494+
1495+
SmallVector<VPValue *> Ops(CR->operands());
1496+
Ops.push_back(&AllOneMask);
1497+
Ops.push_back(&EVL);
1498+
return new VPWidenIntrinsicRecipe(
1499+
VPID, Ops, TypeInfo.inferScalarType(CR), CR->getDebugLoc());
1500+
})
1501+
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1502+
SmallVector<VPValue *> Ops(Sel->operands());
1503+
Ops.push_back(&EVL);
1504+
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1505+
TypeInfo.inferScalarType(Sel),
1506+
Sel->getDebugLoc());
1507+
})
1508+
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1509+
VPValue *LHS, *RHS;
1510+
// Transform select with a header mask condition
1511+
// select(header_mask, LHS, RHS)
1512+
// into vector predication merge.
1513+
// vp.merge(all-true, LHS, RHS, EVL)
1514+
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1515+
m_VPValue(RHS))))
1516+
return nullptr;
1517+
// Use all true as the condition because this transformation is
1518+
// limited to selects whose condition is a header mask.
1519+
return new VPWidenIntrinsicRecipe(
1520+
Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
1521+
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1522+
})
1523+
.Default([&](VPRecipeBase *R) { return nullptr; });
1524+
}
1525+
14451526
/// Replace recipes with their EVL variants.
14461527
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
1447-
using namespace llvm::VPlanPatternMatch;
14481528
Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
14491529
VPTypeAnalysis TypeInfo(CanonicalIVType);
14501530
LLVMContext &Ctx = CanonicalIVType->getContext();
1451-
SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan);
1531+
VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
14521532

14531533
for (VPUser *U : Plan.getVF().users()) {
14541534
if (auto *R = dyn_cast<VPReverseVectorPointerRecipe>(U))
@@ -1460,110 +1540,22 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
14601540
for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
14611541
for (VPUser *U : collectUsersRecursively(HeaderMask)) {
14621542
auto *CurRecipe = cast<VPRecipeBase>(U);
1463-
auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1464-
assert(OrigMask && "Unmasked recipe when folding tail");
1465-
return HeaderMask == OrigMask ? nullptr : OrigMask;
1466-
};
1467-
1468-
VPRecipeBase *NewRecipe =
1469-
TypeSwitch<VPRecipeBase *, VPRecipeBase *>(CurRecipe)
1470-
.Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
1471-
VPValue *NewMask = GetNewMask(L->getMask());
1472-
return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
1473-
})
1474-
.Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
1475-
VPValue *NewMask = GetNewMask(S->getMask());
1476-
return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
1477-
})
1478-
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
1479-
unsigned Opcode = W->getOpcode();
1480-
if (!Instruction::isBinaryOp(Opcode) &&
1481-
!Instruction::isUnaryOp(Opcode))
1482-
return nullptr;
1483-
return new VPWidenEVLRecipe(*W, EVL);
1484-
})
1485-
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
1486-
VPValue *NewMask = GetNewMask(Red->getCondOp());
1487-
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
1488-
})
1489-
.Case<VPWidenIntrinsicRecipe>(
1490-
[&](VPWidenIntrinsicRecipe *CallR) -> VPRecipeBase * {
1491-
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1492-
CallR->getVectorIntrinsicID());
1493-
assert(VPID != Intrinsic::not_intrinsic &&
1494-
"Expected vp.casts Instrinsic");
1495-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1496-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1497-
"Expected VP intrinsic");
1498-
1499-
SmallVector<VPValue *> Ops(CallR->operands());
1500-
VPValue *Mask =
1501-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1502-
Ops.push_back(Mask);
1503-
Ops.push_back(&EVL);
1504-
return new VPWidenIntrinsicRecipe(
1505-
VPID, Ops, TypeInfo.inferScalarType(CallR),
1506-
CallR->getDebugLoc());
1507-
})
1508-
.Case<VPWidenCastRecipe>(
1509-
[&](VPWidenCastRecipe *CastR) -> VPRecipeBase * {
1510-
Intrinsic::ID VPID =
1511-
VPIntrinsic::getForOpcode(CastR->getOpcode());
1512-
assert(VPID != Intrinsic::not_intrinsic &&
1513-
"Expected vp.casts Instrinsic");
1514-
1515-
SmallVector<VPValue *> Ops(CastR->operands());
1516-
assert(VPIntrinsic::getMaskParamPos(VPID) &&
1517-
VPIntrinsic::getVectorLengthParamPos(VPID) &&
1518-
"Expected VP intrinsic");
1519-
VPValue *Mask =
1520-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1521-
Ops.push_back(Mask);
1522-
Ops.push_back(&EVL);
1523-
return new VPWidenIntrinsicRecipe(
1524-
VPID, Ops, TypeInfo.inferScalarType(CastR),
1525-
CastR->getDebugLoc());
1526-
})
1527-
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
1528-
SmallVector<VPValue *> Ops(Sel->operands());
1529-
Ops.push_back(&EVL);
1530-
return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
1531-
TypeInfo.inferScalarType(Sel),
1532-
Sel->getDebugLoc());
1533-
})
1534-
.Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
1535-
VPValue *LHS, *RHS;
1536-
// Transform select with a header mask condition
1537-
// select(header_mask, LHS, RHS)
1538-
// into vector predication merge.
1539-
// vp.merge(all-true, LHS, RHS, EVL)
1540-
if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
1541-
m_VPValue(RHS))))
1542-
return nullptr;
1543-
// Use all true as the condition because this transformation is
1544-
// limited to selects whose condition is a header mask.
1545-
VPValue *AllTrue =
1546-
Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
1547-
return new VPWidenIntrinsicRecipe(
1548-
Intrinsic::vp_merge, {AllTrue, LHS, RHS, &EVL},
1549-
TypeInfo.inferScalarType(LHS), VPI->getDebugLoc());
1550-
})
1551-
.Default([&](VPRecipeBase *R) { return nullptr; });
1552-
1553-
if (!NewRecipe)
1543+
VPRecipeBase *EVLRecipe =
1544+
createEVLRecipe(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
1545+
if (!EVLRecipe)
15541546
continue;
15551547

1556-
[[maybe_unused]] unsigned NumDefVal = NewRecipe->getNumDefinedValues();
1548+
[[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
15571549
assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
15581550
"New recipe must define the same number of values as the "
15591551
"original.");
15601552
assert(
15611553
NumDefVal <= 1 &&
15621554
"Only supports recipes with a single definition or without users.");
1563-
NewRecipe->insertBefore(CurRecipe);
1564-
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(NewRecipe)) {
1555+
EVLRecipe->insertBefore(CurRecipe);
1556+
if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
15651557
VPValue *CurVPV = CurRecipe->getVPSingleValue();
1566-
CurVPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
1558+
CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
15671559
}
15681560
// Defer erasing recipes till the end so that we don't invalidate the
15691561
// VPTypeAnalysis cache.

0 commit comments

Comments
 (0)