[VPlan] Replace VPWidenCastRecipe by VPInstructionWithType (NFC) (WIP). #129712
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
…(NFC) There are some opcodes that currently require specialized recipes, due to their result type not being implied by their operands, including casts. This leads to duplication from defining multiple full recipes. This patch introduces a new VPInstructionWithType subclass that also stores the result type. The general idea is to have opcodes needing to specify a result type to use this general recipe. The current patch replaces VPScalarCastRecipe with VInstructionWithType, a similar patch for VPWidenCastRecipe will follow soon. There are a few proposed opcodes that should also benefit, without the need of workarounds: * llvm#129508 * llvm#119284
WIP as it depends on llvm#129706.
|
@llvm/pr-subscribers-vectorizers Author: Florian Hahn (fhahn) ChangesWIP as it depends on #129706. Patch is 92.14 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/129712.diff 30 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ed3e45dd2c6c8..1f4bef08b81ce 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -246,15 +246,13 @@ class VPBuilder {
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Name));
}
- VPScalarCastRecipe *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy, DebugLoc DL) {
- return tryInsertInstruction(
- new VPScalarCastRecipe(Opcode, Op, ResultTy, DL));
- }
-
- VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy) {
- return tryInsertInstruction(new VPWidenCastRecipe(Opcode, Op, ResultTy));
+ VPInstructionWithType *createCast(Instruction::CastOps Opcode, VPValue *Op,
+ Type *ResultTy, DebugLoc DL = {},
+ const Twine &Name = "",
+ Instruction *CI = nullptr) {
+ auto *VPI = new VPInstructionWithType(Opcode, {Op}, ResultTy, DL, Name);
+ VPI->setUnderlyingValue(CI);
+ return tryInsertInstruction(VPI);
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cb860a472d8f7..6a5d4d3057664 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4434,8 +4434,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
[](const auto *R) { return Instruction::Load; })
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
[](const auto *R) { return Instruction::Call; })
- .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCastRecipe>(
+ .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe>(
[](const auto *R) { return R->getOpcode(); })
.Case<VPInterleaveRecipe>([](const VPInterleaveRecipe *R) {
return R->getStoredValues().empty() ? Instruction::Load
@@ -4496,15 +4495,11 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
if (EphemeralRecipes.contains(&R))
continue;
// Continue early if the recipe is considered to not produce a vector
- // result. Note that this includes VPInstruction where some opcodes may
- // produce a vector, to preserve existing behavior as VPInstructions model
- // aspects not directly mapped to existing IR instructions.
+ // result.
switch (R.getVPDefID()) {
case VPDef::VPDerivedIVSC:
case VPDef::VPScalarIVStepsSC:
- case VPDef::VPScalarCastSC:
case VPDef::VPReplicateSC:
- case VPDef::VPInstructionSC:
case VPDef::VPCanonicalIVPHISC:
case VPDef::VPVectorPointerSC:
case VPDef::VPReverseVectorPointerSC:
@@ -4517,7 +4512,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPActiveLaneMaskPHISC:
case VPDef::VPWidenCallSC:
case VPDef::VPWidenCanonicalIVSC:
- case VPDef::VPWidenCastSC:
case VPDef::VPWidenGEPSC:
case VPDef::VPWidenIntrinsicSC:
case VPDef::VPWidenSC:
@@ -4534,6 +4528,15 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPWidenStoreEVLSC:
case VPDef::VPWidenStoreSC:
break;
+ case VPDef::VPInstructionSC: {
+ // Note that for VPInstruction some opcodes may produce a vector. To
+ // preserve existing behavior only consider them vector-generating if
+ // they are casts with an underlying value.
+ if (Instruction::isCast(cast<VPInstruction>(&R)->getOpcode()) &&
+ R.getVPSingleValue()->getUnderlyingValue())
+ break;
+ continue;
+ }
default:
llvm_unreachable("unhandled recipe");
}
@@ -8938,8 +8941,15 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
}
if (auto *CI = dyn_cast<CastInst>(Instr)) {
- return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
- *CI);
+ auto *VPI =
+ isa<PossiblyNonNegInst>(CI)
+ ? new VPInstructionWithType(CI->getOpcode(), {Operands[0]},
+ CI->getType(), {CI->hasNonNeg()}, {})
+ : new VPInstructionWithType(CI->getOpcode(), {Operands[0]},
+ CI->getType(), {});
+
+ VPI->setUnderlyingValue(CI);
+ return VPI;
}
return tryToWiden(Instr, Operands);
@@ -9061,9 +9071,9 @@ static VPInstruction *addResumePhiRecipeForInduction(
// the widest induction) and thus may be wider than the induction here.
Type *ScalarTypeOfWideIV = TypeInfo.inferScalarType(WideIV);
if (ScalarTypeOfWideIV != TypeInfo.inferScalarType(EndValue)) {
- EndValue = VectorPHBuilder.createScalarCast(Instruction::Trunc, EndValue,
- ScalarTypeOfWideIV,
- WideIV->getDebugLoc());
+ EndValue =
+ VectorPHBuilder.createCast(Instruction::Trunc, EndValue,
+ ScalarTypeOfWideIV, WideIV->getDebugLoc());
}
auto *ResumePhiRecipe =
@@ -9861,12 +9871,12 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
RdxDesc.getRecurrenceKind())) {
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
Type *RdxTy = RdxDesc.getRecurrenceType();
- auto *Trunc =
- new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+ auto *Trunc = new VPInstructionWithType(Instruction::Trunc, NewExitingVPV,
+ RdxTy, {});
auto *Extnd =
RdxDesc.isSigned()
- ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
- : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
+ ? new VPInstructionWithType(Instruction::SExt, Trunc, PhiTy, {})
+ : new VPInstructionWithType(Instruction::ZExt, Trunc, PhiTy, {});
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
Extnd->insertAfter(Trunc);
@@ -10396,8 +10406,10 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
assert(all_of(IV->users(),
[](const VPUser *U) {
return isa<VPScalarIVStepsRecipe>(U) ||
- isa<VPScalarCastRecipe>(U) ||
isa<VPDerivedIVRecipe>(U) ||
+ Instruction::isCast(
+ cast<VPInstruction>(U)->getOpcode()) ||
+
cast<VPInstruction>(U)->getOpcode() ==
Instruction::Add;
}) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b1288c42b20f2..f47109156741a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -519,7 +519,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPReverseVectorPointerSC:
case VPRecipeBase::VPWidenCallSC:
case VPRecipeBase::VPWidenCanonicalIVSC:
- case VPRecipeBase::VPWidenCastSC:
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPWidenIntrinsicSC:
case VPRecipeBase::VPWidenSC:
@@ -533,7 +532,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
case VPRecipeBase::VPReductionPHISC:
- case VPRecipeBase::VPScalarCastSC:
case VPRecipeBase::VPScalarPHISC:
case VPRecipeBase::VPPartialReductionSC:
return true;
@@ -599,13 +597,15 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
DisjointFlagsTy(bool IsDisjoint) : IsDisjoint(IsDisjoint) {}
};
+ struct NonNegFlagsTy {
+ char NonNeg : 1;
+ NonNegFlagsTy(bool IsNonNeg = false) : NonNeg(IsNonNeg) {}
+ };
+
private:
struct ExactFlagsTy {
char IsExact : 1;
};
- struct NonNegFlagsTy {
- char NonNeg : 1;
- };
struct FastMathFlagsTy {
char AllowReassoc : 1;
char NoNaNs : 1;
@@ -699,6 +699,12 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::DisjointOp),
DisjointFlags(DisjointFlags) {}
+ template <typename IterT>
+ VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
+ NonNegFlagsTy NonNegFlags, DebugLoc DL = {})
+ : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::NonNegOp),
+ NonNegFlags(NonNegFlags) {}
+
protected:
template <typename IterT>
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
@@ -711,7 +717,6 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
R->getVPDefID() == VPRecipeBase::VPWidenSC ||
R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
@@ -954,6 +959,12 @@ class VPInstruction : public VPRecipeWithIRFlags,
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ NonNegFlagsTy NonNegFlags, DebugLoc DL = {},
+ const Twine &Name = "")
+ : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, NonNegFlags, DL),
+ Opcode(Opcode), Name(Name.str()) {}
+
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPInstruction *clone() override {
@@ -1026,6 +1037,60 @@ class VPInstruction : public VPRecipeWithIRFlags,
StringRef getName() const { return Name; }
};
+/// A specialization of VPInstruction augmenting it with a dedicated result
+/// type, to be used when the opcode and operands of the VPInstruction don't
+/// directly determine the result type.
+class VPInstructionWithType : public VPInstruction {
+ /// Scalar result type produced by the recipe.
+ Type *ResultTy;
+
+ Value *generate(VPTransformState &State);
+
+public:
+ VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, DebugLoc DL, const Twine &Name = "")
+ : VPInstruction(Opcode, Operands, DL, Name), ResultTy(ResultTy) {}
+
+ VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, NonNegFlagsTy Flags, DebugLoc DL,
+ const Twine &Name = "")
+ : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {}
+
+ static inline bool classof(const VPRecipeBase *R) {
+ auto *VPI = dyn_cast<VPInstruction>(R);
+ return VPI && Instruction::isCast(VPI->getOpcode());
+ }
+
+ static inline bool classof(const VPUser *R) {
+ return isa<VPInstructionWithType>(cast<VPRecipeBase>(R));
+ }
+
+ VPInstruction *clone() override {
+ auto *New =
+ new VPInstructionWithType(getOpcode(), {getOperand(0)}, getResultType(),
+ {}, getDebugLoc(), getName());
+ New->setUnderlyingValue(getUnderlyingValue());
+ New->transferFlags(*this);
+ return New;
+ }
+
+ void execute(VPTransformState &State) override;
+
+ /// Return the cost of this VPIRInstruction.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+ Type *getResultType() const { return ResultTy; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ bool onlyFirstLaneUsed(const VPValue *Op) const override;
+};
+
/// A recipe to wrap on original IR instruction not to be modified during
/// execution, execept for PHIs. For PHIs, a single VPValue operand is allowed,
/// and it is used to add a new incoming value for the single predecessor VPBB.
@@ -1131,106 +1196,6 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
#endif
};
-/// VPWidenCastRecipe is a recipe to create vector cast instructions.
-class VPWidenCastRecipe : public VPRecipeWithIRFlags {
- /// Cast instruction opcode.
- Instruction::CastOps Opcode;
-
- /// Result type for the cast.
- Type *ResultTy;
-
-public:
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
- ResultTy(ResultTy) {
- assert(UI.getOpcode() == Opcode &&
- "opcode of underlying cast doesn't match");
- }
-
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
- ResultTy(ResultTy) {}
-
- ~VPWidenCastRecipe() override = default;
-
- VPWidenCastRecipe *clone() override {
- if (auto *UV = getUnderlyingValue())
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
- *cast<CastInst>(UV));
-
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
- }
-
- VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
-
- /// Produce widened copies of the cast.
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPWidenCastRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- Instruction::CastOps getOpcode() const { return Opcode; }
-
- /// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
-};
-
-/// VPScalarCastRecipe is a recipe to create scalar cast instructions.
-class VPScalarCastRecipe : public VPSingleDefRecipe {
- Instruction::CastOps Opcode;
-
- Type *ResultTy;
-
- Value *generate(VPTransformState &State);
-
-public:
- VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- DebugLoc DL)
- : VPSingleDefRecipe(VPDef::VPScalarCastSC, {Op}, DL), Opcode(Opcode),
- ResultTy(ResultTy) {}
-
- ~VPScalarCastRecipe() override = default;
-
- VPScalarCastRecipe *clone() override {
- return new VPScalarCastRecipe(Opcode, getOperand(0), ResultTy,
- getDebugLoc());
- }
-
- VP_CLASSOF_IMPL(VPDef::VPScalarCastSC)
-
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPScalarCastRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override {
- // TODO: Compute accurate cost after retiring the legacy cost model.
- return 0;
- }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- /// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
-
- bool onlyFirstLaneUsed(const VPValue *Op) const override {
- // At the moment, only uniform codegen is implemented.
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-};
-
/// A recipe for widening vector intrinsics.
class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
/// ID of the vector intrinsic to widen.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 6f6875f0e5e0e..028aebd18cf53 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -252,20 +252,15 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
VPPartialReductionRecipe>([this](const VPRecipeBase *R) {
return inferScalarType(R->getOperand(0));
})
+ .Case<VPInstructionWithType, VPWidenIntrinsicRecipe>(
+ [](const auto *R) { return R->getResultType(); })
.Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
- .Case<VPWidenIntrinsicRecipe>([](const VPWidenIntrinsicRecipe *R) {
- return R->getResultType();
- })
.Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) {
// TODO: Use info from interleave group.
return V->getUnderlyingValue()->getType();
})
- .Case<VPWidenCastRecipe>(
- [](const VPWidenCastRecipe *R) { return R->getResultType(); })
- .Case<VPScalarCastRecipe>(
- [](const VPScalarCastRecipe *R) { return R->getResultType(); })
.Case<VPExpandSCEVRecipe>([](const VPExpandSCEVRecipe *R) {
return R->getSCEV()->getType();
})
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 8c11d93734667..3594b36bdee08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -204,7 +204,7 @@ using UnaryVPInstruction_match =
template <typename Op0_t, unsigned Opcode>
using AllUnaryRecipe_match =
UnaryRecipe_match<Op0_t, Opcode, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCastRecipe, VPInstruction>;
+ VPInstruction>;
template <typename Op0_t, typename Op1_t, unsigned Opcode, bool Commutative,
typename... RecipeTys>
@@ -220,7 +220,7 @@ template <typename Op0_t, typename Op1_t, unsigned Opcode,
bool Commutative = false>
using AllBinaryRecipe_match =
BinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative, VPWidenRecipe,
- VPReplicateRecipe, VPWidenCastRecipe, VPInstruction>;
+ VPReplicateRecipe, VPInstruction>;
template <unsigned Opcode, typename Op0_t>
inline UnaryVPInstruction_match<Op0_t, Opcode>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d154d54c37862..2a8e00dc649fa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -79,7 +79,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPReductionSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenLoadEVLSC:
@@ -126,7 +125,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPReductionSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
@@ -148,7 +146,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
switch (getVPDefID()) {
case VPDerivedIVSC:
case VPPredInstPHISC:
- case VPScalarCastSC:
case VPReverseVectorPointerSC:
return false;
case VPInstructionSC:
@@ -165,7 +162,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPScalarIVStepsSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
@@ -311,7 +307,7 @@ VPPartialReductionRecipe::computeCost(ElementCount VF,
// The extend could come from outside the plan.
if (!R)
return TargetTransformInfo::PR_None;
- auto *WidenCastR = dyn_cast<VPWidenCastRecipe>(R);
+ auto *WidenCastR = dyn_cast<VPInstructionWithType>(R);
if (!WidenCastR)
return TargetTransformInfo::PR_None;
if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
@@ -413,7 +409,...
[truncated]
|
|
@llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesWIP as it depends on #129706. Patch is 92.14 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/129712.diff 30 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ed3e45dd2c6c8..1f4bef08b81ce 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -246,15 +246,13 @@ class VPBuilder {
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Name));
}
- VPScalarCastRecipe *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy, DebugLoc DL) {
- return tryInsertInstruction(
- new VPScalarCastRecipe(Opcode, Op, ResultTy, DL));
- }
-
- VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
- Type *ResultTy) {
- return tryInsertInstruction(new VPWidenCastRecipe(Opcode, Op, ResultTy));
+ VPInstructionWithType *createCast(Instruction::CastOps Opcode, VPValue *Op,
+ Type *ResultTy, DebugLoc DL = {},
+ const Twine &Name = "",
+ Instruction *CI = nullptr) {
+ auto *VPI = new VPInstructionWithType(Opcode, {Op}, ResultTy, DL, Name);
+ VPI->setUnderlyingValue(CI);
+ return tryInsertInstruction(VPI);
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index cb860a472d8f7..6a5d4d3057664 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4434,8 +4434,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
[](const auto *R) { return Instruction::Load; })
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
[](const auto *R) { return Instruction::Call; })
- .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCastRecipe>(
+ .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe>(
[](const auto *R) { return R->getOpcode(); })
.Case<VPInterleaveRecipe>([](const VPInterleaveRecipe *R) {
return R->getStoredValues().empty() ? Instruction::Load
@@ -4496,15 +4495,11 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
if (EphemeralRecipes.contains(&R))
continue;
// Continue early if the recipe is considered to not produce a vector
- // result. Note that this includes VPInstruction where some opcodes may
- // produce a vector, to preserve existing behavior as VPInstructions model
- // aspects not directly mapped to existing IR instructions.
+ // result.
switch (R.getVPDefID()) {
case VPDef::VPDerivedIVSC:
case VPDef::VPScalarIVStepsSC:
- case VPDef::VPScalarCastSC:
case VPDef::VPReplicateSC:
- case VPDef::VPInstructionSC:
case VPDef::VPCanonicalIVPHISC:
case VPDef::VPVectorPointerSC:
case VPDef::VPReverseVectorPointerSC:
@@ -4517,7 +4512,6 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPActiveLaneMaskPHISC:
case VPDef::VPWidenCallSC:
case VPDef::VPWidenCanonicalIVSC:
- case VPDef::VPWidenCastSC:
case VPDef::VPWidenGEPSC:
case VPDef::VPWidenIntrinsicSC:
case VPDef::VPWidenSC:
@@ -4534,6 +4528,15 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPWidenStoreEVLSC:
case VPDef::VPWidenStoreSC:
break;
+ case VPDef::VPInstructionSC: {
+ // Note that for VPInstruction some opcodes may produce a vector. To
+ // preserve existing behavior only consider them vector-generating if
+ // they are casts with an underlying value.
+ if (Instruction::isCast(cast<VPInstruction>(&R)->getOpcode()) &&
+ R.getVPSingleValue()->getUnderlyingValue())
+ break;
+ continue;
+ }
default:
llvm_unreachable("unhandled recipe");
}
@@ -8938,8 +8941,15 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
}
if (auto *CI = dyn_cast<CastInst>(Instr)) {
- return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
- *CI);
+ auto *VPI =
+ isa<PossiblyNonNegInst>(CI)
+ ? new VPInstructionWithType(CI->getOpcode(), {Operands[0]},
+ CI->getType(), {CI->hasNonNeg()}, {})
+ : new VPInstructionWithType(CI->getOpcode(), {Operands[0]},
+ CI->getType(), {});
+
+ VPI->setUnderlyingValue(CI);
+ return VPI;
}
return tryToWiden(Instr, Operands);
@@ -9061,9 +9071,9 @@ static VPInstruction *addResumePhiRecipeForInduction(
// the widest induction) and thus may be wider than the induction here.
Type *ScalarTypeOfWideIV = TypeInfo.inferScalarType(WideIV);
if (ScalarTypeOfWideIV != TypeInfo.inferScalarType(EndValue)) {
- EndValue = VectorPHBuilder.createScalarCast(Instruction::Trunc, EndValue,
- ScalarTypeOfWideIV,
- WideIV->getDebugLoc());
+ EndValue =
+ VectorPHBuilder.createCast(Instruction::Trunc, EndValue,
+ ScalarTypeOfWideIV, WideIV->getDebugLoc());
}
auto *ResumePhiRecipe =
@@ -9861,12 +9871,12 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
RdxDesc.getRecurrenceKind())) {
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
Type *RdxTy = RdxDesc.getRecurrenceType();
- auto *Trunc =
- new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+ auto *Trunc = new VPInstructionWithType(Instruction::Trunc, NewExitingVPV,
+ RdxTy, {});
auto *Extnd =
RdxDesc.isSigned()
- ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
- : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
+ ? new VPInstructionWithType(Instruction::SExt, Trunc, PhiTy, {})
+ : new VPInstructionWithType(Instruction::ZExt, Trunc, PhiTy, {});
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
Extnd->insertAfter(Trunc);
@@ -10396,8 +10406,10 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
assert(all_of(IV->users(),
[](const VPUser *U) {
return isa<VPScalarIVStepsRecipe>(U) ||
- isa<VPScalarCastRecipe>(U) ||
isa<VPDerivedIVRecipe>(U) ||
+ Instruction::isCast(
+ cast<VPInstruction>(U)->getOpcode()) ||
+
cast<VPInstruction>(U)->getOpcode() ==
Instruction::Add;
}) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b1288c42b20f2..f47109156741a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -519,7 +519,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPReverseVectorPointerSC:
case VPRecipeBase::VPWidenCallSC:
case VPRecipeBase::VPWidenCanonicalIVSC:
- case VPRecipeBase::VPWidenCastSC:
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPWidenIntrinsicSC:
case VPRecipeBase::VPWidenSC:
@@ -533,7 +532,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
case VPRecipeBase::VPReductionPHISC:
- case VPRecipeBase::VPScalarCastSC:
case VPRecipeBase::VPScalarPHISC:
case VPRecipeBase::VPPartialReductionSC:
return true;
@@ -599,13 +597,15 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
DisjointFlagsTy(bool IsDisjoint) : IsDisjoint(IsDisjoint) {}
};
+ struct NonNegFlagsTy {
+ char NonNeg : 1;
+ NonNegFlagsTy(bool IsNonNeg = false) : NonNeg(IsNonNeg) {}
+ };
+
private:
struct ExactFlagsTy {
char IsExact : 1;
};
- struct NonNegFlagsTy {
- char NonNeg : 1;
- };
struct FastMathFlagsTy {
char AllowReassoc : 1;
char NoNaNs : 1;
@@ -699,6 +699,12 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
: VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::DisjointOp),
DisjointFlags(DisjointFlags) {}
+ template <typename IterT>
+ VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
+ NonNegFlagsTy NonNegFlags, DebugLoc DL = {})
+ : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::NonNegOp),
+ NonNegFlags(NonNegFlags) {}
+
protected:
template <typename IterT>
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
@@ -711,7 +717,6 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
R->getVPDefID() == VPRecipeBase::VPWidenSC ||
R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
R->getVPDefID() == VPRecipeBase::VPReverseVectorPointerSC ||
@@ -954,6 +959,12 @@ class VPInstruction : public VPRecipeWithIRFlags,
VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ NonNegFlagsTy NonNegFlags, DebugLoc DL = {},
+ const Twine &Name = "")
+ : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, NonNegFlags, DL),
+ Opcode(Opcode), Name(Name.str()) {}
+
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPInstruction *clone() override {
@@ -1026,6 +1037,60 @@ class VPInstruction : public VPRecipeWithIRFlags,
StringRef getName() const { return Name; }
};
+/// A specialization of VPInstruction augmenting it with a dedicated result
+/// type, to be used when the opcode and operands of the VPInstruction don't
+/// directly determine the result type.
+class VPInstructionWithType : public VPInstruction {
+ /// Scalar result type produced by the recipe.
+ Type *ResultTy;
+
+ Value *generate(VPTransformState &State);
+
+public:
+ VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, DebugLoc DL, const Twine &Name = "")
+ : VPInstruction(Opcode, Operands, DL, Name), ResultTy(ResultTy) {}
+
+ VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, NonNegFlagsTy Flags, DebugLoc DL,
+ const Twine &Name = "")
+ : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {}
+
+ static inline bool classof(const VPRecipeBase *R) {
+ auto *VPI = dyn_cast<VPInstruction>(R);
+ return VPI && Instruction::isCast(VPI->getOpcode());
+ }
+
+ static inline bool classof(const VPUser *R) {
+ return isa<VPInstructionWithType>(cast<VPRecipeBase>(R));
+ }
+
+ VPInstruction *clone() override {
+ auto *New =
+ new VPInstructionWithType(getOpcode(), {getOperand(0)}, getResultType(),
+ {}, getDebugLoc(), getName());
+ New->setUnderlyingValue(getUnderlyingValue());
+ New->transferFlags(*this);
+ return New;
+ }
+
+ void execute(VPTransformState &State) override;
+
+ /// Return the cost of this VPIRInstruction.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
+ Type *getResultType() const { return ResultTy; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ bool onlyFirstLaneUsed(const VPValue *Op) const override;
+};
+
/// A recipe to wrap on original IR instruction not to be modified during
/// execution, execept for PHIs. For PHIs, a single VPValue operand is allowed,
/// and it is used to add a new incoming value for the single predecessor VPBB.
@@ -1131,106 +1196,6 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
#endif
};
-/// VPWidenCastRecipe is a recipe to create vector cast instructions.
-class VPWidenCastRecipe : public VPRecipeWithIRFlags {
- /// Cast instruction opcode.
- Instruction::CastOps Opcode;
-
- /// Result type for the cast.
- Type *ResultTy;
-
-public:
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
- ResultTy(ResultTy) {
- assert(UI.getOpcode() == Opcode &&
- "opcode of underlying cast doesn't match");
- }
-
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
- ResultTy(ResultTy) {}
-
- ~VPWidenCastRecipe() override = default;
-
- VPWidenCastRecipe *clone() override {
- if (auto *UV = getUnderlyingValue())
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
- *cast<CastInst>(UV));
-
- return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
- }
-
- VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
-
- /// Produce widened copies of the cast.
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPWidenCastRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- Instruction::CastOps getOpcode() const { return Opcode; }
-
- /// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
-};
-
-/// VPScalarCastRecipe is a recipe to create scalar cast instructions.
-class VPScalarCastRecipe : public VPSingleDefRecipe {
- Instruction::CastOps Opcode;
-
- Type *ResultTy;
-
- Value *generate(VPTransformState &State);
-
-public:
- VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- DebugLoc DL)
- : VPSingleDefRecipe(VPDef::VPScalarCastSC, {Op}, DL), Opcode(Opcode),
- ResultTy(ResultTy) {}
-
- ~VPScalarCastRecipe() override = default;
-
- VPScalarCastRecipe *clone() override {
- return new VPScalarCastRecipe(Opcode, getOperand(0), ResultTy,
- getDebugLoc());
- }
-
- VP_CLASSOF_IMPL(VPDef::VPScalarCastSC)
-
- void execute(VPTransformState &State) override;
-
- /// Return the cost of this VPScalarCastRecipe.
- InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override {
- // TODO: Compute accurate cost after retiring the legacy cost model.
- return 0;
- }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- /// Returns the result type of the cast.
- Type *getResultType() const { return ResultTy; }
-
- bool onlyFirstLaneUsed(const VPValue *Op) const override {
- // At the moment, only uniform codegen is implemented.
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-};
-
/// A recipe for widening vector intrinsics.
class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
/// ID of the vector intrinsic to widen.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 6f6875f0e5e0e..028aebd18cf53 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -252,20 +252,15 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
VPPartialReductionRecipe>([this](const VPRecipeBase *R) {
return inferScalarType(R->getOperand(0));
})
+ .Case<VPInstructionWithType, VPWidenIntrinsicRecipe>(
+ [](const auto *R) { return R->getResultType(); })
.Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
- .Case<VPWidenIntrinsicRecipe>([](const VPWidenIntrinsicRecipe *R) {
- return R->getResultType();
- })
.Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) {
// TODO: Use info from interleave group.
return V->getUnderlyingValue()->getType();
})
- .Case<VPWidenCastRecipe>(
- [](const VPWidenCastRecipe *R) { return R->getResultType(); })
- .Case<VPScalarCastRecipe>(
- [](const VPScalarCastRecipe *R) { return R->getResultType(); })
.Case<VPExpandSCEVRecipe>([](const VPExpandSCEVRecipe *R) {
return R->getSCEV()->getType();
})
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 8c11d93734667..3594b36bdee08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -204,7 +204,7 @@ using UnaryVPInstruction_match =
template <typename Op0_t, unsigned Opcode>
using AllUnaryRecipe_match =
UnaryRecipe_match<Op0_t, Opcode, VPWidenRecipe, VPReplicateRecipe,
- VPWidenCastRecipe, VPInstruction>;
+ VPInstruction>;
template <typename Op0_t, typename Op1_t, unsigned Opcode, bool Commutative,
typename... RecipeTys>
@@ -220,7 +220,7 @@ template <typename Op0_t, typename Op1_t, unsigned Opcode,
bool Commutative = false>
using AllBinaryRecipe_match =
BinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative, VPWidenRecipe,
- VPReplicateRecipe, VPWidenCastRecipe, VPInstruction>;
+ VPReplicateRecipe, VPInstruction>;
template <unsigned Opcode, typename Op0_t>
inline UnaryVPInstruction_match<Op0_t, Opcode>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d154d54c37862..2a8e00dc649fa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -79,7 +79,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPReductionSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenLoadEVLSC:
@@ -126,7 +125,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPReductionSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
@@ -148,7 +146,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
switch (getVPDefID()) {
case VPDerivedIVSC:
case VPPredInstPHISC:
- case VPScalarCastSC:
case VPReverseVectorPointerSC:
return false;
case VPInstructionSC:
@@ -165,7 +162,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPScalarIVStepsSC:
case VPVectorPointerSC:
case VPWidenCanonicalIVSC:
- case VPWidenCastSC:
case VPWidenGEPSC:
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
@@ -311,7 +307,7 @@ VPPartialReductionRecipe::computeCost(ElementCount VF,
// The extend could come from outside the plan.
if (!R)
return TargetTransformInfo::PR_None;
- auto *WidenCastR = dyn_cast<VPWidenCastRecipe>(R);
+ auto *WidenCastR = dyn_cast<VPInstructionWithType>(R);
if (!WidenCastR)
return TargetTransformInfo::PR_None;
if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
@@ -413,7 +409,...
[truncated]
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
WIP as it depends on #129706.