[VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). #142322

fhahn · 2025-06-01T21:30:17Z

Replace RdxDesc with RecurKind in VPReductionPHIRecipe, as
all VPlan analyses and codegen only require the recurrence kind. This
enables creating new VPReductionPHIRecipe directly in LV, without
needing to construction a whole RecurrenceDescriptor object.

Depends on
#141860
#141932
#142290
#142291

llvmbot · 2025-06-01T21:30:52Z

@llvm/pr-subscribers-llvm-transforms
@llvm/pr-subscribers-backend-powerpc

@llvm/pr-subscribers-vectorizers

Author: Florian Hahn (fhahn)

Changes

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as
all VPlan analyses and codegen only require the recurrence kind. This
enables creating new VPReductionPHIRecipe directly in LV, without
needing to construction a whole RecurrenceDescriptor object.

Depends on
#141860
#141932
#142290
#142291

Patch is 65.64 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/142322.diff

18 Files Affected:

(modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+116-52)
(modified) llvm/lib/Transforms/Vectorize/VPlan.h (+18-15)
(modified) llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp (+4)
(modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+25)
(modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+59-86)
(modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+11-2)
(modified) llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp (+21-2)
(modified) llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll (+1-1)
(modified) llvm/test/Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll (+1-1)
(modified) llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll (+3-3)
(modified) llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll (+2-2)
(modified) llvm/test/Transforms/LoopVectorize/X86/cost-model.ll (+2-3)
(modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll (+2-4)
(modified) llvm/test/Transforms/LoopVectorize/if-pred-stores.ll (+1-1)
(modified) llvm/test/Transforms/LoopVectorize/induction.ll (+5-5)
(modified) llvm/test/Transforms/LoopVectorize/reduction-small-size.ll (+3-6)
(modified) llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll (+1-3)
(modified) llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll (+13-7)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e9ace195684b3..5a77ac74cd4d9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7485,6 +7485,13 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
   }
 }
 
+static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
+  using namespace VPlanPatternMatch;
+  VPValue *StartVPV = RdxResult->getOperand(1);
+  match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
+  return StartVPV->getLiveInIRValue();
+}
+
 // If \p R is a ComputeReductionResult when vectorizing the epilog loop,
 // fix the reduction's scalar PHI node by adding the incoming value from the
 // main vector loop.
@@ -7493,39 +7500,41 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
     BasicBlock *BypassBlock) {
   auto *EpiRedResult = dyn_cast<VPInstruction>(R);
   if (!EpiRedResult ||
-      (EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
+      (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
+       EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
        EpiRedResult->getOpcode() != VPInstruction::ComputeFindLastIVResult))
     return;
 
   auto *EpiRedHeaderPhi =
       cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
-  const RecurrenceDescriptor &RdxDesc =
-      EpiRedHeaderPhi->getRecurrenceDescriptor();
+  RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
   Value *MainResumeValue =
       EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
-  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-          RdxDesc.getRecurrenceKind())) {
+  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
+    Value *StartV = EpiRedResult->getOperand(1)->getLiveInIRValue();
+    (void)StartV;
+
     auto *Cmp = cast<ICmpInst>(MainResumeValue);
     assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
            "AnyOf expected to start with ICMP_NE");
-    assert(Cmp->getOperand(1) == RdxDesc.getRecurrenceStartValue() &&
+    assert(Cmp->getOperand(1) == StartV &&
            "AnyOf expected to start by comparing main resume value to original "
            "start value");
     MainResumeValue = Cmp->getOperand(0);
-  } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
-                 RdxDesc.getRecurrenceKind())) {
+  } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind)) {
+    Value *StartV = getStartValueFromReductionResult(EpiRedResult);
+    (void)StartV;
     using namespace llvm::PatternMatch;
     Value *Cmp, *OrigResumeV, *CmpOp;
     bool IsExpectedPattern =
-        match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
-                                        m_Specific(RdxDesc.getSentinelValue()),
-                                        m_Value(OrigResumeV))) &&
+        match(MainResumeValue,
+              m_Select(
+                  m_OneUse(m_Value(Cmp)),
+                  m_Specific(EpiRedResult->getOperand(2)->getLiveInIRValue()),
+                  m_Value(OrigResumeV))) &&
         (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
                                    m_Value(CmpOp))) &&
-         (match(CmpOp,
-                m_Freeze(m_Specific(RdxDesc.getRecurrenceStartValue()))) ||
-          (CmpOp == RdxDesc.getRecurrenceStartValue() &&
-           isGuaranteedNotToBeUndefOrPoison(CmpOp))));
+         ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
     assert(IsExpectedPattern && "Unexpected reduction resume pattern");
     (void)IsExpectedPattern;
     MainResumeValue = OrigResumeV;
@@ -7536,6 +7545,13 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
   // over the incoming values correctly.
   using namespace VPlanPatternMatch;
+  if (EpiRedResult->getNumUsers() == 1 &&
+      isa<VPInstructionWithType>(*EpiRedResult->user_begin())) {
+    EpiRedResult = cast<VPInstructionWithType>(*EpiRedResult->user_begin());
+    assert((EpiRedResult->getOpcode() == Instruction::SExt ||
+            EpiRedResult->getOpcode() == Instruction::ZExt) &&
+           "can only have SExt/ZExt users");
+  }
   assert(count_if(EpiRedResult->users(), IsaPred<VPPhi>) == 1 &&
          "ResumePhi must have a single user");
   auto *EpiResumePhiVPI =
@@ -8552,6 +8568,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
       // If the PHI is used by a partial reduction, set the scale factor.
       unsigned ScaleFactor =
           getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
+
       PhiRecipe = new VPReductionPHIRecipe(
           Phi, RdxDesc, *StartV, CM.isInLoopReduction(Phi),
           CM.useOrderedReductions(RdxDesc), ScaleFactor);
@@ -9301,8 +9318,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind Kind = RdxDesc.getRecurrenceKind();
+    RecurKind Kind = PhiR->getRecurrenceKind();
     assert(
         !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
@@ -9408,6 +9424,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
         CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
 
+      const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+          cast<PHINode>(PhiR->getUnderlyingInstr()));
       // Non-FP RdxDescs will have all fast math flags set, so clear them.
       FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
                                ? RdxDesc.getFastMathFlags()
@@ -9438,8 +9456,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR)
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    Type *PhiTy = PhiR->getOperand(0)->getLiveInIRValue()->getType();
+    const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+        cast<PHINode>(PhiR->getUnderlyingInstr()));
+    Type *PhiTy = PhiR->getUnderlyingValue()->getType();
     // If tail is folded by masking, introduce selects between the phi
     // and the users outside the vector region of each reduction, at the
     // beginning of the dedicated latch block.
@@ -9460,7 +9479,10 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
         return isa<VPInstruction>(&U) &&
                (cast<VPInstruction>(&U)->getOpcode() ==
+                    VPInstruction::ComputeAnyOfResult ||
+                cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeReductionResult ||
+
                 cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeFindLastIVResult);
       });
@@ -9468,28 +9490,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
         PhiR->setOperand(1, NewExitingVPV);
     }
 
-    // If the vector reduction can be performed in a smaller type, we truncate
-    // then extend the loop exit value to enable InstCombine to evaluate the
-    // entire expression in the smaller type.
-    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
-        !RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
-      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
-      Type *RdxTy = RdxDesc.getRecurrenceType();
-      auto *Trunc =
-          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
-      auto *Extnd =
-          RdxDesc.isSigned()
-              ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
-              : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
-
-      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
-      Extnd->insertAfter(Trunc);
-      if (PhiR->getOperand(1) == NewExitingVPV)
-        PhiR->setOperand(1, Extnd->getVPSingleValue());
-      NewExitingVPV = Extnd;
-    }
-
     // We want code in the middle block to appear to execute on the location of
     // the scalar loop's latch terminator because: (a) it is all compiler
     // generated, (b) these instructions are always executed after evaluating
@@ -9511,6 +9511,14 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       VPValue *Start = PhiR->getStartValue();
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
+          {PhiR, Start, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()),
+           NewExitingVPV},
+          ExitDL);
+    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
+                   RdxDesc.getRecurrenceKind())) {
+      VPValue *Start = PhiR->getStartValue();
+      FinalReductionResult =
+          Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
                                {PhiR, Start, NewExitingVPV}, ExitDL);
     } else {
       VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind(
@@ -9521,6 +9529,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
           Builder.createNaryOp(VPInstruction::ComputeReductionResult,
                                {PhiR, NewExitingVPV}, Flags, ExitDL);
     }
+    // If the vector reduction can be performed in a smaller type, we truncate
+    // then extend the loop exit value to enable InstCombine to evaluate the
+    // entire expression in the smaller type.
+    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
+        !RecurrenceDescriptor::isAnyOfRecurrenceKind(
+            RdxDesc.getRecurrenceKind())) {
+      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
+      Type *RdxTy = RdxDesc.getRecurrenceType();
+      auto *Trunc =
+          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+      Instruction::CastOps ExtendOpc =
+          RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
+      auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
+      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
+      Extnd->insertAfter(Trunc);
+      if (PhiR->getOperand(1) == NewExitingVPV)
+        PhiR->setOperand(1, Extnd->getVPSingleValue());
+
+      // Update ComputeReductionResult with the truncated exiting value and
+      // extend its result.
+      FinalReductionResult->setOperand(1, Trunc);
+      FinalReductionResult =
+          Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
+    }
+
     // Update all users outside the vector region.
     OrigExitingVPV->replaceUsesWithIf(
         FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {
@@ -9569,6 +9602,27 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       // start value.
       PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
     }
+    RecurKind RK = RdxDesc.getRecurrenceKind();
+    if (PhiR->isOrdered() || PhiR->isInLoop() ||
+        (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))) {
+      VPBuilder PHBuilder(Plan->getVectorPreheader());
+      VPValue *Iden = Plan->getOrAddLiveIn(
+          getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
+      // If the PHI is used by a partial reduction, set the scale factor.
+      unsigned ScaleFactor =
+          RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
+              .value_or(1);
+      Type *I32Ty = IntegerType::getInt32Ty(PhiTy->getContext());
+      auto *ScalarFactorVPV =
+          Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
+      VPValue *StartV =
+          PHBuilder.createNaryOp(VPInstruction::ReductionStartVector,
+                                 {PhiR->getStartValue(), Iden, ScalarFactorVPV},
+                                 RdxDesc.getFastMathFlags());
+      PhiR->setOperand(0, StartV);
+    }
   }
   for (VPRecipeBase *R : ToDelete)
     R->eraseFromParent();
@@ -10040,23 +10094,28 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
     Value *ResumeV = nullptr;
     // TODO: Move setting of resume values to prepareToExecute.
     if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+      auto *RdxResult =
+          cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
+            auto *VPI = dyn_cast<VPInstruction>(U);
+            return VPI &&
+                   (VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
+                    VPI->getOpcode() == VPInstruction::ComputeFindLastIVResult);
+          }));
       ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
                     ->getIncomingValueForBlock(L->getLoopPreheader());
-      const RecurrenceDescriptor &RdxDesc =
-          ReductionPhi->getRecurrenceDescriptor();
-      RecurKind RK = RdxDesc.getRecurrenceKind();
+      RecurKind RK = ReductionPhi->getRecurrenceKind();
       if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+        Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
         // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
         // start value; compare the final value from the main vector loop
         // to the start value.
         BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
-        ResumeV =
-            Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
+        ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
       } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
-        ToFrozen[RdxDesc.getRecurrenceStartValue()] =
-            cast<PHINode>(ResumeV)->getIncomingValueForBlock(
-                EPI.MainLoopIterationCountCheck);
+        Value *StartV = getStartValueFromReductionResult(RdxResult);
+        ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
+            EPI.MainLoopIterationCountCheck);
 
         // VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
         // to the resume value. The resume value is adjusted to the sentinel
@@ -10066,10 +10125,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
         // variable.
         BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
-        Value *Cmp = Builder.CreateICmpEQ(
-            ResumeV, ToFrozen[RdxDesc.getRecurrenceStartValue()]);
+        Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
         ResumeV =
-            Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
+            Builder.CreateSelect(Cmp, RdxResult->getOperand(2)->getLiveInIRValue(), ResumeV);
       }
     } else {
       // Retrieve the induction resume values for wide inductions from
@@ -10081,6 +10139,12 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
     }
     assert(ResumeV && "Must have a resume value");
     VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
+    if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R)) {
+      if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
+        VPI->setOperand(0, StartVal);
+        continue;
+      }
+    }
     cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
   }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 44f0b6d964a6e..e5e6b1ad6fb58 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -907,6 +907,11 @@ class VPInstruction : public VPRecipeWithIRFlags,
     BranchOnCount,
     BranchOnCond,
     Broadcast,
+    /// Start vector for reductions with 3 operands: the original start value,
+    /// the identity value for the reduction and an integer indicating the
+    /// scaling factor.
+    ReductionStartVector,
+    ComputeAnyOfResult,
     ComputeFindLastIVResult,
     ComputeReductionResult,
     // Extracts the last lane from its operand if it is a vector, or the last
@@ -2168,7 +2173,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
 class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
                              public VPUnrollPartAccessor<2> {
   /// Descriptor for the reduction.
-  const RecurrenceDescriptor &RdxDesc;
+  const RecurKind Kind;
 
   /// The phi is part of an in-loop reduction.
   bool IsInLoop;
@@ -2187,8 +2192,15 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
                        VPValue &Start, bool IsInLoop = false,
                        bool IsOrdered = false, unsigned VFScaleFactor = 1)
       : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
-        RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
-        VFScaleFactor(VFScaleFactor) {
+        Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop),
+        IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
+    assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
+  }
+  VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
+                       bool IsInLoop = false, bool IsOrdered = false,
+                       unsigned VFScaleFactor = 1)
+      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
+        IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
     assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
   }
 
@@ -2196,8 +2208,8 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
 
   VPReductionPHIRecipe *clone() override {
     auto *R = new VPReductionPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
-                                       RdxDesc, *getOperand(0), IsInLoop,
-                                       IsOrdered, VFScaleFactor);
+                                       getRecurrenceKind(), *getOperand(0),
+                                       IsInLoop, IsOrdered, VFScaleFactor);
     R->addOperand(getBackedgeValue());
     return R;
   }
@@ -2216,22 +2228,13 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
              VPSlotTracker &SlotTracker) const override;
 #endif
 
-  const RecurrenceDescriptor &getRecurrenceDescriptor() const {
-    return RdxDesc;
-  }
+  RecurKind getRecurrenceKind() const { return Kind; }
 
   /// Returns true, if the phi is part of an ordered reduction.
   bool isOrdered() const { return IsOrdered; }
 
   /// Returns true, if the phi is part of an in-loop reduction.
   bool isInLoop() const { return IsInLoop; }
-
-  /// Returns true if the recipe only uses the first lane of operand \p Op.
-  bool onlyFirstLaneUsed(const VPValue *Op) const override {
-    assert(is_contained(operands(), Op) &&
-           "Op must be an operand of the recipe");
-    return Op == getStartValue();
-  }
 };
 
 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 926490bfad7d0..0287cf36a3cda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -87,6 +87,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
                inferScalarType(R->getOperand(1)) &&
            "different types inferred for different operands");
     return IntegerType::get(Ctx, 1);
+  case VPInstruction::ReductionStartVector:
+    return inferScalarType(R->getOperand(0));
+  case VPInstruction::ComputeAnyOfResult:
+    return inferScalarType(R->getOperand(1));
   case VPInstruction::ComputeFindLastIVResult:
   case VPInstruction::ComputeReductionResult: {
     auto *PhiR = cast<VPReductionPHIRecipe>(R->getOperand(0));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index f2a7f16e19a79..b2535fe3aa578 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -318,6 +318,31 @@ m_VPInstruction(const Op0_t &Op0, const Op1...
[truncated]

github-actions · 2025-06-01T21:32:39Z

✅ With the latest revision this PR passed the C/C++ code formatter.

david-arm · 2025-06-09T12:38:29Z

In the commit message should this be Replace RdxDesc with RecurKind in VPReductionPHIRecipe instead?

fhahn

In the commit message should this be Replace RdxDesc with RecurKind in VPReductionPHIRecipe instead?

Ah yes, should be updated, thanks

fhahn

ping :)

all dependencies have now landed

artagnon · 2025-07-01T09:13:34Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+        cast<PHINode>(PhiR->getUnderlyingInstr()));


Can you comment on why this is a good change? Are we planning to never use Recurrence descriptors in the future?

Currently all information needed is already available in the recipes together with the recurrence kind. Having just the recurrence kind makes it easier to create new VPReductionPHIRecipes directly in VPlan, as we do not need to create new RecurrenceDescriptor objects.

One use case enabled by this is #141431, and I'll share another one soon.

Better to use getRecurrenceDescriptor you created.

Done thanks

artagnon · 2025-07-01T09:14:32Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

-              Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
-                                  RdxPart, ReducedPartRdx, "bin.rdx");
+          ReducedPartRdx = Builder.CreateBinOp(
+              (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),


Change this C-style cast to a static_cast?

Updated thanks

+1, can be done independently, keeping this patch focused on the RdxDesc.getOpcode() --> getOpcode(RK) transition.

Will do separately, thanks

ayalz

Sorry, wrong window :-(

Mel-Chen · 2025-07-02T08:37:09Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+      const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+          cast<PHINode>(PhiR->getUnderlyingInstr()));


Could we have a wrap function to get RecurrenceDescriptor?
I think we may need an assertion to check if the RecurrenceDescriptor is existing.

Done, thanks

Wrap function can also be useful elsewhere, independently.

Mel-Chen · 2025-07-03T15:44:10Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+        cast<PHINode>(PhiR->getUnderlyingInstr()));


Better to use getRecurrenceDescriptor you created.

Mel-Chen · 2025-07-03T15:45:35Z

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -2211,16 +2211,24 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
                       VPValue &Start, bool IsInLoop = false,
                       bool IsOrdered = false, unsigned VFScaleFactor = 1)
      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
-        RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
-        VFScaleFactor(VFScaleFactor) {
+        Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop),


Why we still need constructor with argument RecurrenceDescriptor?

No, removed, thanks

Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm#141860 llvm#141932 llvm#142290 llvm#142291

ayalz

Review after (my earlier premature) approval.

ayalz · 2025-07-06T09:12:10Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

-              Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
-                                  RdxPart, ReducedPartRdx, "bin.rdx");
+          ReducedPartRdx = Builder.CreateBinOp(
+              (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),


+1, can be done independently, keeping this patch focused on the RdxDesc.getOpcode() --> getOpcode(RK) transition.

ayalz · 2025-07-06T09:18:43Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -751,8 +751,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
    // and will be removed by breaking up the recipe further.
    auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
    // Get its reduction variable descriptor.
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind RK = RdxDesc.getRecurrenceKind();
+    [[maybe_unused]] RecurKind RK = PhiR->getRecurrenceKind();


Suggested change

[[maybe_unused]] RecurKind RK = PhiR->getRecurrenceKind();

RecurKind RK = PhiR->getRecurrenceKind();

current RK is set w/o [[maybe_unused]], surely used to set IsSigned below?

Yep dropped, thanks

ayalz · 2025-07-06T09:20:02Z

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

+          static_cast<Instruction::BinaryOps>(
+              RecurrenceDescriptor::getOpcode(RecurKind::AnyOf)),


Can be done independently.

Sounds good , will do separately, thanks

ayalz · 2025-07-06T09:21:49Z

llvm/lib/Transforms/Vectorize/VPlan.h

-  const RecurrenceDescriptor &getRecurrenceDescriptor() const {
-    return RdxDesc;
-  }
+  RecurKind getRecurrenceKind() const { return Kind; }


Suggested change

RecurKind getRecurrenceKind() const { return Kind; }

/// Returns the recurrence kind of the reduction.

RecurKind getRecurrenceKind() const { return Kind; }

Added thanks

ayalz · 2025-07-06T09:22:06Z

llvm/lib/Transforms/Vectorize/VPlan.h

    assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
  }

  ~VPReductionPHIRecipe() override = default;

  VPReductionPHIRecipe *clone() override {
+


Suggested change

stripped thanks

ayalz · 2025-07-06T12:14:49Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -8310,7 +8307,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
      unsigned ScaleFactor =


Above

const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().find(Phi)->second;

can be replaced (independently) with

const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);

?

Similar candidates in
LoopVectorizationCostModel::collectElementTypesForWidening(), LoopVectorizationCostModel::getReductionPatternCost().

Updated in c5fff13, thanks

ayalz · 2025-07-06T12:26:50Z

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

+    assert(Reductions.contains(PN) && "no recurrence descriptor for phi");
+    return Reductions.lookup(PN);
+  }
+


Independent: can update below
bool isReductionVariable(PHINode *PN) const { return Reductions.count(PN); }
to
bool isReductionVariable(PHINode *PN) const { return Reductions.contains(PN); }

ayalz · 2025-07-06T12:28:06Z

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

@@ -301,6 +301,11 @@ class LoopVectorizationLegality {
  /// Returns the reduction variables found in the loop.
  const ReductionList &getReductionVars() const { return Reductions; }

+  RecurrenceDescriptor getRecurrenceDescriptor(PHINode *PN) const {
+    assert(Reductions.contains(PN) && "no recurrence descriptor for phi");


Suggested change

assert(Reductions.contains(PN) && "no recurrence descriptor for phi");

assert(isReductionVariable(PN) && "only reductions have recurrence descriptors");

Done in split off c5fff13, thanks

ayalz · 2025-07-06T14:25:43Z

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

@@ -301,6 +301,11 @@ class LoopVectorizationLegality {
  /// Returns the reduction variables found in the loop.
  const ReductionList &getReductionVars() const { return Reductions; }

+  RecurrenceDescriptor getRecurrenceDescriptor(PHINode *PN) const {
+    assert(Reductions.contains(PN) && "no recurrence descriptor for phi");
+    return Reductions.lookup(PN);


Independent: can search and replace several usages.

Done in split off c5fff13, thanks

ayalz · 2025-07-06T16:30:16Z

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

+      const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
+          cast<PHINode>(PhiR->getUnderlyingInstr()));


Wrap function can also be useful elsewhere, independently.

…-with-recurkind

Split off adding helper to retrieve RecurrenceDescriptor as suggested from #142322.

…-with-recurkind

Split off adding helper to retrieve RecurrenceDescriptor as suggested from llvm/llvm-project#142322.

Directly retrieve isOrdered from the reduction phi recipe instead of going through the legacy cost model. Split off as suggested in #142322.

…-with-recurkind

…(NFC). Directly retrieve isOrdered from the reduction phi recipe instead of going through the legacy cost model. Split off as suggested in llvm/llvm-project#142322.

…cipe (NFC). (#142322) Replace RdxDesc with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on llvm/llvm-project#141860 llvm/llvm-project#141932 llvm/llvm-project#142290 llvm/llvm-project#142291 PR: llvm/llvm-project#142322

fhahn requested review from rengolin, Mel-Chen, ayalz and aniragil June 1, 2025 21:30

llvmbot added backend:PowerPC vectorizers llvm:transforms labels Jun 1, 2025

fhahn mentioned this pull request Jun 1, 2025

[LV] Vectorize selecting last IV of min/max element. #141431

Open

fhahn force-pushed the vplan-replace-rdxdesc-with-recurkind branch from 0a5bd5f to 4cba7de Compare June 25, 2025 09:50

fhahn commented Jun 25, 2025

View reviewed changes

fhahn force-pushed the vplan-replace-rdxdesc-with-recurkind branch 3 times, most recently from 5c79627 to 629af4f Compare June 30, 2025 21:45

fhahn commented Jun 30, 2025

View reviewed changes

artagnon reviewed Jul 1, 2025

View reviewed changes

ayalz approved these changes Jul 1, 2025

View reviewed changes

fhahn force-pushed the vplan-replace-rdxdesc-with-recurkind branch from 629af4f to 85a044b Compare July 2, 2025 08:22

Mel-Chen reviewed Jul 2, 2025

View reviewed changes

fhahn force-pushed the vplan-replace-rdxdesc-with-recurkind branch from 85a044b to 8949101 Compare July 2, 2025 14:45

fhahn mentioned this pull request Jul 2, 2025

[LV] Vectorize FMax w/o fast-math flags. #146711

Open

Mel-Chen reviewed Jul 3, 2025

View reviewed changes

fhahn added 3 commits July 3, 2025 17:25

!fixup use static cast, add helper.

5d0c744

!fixup unify constructors, use helper

dca0d4b

fhahn force-pushed the vplan-replace-rdxdesc-with-recurkind branch from 8949101 to dca0d4b Compare July 3, 2025 17:41

ayalz approved these changes Jul 6, 2025

View reviewed changes

Merge remote-tracking branch 'origin/main' into vplan-replace-rdxdesc…

4c2e494

…-with-recurkind

fhahn added a commit that referenced this pull request Jul 6, 2025

[LV] Add LVL::getRecurrenceDescriptor (NFC).

c5fff13

Split off adding helper to retrieve RecurrenceDescriptor as suggested from #142322.

Merge remote-tracking branch 'origin/main' into vplan-replace-rdxdesc…

0ab71f6

…-with-recurkind

llvm-sync bot pushed a commit to arm/arm-toolchain that referenced this pull request Jul 6, 2025

Automerge: [LV] Add LVL::getRecurrenceDescriptor (NFC).

02d5ab1

Split off adding helper to retrieve RecurrenceDescriptor as suggested from llvm/llvm-project#142322.

!fixup address comments, thanks

cb4b75d

fhahn added a commit that referenced this pull request Jul 6, 2025

[VPlan] Use VPReductionPHIRecipe::isOrdered instead of CM (NFC).

c35fbb5

Directly retrieve isOrdered from the reduction phi recipe instead of going through the legacy cost model. Split off as suggested in #142322.

fhahn added 2 commits July 6, 2025 20:52

Merge remote-tracking branch 'origin/main' into vplan-replace-rdxdesc…

db65c6f

…-with-recurkind

!fixup remove unrelated change

0bd4d2a

fhahn merged commit 6a9a16d into llvm:main Jul 6, 2025
8 of 9 checks passed

fhahn deleted the vplan-replace-rdxdesc-with-recurkind branch July 6, 2025 20:40

		const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup(
		cast<PHINode>(PhiR->getUnderlyingInstr()));

	[[maybe_unused]] RecurKind RK = PhiR->getRecurrenceKind();
	RecurKind RK = PhiR->getRecurrenceKind();

		static_cast<Instruction::BinaryOps>(
		RecurrenceDescriptor::getOpcode(RecurKind::AnyOf)),

	RecurKind getRecurrenceKind() const { return Kind; }
	/// Returns the recurrence kind of the reduction.
	RecurKind getRecurrenceKind() const { return Kind; }

		@@ -8310,7 +8307,7 @@ VPRecipeBase VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe R,
		unsigned ScaleFactor =

	assert(Reductions.contains(PN) && "no recurrence descriptor for phi");
	assert(isReductionVariable(PN) && "only reductions have recurrence descriptors");

[VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). #142322

[VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). #142322

Conversation

fhahn commented Jun 1, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented Jun 1, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

github-actions bot commented Jun 1, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

david-arm commented Jun 9, 2025

Uh oh!

fhahn left a comment

Choose a reason for hiding this comment

Uh oh!

fhahn left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

ayalz left a comment • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

ayalz left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

fhahn commented Jun 1, 2025 •

edited

Loading

llvmbot commented Jun 1, 2025 •

edited

Loading

github-actions bot commented Jun 1, 2025 •

edited

Loading

ayalz left a comment •

edited

Loading