Address comments and split off abstract recipes creation from adjustR…

…ecipesForReduction.

llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

@@ -509,7 +509,7 @@ class LoopVectorizationPlanner {
   // between the phi and users outside the vector region when folding the tail.
   void adjustRecipesForReductions(VPlanPtr &Plan,
                                   VPRecipeBuilder &RecipeBuilder,
-                                  VFRange &Range);
+                                  ElementCount MinVF);
 
 #ifndef NDEBUG
   /// \return The most profitable vectorization factor for the available VPlans

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -233,7 +233,7 @@ static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
                          "prefers tail-folding, don't attempt vectorization if "
                          "tail-folding fails.")));
 
-static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
+cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
     "force-tail-folding-style", cl::desc("Force the tail folding style"),
     cl::init(TailFoldingStyle::None),
     cl::values(
@@ -9313,7 +9313,12 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   // ---------------------------------------------------------------------------
 
   // Adjust the recipes for any inloop reductions.
-  adjustRecipesForReductions(Plan, RecipeBuilder, Range);
+  adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+
+  // Transform recipes to abstract recipes if it is beneficial and clamp
+  // the range.
+  VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM);
+  VPlanTransforms::convertToAbstractRecipes(*Plan, CostCtx, Range);
 
   // Interleave memory: for each Interleave Group we marked earlier as relevant
   // for this VPlan, replace the Recipes widening its memory instructions with a
@@ -9435,9 +9440,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
 // with a boolean reduction phi node to check if the condition is true in any
 // iteration. The final value is selected by the final ComputeReductionResult.
 void LoopVectorizationPlanner::adjustRecipesForReductions(
-    VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, VFRange &Range) {
+    VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
   using namespace VPlanPatternMatch;
-  ElementCount MinVF = Range.Start;
   VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
   VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
   VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
@@ -9555,30 +9559,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       if (CM.blockNeedsPredicationForAnyReason(BB))
         CondOp = RecipeBuilder.getBlockInMask(BB);
 
-      VPReductionRecipe *RedRecipe;
-      VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
-                            CM);
-      bool IsOrderedRed = CM.useOrderedReductions(RdxDesc);
-      // TODO: Remove EVL check when we support EVL version of
-      // VPExtendedReductionRecipe and VPMulAccumulateReductionRecipe.
-      if (ForceTailFoldingStyle == TailFoldingStyle::DataWithEVL) {
-        RedRecipe = new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink,
-                                          VecOp, CondOp, IsOrderedRed);
-      } else {
-        if (auto *MulAcc =
-                VPlanTransforms::tryToMatchAndCreateMulAccumulateReduction(
-                    RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp,
-                    IsOrderedRed, CostCtx, Range))
-          RedRecipe = MulAcc;
-        else if (auto *ExtRed =
-                     VPlanTransforms::tryToMatchAndCreateExtendedReduction(
-                         RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp,
-                         IsOrderedRed, CostCtx, Range))
-          RedRecipe = ExtRed;
-        else
-          RedRecipe = new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink,
-                                            VecOp, CondOp, IsOrderedRed);
-      }
+      VPReductionRecipe *RedRecipe =
+          new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink, VecOp,
+                                CondOp, CM.useOrderedReductions(RdxDesc));
       // Append the recipe to the end of the VPBasicBlock because we need to
       // ensure that it comes after all of it's inputs, including CondOp.
       // Note that this transformation may leave over dead recipes (including

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1578,7 +1578,7 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags {
         ResultTy(ResultTy) {}
 
   VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
-                    bool IsNonNeg, DebugLoc DL)
+                    bool IsNonNeg, DebugLoc DL = {})
       : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, NonNegFlagsTy(IsNonNeg),
                             DL),
         Opcode(Opcode), ResultTy(ResultTy) {}
@@ -2643,8 +2643,8 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
     }
   }
 
-  // For VPExtendedReductionRecipe.
-  // Note that IsNonNeg flag and the debug location are for extend instruction.
+  /// For VPExtendedReductionRecipe.
+  /// Note that IsNonNeg flag and the debug location are for extend instruction.
   VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
                     ArrayRef<VPValue *> Operands, VPValue *CondOp,
                     bool IsOrdered, bool IsNonNeg, DebugLoc DL)
@@ -2656,8 +2656,8 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
     }
   }
 
-  // For VPMulAccRecipe.
-  // Note that the NUW/NSW and DL are for mul instruction.
+  /// For VPMulAccumulateReductionRecipe.
+  /// Note that the NUW/NSW and DL are for mul instruction.
   VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
                     ArrayRef<VPValue *> Operands, VPValue *CondOp,
                     bool IsOrdered, bool NUW, bool NSW, DebugLoc DL)
@@ -2797,22 +2797,19 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
   DebugLoc RedDL;
 
 public:
-  VPExtendedReductionRecipe(const RecurrenceDescriptor &R, Instruction *RedI,
-                            VPValue *ChainOp, VPWidenCastRecipe *Ext,
-                            VPValue *CondOp, bool IsOrdered)
-      : VPReductionRecipe(VPDef::VPExtendedReductionSC, R,
-                          ArrayRef<VPValue *>({ChainOp, Ext->getOperand(0)}),
-                          CondOp, IsOrdered, Ext->isNonNeg(),
-                          Ext->getDebugLoc()),
-        ExtOp(Ext->getOpcode()), RedDL(RedI->getDebugLoc()) {}
-
-  /// Contructor for cloning VPExtendedReductionRecipe.
+  VPExtendedReductionRecipe(VPReductionRecipe *R, VPWidenCastRecipe *Ext)
+      : VPReductionRecipe(VPDef::VPExtendedReductionSC,
+                          R->getRecurrenceDescriptor(),
+                          {R->getChainOp(), Ext->getOperand(0)}, R->getCondOp(),
+                          R->isOrdered(), Ext->isNonNeg(), Ext->getDebugLoc()),
+        ExtOp(Ext->getOpcode()), RedDL(R->getDebugLoc()) {}
+
+  /// For cloning VPExtendedReductionRecipe.
   VPExtendedReductionRecipe(VPExtendedReductionRecipe *ExtRed)
       : VPReductionRecipe(
             VPDef::VPExtendedReductionSC, ExtRed->getRecurrenceDescriptor(),
-            ArrayRef<VPValue *>({ExtRed->getChainOp(), ExtRed->getVecOp()}),
-            ExtRed->getCondOp(), ExtRed->isOrdered(), ExtRed->isNonNeg(),
-            ExtRed->getExtDebugLoc()),
+            {ExtRed->getChainOp(), ExtRed->getVecOp()}, ExtRed->getCondOp(),
+            ExtRed->isOrdered(), ExtRed->isNonNeg(), ExtRed->getExtDebugLoc()),
         ExtOp(ExtRed->getExtOpcode()), RedDL(ExtRed->getRedDebugLoc()) {}
 
   ~VPExtendedReductionRecipe() override = default;
@@ -2843,15 +2840,16 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
     return getRecurrenceDescriptor().getRecurrenceType();
   }
 
+  /// Is the extend ZExt?
   bool isZExt() const { return getExtOpcode() == Instruction::ZExt; }
 
-  /// The Opcode of extend instruction.
+  /// The Opcode of extend recipe.
   Instruction::CastOps getExtOpcode() const { return ExtOp; }
 
-  /// Return the debug location of the extend instruction.
+  /// Return the debug location of the extend recipe.
   DebugLoc getExtDebugLoc() const { return getDebugLoc(); }
 
-  /// Return the debug location of the reduction instruction.
+  /// Return the debug location of the reduction recipe.
   DebugLoc getRedDebugLoc() const { return RedDL; }
 };
 
@@ -2865,7 +2863,7 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
   Instruction::CastOps ExtOp;
 
   /// Non-neg flag of the extend recipe.
-  bool IsNonNeg;
+  bool IsNonNeg = false;
 
   /// Debug location of extend recipes will be lowered to.
   DebugLoc Ext0DL;
@@ -2878,47 +2876,41 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
   bool IsExtended = false;
 
 public:
-  VPMulAccumulateReductionRecipe(const RecurrenceDescriptor &R,
-                                 Instruction *RedI, VPValue *ChainOp,
-                                 VPValue *CondOp, bool IsOrdered,
-                                 VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0,
+  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul,
+                                 VPWidenCastRecipe *Ext0,
                                  VPWidenCastRecipe *Ext1)
-      : VPReductionRecipe(VPDef::VPMulAccumulateReductionSC, R,
-                          ArrayRef<VPValue *>({ChainOp, Ext0->getOperand(0),
-                                               Ext1->getOperand(0)}),
-                          CondOp, IsOrdered, Mul->hasNoUnsignedWrap(),
-                          Mul->hasNoSignedWrap(), Mul->getDebugLoc()),
+      : VPReductionRecipe(
+            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceDescriptor(),
+            {R->getChainOp(), Ext0->getOperand(0), Ext1->getOperand(0)},
+            R->getCondOp(), R->isOrdered(), Mul->hasNoUnsignedWrap(),
+            Mul->hasNoSignedWrap(), Mul->getDebugLoc()),
         ExtOp(Ext0->getOpcode()), IsNonNeg(Ext0->isNonNeg()),
         Ext0DL(Ext0->getDebugLoc()), Ext1DL(Ext1->getDebugLoc()),
-        RedDL(RedI->getDebugLoc()) {
-    assert(R.getOpcode() == Instruction::Add &&
+        RedDL(R->getDebugLoc()) {
+    assert(getRecurrenceDescriptor().getOpcode() == Instruction::Add &&
            "The reduction instruction in MulAccumulateteReductionRecipe must "
            "be Add");
     IsExtended = true;
   }
 
-  VPMulAccumulateReductionRecipe(const RecurrenceDescriptor &R,
-                                 Instruction *RedI, VPValue *ChainOp,
-                                 VPValue *CondOp, bool IsOrdered,
-                                 VPWidenRecipe *Mul)
-      : VPReductionRecipe(VPDef::VPMulAccumulateReductionSC, R,
-                          ArrayRef<VPValue *>({ChainOp, Mul->getOperand(0),
-                                               Mul->getOperand(1)}),
-                          CondOp, IsOrdered, Mul->hasNoUnsignedWrap(),
-                          Mul->hasNoSignedWrap(), Mul->getDebugLoc()),
-        RedDL(RedI->getDebugLoc()) {
-    assert(R.getOpcode() == Instruction::Add &&
+  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul)
+      : VPReductionRecipe(
+            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceDescriptor(),
+            {R->getChainOp(), Mul->getOperand(0), Mul->getOperand(1)},
+            R->getCondOp(), R->isOrdered(), Mul->hasNoUnsignedWrap(),
+            Mul->hasNoSignedWrap(), Mul->getDebugLoc()),
+        RedDL(R->getDebugLoc()) {
+    assert(getRecurrenceDescriptor().getOpcode() == Instruction::Add &&
            "The reduction instruction in MulAccumulateReductionRecipe must be "
            "Add");
   }
 
-  /// Constructor for cloning VPMulAccumulateReductionRecipe.
+  /// For cloning VPMulAccumulateReductionRecipe.
   VPMulAccumulateReductionRecipe(VPMulAccumulateReductionRecipe *MulAcc)
       : VPReductionRecipe(
             VPDef::VPMulAccumulateReductionSC,
             MulAcc->getRecurrenceDescriptor(),
-            ArrayRef<VPValue *>({MulAcc->getChainOp(), MulAcc->getVecOp0(),
-                                 MulAcc->getVecOp1()}),
+            {MulAcc->getChainOp(), MulAcc->getVecOp0(), MulAcc->getVecOp1()},
             MulAcc->getCondOp(), MulAcc->isOrdered(),
             MulAcc->hasNoUnsignedWrap(), MulAcc->hasNoSignedWrap(),
             MulAcc->getMulDebugLoc()),
@@ -2970,7 +2962,7 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
     return ExtOp == Instruction::CastOps::ZExt;
   }
 
-  /// Return the non negative flag for the ext instruction.
+  /// Return the non negative flag of the ext recipe.
   bool isNonNeg() const { return IsNonNeg; }
 
   /// Drop flags in this recipe.
@@ -2980,14 +2972,14 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
     this->IsNonNeg = false;
   }
 
-  /// Return debug location of mul instruction.
+  /// Return debug location of mul recipe.
   DebugLoc getMulDebugLoc() const { return getDebugLoc(); }
 
-  /// Return debug location of extend instructions.
+  /// Return debug location of extend recipe.
   DebugLoc getExt0DebugLoc() const { return Ext0DL; }
   DebugLoc getExt1DebugLoc() const { return Ext1DL; }
 
-  /// Return the debug location of reduction instruction.
+  /// Return the debug location of reduction recipe.
   DebugLoc getRedDebugLoc() const { return RedDL; }
 };
 

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -1523,50 +1523,49 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
     setFlags(CastOp);
 }
 
-/// Computes the CastContextHint for a recipe.
-static TTI::CastContextHint computeCCH(const VPRecipeBase *R, ElementCount VF) {
-  if (VF.isScalar())
-    return TTI::CastContextHint::Normal;
-  if (isa<VPInterleaveRecipe>(R))
-    return TTI::CastContextHint::Interleave;
-  if (const auto *ReplicateRecipe = dyn_cast<VPReplicateRecipe>(R))
-    return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked
-                                           : TTI::CastContextHint::Normal;
-  const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
-  if (WidenMemoryRecipe == nullptr)
-    return TTI::CastContextHint::None;
-  if (!WidenMemoryRecipe->isConsecutive())
-    return TTI::CastContextHint::GatherScatter;
-  if (WidenMemoryRecipe->isReverse())
-    return TTI::CastContextHint::Reversed;
-  if (WidenMemoryRecipe->isMasked())
-    return TTI::CastContextHint::Masked;
-  return TTI::CastContextHint::Normal;
-}
-
 InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF,
                                                VPCostContext &Ctx) const {
   // TODO: In some cases, VPWidenCastRecipes are created but not considered in
   // the legacy cost model, including truncates/extends when evaluating a
   // reduction in a smaller type.
   if (!getUnderlyingValue())
     return 0;
+  // Computes the CastContextHint from a recipes that may access memory.
+  auto ComputeCCH = [&](const VPRecipeBase *R) -> TTI::CastContextHint {
+    if (VF.isScalar())
+      return TTI::CastContextHint::Normal;
+    if (isa<VPInterleaveRecipe>(R))
+      return TTI::CastContextHint::Interleave;
+    if (const auto *ReplicateRecipe = dyn_cast<VPReplicateRecipe>(R))
+      return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked
+                                             : TTI::CastContextHint::Normal;
+    const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
+    if (WidenMemoryRecipe == nullptr)
+      return TTI::CastContextHint::None;
+    if (!WidenMemoryRecipe->isConsecutive())
+      return TTI::CastContextHint::GatherScatter;
+    if (WidenMemoryRecipe->isReverse())
+      return TTI::CastContextHint::Reversed;
+    if (WidenMemoryRecipe->isMasked())
+      return TTI::CastContextHint::Masked;
+    return TTI::CastContextHint::Normal;
+  };
 
   VPValue *Operand = getOperand(0);
   TTI::CastContextHint CCH = TTI::CastContextHint::None;
   // For Trunc/FPTrunc, get the context from the only user.
   if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&
       !hasMoreThanOneUniqueUser() && getNumUsers() > 0) {
     if (auto *StoreRecipe = dyn_cast<VPRecipeBase>(*user_begin()))
-      CCH = computeCCH(StoreRecipe, VF);
+      CCH = ComputeCCH(StoreRecipe);
   }
   // For Z/Sext, get the context from the operand.
   else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
            Opcode == Instruction::FPExt) {
     if (Operand->isLiveIn())
       CCH = TTI::CastContextHint::Normal;
     else if (Operand->getDefiningRecipe())
-      CCH = computeCCH(Operand->getDefiningRecipe(), VF);
+      CCH = ComputeCCH(Operand->getDefiningRecipe());
   }
 
   auto *SrcTy =