Rebase and updated

Author: Mel-Chen

llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

@@ -61,7 +61,7 @@ struct specificval_ty {
 
   specificval_ty(const VPValue *V) : Val(V) {}
 
-  bool match(VPValue *VPV) { return VPV == Val; }
+  bool match(VPValue *VPV) const { return VPV == Val; }
 };
 
 inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
@@ -227,16 +227,16 @@ struct TernaryRecipe_match {
   TernaryRecipe_match(Op0_t Op0, Op1_t Op1, Op2_t Op2)
       : Op0(Op0), Op1(Op1), Op2(Op2) {}
 
-  bool match(const VPValue *V) {
+  bool match(const VPValue *V) const {
     auto *DefR = V->getDefiningRecipe();
     return DefR && match(DefR);
   }
 
-  bool match(const VPSingleDefRecipe *R) {
+  bool match(const VPSingleDefRecipe *R) const {
     return match(static_cast<const VPRecipeBase *>(R));
   }
 
-  bool match(const VPRecipeBase *R) {
+  bool match(const VPRecipeBase *R) const {
     if (!detail::MatchRecipeAndOpcode<Opcode, RecipeTys...>::match(R))
       return false;
     assert(R->getNumOperands() == 3 &&

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -674,11 +674,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return NewPhi;
   }
   case VPInstruction::MergeUntilPivot: {
-    assert(Part == 0 && "No unrolling expected for predicated vectorization.");
-    Value *Cond = State.get(getOperand(0), Part);
-    Value *OnTrue = State.get(getOperand(1), Part);
-    Value *OnFalse = State.get(getOperand(2), Part);
-    Value *Pivot = State.get(getOperand(3), VPIteration(0, 0));
+    Value *Cond = State.get(getOperand(0), 0);
+    Value *OnTrue = State.get(getOperand(1), 0);
+    Value *OnFalse = State.get(getOperand(2), 0);
+    Value *Pivot = State.get(getOperand(3), /* IsScalar */ true);
     assert(Pivot->getType()->isIntegerTy() && "Pivot should be an integer.");
     return Builder.CreateIntrinsic(Intrinsic::vp_merge, {OnTrue->getType()},
                                    {Cond, OnTrue, OnFalse, Pivot}, nullptr,

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -1351,7 +1351,8 @@ void VPlanTransforms::addActiveLaneMask(
 }
 
 /// Replace recipes with their EVL variants.
-static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
+static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL,
+                                         LLVMContext &Ctx) {
   using namespace llvm::VPlanPatternMatch;
   SmallVector<VPValue *> HeaderMasks = collectAllHeaderMasks(Plan);
   VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
@@ -1393,15 +1394,14 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                                                   TypeInfo.inferScalarType(Sel),
                                                   false, false, false);
               })
-              .Case<VPInstruction>([&](VPInstruction *VPI) {
+              .Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
                 VPValue *LHS, *RHS;
                 if (!match(VPI, m_Select(m_Specific(HeaderMask), m_VPValue(LHS),
                                          m_VPValue(RHS))))
                   return nullptr;
-                VPValue *Cond = Plan.getOrAddLiveIn(ConstantInt::getTrue(
-                    CanonicalIVPHI->getScalarType()->getContext()));
+                VPValue *Cond = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
                 return new VPInstruction(VPInstruction::MergeUntilPivot,
-                                         {Cond, LHS, RHS, EVL},
+                                         {Cond, LHS, RHS, &EVL},
                                          VPI->getDebugLoc());
               })
               .Default([&](VPRecipeBase *R) { return nullptr; });
@@ -1495,7 +1495,8 @@ bool VPlanTransforms::tryAddExplicitVectorLength(VPlan &Plan) {
   NextEVLIV->insertBefore(CanonicalIVIncrement);
   EVLPhi->addOperand(NextEVLIV);
 
-  transformRecipestoEVLRecipes(Plan, *VPEVL);
+  LLVMContext &Ctx = CanonicalIVPHI->getScalarType()->getContext();
+  transformRecipestoEVLRecipes(Plan, *VPEVL, Ctx);
 
   // Replace all uses of VPCanonicalIVPHIRecipe by
   // VPEVLBasedIVPHIRecipe except for the canonical IV increment.