!fixup address latest comments, thanks

Author: fhahn

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -775,6 +775,7 @@ VPRegionBlock *VPRegionBlock::clone() {
 
   if (CanIVInfo.CanIV) {
     NewRegion->CanIVInfo.CanIV = new VPRegionValue();
+    NewRegion->CanIVInfo.Ty = CanIVInfo.Ty;
     NewRegion->CanIVInfo.HasNUW = CanIVInfo.HasNUW;
     NewRegion->CanIVInfo.DL = CanIVInfo.DL;
   }
@@ -901,11 +902,11 @@ void VPRegionBlock::dissolveToCFGLoop() {
                          Instruction::Add, {CanIV, &Plan.getVFxUF()},
                          {CanIVInfo.HasNUW, false}, CanIVInfo.DL, "index.next");
     }
-    Type *CanIVTy = VPTypeAnalysis(Plan).inferScalarType(CanIV);
     auto *ScalarR =
         VPBuilder(Header, Header->begin())
             .createScalarPhi(
-                {Plan.getOrAddLiveIn(ConstantInt::get(CanIVTy, 0)), CanIVInc},
+                {Plan.getOrAddLiveIn(ConstantInt::get(CanIVInfo.Ty, 0)),
+                 CanIVInc},
                 CanIVInfo.DL, "index");
     CanIV->replaceAllUsesWith(ScalarR);
   }
@@ -1555,9 +1556,9 @@ void VPSlotTracker::assignNames(const VPlan &Plan) {
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<const VPBlockBase *>>
       RPOT(VPBlockDeepTraversalWrapper<const VPBlockBase *>(Plan.getEntry()));
   for (const VPBlockBase *VPB : RPOT) {
-    if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB)) {
+    if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB))
       assignNames(VPBB);
-    } else if (auto *CanIV = cast<VPRegionBlock>(VPB)->getCanonicalIV())
+    else if (auto *CanIV = cast<VPRegionBlock>(VPB)->getCanonicalIV())
       assignName(CanIV);
   }
 }

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -3900,11 +3900,12 @@ class VPIRBasicBlock : public VPBasicBlock {
 /// Track information about the canonical IV value of a region.
 struct VPCanonicalIVInfo {
   VPRegionValue *CanIV = nullptr;
+  Type *Ty = nullptr;
   bool HasNUW = true;
   DebugLoc DL = DebugLoc::getUnknown();
 
-  VPCanonicalIVInfo(VPRegionValue *CanIV, bool HasNUW, DebugLoc DL)
-      : CanIV(CanIV), HasNUW(HasNUW), DL(DL) {}
+  VPCanonicalIVInfo(VPRegionValue *CanIV, Type *Ty, bool HasNUW, DebugLoc DL)
+      : CanIV(CanIV), Ty(Ty), HasNUW(HasNUW), DL(DL) {}
 
   VPCanonicalIVInfo() {}
 
@@ -3958,9 +3959,9 @@ class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
     Exiting->setParent(this);
   }
 
-  VPRegionBlock(DebugLoc DL, const std::string &Name = "")
+  VPRegionBlock(Type *CanIVTy, DebugLoc DL, const std::string &Name = "")
       : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
-        CanIVInfo(new VPRegionValue(), true, DL) {}
+        CanIVInfo(new VPRegionValue(), CanIVTy, true, DL) {}
 
 public:
   ~VPRegionBlock() override {}
@@ -4036,6 +4037,9 @@ class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
   VPValue *getCanonicalIV() { return CanIVInfo.CanIV; }
   const VPValue *getCanonicalIV() const { return CanIVInfo.CanIV; }
 
+  Type *getCanonicalIVType() { return CanIVInfo.Ty; }
+  const Type *getCanonicalIVType() const { return CanIVInfo.Ty; }
+
   VPCanonicalIVInfo &getCanonicalIVInfo() { return CanIVInfo; }
 };
 
@@ -4390,9 +4394,9 @@ class VPlan {
   /// Create a new loop VPRegionBlock with \p StartV and \p Name, and entry and
   /// exiting blocks set to nullptr. The returned block is owned by the VPlan
   /// and deleted once the VPlan is destroyed.
-  VPRegionBlock *createVPRegionBlock(DebugLoc DL,
+  VPRegionBlock *createVPRegionBlock(Type *CanIVTy, DebugLoc DL,
                                      const std::string &Name = "") {
-    auto *VPB = new VPRegionBlock(DL, Name);
+    auto *VPB = new VPRegionBlock(CanIVTy, DL, Name);
     CreatedBlocks.push_back(VPB);
     return VPB;
   }

llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

@@ -494,9 +494,9 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
     return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
   };
 
-  if (auto *CanIV = LoopRegion->getCanonicalIV())
-    if (CanIV->getNumUsers() != 0)
-      OpenIntervals.insert(CanIV);
+  VPValue *CanIV = LoopRegion->getCanonicalIV();
+  if (CanIV->getNumUsers() != 0)
+    OpenIntervals.insert(CanIV);
 
   // We scan the instructions linearly and record each time that a new interval
   // starts, by placing it in a set. If we find this value in TransposEnds then

llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp

@@ -395,7 +395,7 @@ static bool canonicalHeaderAndLatch(VPBlockBase *HeaderVPB,
 /// Create a new VPRegionBlock for the loop starting at \p HeaderVPB.
 static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
   auto *PreheaderVPBB = HeaderVPB->getPredecessors()[0];
-  auto *LatchVPBB = cast<VPBasicBlock>(HeaderVPB->getPredecessors()[1]);
+  auto *LatchVPBB = HeaderVPB->getPredecessors()[1];
 
   VPBlockUtils::disconnectBlocks(PreheaderVPBB, HeaderVPB);
   VPBlockUtils::disconnectBlocks(LatchVPBB, HeaderVPB);
@@ -409,9 +409,10 @@ static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
   VPPhi *ScalarCanIV = nullptr;
   if (PreheaderVPBB->getSinglePredecessor() == Plan.getEntry())
     ScalarCanIV = cast<VPPhi>(&*cast<VPBasicBlock>(HeaderVPB)->begin());
-  auto *R =
-      Plan.createVPRegionBlock(ScalarCanIV ? ScalarCanIV->getDebugLoc()
-                                           : DebugLoc::getCompilerGenerated());
+  auto *R = Plan.createVPRegionBlock(
+      ScalarCanIV->getOperand(0)->getLiveInIRValue()->getType(),
+      ScalarCanIV ? ScalarCanIV->getDebugLoc()
+                  : DebugLoc::getCompilerGenerated());
   VPBlockUtils::insertOnEdge(LatchVPBB, LatchExitVPB, R);
   VPBlockUtils::disconnectBlocks(LatchVPBB, R);
   VPBlockUtils::connectBlocks(PreheaderVPBB, R);
@@ -447,7 +448,10 @@ static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
   }
 
   VPBuilder Builder(LatchVPBB);
-  auto CanonicalIVIncrement = Builder.createOverflowingOp(
+  // Add a VPInstruction to increment the scalar canonical IV by VF * UF.
+  // Initially the induction increment is guaranteed to not wrap, but that may
+  // change later, e.g. when tail-folding, when the flags need to be dropped.
+  auto *CanonicalIVIncrement = Builder.createOverflowingOp(
       Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {true, false}, DL,
       "index.next");
   // Add the BranchOnCount VPInstruction to the latch.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -2341,11 +2341,9 @@ bool VPWidenIntOrFpInductionRecipe::isCanonical() const {
   if (getStepValue()->getDefiningRecipe())
     return false;
   auto *StepC = dyn_cast<ConstantInt>(getStepValue()->getLiveInIRValue());
-  auto *CanIV = getParent()->getParent()->getCanonicalIV();
   auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
   return StartC && StartC->isZero() && StepC && StepC->isOne() &&
-         getScalarType() ==
-             VPTypeAnalysis(*getParent()->getPlan()).inferScalarType(CanIV);
+         getScalarType() == getParent()->getParent()->getCanonicalIVType();
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -1710,9 +1710,8 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
       HeaderR.getVPSingleValue()->replaceAllUsesWith(Phi->getIncomingValue(0));
       HeaderR.eraseFromParent();
     }
-    Plan.getCanonicalIV()->replaceAllUsesWith(
-        Plan.getOrAddLiveIn(ConstantInt::getNullValue(
-            VPTypeAnalysis(Plan).inferScalarType(Plan.getCanonicalIV()))));
+    VectorRegion->getCanonicalIV()->replaceAllUsesWith(Plan.getOrAddLiveIn(
+        ConstantInt::getNullValue(VectorRegion->getCanonicalIVType())));
 
     VPBlockBase *Preheader = VectorRegion->getSinglePredecessor();
     VPBlockBase *Exit = VectorRegion->getSingleSuccessor();
@@ -2319,16 +2318,15 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
     VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck) {
   VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
-  VPValue *CanonicalIV = Plan.getCanonicalIV();
-  VPValue *StartV = Plan.getOrAddLiveIn(Constant::getNullValue(
-      VPTypeAnalysis(Plan).inferScalarType(CanonicalIV)));
+  auto &CanIVInfo = Plan.getCanonicalIVInfo();
+  VPValue *CanonicalIV = CanIVInfo.CanIV;
+  VPValue *StartV = Plan.getOrAddLiveIn(Constant::getNullValue(CanIVInfo.Ty));
 
   auto *CanonicalIVIncrement =
       cast<VPInstruction>(EB->getTerminator()->getOperand(0));
   // TODO: Check if dropping the flags is needed if
   // !DataAndControlFlowWithoutRuntimeCheck.
   CanonicalIVIncrement->dropPoisonGeneratingFlags();
-  auto &CanIVInfo = Plan.getCanonicalIVInfo();
   CanIVInfo.HasNUW = false;
   DebugLoc DL = CanIVInfo.DL;
   // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
@@ -2360,8 +2358,8 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
       "index.part.next");
 
   // Create the active lane mask instruction in the VPlan preheader.
-  VPValue *ALMMultiplier = Plan.getOrAddLiveIn(
-      ConstantInt::get(VPTypeAnalysis(Plan).inferScalarType(CanonicalIV), 1));
+  VPValue *ALMMultiplier =
+      Plan.getOrAddLiveIn(ConstantInt::get(CanIVInfo.Ty, 1));
   auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
                                         {EntryIncrement, TC, ALMMultiplier}, DL,
                                         "active.lane.mask.entry");
@@ -2458,8 +2456,8 @@ void VPlanTransforms::addActiveLaneMask(
         Plan, DataAndControlFlowWithoutRuntimeCheck);
   } else {
     VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
-    VPValue *ALMMultiplier = Plan.getOrAddLiveIn(ConstantInt::get(
-        VPTypeAnalysis(Plan).inferScalarType(Plan.getCanonicalIV()), 1));
+    VPValue *ALMMultiplier =
+        Plan.getOrAddLiveIn(ConstantInt::get(Plan.getCanonicalIVInfo().Ty, 1));
     LaneMask =
         B.createNaryOp(VPInstruction::ActiveLaneMask,
                        {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier},
@@ -2730,7 +2728,7 @@ void VPlanTransforms::addExplicitVectorLength(
 
   auto *CanonicalIV = Plan.getCanonicalIV();
   auto &CanIVInfo = Plan.getCanonicalIVInfo();
-  auto *CanIVTy = VPTypeAnalysis(Plan).inferScalarType(CanonicalIV);
+  auto *CanIVTy = CanIVInfo.Ty;
   VPValue *StartV = Plan.getOrAddLiveIn(ConstantInt::getNullValue(CanIVTy));
   auto *CanonicalIVIncrement = cast<VPInstruction>(Plan.getVectorLoopRegion()
                                                        ->getExitingBasicBlock()
@@ -4174,8 +4172,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
 
   // Adjust induction to reflect that the transformed plan only processes one
   // original iteration.
-  auto *CanIV = Plan.getCanonicalIV();
-  Type *CanIVTy = TypeInfo.inferScalarType(CanIV);
+  Type *CanIVTy = Plan.getCanonicalIVInfo().Ty;
   auto *Inc = cast<VPInstruction>(
       VectorLoop->getExitingBasicBlock()->getTerminator()->getOperand(0));
   VPBuilder PHBuilder(Plan.getVectorPreheader());

llvm/lib/Transforms/Vectorize/VPlanValue.h

@@ -169,7 +169,9 @@ class LLVM_ABI_FOR_TEST VPValue {
   const VPRecipeBase *getDefiningRecipe() const;
 
   /// Returns true if this VPValue is defined by a recipe.
-  bool hasDefiningRecipe() const { return getDefiningRecipe(); }
+  bool hasDefiningRecipe() const {
+    return SubclassID != VPRegionValueSC && getDefiningRecipe();
+  }
 
   /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
   bool isLiveIn() const {
@@ -196,8 +198,7 @@ class LLVM_ABI_FOR_TEST VPValue {
 };
 
 /// VPValues defined by a VPRegionBlock, like the canonical IV.
-class VPRegionValue : public VPValue {
-public:
+struct VPRegionValue : public VPValue {
   VPRegionValue() : VPValue(VPValue::VPRegionValueSC) {}
 
   ~VPRegionValue() override = default;