|
56 | 56 | #include "llvm/Transforms/Vectorize/LoopVectorize.h"
|
57 | 57 | #include "LoopVectorizationPlanner.h"
|
58 | 58 | #include "VPRecipeBuilder.h"
|
59 |
| -#include "VPlanPatternMatch.h" |
60 | 59 | #include "VPlan.h"
|
61 | 60 | #include "VPlanAnalysis.h"
|
62 | 61 | #include "VPlanHCFGBuilder.h"
|
| 62 | +#include "VPlanPatternMatch.h" |
63 | 63 | #include "VPlanTransforms.h"
|
64 | 64 | #include "VPlanVerifier.h"
|
65 | 65 | #include "llvm/ADT/APInt.h"
|
@@ -607,8 +607,8 @@ class InnerLoopVectorizer {
|
607 | 607 | BasicBlock *MiddleBlock, BasicBlock *VectorHeader,
|
608 | 608 | VPlan &Plan, VPTransformState &State);
|
609 | 609 |
|
610 |
| - /// Create the phi node for the resume value of first order recurrences in the scalar preheader and |
611 |
| - /// update the users in the scalar loop. |
| 610 | + /// Create the phi node for the resume value of first order recurrences in the |
| 611 | + /// scalar preheader and update the users in the scalar loop. |
612 | 612 | void fixFixedOrderRecurrence(VPLiveOut *LO, VPTransformState &State);
|
613 | 613 |
|
614 | 614 | /// Iteratively sink the scalarized operands of a predicated instruction into
|
@@ -3475,22 +3475,26 @@ void InnerLoopVectorizer::fixFixedOrderRecurrence(VPLiveOut *LO,
|
3475 | 3475 | // Extract the last vector element in the middle block. This will be the
|
3476 | 3476 | // initial value for the recurrence when jumping to the scalar loop.
|
3477 | 3477 | VPValue *VPExtract = LO->getOperand(0);
|
3478 |
| -using namespace llvm::VPlanPatternMatch; |
3479 |
| - assert(match(VPExtract, m_VPInstruction<VPInstruction::ExtractFromEnd>(m_VPValue(), m_VPValue())) && "FOR LiveOut expects to use an extract from end."); |
| 3478 | + using namespace llvm::VPlanPatternMatch; |
| 3479 | + assert(match(VPExtract, m_VPInstruction<VPInstruction::ExtractFromEnd>( |
| 3480 | + m_VPValue(), m_VPValue())) && |
| 3481 | + "FOR LiveOut expects to use an extract from end."); |
3480 | 3482 | Value *ResumeScalarFOR = State.get(VPExtract, UF - 1, true);
|
3481 | 3483 |
|
3482 | 3484 | // Fix the initial value of the original recurrence in the scalar loop.
|
3483 | 3485 | Builder.SetInsertPoint(LoopScalarPreHeader, LoopScalarPreHeader->begin());
|
3484 | 3486 | PHINode *ScalarHeaderPhi = LO->getPhi();
|
3485 |
| - auto *NewScalarHeaderPhi = Builder.CreatePHI(ScalarHeaderPhi->getType(), 2, "scalar.recur.init"); |
| 3487 | + auto *NewScalarHeaderPhi = |
| 3488 | + Builder.CreatePHI(ScalarHeaderPhi->getType(), 2, "scalar.recur.init"); |
3486 | 3489 | auto *InitScalarFOR =
|
3487 | 3490 | ScalarHeaderPhi->getIncomingValueForBlock(LoopScalarPreHeader);
|
3488 | 3491 | for (auto *BB : predecessors(LoopScalarPreHeader)) {
|
3489 | 3492 | auto *Incoming = BB == LoopMiddleBlock ? ResumeScalarFOR : InitScalarFOR;
|
3490 |
| - NewScalarHeaderPhi ->addIncoming(Incoming, BB); |
| 3493 | + NewScalarHeaderPhi->addIncoming(Incoming, BB); |
3491 | 3494 | }
|
3492 | 3495 |
|
3493 |
| - ScalarHeaderPhi->setIncomingValueForBlock(LoopScalarPreHeader, NewScalarHeaderPhi ); |
| 3496 | + ScalarHeaderPhi->setIncomingValueForBlock(LoopScalarPreHeader, |
| 3497 | + NewScalarHeaderPhi); |
3494 | 3498 | ScalarHeaderPhi->setName("scalar.recur");
|
3495 | 3499 | }
|
3496 | 3500 |
|
|
0 commit comments