Skip to content

Commit

Permalink
Merged master:07239c736a5 into amd-gfx:5df6a2a9b9a
Browse files Browse the repository at this point in the history 
Local branch amd-gfx 5df6a2a Merged master:801d823bdec into amd-gfx:0194603d562
Remote branch master 07239c7 [BrachProbablityInfo] Proportional distribution of reachable probabilities
  • Loading branch information
Sw authored and Sw committed Jun 2, 2020
2 parents 5df6a2a + 07239c7 commit 652d2b7
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 61 deletions.
2 changes: 0 additions & 2 deletions clang/lib/Analysis/CFG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,6 @@ class AddStmtChoice {
///
class LocalScope {
public:
friend class const_iterator;

using AutomaticVarsTy = BumpVector<VarDecl *>;

/// const_iterator - Iterates local scope backwards and jumps to previous
Expand Down
2 changes: 0 additions & 2 deletions clang/utils/TableGen/NeonEmitter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,8 +303,6 @@ class Variable {
/// The main grunt class. This represents an instantiation of an intrinsic with
/// a particular typespec and prototype.
class Intrinsic {
friend class DagEmitter;

/// The Record this intrinsic was created from.
Record *R;
/// The unmangled name.
Expand Down
116 changes: 75 additions & 41 deletions llvm/lib/Analysis/BranchProbabilityInfo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ static const uint32_t LBH_UNLIKELY_WEIGHT = 62;
///
/// This is the probability for a branch being taken to a block that terminates
/// (eventually) in unreachable. These are predicted as unlikely as possible.
/// All reachable probability will equally share the remaining part.
/// All reachable probability will proportionally share the remaining part.
static const BranchProbability UR_TAKEN_PROB = BranchProbability::getRaw(1);

/// Weight for a branch taken going into a cold block.
Expand Down Expand Up @@ -305,19 +305,19 @@ bool BranchProbabilityInfo::calcMetadataWeights(const BasicBlock *BB) {
SmallVector<unsigned, 2> UnreachableIdxs;
SmallVector<unsigned, 2> ReachableIdxs;
Weights.reserve(TI->getNumSuccessors());
for (unsigned i = 1, e = WeightsNode->getNumOperands(); i != e; ++i) {
for (unsigned I = 1, E = WeightsNode->getNumOperands(); I != E; ++I) {
ConstantInt *Weight =
mdconst::dyn_extract<ConstantInt>(WeightsNode->getOperand(i));
mdconst::dyn_extract<ConstantInt>(WeightsNode->getOperand(I));
if (!Weight)
return false;
assert(Weight->getValue().getActiveBits() <= 32 &&
"Too many bits for uint32_t");
Weights.push_back(Weight->getZExtValue());
WeightSum += Weights.back();
if (PostDominatedByUnreachable.count(TI->getSuccessor(i - 1)))
UnreachableIdxs.push_back(i - 1);
if (PostDominatedByUnreachable.count(TI->getSuccessor(I - 1)))
UnreachableIdxs.push_back(I - 1);
else
ReachableIdxs.push_back(i - 1);
ReachableIdxs.push_back(I - 1);
}
assert(Weights.size() == TI->getNumSuccessors() && "Checked above");

Expand All @@ -328,56 +328,90 @@ bool BranchProbabilityInfo::calcMetadataWeights(const BasicBlock *BB) {

if (ScalingFactor > 1) {
WeightSum = 0;
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
Weights[i] /= ScalingFactor;
WeightSum += Weights[i];
for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
Weights[I] /= ScalingFactor;
WeightSum += Weights[I];
}
}
assert(WeightSum <= UINT32_MAX &&
"Expected weights to scale down to 32 bits");

if (WeightSum == 0 || ReachableIdxs.size() == 0) {
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
Weights[i] = 1;
for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I)
Weights[I] = 1;
WeightSum = TI->getNumSuccessors();
}

// Set the probability.
SmallVector<BranchProbability, 2> BP;
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
BP.push_back({ Weights[i], static_cast<uint32_t>(WeightSum) });
for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I)
BP.push_back({ Weights[I], static_cast<uint32_t>(WeightSum) });

// Examine the metadata against unreachable heuristic.
// If the unreachable heuristic is more strong then we use it for this edge.
if (UnreachableIdxs.size() > 0 && ReachableIdxs.size() > 0) {
auto UnreachableProb = UR_TAKEN_PROB;
for (auto i : UnreachableIdxs)
if (UnreachableProb < BP[i]) {
BP[i] = UnreachableProb;
}
if (UnreachableIdxs.size() == 0 || ReachableIdxs.size() == 0) {
setEdgeProbability(BB, BP);
return true;
}

auto UnreachableProb = UR_TAKEN_PROB;
for (auto I : UnreachableIdxs)
if (UnreachableProb < BP[I]) {
BP[I] = UnreachableProb;
}

// Because of possible rounding errors and the above fix up for
// the unreachable heuristic the sum of probabilities of all edges may be
// less than 1.0. Distribute the remaining probability (calculated as
// 1.0 - (sum of BP[i])) evenly among all the reachable edges.
auto ToDistribute = BranchProbability::getOne();
for (auto &P : BP)
ToDistribute -= P;

// If we modified the probability of some edges then we must distribute
// the difference between reachable blocks.
// TODO: This spreads ToDistribute evenly upon the reachable edges. A better
// distribution would be proportional. So the relation between weights of
// the reachable edges would be kept unchanged. That is for any reachable
// edges i and j:
// newBP[i] / newBP[j] == oldBP[i] / oldBP[j]
// newBP[i] / oldBP[i] == newBP[j] / oldBP[j] ==
// == Denominator / (Denominator - ToDistribute)
// newBP[i] = oldBP[i] * Denominator / (Denominator - ToDistribute)
BranchProbability PerEdge = ToDistribute / ReachableIdxs.size();
if (PerEdge > BranchProbability::getZero())
for (auto i : ReachableIdxs)
BP[i] += PerEdge;
// Sum of all edge probabilities must be 1.0. If we modified the probability
// of some edges then we must distribute the introduced difference over the
// reachable blocks.
//
// Proportional distribution: the relation between probabilities of the
// reachable edges is kept unchanged. That is for any reachable edges i and j:
// newBP[i] / newBP[j] == oldBP[i] / oldBP[j] =>
// newBP[i] / oldBP[i] == newBP[j] / oldBP[j] == K
// Where K is independent of i,j.
// newBP[i] == oldBP[i] * K
// We need to find K.
// Make sum of all reachables of the left and right parts:
// sum_of_reachable(newBP) == K * sum_of_reachable(oldBP)
// Sum of newBP must be equal to 1.0:
// sum_of_reachable(newBP) + sum_of_unreachable(newBP) == 1.0 =>
// sum_of_reachable(newBP) = 1.0 - sum_of_unreachable(newBP)
// Where sum_of_unreachable(newBP) is what has been just changed.
// Finally:
// K == sum_of_reachable(newBP) / sum_of_reachable(oldBP) =>
// K == (1.0 - sum_of_unreachable(newBP)) / sum_of_reachable(oldBP)
BranchProbability NewUnreachableSum = BranchProbability::getZero();
for (auto I : UnreachableIdxs)
NewUnreachableSum += BP[I];

BranchProbability NewReachableSum =
BranchProbability::getOne() - NewUnreachableSum;

BranchProbability OldReachableSum = BranchProbability::getZero();
for (auto I : ReachableIdxs)
OldReachableSum += BP[I];

if (OldReachableSum != NewReachableSum) { // Anything to dsitribute?
if (OldReachableSum.isZero()) {
// If all oldBP[i] are zeroes then the proportional distribution results
// in all zero probabilities and the error stays big. In this case we
// evenly spread NewReachableSum over the reachable edges.
BranchProbability PerEdge = NewReachableSum / ReachableIdxs.size();
for (auto I : ReachableIdxs)
BP[I] = PerEdge;
} else {
for (auto I : ReachableIdxs) {
// We use uint64_t to avoid double rounding error of the following
// calculation: BP[i] = BP[i] * NewReachableSum / OldReachableSum
// The formula is taken from the private constructor
// BranchProbability(uint32_t Numerator, uint32_t Denominator)
uint64_t Mul = static_cast<uint64_t>(NewReachableSum.getNumerator()) *
BP[I].getNumerator();
uint32_t Div = static_cast<uint32_t>(
divideNearest(Mul, OldReachableSum.getNumerator()));
BP[I] = BranchProbability::getRaw(Div);
}
}
}

setEdgeProbability(BB, BP);
Expand Down
1 change: 0 additions & 1 deletion llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -640,7 +640,6 @@ class DwarfDebug : public DebugHandlerBase {
void addDwarfTypeUnitType(DwarfCompileUnit &CU, StringRef Identifier,
DIE &Die, const DICompositeType *CTy);

friend class NonTypeUnitContext;
class NonTypeUnitContext {
DwarfDebug *DD;
decltype(DwarfDebug::TypeUnitsUnderConstruction) TypeUnitsUnderConstruction;
Expand Down
2 changes: 0 additions & 2 deletions llvm/lib/CodeGen/InterferenceCache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,6 @@ class LLVM_LIBRARY_VISIBILITY InterferenceCache {
Entry *get(unsigned PhysReg);

public:
friend class Cursor;

InterferenceCache() = default;

~InterferenceCache() {
Expand Down
6 changes: 3 additions & 3 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35705,7 +35705,7 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
LN->getMemOperand()->getFlags());
DCI.CombineTo(N.getNode(), BcastLd);
DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
DCI.recursivelyDeleteUnusedNodes(LN);
DCI.recursivelyDeleteUnusedNodes(Src.getNode());
return N; // Return N so it doesn't get rechecked!
}
}
Expand All @@ -35722,7 +35722,7 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
LN->getMemoryVT(), LN->getMemOperand());
DCI.CombineTo(N.getNode(), BcastLd);
DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
DCI.recursivelyDeleteUnusedNodes(LN);
DCI.recursivelyDeleteUnusedNodes(Src.getNode());
return N; // Return N so it doesn't get rechecked!
}
}
Expand Down Expand Up @@ -35750,7 +35750,7 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
LN->getMemOperand()->getFlags());
DCI.CombineTo(N.getNode(), BcastLd);
DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
DCI.recursivelyDeleteUnusedNodes(LN);
DCI.recursivelyDeleteUnusedNodes(Src.getNode());
return N; // Return N so it doesn't get rechecked!
}
}
Expand Down
24 changes: 14 additions & 10 deletions llvm/test/Analysis/BranchProbabilityInfo/basic.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,11 +469,12 @@ entry:
i32 2, label %case_c
i32 3, label %case_d
i32 4, label %case_e ], !prof !8
; Reachable probabilities keep their relation: 4/64/4/4 = 5.26% / 84.21% / 5.26% / 5.26%.
; CHECK: edge entry -> case_a probability is 0x00000001 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_b probability is 0x07ffffff / 0x80000000 = 6.25%
; CHECK: edge entry -> case_c probability is 0x67ffffff / 0x80000000 = 81.25% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x07ffffff / 0x80000000 = 6.25%
; CHECK: edge entry -> case_e probability is 0x07ffffff / 0x80000000 = 6.25%
; CHECK: edge entry -> case_b probability is 0x06bca1af / 0x80000000 = 5.26%
; CHECK: edge entry -> case_c probability is 0x6bca1af3 / 0x80000000 = 84.21% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x06bca1af / 0x80000000 = 5.26%
; CHECK: edge entry -> case_e probability is 0x06bca1af / 0x80000000 = 5.26%

case_a:
unreachable
Expand Down Expand Up @@ -511,11 +512,13 @@ entry:
i32 2, label %case_c
i32 3, label %case_d
i32 4, label %case_e ], !prof !9
; Reachable probabilities keep their relation: 64/4/4 = 88.89% / 5.56% / 5.56%.
; CHECK: edge entry -> case_a probability is 0x00000001 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_b probability is 0x00000001 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_c probability is 0x6aaaaaaa / 0x80000000 = 83.33% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x0aaaaaaa / 0x80000000 = 8.33%
; CHECK: edge entry -> case_e probability is 0x0aaaaaaa / 0x80000000 = 8.33%
; CHECK: edge entry -> case_c probability is 0x71c71c71 / 0x80000000 = 88.89% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x071c71c7 / 0x80000000 = 5.56%
; CHECK: edge entry -> case_e probability is 0x071c71c7 / 0x80000000 = 5.56%


case_a:
unreachable
Expand Down Expand Up @@ -551,11 +554,12 @@ entry:
i32 2, label %case_c
i32 3, label %case_d
i32 4, label %case_e ], !prof !10
; Reachable probabilities keep their relation: 64/4/4 = 88.89% / 5.56% / 5.56%.
; CHECK: edge entry -> case_a probability is 0x00000000 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_b probability is 0x00000001 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_c probability is 0x6e08fb82 / 0x80000000 = 85.96% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x08fb823e / 0x80000000 = 7.02%
; CHECK: edge entry -> case_e probability is 0x08fb823e / 0x80000000 = 7.02%
; CHECK: edge entry -> case_c probability is 0x71c71c71 / 0x80000000 = 88.89% [HOT edge]
; CHECK: edge entry -> case_d probability is 0x071c71c7 / 0x80000000 = 5.56%
; CHECK: edge entry -> case_e probability is 0x071c71c7 / 0x80000000 = 5.56%

case_a:
unreachable
Expand Down

0 comments on commit 652d2b7

Please sign in to comment.