Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Hot sche check dir/rw pending #4

Open
wants to merge 6 commits into
base: hot-sche-check
Choose a base branch
from
Open

Hot sche check dir/rw pending #4

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5ae0450
use pending rate
HunDunDM Aug 5, 2022
f12faf5
no pending
HunDunDM Aug 10, 2022
b878928
Revert "no pending"
HunDunDM Aug 28, 2022
7f4690f
Merge branch 'hot-sche-check' into hot-sche-check-dir/rw-pending
HunDunDM Aug 30, 2022
18a2be4
update calcProgressiveRank algo
HunDunDM Aug 29, 2022
61bbcd4
Use different ratios for different dimensions
HunDunDM Aug 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 86 additions & 64 deletions server/schedulers/hot_region.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ type solution struct {

// getExtremeLoad returns the min load of the src store and the max load of the dst store.
// If peersRate is negative, the direction is reversed.
func (s *solution) getExtremeLoad(dim int) (src float64, dst float64) {
func (s *solution) getExtremeLoad(dim int) (srcLoad, dstLoad, maxPendingLoad float64) {
srcCurrentLoad := s.srcStore.LoadPred.Current.Loads[dim]
srcFutureLoad := s.srcStore.LoadPred.Future.Loads[dim]
dstCurrentLoad := s.dstStore.LoadPred.Current.Loads[dim]
Expand All @@ -374,10 +374,15 @@ func (s *solution) getExtremeLoad(dim int) (src float64, dst float64) {
fmt.Sprintf("src-cur-load: %.0f, src-fut-load: %.0f, dst-cur-load: %.0f, dst-fut-load: %.0f, peersRate: %0.f",
srcCurrentLoad, srcFutureLoad, dstCurrentLoad, dstFutureLoad, peersRate))

if peersRate >= 0 {
return s.srcStore.LoadPred.Min().Loads[dim], s.dstStore.LoadPred.Max().Loads[dim]
maxPendingLoad = math.Abs(srcFutureLoad-srcCurrentLoad) + math.Abs(dstFutureLoad-dstCurrentLoad)
if srcCurrentLoad-peersRate >= dstCurrentLoad+peersRate {
srcLoad = math.Min(srcCurrentLoad, srcFutureLoad)
dstLoad = math.Max(dstCurrentLoad, dstFutureLoad)
} else {
srcLoad = math.Max(srcCurrentLoad, srcFutureLoad)
dstLoad = math.Min(dstCurrentLoad, dstFutureLoad)
}
return s.srcStore.LoadPred.Max().Loads[dim], s.dstStore.LoadPred.Min().Loads[dim]
return
}

// getCurrentLoad returns the current load of the src store and the dst store.
Expand Down Expand Up @@ -410,9 +415,32 @@ func (s *solution) getPeersRateFromCache(dim int) float64 {

// isAvailable returns the solution is available.
// If the solution has no revertRegion, progressiveRank should < 0.
// If the solution has some revertRegion, progressiveRank should < -1.
// If the solution has some revertRegion, progressiveRank should == -4/-3/-1.
func (s *solution) isAvailable() bool {
return s.progressiveRank < -1 || (s.progressiveRank < 0 && s.revertRegion == nil)
return s.progressiveRank < -2 || s.progressiveRank == -1 || (s.progressiveRank < 0 && s.revertRegion == nil)
}

type ratioSet struct {
preBalancedRatio float64
balancedRatio float64
preBalancedCheckRatio float64
balancedCheckRatio float64
perceivedRatio float64
}

func newRatioSet(balancedRatio float64) *ratioSet {
if balancedRatio < 0.7 {
balancedRatio = 0.7
}
if balancedRatio > 0.95 {
balancedRatio = 0.95
}
rs := &ratioSet{balancedRatio: balancedRatio}
rs.preBalancedRatio = math.Max(2.0*balancedRatio-1.0, balancedRatio-0.15)
rs.balancedCheckRatio = balancedRatio - 0.02
rs.preBalancedCheckRatio = rs.preBalancedRatio - 0.03
rs.perceivedRatio = math.Min(2.0-rs.preBalancedRatio*2, 0.5)
return rs
}

type balanceSolver struct {
Expand All @@ -438,11 +466,8 @@ type balanceSolver struct {
firstPriority int
secondPriority int

preBalancedRatio float64
balancedRatio float64
preBalancedCheckRatio float64
balancedCheckRatio float64
perceivedRatio float64
firstPriorityRatioSet *ratioSet
secondPriorityRatioSet *ratioSet

maxPeerNum int
minPerceivedLoadIndex int
Expand Down Expand Up @@ -486,11 +511,8 @@ func (bs *balanceSolver) init() {
}

bs.firstPriority, bs.secondPriority = prioritiesToDim(bs.getPriorities())
bs.balancedRatio = bs.sche.conf.GetGreatDecRatio()
bs.preBalancedRatio = 2.0*bs.balancedRatio - 1.0 // 1.0 - (1.0-bs.balancedRatio)*2
bs.balancedCheckRatio = bs.balancedRatio - 0.02
bs.preBalancedCheckRatio = bs.preBalancedRatio - 0.03
bs.perceivedRatio = (1.0 - bs.preBalancedRatio) * 2
bs.firstPriorityRatioSet = newRatioSet(bs.sche.conf.GetGreatDecRatio())
bs.secondPriorityRatioSet = newRatioSet(bs.firstPriorityRatioSet.preBalancedRatio)
bs.maxPeerNum = bs.sche.conf.GetMaxPeerNumber()
bs.minPerceivedLoadIndex = bs.maxPeerNum/100 - 1
if bs.minPerceivedLoadIndex < 0 {
Expand Down Expand Up @@ -549,12 +571,12 @@ func (bs *balanceSolver) filterUniformStore() (string, bool) {
// If both dims are enough uniform, any schedule is unnecessary.
return "all-dim", true
}
if isUniformFirstPriority && (bs.cur.progressiveRank == -1 || bs.cur.progressiveRank == -3) {
// If first priority dim is enough uniform, -1 is unnecessary and maybe lead to worse balance for second priority dim
if isUniformFirstPriority && (bs.cur.progressiveRank == -2 || bs.cur.progressiveRank == -3) {
// If first priority dim is enough uniform, -2 is unnecessary and maybe lead to worse balance for second priority dim
return dimToString(bs.firstPriority), true
}
if isUniformSecondPriority && bs.cur.progressiveRank == -2 {
// If second priority dim is enough uniform, -2 is unnecessary and maybe lead to worse balance for first priority dim
if isUniformSecondPriority && bs.cur.progressiveRank == -1 {
// If second priority dim is enough uniform, -1 is unnecessary and maybe lead to worse balance for first priority dim
return dimToString(bs.secondPriority), true
}
return "", false
Expand Down Expand Up @@ -615,14 +637,14 @@ func (bs *balanceSolver) solve() []*operator.Operator {
bs.calcProgressiveRank()
tryUpdateBestSolution()

if searchRevertRegions && (bs.cur.progressiveRank >= -1 && bs.cur.progressiveRank <= 0) &&
(bs.best == nil || bs.best.progressiveRank >= -1 || bs.best.revertRegion != nil) {
if searchRevertRegions && (bs.cur.progressiveRank == -2 || bs.cur.progressiveRank == 0) &&
(bs.best == nil || bs.best.progressiveRank >= -2 || bs.best.revertRegion != nil) {
// The search-revert-regions is performed only when the following conditions are met to improve performance.
// * `searchRevertRegions` is true. It depends on the result of the last `solve`.
// * `IsStrictPickingStoreEnabled` is false.
// * The current solution is not good enough. -1 <= progressiveRank <= 0
// * The current solution is not good enough. progressiveRank == -2/0
// * The current best solution is not good enough.
// * The current best solution has progressiveRank < -1 and does not contain revert regions.
// * The current best solution has progressiveRank < -2 and does not contain revert regions.
// * The current best solution contain revert regions.
schedulerCounter.WithLabelValues(bs.sche.GetName(), "search-revert-regions").Inc()
dstStoreID := dstStore.GetID()
Expand Down Expand Up @@ -657,9 +679,9 @@ func (bs *balanceSolver) solve() []*operator.Operator {
func (bs *balanceSolver) allowSearchRevertRegions() bool {
// The next solve is allowed to search-revert-regions only when the following conditions are met.
// * No best solution was found this time.
// * The progressiveRank of the best solution is -1.
// * The progressiveRank of the best solution is -2.
// * The best solution contain revert regions.
return bs.best == nil || bs.best.progressiveRank >= -1 || bs.best.revertRegion != nil
return bs.best == nil || bs.best.progressiveRank >= -2 || bs.best.revertRegion != nil
}

func (bs *balanceSolver) tryAddPendingInfluence() bool {
Expand Down Expand Up @@ -957,8 +979,8 @@ func (bs *balanceSolver) isUniformSecondPriority(store *statistics.StoreLoadDeta
// calcProgressiveRank calculates `bs.cur.progressiveRank`.
// See the comments of `solution.progressiveRank` for more about progressive rank.
// | ↓ firstPriority \ secondPriority → | isBetter | isNotWorsened | Worsened |
// | isBetter | -4 | -3 | -1 / 0 |
// | isNotWorsened | -2 | 1 | 1 |
// | isBetter | -4 | -3 | -2 |
// | isNotWorsened | -1 | 1 | 1 |
// | Worsened | 0 | 1 | 1 |
func (bs *balanceSolver) calcProgressiveRank() {
bs.cur.progressiveRank = 1
Expand All @@ -973,25 +995,25 @@ func (bs *balanceSolver) calcProgressiveRank() {
// For write leader, only compare the first priority.
// If the first priority is better, the progressiveRank is -3.
// Because it is not a solution that needs to be optimized.
if bs.getBalanceBoostByPriorities(bs.firstPriority) == 1 {
if bs.getBalanceBoostByPriorities(bs.firstPriority, bs.firstPriorityRatioSet) == 1 {
bs.cur.progressiveRank = -3
}
return
}
firstCmp := bs.getBalanceBoostByPriorities(bs.firstPriority)
secondCmp := bs.getBalanceBoostByPriorities(bs.secondPriority)
firstCmp := bs.getBalanceBoostByPriorities(bs.firstPriority, bs.firstPriorityRatioSet)
secondCmp := bs.getBalanceBoostByPriorities(bs.secondPriority, bs.secondPriorityRatioSet)
switch {
case firstCmp == 1 && secondCmp == 1:
// If belonging to the case, all two dim will be more balanced, the best choice.
bs.cur.progressiveRank = -4
case firstCmp == 1 && secondCmp == 0:
// If belonging to the case, the first priority dim will be more balanced, the second priority dim will be not worsened.
bs.cur.progressiveRank = -3
case firstCmp == 0 && secondCmp == 1:
// If belonging to the case, the first priority dim will be not worsened, the second priority dim will be more balanced.
bs.cur.progressiveRank = -2
case firstCmp == 1:
// If belonging to the case, the first priority dim will be more balanced, ignore the second priority dim.
bs.cur.progressiveRank = -2
case firstCmp == 0 && secondCmp == 1:
// If belonging to the case, the first priority dim will be not worsened, the second priority dim will be more balanced.
bs.cur.progressiveRank = -1
case secondCmp == 1:
// If belonging to the case, the second priority dim will be more balanced, ignore the first priority dim.
Expand Down Expand Up @@ -1040,17 +1062,17 @@ func (bs *balanceSolver) getHotDecRatioByPriorities(dim int) (isHot bool, decRat
return
}*/

func (bs *balanceSolver) getBalanceBoostByPriorities(dim int) (cmp int) {
func (bs *balanceSolver) getBalanceBoostByPriorities(dim int, rs *ratioSet) (cmp int) {
// Four values minNotWorsenedRate, minBetterRate, maxBetterRate, maxNotWorsenedRate can be determined from src and dst.
// peersRate < minNotWorsenedRate ====> cmp == -1
// peersRate < minNotWorsenedRate ====> cmp == -2
// minNotWorsenedRate <= peersRate < minBetterRate ====> cmp == 0
// minBetterRate <= peersRate <= maxBetterRate ====> cmp == 1
// maxBetterRate < peersRate <= maxNotWorsenedRate ====> cmp == 0
// peersRate > maxNotWorsenedRate ====> cmp == -1
// maxBetterRate < peersRate <= maxNotWorsenedRate ====> cmp == -1
// peersRate > maxNotWorsenedRate ====> cmp == -2
bs.cur.debugMessage = append(bs.cur.debugMessage, fmt.Sprintf("%s-dim, %s-type, %s-type",
dimToString(dim), bs.rwTy.String(), bs.opTy.String()))

srcRate, dstRate := bs.cur.getExtremeLoad(dim)
srcRate, dstRate, maxPendingRate := bs.cur.getExtremeLoad(dim)
peersRate := bs.cur.getPeersRateFromCache(dim)
highRate, lowRate := srcRate, dstRate
reverse := false
Expand All @@ -1063,15 +1085,15 @@ func (bs *balanceSolver) getBalanceBoostByPriorities(dim int) (cmp int) {
bs.cur.debugMessage = append(bs.cur.debugMessage, fmt.Sprintf("high-rate: %.0f, low-rate: %.0f, peersRate: %.0f, reverse: %t",
highRate, lowRate, peersRate, reverse))

if highRate*bs.balancedCheckRatio <= lowRate {
if highRate*rs.balancedCheckRatio <= lowRate {
// At this time, it is considered to be in the balanced state, and cmp = 1 will not be judged.
// If the balanced state is not broken, cmp = 0.
// If the balanced state is broken, cmp = -1.

// highRate - (highRate+lowRate)/(1.0+bs.balancedRatio)
minNotWorsenedRate := (highRate*bs.balancedRatio - lowRate) / (1.0 + bs.balancedRatio)
// highRate - (highRate+lowRate)/(1.0+bs.balancedRatio)*bs.balancedRatio
maxNotWorsenedRate := (highRate - lowRate*bs.balancedRatio) / (1.0 + bs.balancedRatio)
// highRate - (highRate+lowRate)/(1.0+balancedRatio)
minNotWorsenedRate := (highRate*rs.balancedRatio - lowRate) / (1.0 + rs.balancedRatio)
// highRate - (highRate+lowRate)/(1.0+balancedRatio)*balancedRatio
maxNotWorsenedRate := (highRate - lowRate*rs.balancedRatio) / (1.0 + rs.balancedRatio)
if minNotWorsenedRate > 0 {
minNotWorsenedRate = 0
}
Expand All @@ -1081,19 +1103,19 @@ func (bs *balanceSolver) getBalanceBoostByPriorities(dim int) (cmp int) {
return 0
}
bs.cur.debugMessage = append(bs.cur.debugMessage, "balanced-state, cmp: -1")
return -1
return -2
}

var minNotWorsenedRate, minBetterRate, maxBetterRate, maxNotWorsenedRate float64
var state string
if highRate*bs.preBalancedCheckRatio <= lowRate {
if highRate*rs.preBalancedCheckRatio <= lowRate {
// At this time, it is considered to be in pre-balanced state.
// Only the schedules that reach the balanced state will be judged as 1,
// and the schedules that do not destroy the pre-balanced state will be judged as 0.
minNotWorsenedRate = (highRate*bs.preBalancedRatio - lowRate) / (1.0 + bs.preBalancedRatio)
minBetterRate = (highRate*bs.balancedRatio - lowRate) / (1.0 + bs.balancedRatio)
maxBetterRate = (highRate - lowRate*bs.balancedRatio) / (1.0 + bs.balancedRatio)
maxNotWorsenedRate = (highRate - lowRate*bs.preBalancedRatio) / (1.0 + bs.preBalancedRatio)
minNotWorsenedRate = (highRate*rs.preBalancedRatio - lowRate) / (1.0 + rs.preBalancedRatio)
minBetterRate = (highRate*rs.balancedRatio - lowRate) / (1.0 + rs.balancedRatio)
maxBetterRate = (highRate - lowRate*rs.balancedRatio) / (1.0 + rs.balancedRatio)
maxNotWorsenedRate = (highRate - lowRate*rs.preBalancedRatio) / (1.0 + rs.preBalancedRatio)
if minNotWorsenedRate > 0 {
minNotWorsenedRate = 0
}
Expand All @@ -1103,13 +1125,13 @@ func (bs *balanceSolver) getBalanceBoostByPriorities(dim int) (cmp int) {
// As long as the balance is significantly improved, it is judged as 1.
// If the balance is not reduced, it is judged as 0.
// If the rate relationship between src and dst is reversed, there will be a certain penalty.
minBalancedRate := (highRate*bs.balancedRatio - lowRate) / (1.0 + bs.balancedRatio)
maxBalancedRate := (highRate - lowRate*bs.balancedRatio) / (1.0 + bs.balancedRatio)
minBalancedRate := (highRate*rs.balancedRatio - lowRate) / (1.0 + rs.balancedRatio)
maxBalancedRate := (highRate - lowRate*rs.balancedRatio) / (1.0 + rs.balancedRatio)

minNotWorsenedRate = -bs.getMinRate(dim)
minBetterRate = math.Min(minBalancedRate*bs.perceivedRatio, bs.minPerceivedLoads[dim])
maxBetterRate = maxBalancedRate + (highRate-lowRate-minBetterRate-maxBalancedRate)*bs.perceivedRatio
maxNotWorsenedRate = maxBalancedRate + (highRate-lowRate-minNotWorsenedRate-maxBalancedRate)*bs.perceivedRatio
minBetterRate = math.Min(minBalancedRate*rs.perceivedRatio, bs.minPerceivedLoads[dim])
maxBetterRate = maxBalancedRate + (highRate-lowRate-minBetterRate-maxBalancedRate)*rs.perceivedRatio
maxNotWorsenedRate = maxBalancedRate + (highRate-lowRate-minNotWorsenedRate-maxBalancedRate)*rs.perceivedRatio
if maxBetterRate < minBetterRate {
maxBetterRate = minBetterRate
}
Expand All @@ -1125,17 +1147,17 @@ func (bs *balanceSolver) getBalanceBoostByPriorities(dim int) (cmp int) {
greaterMinRate := peersRate >= bs.getMinRate(dim)
isTolerance := bs.isTolerance(dim, reverse)
otherMessage = fmt.Sprintf(", >=min-rate: %t, is-tolerance: %t", greaterMinRate, isTolerance)
if peersRate >= bs.getMinRate(dim) && bs.isTolerance(dim, reverse) {
if peersRate >= bs.getMinRate(dim) && bs.isTolerance(dim, reverse) && (state == "non-balanced" || maxPendingRate < 1) {
cmp = 1
} else {
cmp = 0
}
case minNotWorsenedRate <= peersRate && peersRate < minBetterRate:
cmp = 0
case maxBetterRate < peersRate && peersRate <= maxNotWorsenedRate:
cmp = 0
default:
cmp = -1
default:
cmp = -2
}
bs.cur.debugMessage = append(bs.cur.debugMessage, fmt.Sprintf("%s-state, cmp: %d%s", state, cmp, otherMessage))
return
Expand Down Expand Up @@ -1217,15 +1239,15 @@ func (bs *balanceSolver) betterThan(old *solution) bool {
}
// prefer smaller second priority rate, to reduce oscillation
return secondCmp < 0
case -2: // isNotWorsened(firstPriority) && isBetter(secondPriority)
case -2: // isBetter(firstPriority)
return firstCmp > 0
// TODO: The smaller the difference between the value and the expectation, the better.
case -1: // isNotWorsened(firstPriority) && isBetter(secondPriority)
if secondCmp != 0 {
return secondCmp > 0
}
// prefer smaller first priority rate, to reduce oscillation
return firstCmp < 0
case -1: // isBetter(firstPriority)
return firstCmp > 0
// TODO: The smaller the difference between the value and the expectation, the better.
}
}

Expand Down Expand Up @@ -1353,9 +1375,9 @@ func (bs *balanceSolver) buildOperators() (ops []*operator.Operator) {
case -3:
dim = dimToString(bs.firstPriority)
case -2:
dim = dimToString(bs.secondPriority)
case -1:
dim = dimToString(bs.firstPriority) + "-only"
case -1:
dim = dimToString(bs.secondPriority)
}

var createOperator func(region *core.RegionInfo, srcStoreID, dstStoreID uint64) (op *operator.Operator, typ string, err error)
Expand Down