Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: cn in the Bound phase should not be scaled in first #544

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/controllers/cnstore/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,7 @@ func (c *withCNSet) syncStats(ctx *recon.Context[*corev1.Pod]) error {
// clean previously recorded score and update startTime if CN is restarted
sc.Restarted(startedTime)
}
sc.Phase = pod.Labels[v1alpha1.CNPodPhaseLabel]

uid := v1alpha1.GetCNPodUUID(pod)
moVersion := common.GetSemanticVersion(&pod.ObjectMeta)
Expand Down
21 changes: 20 additions & 1 deletion pkg/controllers/common/cnstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ const (
ReclaimedAt = "matrixorigin.io/reclaimed-at"

SemanticVersionAnno = "matrixorigin.io/semantic-version"

sessionWeight = 1
pipelineWeight = 0
phaseWeight = 100
maxCost = 1000
)

func AddReadinessGate(podSpec *corev1.PodSpec, ct corev1.PodConditionType) {
Expand Down Expand Up @@ -151,10 +156,24 @@ type StoreScore struct {
PipelineCount int `json:"pipelineCount"`

StartedTime *time.Time `json:"startedTime,omitempty"`
Phase string `json:"phase,omitempty"`
}

func (s *StoreScore) GenDeletionCost() int {
return s.SessionCount
deletionCost := s.SessionCount*sessionWeight + s.PipelineCount*pipelineWeight
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sessionCount and pipelineCount can go easily over phaseWeight, e.g. serving 1000 sessions in one host is a common case

switch s.Phase {
case v1alpha1.CNPodPhaseBound:
deletionCost += phaseWeight * 2 // Bound should not be deleted
case v1alpha1.CNPodPhaseIdle, v1alpha1.CNPodPhaseTerminating:
deletionCost += phaseWeight * 0 // Idle and Terminating should be deleted first
case v1alpha1.CNPodPhaseDraining:
deletionCost += phaseWeight / 2 // Draining may be deleted later
}
// if deletion cost is larger than maxScore, normalize it from [0, maxCost]
for deletionCost > maxCost {
deletionCost >>= 1
Comment on lines +173 to +174
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buggy: maxCost would be larger than maxCost+1 after normalization

}
return deletionCost
}

func (s *StoreScore) IsSafeToReclaim() bool {
Expand Down
70 changes: 70 additions & 0 deletions pkg/controllers/common/cnstore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
package common

import (
"github.com/matrixorigin/matrixone-operator/api/core/v1alpha1"
corev1 "k8s.io/api/core/v1"
"reflect"
"sort"
"testing"
)

Expand All @@ -38,3 +41,70 @@ func TestNeedUpdateImage_DifferentImages(t *testing.T) {
t.Errorf("Expected true, got false")
}
}

func TestGenDeletionCostSorting(t *testing.T) {
type namedStoreScore struct {
Name string
Score StoreScore
}
tests := []struct {
name string
scores []namedStoreScore
want []string
}{
{
name: "Basic sorting",
scores: []namedStoreScore{
{"A", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"B", StoreScore{SessionCount: 2, PipelineCount: 1, Phase: v1alpha1.CNPodPhaseIdle}},
{"C", StoreScore{SessionCount: 10, PipelineCount: 5, Phase: v1alpha1.CNPodPhaseDraining}},
},
want: []string{"B", "C", "A"},
},
{
name: "Sorting with same scores",
scores: []namedStoreScore{
{"A", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"B", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"C", StoreScore{SessionCount: 2, PipelineCount: 1, Phase: v1alpha1.CNPodPhaseIdle}},
},
want: []string{"C", "A", "B"},
},
{
name: "Sorting with different phases",
scores: []namedStoreScore{
{"A", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"B", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseIdle}},
{"C", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseDraining}},
{"D", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseTerminating}},
},
want: []string{"B", "D", "C", "A"},
},
{
name: "Sorting with different session count",
scores: []namedStoreScore{
{"A", StoreScore{SessionCount: 2, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"B", StoreScore{SessionCount: 3, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
{"C", StoreScore{SessionCount: 5, PipelineCount: 3, Phase: v1alpha1.CNPodPhaseBound}},
},
want: []string{"A", "B", "C"},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sort.Slice(tt.scores, func(i, j int) bool {
return tt.scores[i].Score.GenDeletionCost() < tt.scores[j].Score.GenDeletionCost()
})

got := make([]string, len(tt.scores))
for i, score := range tt.scores {
got[i] = score.Name
}

if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GenDeletionCost sorting = %v, want %v", got, tt.want)
}
})
}
}
Loading