Add subset capacity planning for UnitedDeployment (#1428)

* add subset minReplicas&maxReplicas api for UnitedDeployment Signed-off-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com> * add subset capacity planning for UnitiedDeployment Signed-off-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com> --------- Signed-off-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com> Co-authored-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com>
openkruise · Oct 25, 2023 · 18b15d5 · 18b15d5
1 parent b9484d6
commit 18b15d5
Show file tree

Hide file tree

Showing 11 changed files with 698 additions and 67 deletions.
diff --git a/apis/apps/defaults/v1alpha1.go b/apis/apps/defaults/v1alpha1.go
@@ -225,6 +225,25 @@ func SetDefaultsUnitedDeployment(obj *v1alpha1.UnitedDeployment, injectTemplateD
 			}
 		}
 	}
+
+	hasReplicasSettings := false
+	hasCapacitySettings := false
+	for _, subset := range obj.Spec.Topology.Subsets {
+		if subset.Replicas != nil {
+			hasReplicasSettings = true
+		}
+		if subset.MinReplicas != nil || subset.MaxReplicas != nil {
+			hasCapacitySettings = true
+		}
+	}
+	if hasCapacitySettings && !hasReplicasSettings {
+		for i := range obj.Spec.Topology.Subsets {
+			subset := &obj.Spec.Topology.Subsets[i]
+			if subset.MinReplicas == nil {
+				subset.MinReplicas = &intstr.IntOrString{Type: intstr.Int, IntVal: 0}
+			}
+		}
+	}
 }
 
 // SetDefaults_CloneSet set default values for CloneSet.

diff --git a/apis/apps/v1alpha1/uniteddeployment_types.go b/apis/apps/v1alpha1/uniteddeployment_types.go
@@ -189,8 +189,26 @@ type Subset struct {
 	// percentage like '10%', which means 10% of UnitedDeployment replicas of pods will be distributed
 	// under this subset. If nil, the number of replicas in this subset is determined by controller.
 	// Controller will try to keep all the subsets with nil replicas have average pods.
+	// Replicas and MinReplicas/MaxReplicas are mutually exclusive in a UnitedDeployment.
 	// +optional
 	Replicas *intstr.IntOrString `json:"replicas,omitempty"`
+
+	// Indicates the lower bounded replicas of the subset.
+	// MinReplicas must be more than or equal to 0 if it is set.
+	// Controller will prioritize satisfy minReplicas for each subset
+	// according to the order of Topology.Subsets.
+	// Defaults to 0.
+	// +optional
+	MinReplicas *intstr.IntOrString `json:"minReplicas,omitempty"`
+
+	// Indicates the upper bounded replicas of the subset.
+	// MaxReplicas must be more than or equal to MinReplicas.
+	// MaxReplicas == nil means no limitation.
+	// Please ensure that at least one subset has empty MaxReplicas(no limitation) to avoid stuck scaling.
+	// Defaults to nil.
+	// +optional
+	MaxReplicas *intstr.IntOrString `json:"maxReplicas,omitempty"`
+
 	// Patch indicates patching to the templateSpec.
 	// Patch takes precedence over other fields
 	// If the Patch also modifies the Replicas, NodeSelectorTerm or Tolerations, use value in the Patch

diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml
@@ -975,6 +975,26 @@ spec:
                     items:
                       description: Subset defines the detail of a subset.
                       properties:
+                        maxReplicas:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          description: Indicates the upper bounded replicas of the
+                            subset. MaxReplicas must be more than or equal to MinReplicas.
+                            MaxReplicas == nil means no limitation. Please ensure
+                            that at least one subset has empty MaxReplicas(no limitation)
+                            to avoid stuck scaling. Defaults to nil.
+                          x-kubernetes-int-or-string: true
+                        minReplicas:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          description: Indicates the lower bounded replicas of the
+                            subset. MinReplicas must be more than or equal to 0 if
+                            it is set. Controller will prioritize satisfy minReplicas
+                            for each subset according to the order of Topology.Subsets.
+                            Defaults to 0.
+                          x-kubernetes-int-or-string: true
                         name:
                           description: Indicates subset name as a DNS_LABEL, which
                             will be used to generate subset workload name prefix in
@@ -1072,7 +1092,8 @@ spec:
                             pods will be distributed under this subset. If nil, the
                             number of replicas in this subset is determined by controller.
                             Controller will try to keep all the subsets with nil replicas
-                            have average pods.
+                            have average pods. Replicas and MinReplicas/MaxReplicas
+                            are mutually exclusive in a UnitedDeployment.
                           x-kubernetes-int-or-string: true
                         tolerations:
                           description: Indicates the tolerations the pods under this

diff --git a/pkg/controller/uniteddeployment/allocator.go b/pkg/controller/uniteddeployment/allocator.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/utils/integer"
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 )
@@ -54,33 +55,43 @@ func (n subsetInfos) Swap(i, j int) {
 	n[i], n[j] = n[j], n[i]
 }
 
-// GetAllocatedReplicas returns a mapping from subset to next replicas.
-// Next replicas is allocated by replicasAllocator, which will consider the current replicas of each subset and
-// new replicas indicated from UnitedDeployment.Spec.Topology.Subsets.
-func GetAllocatedReplicas(nameToSubset *map[string]*Subset, ud *appsv1alpha1.UnitedDeployment) (*map[string]int32, error) {
-	subsetInfos := getSubsetInfos(nameToSubset, ud)
+type ReplicaAllocator interface {
+	Alloc(nameToSubset *map[string]*Subset) (*map[string]int32, error)
+}
 
-	var expectedReplicas int32 = -1
-	if ud.Spec.Replicas != nil {
-		expectedReplicas = *ud.Spec.Replicas
+func NewReplicaAllocator(ud *appsv1alpha1.UnitedDeployment) ReplicaAllocator {
+	for _, subset := range ud.Spec.Topology.Subsets {
+		if subset.MinReplicas != nil || subset.MaxReplicas != nil {
+			return &elasticAllocator{ud}
+		}
 	}
-
-	specifiedReplicas := getSpecifiedSubsetReplicas(expectedReplicas, ud)
-	klog.V(4).Infof("UnitedDeployment %s/%s specifiedReplicas: %v", ud.Namespace, ud.Name, specifiedReplicas)
-	// call SortToAllocator to sort all subset by subset.Replicas in order of increment
-	return subsetInfos.SortToAllocator().AllocateReplicas(expectedReplicas, specifiedReplicas)
+	return &specificAllocator{UnitedDeployment: ud}
 }
 
-func (n subsetInfos) SortToAllocator() *replicasAllocator {
-	sort.Sort(n)
-	return &replicasAllocator{subsets: &n}
+type specificAllocator struct {
+	*appsv1alpha1.UnitedDeployment
+	subsets *subsetInfos
 }
 
-type replicasAllocator struct {
-	subsets *subsetInfos
+// Alloc returns a mapping from subset to next replicas.
+// Next replicas is allocated by realReplicasAllocator, which will consider the current replicas of each subset and
+// new replicas indicated from UnitedDeployment.Spec.Topology.Subsets.
+func (s *specificAllocator) Alloc(nameToSubset *map[string]*Subset) (*map[string]int32, error) {
+	// SortToAllocator to sort all subset by subset.Replicas in order of increment
+	s.subsets = getSubsetInfos(nameToSubset, s.UnitedDeployment)
+	sort.Sort(s.subsets)
+
+	var expectedReplicas int32 = -1
+	if s.Spec.Replicas != nil {
+		expectedReplicas = *s.Spec.Replicas
+	}
+
+	specifiedReplicas := getSpecifiedSubsetReplicas(expectedReplicas, s.UnitedDeployment)
+	klog.V(4).Infof("UnitedDeployment %s/%s specifiedReplicas: %v", s.Namespace, s.Name, specifiedReplicas)
+	return s.AllocateReplicas(expectedReplicas, specifiedReplicas)
 }
 
-func (s *replicasAllocator) validateReplicas(replicas int32, subsetReplicasLimits *map[string]int32) error {
+func (s *specificAllocator) validateReplicas(replicas int32, subsetReplicasLimits *map[string]int32) error {
 	if subsetReplicasLimits == nil {
 		return nil
 	}
@@ -150,7 +161,7 @@ func getSubsetInfos(nameToSubset *map[string]*Subset, ud *appsv1alpha1.UnitedDep
 // AllocateReplicas will first try to check the specifiedSubsetReplicas is valid or not.
 // If valid , normalAllocate will be called. It will apply these specified replicas, then average the rest replicas to left unspecified subsets.
 // If not, it will return error
-func (s *replicasAllocator) AllocateReplicas(replicas int32, specifiedSubsetReplicas *map[string]int32) (
+func (s *specificAllocator) AllocateReplicas(replicas int32, specifiedSubsetReplicas *map[string]int32) (
 	*map[string]int32, error) {
 	if err := s.validateReplicas(replicas, specifiedSubsetReplicas); err != nil {
 		return nil, err
@@ -159,7 +170,7 @@ func (s *replicasAllocator) AllocateReplicas(replicas int32, specifiedSubsetRepl
 	return s.normalAllocate(replicas, specifiedSubsetReplicas), nil
 }
 
-func (s *replicasAllocator) normalAllocate(expectedReplicas int32, specifiedSubsetReplicas *map[string]int32) *map[string]int32 {
+func (s *specificAllocator) normalAllocate(expectedReplicas int32, specifiedSubsetReplicas *map[string]int32) *map[string]int32 {
 	var specifiedReplicas int32
 	specifiedSubsetCount := 0
 	// Step 1: apply replicas to specified subsets, and mark them as specified = true.
@@ -203,7 +214,7 @@ func (s *replicasAllocator) normalAllocate(expectedReplicas int32, specifiedSubs
 	return s.toSubsetReplicaMap()
 }
 
-func (s *replicasAllocator) toSubsetReplicaMap() *map[string]int32 {
+func (s *specificAllocator) toSubsetReplicaMap() *map[string]int32 {
 	allocatedReplicas := map[string]int32{}
 	for _, subset := range *s.subsets {
 		allocatedReplicas[subset.SubsetName] = subset.Replicas
@@ -212,7 +223,7 @@ func (s *replicasAllocator) toSubsetReplicaMap() *map[string]int32 {
 	return &allocatedReplicas
 }
 
-func (s *replicasAllocator) String() string {
+func (s *specificAllocator) String() string {
 	result := ""
 	sort.Sort(s.subsets)
 	for _, subset := range *s.subsets {
@@ -221,3 +232,88 @@ func (s *replicasAllocator) String() string {
 
 	return result
 }
+
+type elasticAllocator struct {
+	*appsv1alpha1.UnitedDeployment
+}
+
+// Alloc returns a mapping from subset to next replicas.
+// Next replicas is allocated by elasticAllocator, which will consider the current minReplicas and maxReplicas
+// of each subset and spec.replicas of UnitedDeployment. For example:
+// spec.replicas: 5
+// subsets:
+//   - name: subset-a
+//     minReplicas: 2    # will be satisfied with 1st priority
+//     maxReplicas: 4    # will be satisfied with 3rd priority
+//   - name: subset-b
+//     minReplicas: 2    # will be satisfied with 2nd priority
+//     maxReplicas: nil  # will be satisfied with 4th priority
+//
+// the results of map will be: {"subset-a": 3, "subset-b": 2}
+func (ac *elasticAllocator) Alloc(_ *map[string]*Subset) (*map[string]int32, error) {
+	replicas := int32(1)
+	if ac.Spec.Replicas != nil {
+		replicas = *ac.Spec.Replicas
+	}
+
+	minReplicasMap, maxReplicasMap, err := ac.validateAndCalculateMinMaxMap(replicas)
+	if err != nil {
+		return nil, err
+	}
+	return ac.alloc(replicas, minReplicasMap, maxReplicasMap), nil
+}
+
+func (ac *elasticAllocator) validateAndCalculateMinMaxMap(replicas int32) (map[string]int32, map[string]int32, error) {
+	totalMin, totalMax := int64(0), int64(0)
+	numSubset := len(ac.Spec.Topology.Subsets)
+	minReplicasMap := make(map[string]int32, numSubset)
+	maxReplicasMap := make(map[string]int32, numSubset)
+	for index, subset := range ac.Spec.Topology.Subsets {
+		minReplicas := int32(0)
+		if subset.MinReplicas != nil {
+			minReplicas, _ = ParseSubsetReplicas(replicas, *subset.MinReplicas)
+		}
+		totalMin += int64(minReplicas)
+		minReplicasMap[subset.Name] = minReplicas
+
+		maxReplicas := int32(1000000)
+		if subset.MaxReplicas != nil {
+			maxReplicas, _ = ParseSubsetReplicas(replicas, *subset.MaxReplicas)
+		}
+		totalMax += int64(maxReplicas)
+		maxReplicasMap[subset.Name] = maxReplicas
+
+		if minReplicas > maxReplicas {
+			return nil, nil, fmt.Errorf("subset[%d].maxReplicas must be more than or equal to minReplicas", index)
+		}
+	}
+	return minReplicasMap, maxReplicasMap, nil
+}
+
+func (ac *elasticAllocator) alloc(replicas int32, minReplicasMap, maxReplicasMap map[string]int32) *map[string]int32 {
+	allocated := int32(0)
+	// Step 1: satisfy the minimum replicas of each subset firstly.
+	subsetReplicas := make(map[string]int32, len(ac.Spec.Topology.Subsets))
+	for _, subset := range ac.Spec.Topology.Subsets {
+		minReplicas := minReplicasMap[subset.Name]
+		addReplicas := integer.Int32Min(minReplicas, replicas-allocated)
+		addReplicas = integer.Int32Max(addReplicas, 0)
+		subsetReplicas[subset.Name] = addReplicas
+		allocated += addReplicas
+	}
+
+	if allocated >= replicas { // no quota to allocate.
+		return &subsetReplicas
+	}
+
+	// Step 2: satisfy the maximum replicas of each subset.
+	for _, subset := range ac.Spec.Topology.Subsets {
+		maxReplicas := maxReplicasMap[subset.Name]
+		minReplicas := minReplicasMap[subset.Name]
+		addReplicas := integer.Int32Min(maxReplicas-minReplicas, replicas-allocated)
+		addReplicas = integer.Int32Max(addReplicas, 0)
+		subsetReplicas[subset.Name] += addReplicas
+		allocated += addReplicas
+	}
+	return &subsetReplicas
+}