Skip to content

Commit

Permalink
Support setting maxHealthyPercentage to configure ASG instance refresh
Browse files Browse the repository at this point in the history
  • Loading branch information
fiunchinho committed Oct 8, 2024
1 parent 11cb106 commit 893c671
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -987,6 +987,22 @@ spec:
The default is to use the value for the health check grace period defined for the group.
format: int64
type: integer
maxHealthyPercentage:
description: |-
The amount of capacity as a percentage in ASG that can be in service and healthy, or pending,
to support your workload when replacing instances.
The value is expressed as a percentage of the desired capacity of the ASG. Value range is 100 to 200.
If you specify MaxHealthyPercentage , you must also specify MinHealthyPercentage , and the difference between
them cannot be greater than 100.
A larger range increases the number of instances that can be replaced at the same time.
If you do not specify this property, the default is 100 percent, or the percentage set in the instance
maintenance policy for the Auto Scaling group, if defined.
format: int64
type: integer
minHealthyPercentage:
description: |-
The amount of capacity as a percentage in ASG that must remain healthy
Expand Down
3 changes: 2 additions & 1 deletion exp/api/v1beta1/conversion.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
if restored.Spec.SuspendProcesses != nil {
dst.Spec.SuspendProcesses = restored.Spec.SuspendProcesses
}
if dst.Spec.RefreshPreferences != nil && restored.Spec.RefreshPreferences != nil {
if restored.Spec.RefreshPreferences != nil {
dst.Spec.RefreshPreferences.Disable = restored.Spec.RefreshPreferences.Disable
dst.Spec.RefreshPreferences.MaxHealthyPercentage = restored.Spec.RefreshPreferences.MaxHealthyPercentage
}
if restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions != nil {
dst.Spec.AWSLaunchTemplate.InstanceMetadataOptions = restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions
Expand Down
1 change: 1 addition & 0 deletions exp/api/v1beta1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions exp/api/v1beta2/awsmachinepool_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,19 @@ type RefreshPreferences struct {
// during an instance refresh. The default is 90.
// +optional
MinHealthyPercentage *int64 `json:"minHealthyPercentage,omitempty"`

// The amount of capacity as a percentage in ASG that can be in service and healthy, or pending,
// to support your workload when replacing instances.
// The value is expressed as a percentage of the desired capacity of the ASG. Value range is 100 to 200.
// If you specify MaxHealthyPercentage , you must also specify MinHealthyPercentage , and the difference between
// them cannot be greater than 100.
//
// A larger range increases the number of instances that can be replaced at the same time.
//
// If you do not specify this property, the default is 100 percent, or the percentage set in the instance
// maintenance policy for the Auto Scaling group, if defined.
// +optional
MaxHealthyPercentage *int64 `json:"maxHealthyPercentage,omitempty"`
}

// AWSMachinePoolStatus defines the observed state of AWSMachinePool.
Expand Down
23 changes: 23 additions & 0 deletions exp/api/v1beta2/awsmachinepool_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ func (r *AWSMachinePool) validateAdditionalSecurityGroups() field.ErrorList {
}
return allErrs
}

func (r *AWSMachinePool) validateSpotInstances() field.ErrorList {
var allErrs field.ErrorList
if r.Spec.AWSLaunchTemplate.SpotMarketOptions != nil && r.Spec.MixedInstancesPolicy != nil {
Expand All @@ -133,6 +134,26 @@ func (r *AWSMachinePool) validateIgnition() field.ErrorList {
return allErrs
}

func (r *AWSMachinePool) validateRefreshPreferences() field.ErrorList {
var allErrs field.ErrorList

if r.Spec.RefreshPreferences == nil {
return allErrs
}

if r.Spec.RefreshPreferences.MaxHealthyPercentage != nil && r.Spec.RefreshPreferences.MinHealthyPercentage == nil {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec.refreshPreferences.maxHealthyPercentage"), "If you specify spec.refreshPreferences.maxHealthyPercentage, you must also specify spec.refreshPreferences.minHealthyPercentage"))
}

if r.Spec.RefreshPreferences.MaxHealthyPercentage != nil && r.Spec.RefreshPreferences.MinHealthyPercentage != nil {
if *r.Spec.RefreshPreferences.MaxHealthyPercentage-*r.Spec.RefreshPreferences.MinHealthyPercentage > 100 {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec.refreshPreferences.maxHealthyPercentage"), "the difference between spec.refreshPreferences.maxHealthyPercentage and spec.refreshPreferences.minHealthyPercentage cannot be greater than 100"))
}
}

return allErrs
}

// ValidateCreate will do any extra validation when creating a AWSMachinePool.
func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
Expand All @@ -146,6 +167,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
allErrs = append(allErrs, r.validateSpotInstances()...)
allErrs = append(allErrs, r.validateIgnition()...)
allErrs = append(allErrs, r.validateRefreshPreferences()...)

if len(allErrs) == 0 {
return nil, nil
Expand All @@ -167,6 +189,7 @@ func (r *AWSMachinePool) ValidateUpdate(_ runtime.Object) (admission.Warnings, e
allErrs = append(allErrs, r.validateSubnets()...)
allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
allErrs = append(allErrs, r.validateSpotInstances()...)
allErrs = append(allErrs, r.validateRefreshPreferences()...)

if len(allErrs) == 0 {
return nil, nil
Expand Down
42 changes: 42 additions & 0 deletions exp/api/v1beta2/awsmachinepool_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,27 @@ func TestAWSMachinePoolValidateCreate(t *testing.T) {
},
wantErr: true,
},
{
name: "Should fail if MaxHealthyPercentage is set, but MinHealthyPercentage is not set",
pool: &AWSMachinePool{
Spec: AWSMachinePoolSpec{
RefreshPreferences: &RefreshPreferences{MaxHealthyPercentage: aws.Int64(100)},
},
},
wantErr: true,
},
{
name: "Should fail if the difference between MaxHealthyPercentage and MinHealthyPercentage is greater than 100",
pool: &AWSMachinePool{
Spec: AWSMachinePoolSpec{
RefreshPreferences: &RefreshPreferences{
MaxHealthyPercentage: aws.Int64(150),
MinHealthyPercentage: aws.Int64(25),
},
},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -287,6 +308,27 @@ func TestAWSMachinePoolValidateUpdate(t *testing.T) {
},
wantErr: true,
},
{
name: "Should fail if MaxHealthyPercentage is set, but MinHealthyPercentage is not set",
new: &AWSMachinePool{
Spec: AWSMachinePoolSpec{
RefreshPreferences: &RefreshPreferences{MaxHealthyPercentage: aws.Int64(100)},
},
},
wantErr: true,
},
{
name: "Should fail if the difference between MaxHealthyPercentage and MinHealthyPercentage is greater than 100",
new: &AWSMachinePool{
Spec: AWSMachinePoolSpec{
RefreshPreferences: &RefreshPreferences{
MaxHealthyPercentage: aws.Int64(150),
MinHealthyPercentage: aws.Int64(25),
},
},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
5 changes: 5 additions & 0 deletions exp/api/v1beta2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion pkg/cloud/services/autoscaling/autoscalinggroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ func (s *Service) CancelASGInstanceRefresh(scope *scope.MachinePoolScope) error
// StartASGInstanceRefresh will start an ASG instance refresh.
func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
strategy := ptr.To[string](autoscaling.RefreshStrategyRolling)
var minHealthyPercentage, instanceWarmup *int64
var minHealthyPercentage, maxHealthyPercentage, instanceWarmup *int64
if scope.AWSMachinePool.Spec.RefreshPreferences != nil {
if scope.AWSMachinePool.Spec.RefreshPreferences.Strategy != nil {
strategy = scope.AWSMachinePool.Spec.RefreshPreferences.Strategy
Expand All @@ -384,6 +384,9 @@ func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
if scope.AWSMachinePool.Spec.RefreshPreferences.MinHealthyPercentage != nil {
minHealthyPercentage = scope.AWSMachinePool.Spec.RefreshPreferences.MinHealthyPercentage
}
if scope.AWSMachinePool.Spec.RefreshPreferences.MaxHealthyPercentage != nil {
maxHealthyPercentage = scope.AWSMachinePool.Spec.RefreshPreferences.MaxHealthyPercentage
}
}

input := &autoscaling.StartInstanceRefreshInput{
Expand All @@ -392,6 +395,7 @@ func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
Preferences: &autoscaling.RefreshPreferences{
InstanceWarmup: instanceWarmup,
MinHealthyPercentage: minHealthyPercentage,
MaxHealthyPercentage: maxHealthyPercentage,
},
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/cloud/services/autoscaling/autoscalinggroup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1207,6 +1207,7 @@ func TestServiceStartASGInstanceRefresh(t *testing.T) {
Preferences: &autoscaling.RefreshPreferences{
InstanceWarmup: aws.Int64(100),
MinHealthyPercentage: aws.Int64(80),
MaxHealthyPercentage: aws.Int64(100),
},
})).
Return(nil, awserrors.NewNotFound("not found"))
Expand All @@ -1222,6 +1223,7 @@ func TestServiceStartASGInstanceRefresh(t *testing.T) {
Preferences: &autoscaling.RefreshPreferences{
InstanceWarmup: aws.Int64(100),
MinHealthyPercentage: aws.Int64(80),
MaxHealthyPercentage: aws.Int64(100),
},
})).
Return(&autoscaling.StartInstanceRefreshOutput{}, nil)
Expand Down Expand Up @@ -1316,6 +1318,7 @@ func getMachinePoolScope(client client.Client, clusterScope *scope.ClusterScope)
Strategy: aws.String("Rolling"),
InstanceWarmup: aws.Int64(100),
MinHealthyPercentage: aws.Int64(80),
MaxHealthyPercentage: aws.Int64(100),
},
MixedInstancesPolicy: &expinfrav1.MixedInstancesPolicy{
InstancesDistribution: &expinfrav1.InstancesDistribution{
Expand Down

0 comments on commit 893c671

Please sign in to comment.