Skip to content

Commit

Permalink
Use Replica Override to manually scale a component (#665)
Browse files Browse the repository at this point in the history
* Use Replica Override to manually scale a component

* parse component status

* make readable

* handle missing k8s deployment

* simplify code

* use component pods instead of all pods

* Deprecate and add reset-scale endpoints

* refactor component status

* reafctor environment, aux resources, component spec

* bugfixes

* bugfixesuse status from component, not auxiliar service

* upadte tests

* Update api/deployments/models/component_deployment.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/environments/component_handler.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/deployments/models/component_status.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/deployments/models/component_status.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/deployments/models/component_status.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/deployments/models/component_status.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* fix panic in tests

* fix lint bugs

* Update api/environments/environment_handler.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* Update api/environments/environment_handler.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* remove unused function

* Add documentation

* return err if deployment not found

* Update api/utils/predicate/kubernetes.go

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>

* update swagger

* remove outdated null check

* remove outdated test, introduce WithComponentStatuserFunc

* Test component actions with status

* revert go 1.23

* cleanup unused code

* update radix-operator

---------

Co-authored-by: Sergey Smolnikov <ssmol@equinor.com>
  • Loading branch information
Richard87 and satr authored Sep 11, 2024
1 parent 2a61afe commit b1de696
Show file tree
Hide file tree
Showing 28 changed files with 801 additions and 1,055 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
version: v1.59.1
version: v1.60.3

test:
name: Unit Test
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ ifndef HAS_SWAGGER
go install github.com/go-swagger/go-swagger/cmd/swagger@v0.31.0
endif
ifndef HAS_GOLANGCI_LINT
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.58.2
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3
endif
ifndef HAS_MOCKGEN
go install github.com/golang/mock/mockgen@v1.6.0
Expand Down
17 changes: 8 additions & 9 deletions api/deployments/deployment_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"github.com/equinor/radix-api/models"
"github.com/equinor/radix-common/utils/slice"
"github.com/equinor/radix-operator/pkg/apis/kube"
v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
operatorUtils "github.com/equinor/radix-operator/pkg/apis/utils"
radixlabels "github.com/equinor/radix-operator/pkg/apis/utils/labels"
"k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -28,7 +28,6 @@ type DeployHandler interface {
GetDeploymentsForApplicationEnvironment(ctx context.Context, appName, environment string, latest bool) ([]*deploymentModels.DeploymentSummary, error)
GetComponentsForDeploymentName(ctx context.Context, appName, deploymentID string) ([]*deploymentModels.Component, error)
GetComponentsForDeployment(ctx context.Context, appName, deploymentName, envName string) ([]*deploymentModels.Component, error)
GetLatestDeploymentForApplicationEnvironment(ctx context.Context, appName, environment string) (*deploymentModels.DeploymentSummary, error)
GetDeploymentsForPipelineJob(context.Context, string, string) ([]*deploymentModels.DeploymentSummary, error)
GetJobComponentDeployments(context.Context, string, string, string) ([]*deploymentModels.DeploymentItem, error)
}
Expand Down Expand Up @@ -64,6 +63,7 @@ func (deploy *deployHandler) GetLogs(ctx context.Context, appName, podName strin

return log, nil
}

return nil, deploymentModels.NonExistingPod(appName, podName)
}

Expand Down Expand Up @@ -198,7 +198,7 @@ func (deploy *deployHandler) GetDeploymentWithName(ctx context.Context, appName,
return dep, nil
}

func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, appName string, rd *v1.RadixDeployment) (*v1.RadixJob, error) {
func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, appName string, rd *radixv1.RadixDeployment) (*radixv1.RadixJob, error) {
jobName := rd.GetLabels()[kube.RadixJobNameLabel]
radixJob, err := kubequery.GetRadixJob(ctx, deploy.accounts.UserAccount.RadixClient, appName, jobName)
if err != nil {
Expand All @@ -211,15 +211,14 @@ func (deploy *deployHandler) getRadixDeploymentRadixJob(ctx context.Context, app
}

func (deploy *deployHandler) getEnvironmentNames(ctx context.Context, appName string) ([]string, error) {
radixlabels.ForApplicationName(appName).AsSelector()
labelSelector := radixlabels.ForApplicationName(appName).AsSelector()

reList, err := deploy.accounts.ServiceAccount.RadixClient.RadixV1().RadixEnvironments().List(ctx, metav1.ListOptions{LabelSelector: labelSelector.String()})
if err != nil {
return nil, err
}

return slice.Map(reList.Items, func(re v1.RadixEnvironment) string {
return slice.Map(reList.Items, func(re radixv1.RadixEnvironment) string {
return re.Spec.EnvName
}), nil
}
Expand All @@ -239,7 +238,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string,
rdLabelSelector = rdLabelSelector.Add(*jobNameLabel)
}

var radixDeploymentList []v1.RadixDeployment
var radixDeploymentList []radixv1.RadixDeployment
namespaces := slice.Map(environments, func(env string) string { return operatorUtils.GetEnvironmentNamespace(appName, env) })
for _, ns := range namespaces {
rdList, err := deploy.accounts.UserAccount.RadixClient.RadixV1().RadixDeployments(ns).List(ctx, metav1.ListOptions{LabelSelector: rdLabelSelector.String()})
Expand All @@ -250,7 +249,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string,
}

appNamespace := operatorUtils.GetAppNamespace(appName)
radixJobMap := make(map[string]*v1.RadixJob)
radixJobMap := make(map[string]*radixv1.RadixJob)

if jobName != "" {
radixJob, err := deploy.accounts.UserAccount.RadixClient.RadixV1().RadixJobs(appNamespace).Get(ctx, jobName, metav1.GetOptions{})
Expand Down Expand Up @@ -278,7 +277,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string,
rds := sortRdsByActiveFromDesc(radixDeploymentList)
var deploymentSummaries []*deploymentModels.DeploymentSummary
for _, rd := range rds {
if latest && rd.Status.Condition == v1.DeploymentInactive {
if latest && rd.Status.Condition == radixv1.DeploymentInactive {
continue
}

Expand All @@ -298,7 +297,7 @@ func (deploy *deployHandler) getDeployments(ctx context.Context, appName string,
return deploymentSummaries, nil
}

func sortRdsByActiveFromDesc(rds []v1.RadixDeployment) []v1.RadixDeployment {
func sortRdsByActiveFromDesc(rds []radixv1.RadixDeployment) []radixv1.RadixDeployment {
sort.Slice(rds, func(i, j int) bool {
if rds[j].Status.ActiveFrom.IsZero() {
return true
Expand Down
15 changes: 0 additions & 15 deletions api/deployments/mock/deployment_handler_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions api/deployments/models/component_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type componentBuilder struct {
gitTags string
resources *radixv1.ResourceRequirements
runtime *radixv1.Runtime
replicasOverride *int
}

func (b *componentBuilder) WithStatus(status ComponentStatus) ComponentBuilder {
Expand Down Expand Up @@ -99,6 +100,7 @@ func (b *componentBuilder) WithComponent(component radixv1.RadixCommonDeployComp
b.commitID = component.GetEnvironmentVariables()[defaults.RadixCommitHashEnvironmentVariable]
b.gitTags = component.GetEnvironmentVariables()[defaults.RadixGitTagsEnvironmentVariable]
b.runtime = component.GetRuntime()
b.replicasOverride = component.GetReplicasOverride()

ports := []Port{}
if component.GetPorts() != nil {
Expand Down Expand Up @@ -252,6 +254,7 @@ func (b *componentBuilder) BuildComponent() (*Component, error) {
Variables: variables,
Replicas: b.podNames,
ReplicaList: b.replicaSummaryList,
ReplicasOverride: b.replicasOverride,
SchedulerPort: b.schedulerPort,
ScheduledJobPayloadPath: b.scheduledJobPayloadPath,
AuxiliaryResource: b.auxResource,
Expand Down
10 changes: 9 additions & 1 deletion api/deployments/models/component_deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,14 @@ type Component struct {
// required: false
ReplicaList []ReplicaSummary `json:"replicaList"`

// Set if manual control of replicas is in place. Not set means automatic control, 0 means stopped and >= 1 is manually scaled.
//
// required: false
// example: 5
// Extensions:
// x-nullable: true
ReplicasOverride *int `json:"replicasOverride"`

// HorizontalScaling defines horizontal scaling summary for this component
//
// required: false
Expand Down Expand Up @@ -585,7 +593,7 @@ func getReplicaType(pod corev1.Pod) ReplicaType {
switch {
case pod.GetLabels()[kube.RadixPodIsJobSchedulerLabel] == "true":
return JobManager
case pod.GetLabels()[kube.RadixPodIsJobAuxObjectLabel] == "true":
case pod.GetLabels()[kube.RadixAuxiliaryComponentTypeLabel] == kube.RadixJobTypeManagerAux:
return JobManagerAux
case pod.GetLabels()[kube.RadixAuxiliaryComponentTypeLabel] == "oauth":
return OAuth2
Expand Down
58 changes: 49 additions & 9 deletions api/deployments/models/component_status.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
package models

import appsv1 "k8s.io/api/apps/v1"
import (
"github.com/equinor/radix-api/api/utils/owner"
commonutils "github.com/equinor/radix-common/utils"
"github.com/equinor/radix-common/utils/pointers"
operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults"
"github.com/equinor/radix-operator/pkg/apis/kube"
radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
)

// ComponentStatus Enumeration of the statuses of component
type ComponentStatus int
Expand Down Expand Up @@ -31,15 +40,46 @@ func (p ComponentStatus) String() string {
return [...]string{"Stopped", "Consistent", "Reconciling", "Restarting", "Outdated"}[p]
}

func ComponentStatusFromDeployment(deployment *appsv1.Deployment) ComponentStatus {
status := ConsistentComponent
type ComponentStatuserFunc func(component radixv1.RadixCommonDeployComponent, kd *appsv1.Deployment, rd *radixv1.RadixDeployment) ComponentStatus

func ComponentStatusFromDeployment(component radixv1.RadixCommonDeployComponent, kd *appsv1.Deployment, rd *radixv1.RadixDeployment) ComponentStatus {
if kd == nil || kd.GetName() == "" {
return ComponentReconciling
}
replicasUnavailable := kd.Status.UnavailableReplicas
replicasReady := kd.Status.ReadyReplicas
replicas := pointers.Val(kd.Spec.Replicas)

if isComponentRestarting(component, rd) {
return ComponentRestarting
}

if !owner.VerifyCorrectObjectGeneration(rd, kd, kube.RadixDeploymentObservedGeneration) {
return ComponentOutdated
}

switch {
case deployment.Status.Replicas == 0:
status = StoppedComponent
case deployment.Status.UnavailableReplicas > 0:
status = ComponentReconciling
if replicas == 0 {
return StoppedComponent
}

return status
// Check if component is scaling up or down
if replicasUnavailable > 0 || replicas < replicasReady {
return ComponentReconciling
}

return ConsistentComponent
}

func isComponentRestarting(component radixv1.RadixCommonDeployComponent, rd *radixv1.RadixDeployment) bool {
restarted := component.GetEnvironmentVariables()[operatordefaults.RadixRestartEnvironmentVariable]
if restarted == "" {
return false
}
restartedTime, err := commonutils.ParseTimestamp(restarted)
if err != nil {
log.Logger.Warn().Err(err).Msgf("unable to parse restarted time %v, component: %s", restarted, component.GetName())
return false
}
reconciledTime := rd.Status.Reconciled
return reconciledTime.IsZero() || restartedTime.After(reconciledTime.Time)
}
73 changes: 73 additions & 0 deletions api/deployments/models/component_status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package models_test

import (
"testing"
"time"

"github.com/equinor/radix-api/api/deployments/models"
radixutils "github.com/equinor/radix-common/utils"
"github.com/equinor/radix-common/utils/pointers"
operatordefaults "github.com/equinor/radix-operator/pkg/apis/defaults"
"github.com/equinor/radix-operator/pkg/apis/kube"
radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestNoKubeDeployments_IsReconciling(t *testing.T) {
status := models.ComponentStatusFromDeployment(&radixv1.RadixDeployComponent{}, nil, nil)
assert.Equal(t, models.ComponentReconciling, status)
}

func TestKubeDeploymentsWithRestartLabel_IsRestarting(t *testing.T) {
status := models.ComponentStatusFromDeployment(
&radixv1.RadixDeployComponent{EnvironmentVariables: map[string]string{operatordefaults.RadixRestartEnvironmentVariable: radixutils.FormatTimestamp(time.Now())}},
createKubeDeployment(0),
&radixv1.RadixDeployment{
ObjectMeta: metav1.ObjectMeta{Generation: 2},
Status: radixv1.RadixDeployStatus{Reconciled: metav1.NewTime(time.Now().Add(-10 * time.Minute))},
})

assert.Equal(t, models.ComponentRestarting, status)
}

func TestKubeDeploymentsWithoutReplicas_IsStopped(t *testing.T) {
status := models.ComponentStatusFromDeployment(
&radixv1.RadixDeployComponent{},
createKubeDeployment(0),
&radixv1.RadixDeployment{
ObjectMeta: metav1.ObjectMeta{Generation: 1},
})
assert.Equal(t, models.StoppedComponent, status)
}

func TestKubeDeployment_IsConsistent(t *testing.T) {
status := models.ComponentStatusFromDeployment(
&radixv1.RadixDeployComponent{},
createKubeDeployment(1),
&radixv1.RadixDeployment{
ObjectMeta: metav1.ObjectMeta{Generation: 1},
})
assert.Equal(t, models.ConsistentComponent, status)
}

func TestKubeDeployment_IsOutdated(t *testing.T) {
status := models.ComponentStatusFromDeployment(
&radixv1.RadixDeployComponent{},
createKubeDeployment(1),
&radixv1.RadixDeployment{
ObjectMeta: metav1.ObjectMeta{Generation: 2},
})
assert.Equal(t, models.ComponentOutdated, status)
}

func createKubeDeployment(replicas int32) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "helloworld",
Annotations: map[string]string{kube.RadixDeploymentObservedGeneration: "1"},
OwnerReferences: []metav1.OwnerReference{{Controller: pointers.Ptr(true)}}},
Spec: appsv1.DeploymentSpec{Replicas: pointers.Ptr[int32](replicas)},
}
}
Loading

0 comments on commit b1de696

Please sign in to comment.