Skip to content

Commit d1dc59e

Browse files
faiqfaiq
committed
fix: use nodeadm mutations
1 parent bced2a6 commit d1dc59e

File tree

11 files changed

+134
-40
lines changed

11 files changed

+134
-40
lines changed

‎hack/examples/bases/aws/cluster/kustomization.yaml.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
55
kind: Kustomization
66

77
resources:
8-
- https://github.com/nutanix-cloud-native/cluster-api-provider-aws/releases/download/${CAPA_VERSION}-ncn.0/cluster-template-simple-clusterclass.yaml
8+
- https://github.com/nutanix-cloud-native/cluster-api-provider-aws/releases/download/${CAPA_VERSION}/cluster-template-simple-clusterclass.yaml
99

1010
sortOptions:
1111
order: fifo

‎hack/examples/bases/aws/clusterclass/kustomization.yaml.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
55
kind: Kustomization
66

77
resources:
8-
- https://github.com/nutanix-cloud-native/cluster-api-provider-aws/releases/download/${CAPA_VERSION}-ncn.0/cluster-template-simple-clusterclass.yaml
8+
- https://github.com/nutanix-cloud-native/cluster-api-provider-aws/releases/download/${CAPA_VERSION}/cluster-template-simple-clusterclass.yaml
99

1010
configurations:
1111
- kustomizeconfig.yaml

‎make/clusterctl.mk‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@
33

44
export CAPI_VERSION := $(shell GOWORK=off go list -m -f '{{ .Version }}' sigs.k8s.io/cluster-api)
55
export CAPD_VERSION := $(shell GOWORK=off go list -m -f '{{ .Version }}' sigs.k8s.io/cluster-api/test)
6-
export CAPA_VERSION := $(shell cd hack/third-party/capa && GOWORK=off go list -m -f '{{ .Version }}' sigs.k8s.io/cluster-api-provider-aws/v2)
6+
export CAPA_VERSION := $(shell cd hack/third-party/capa && GOWORK=off go list -m -f '{{ .Version }}' sigs.k8s.io/cluster-api-provider-aws/v2)-ncn.0
77
export CAPX_VERSION := $(shell cd hack/third-party/capx && GOWORK=off go list -m -f '{{ .Version }}' github.com/nutanix-cloud-native/cluster-api-provider-nutanix)
88
export CAAPH_VERSION := $(shell cd hack/third-party/caaph && GOWORK=off go list -m -f '{{ .Version }}' sigs.k8s.io/cluster-api-addon-provider-helm)
9-
CAPA_PATCHED_IMAGE := ghcr.io/nutanix-cloud-native/cluster-api-aws-controller:$(CAPA_VERSION)-ncn.0
109

1110
# Leave Nutanix credentials empty here and set it when creating the clusters
1211
.PHONY: clusterctl.init
@@ -22,7 +21,7 @@ clusterctl.init:
2221
--core cluster-api:$(CAPI_VERSION) \
2322
--bootstrap kubeadm:$(CAPI_VERSION) \
2423
--control-plane kubeadm:$(CAPI_VERSION) \
25-
--infrastructure docker:$(CAPD_VERSION),aws:$(CAPA_VERSION)-ncn.0,nutanix:$(CAPX_VERSION) \
24+
--infrastructure docker:$(CAPD_VERSION),aws:$(CAPA_VERSION),nutanix:$(CAPX_VERSION) \
2625
--addon helm:$(CAAPH_VERSION) \
2726
--config clusterctl.yaml \
2827
--wait-providers

‎pkg/handlers/eks/mutation/testutils/request.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,28 +47,28 @@ func NewEKSControlPlaneRequestItem(
4747
)
4848
}
4949

50-
func NewEKSConfigTemplateRequestItem(
50+
func NewNodeadmConfigTemplateRequestItem(
5151
uid types.UID,
52-
existingSpec ...eksbootstrapv1.EKSConfigTemplateSpec,
52+
existingSpec ...eksbootstrapv1.NodeadmConfigTemplateSpec,
5353
) runtimehooksv1.GeneratePatchesRequestItem {
54-
eksConfigTemplate := &eksbootstrapv1.EKSConfigTemplate{
54+
nodeadmConfigTemplate := &eksbootstrapv1.NodeadmConfigTemplate{
5555
TypeMeta: metav1.TypeMeta{
5656
APIVersion: eksbootstrapv1.GroupVersion.String(),
57-
Kind: "EKSConfigTemplate",
57+
Kind: "NodeadmConfigTemplate",
5858
},
5959
}
6060

6161
switch len(existingSpec) {
6262
case 0:
6363
// Do nothing.
6464
case 1:
65-
eksConfigTemplate.Spec = existingSpec[0]
65+
nodeadmConfigTemplate.Spec = existingSpec[0]
6666
default:
6767
panic("can only take at most one existing spec")
6868
}
6969

7070
return request.NewRequestItem(
71-
eksConfigTemplate,
71+
nodeadmConfigTemplate,
7272
&runtimehooksv1.HolderReference{
7373
Kind: "MachineDeployment",
7474
FieldPath: "spec.template.spec.bootstrap.configRef",

‎pkg/handlers/generic/mutation/ntp/inject.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,12 @@ func (h *ntpPatchHandler) Mutate(
124124

125125
if err := patches.MutateIfApplicable(
126126
obj, vars, &holderRef,
127-
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log,
128-
func(obj *eksbootstrapv1.EKSConfigTemplate) error {
127+
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log,
128+
func(obj *eksbootstrapv1.NodeadmConfigTemplate) error {
129129
log.WithValues(
130130
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
131131
"patchedObjectName", client.ObjectKeyFromObject(obj),
132-
).Info("setting users in worker node EKS config template")
132+
).Info("setting users in worker node NodeadmConfig template")
133133
obj.Spec.Template.Spec.NTP = &eksbootstrapv1.NTP{
134134
Enabled: ptr.To(true),
135135
Servers: ntp.Servers,

‎pkg/handlers/generic/mutation/ntp/inject_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,8 @@ var _ = Describe("Generate NTP patches", func() {
160160
},
161161
},
162162
{
163-
Name: "NTP configuration is set for worker nodes with single server for EKSConfigTemplate",
164-
RequestItem: testutils.NewEKSConfigTemplateRequestItem(""),
163+
Name: "NTP configuration is set for worker nodes with single server for NodeadmConfigTemplate",
164+
RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""),
165165
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
166166
{
167167
Operation: "add",

‎pkg/handlers/generic/mutation/taints/inject_worker.go‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ package taints
55

66
import (
77
"context"
8+
"fmt"
89
"strings"
910

1011
"github.com/samber/lo"
1112
v1 "k8s.io/api/core/v1"
1213
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1314
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
15+
"k8s.io/apimachinery/pkg/runtime"
1416
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
1517
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
1618
ctrl "sigs.k8s.io/controller-runtime"
@@ -48,6 +50,10 @@ func newTaintsWorkerPatchHandler(
4850
}
4951
}
5052

53+
type KubeletRegisterOptions struct {
54+
RegisterWithTaints []v1.Taint `json:"registerWithTaints,omitempty"`
55+
}
56+
5157
func (h *taintsWorkerPatchHandler) Mutate(
5258
ctx context.Context,
5359
obj *unstructured.Unstructured,
@@ -103,25 +109,40 @@ func (h *taintsWorkerPatchHandler) Mutate(
103109

104110
if err := patches.MutateIfApplicable(
105111
obj, vars, &holderRef,
106-
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log,
107-
func(obj *eksbootstrapv1.EKSConfigTemplate) error {
112+
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log,
113+
func(obj *eksbootstrapv1.NodeadmConfigTemplate) error {
108114
log.WithValues(
109115
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
110116
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
111-
).Info("adding taints to worker node EKS config template")
112-
if obj.Spec.Template.Spec.KubeletExtraArgs == nil {
113-
obj.Spec.Template.Spec.KubeletExtraArgs = make(map[string]string, 1)
117+
).Info("adding taints to worker NodeadmConfig template")
118+
kubeletOptions := obj.Spec.Template.Spec.Kubelet
119+
var runtimeConfigFromNodeadm *runtime.RawExtension
120+
var flags []string
121+
newTaints := toEKSConfigTaints(taintsVar)
122+
hasRegisterTaintsFlag := false
123+
if kubeletOptions != nil {
124+
runtimeConfigFromNodeadm = kubeletOptions.Config
125+
for _, flag := range kubeletOptions.Flags {
126+
if strings.HasPrefix(flag, "--register-with-taints=") {
127+
hasRegisterTaintsFlag = true
128+
existingTaints := strings.Split(flag, "--register-with-taints=")
129+
if len(existingTaints) != 2 {
130+
return fmt.Errorf("expected flag register-with-taints to be able to split got %v", existingTaints)
131+
}
132+
taintsFromFlags := existingTaints[1]
133+
flags = append(flags, fmt.Sprintf("--register-with-taints=%s,%s", taintsFromFlags, newTaints))
134+
continue
135+
}
136+
flags = append(flags, flag)
137+
}
114138
}
115-
116-
existingTaintsFlagValue := obj.Spec.Template.Spec.KubeletExtraArgs["register-with-taints"]
117-
118-
newTaintsFlagValue := toEKSConfigTaints(taintsVar)
119-
120-
if existingTaintsFlagValue != "" {
121-
newTaintsFlagValue = existingTaintsFlagValue + "," + newTaintsFlagValue
139+
if !hasRegisterTaintsFlag {
140+
flags = append(flags, fmt.Sprintf("--register-with-taints=%s", newTaints))
141+
}
142+
obj.Spec.Template.Spec.Kubelet = &eksbootstrapv1.KubeletOptions{
143+
Flags: flags,
144+
Config: runtimeConfigFromNodeadm,
122145
}
123-
124-
obj.Spec.Template.Spec.KubeletExtraArgs["register-with-taints"] = newTaintsFlagValue
125146
return nil
126147
}); err != nil {
127148
return err

‎pkg/handlers/generic/mutation/taints/inject_worker_test.go‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1414
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
1515

16+
eksbootstrapv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/api/v1beta2"
1617
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
1718
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
1819
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
@@ -59,7 +60,7 @@ var _ = Describe("Generate taints patches for Worker", func() {
5960
}},
6061
},
6162
{
62-
Name: "taints for workers set for EKSConfigTemplate",
63+
Name: "taints for workers set for NodeadmConfigTemplate",
6364
Vars: []runtimehooksv1.Variable{
6465
capitest.VariableWithValue(
6566
v1alpha1.WorkerConfigVariableName,
@@ -77,15 +78,88 @@ var _ = Describe("Generate taints patches for Worker", func() {
7778
},
7879
),
7980
},
80-
RequestItem: testutils.NewEKSConfigTemplateRequestItem(""),
81+
RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""),
8182
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
8283
Operation: "add",
83-
Path: "/spec/template/spec/kubeletExtraArgs",
84+
Path: "/spec/template/spec/kubelet",
8485
ValueMatcher: gomega.HaveKeyWithValue(
85-
"register-with-taints", "key=value:NoExecute",
86+
"flags",
87+
gomega.ContainElement("--register-with-taints=key=value:NoExecute"),
8688
),
8789
}},
8890
},
91+
{
92+
Name: "taints for workers set for NodeadmConfigTemplate with existing flags argument",
93+
Vars: []runtimehooksv1.Variable{
94+
capitest.VariableWithValue(
95+
v1alpha1.WorkerConfigVariableName,
96+
[]v1alpha1.Taint{{
97+
Key: "key",
98+
Effect: v1alpha1.TaintEffectNoExecute,
99+
Value: "value",
100+
}},
101+
VariableName,
102+
),
103+
capitest.VariableWithValue(
104+
"builtin",
105+
apiextensionsv1.JSON{
106+
Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`),
107+
},
108+
),
109+
},
110+
RequestItem: testutils.NewNodeadmConfigTemplateRequestItem("", eksbootstrapv1.NodeadmConfigTemplateSpec{
111+
Template: eksbootstrapv1.NodeadmConfigTemplateResource{
112+
Spec: eksbootstrapv1.NodeadmConfigSpec{
113+
Kubelet: &eksbootstrapv1.KubeletOptions{
114+
Flags: []string{
115+
"--max-pods=110",
116+
},
117+
},
118+
},
119+
},
120+
}),
121+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
122+
Operation: "add",
123+
Path: "/spec/template/spec/kubelet/flags/1",
124+
ValueMatcher: gomega.Equal("--register-with-taints=key=value:NoExecute"),
125+
}},
126+
},
127+
{
128+
Name: "taints for workers set for NodeadmConfigTemplate with existing flags with register-with-taints ",
129+
Vars: []runtimehooksv1.Variable{
130+
capitest.VariableWithValue(
131+
v1alpha1.WorkerConfigVariableName,
132+
[]v1alpha1.Taint{{
133+
Key: "key",
134+
Effect: v1alpha1.TaintEffectNoExecute,
135+
Value: "value",
136+
}},
137+
VariableName,
138+
),
139+
capitest.VariableWithValue(
140+
"builtin",
141+
apiextensionsv1.JSON{
142+
Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`),
143+
},
144+
),
145+
},
146+
RequestItem: testutils.NewNodeadmConfigTemplateRequestItem("", eksbootstrapv1.NodeadmConfigTemplateSpec{
147+
Template: eksbootstrapv1.NodeadmConfigTemplateResource{
148+
Spec: eksbootstrapv1.NodeadmConfigSpec{
149+
Kubelet: &eksbootstrapv1.KubeletOptions{
150+
Flags: []string{
151+
"--register-with-taints=key1=value1:NoSchedule",
152+
},
153+
},
154+
},
155+
},
156+
}),
157+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
158+
Operation: "replace",
159+
Path: "/spec/template/spec/kubelet/flags/0",
160+
ValueMatcher: gomega.Equal("--register-with-taints=key1=value1:NoSchedule,key=value:NoExecute"),
161+
}},
162+
},
89163
}
90164

91165
// create test node for each case

‎pkg/handlers/generic/mutation/users/inject.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,12 @@ func (h *usersPatchHandler) Mutate(
115115

116116
if err := patches.MutateIfApplicable(
117117
obj, vars, &holderRef,
118-
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log,
119-
func(obj *eksbootstrapv1.EKSConfigTemplate) error {
118+
selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log,
119+
func(obj *eksbootstrapv1.NodeadmConfigTemplate) error {
120120
log.WithValues(
121121
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
122122
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
123-
).Info("setting users in worker node EKS config template")
123+
).Info("setting users in worker node NodeadmConfig template")
124124
eksBootstrapUsers := make([]eksbootstrapv1.User, 0, len(bootstrapUsers))
125125
for _, user := range bootstrapUsers {
126126
var passwdFrom *eksbootstrapv1.PasswdSource

‎pkg/handlers/generic/mutation/users/inject_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ var _ = Describe("Generate Users patches", func() {
194194
}},
195195
},
196196
{
197-
Name: "users set for EKSConfigTemplate generic worker",
197+
Name: "users set for NodeadmConfigTemplate generic worker",
198198
Vars: []runtimehooksv1.Variable{
199199
capitest.VariableWithValue(
200200
v1alpha1.ClusterConfigVariableName,
@@ -210,7 +210,7 @@ var _ = Describe("Generate Users patches", func() {
210210
},
211211
),
212212
},
213-
RequestItem: testutils.NewEKSConfigTemplateRequestItem(""),
213+
RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""),
214214
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
215215
Operation: "add",
216216
Path: "/spec/template/spec/users",

0 commit comments

Comments
 (0)