Skip to content

Commit 0c39366

Browse files
togashidmgcgirish
togashidm
and
gcgirish
authored
feat: K3s controlplane and intel providers (#78)
Co-authored-by: girish <girish.gowdra@intel.com>
1 parent 78ab5ab commit 0c39366

File tree

13 files changed

+275
-130
lines changed

13 files changed

+275
-130
lines changed

‎Makefile

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ VERSION_DEV_SUFFIX := ${GIT_HASH_SHORT}
1414
CLUSTERCTL_VERSION ?= v1.9.5
1515
KUBEADM_VERSION ?= v1.9.0
1616
RKE2_VERSION ?= v0.12.0
17+
K3s_VERSION ?= v0.2.1
1718
DOCKER_INFRA_VERSION ?= v1.8.5
1819
CLUSTERCTL := $(shell command -v clusterctl 2> /dev/null)
1920

@@ -483,7 +484,8 @@ kind-create: ## Create a development kind cluster with CAPI enabled
483484
fi
484485
echo "Creating a Kind cluster with CAPI enabled..."
485486
kind create cluster --name $(KIND_CLUSTER) --config $(KIND_CONFIG)
486-
CLUSTER_TOPOLOGY=true clusterctl init --core cluster-api:${CLUSTERCTL_VERSION} --bootstrap kubeadm:${KUBEADM_VERSION},rke2:${RKE2_VERSION} --control-plane kubeadm:${KUBEADM_VERSION},rke2:${RKE2_VERSION} --infrastructure docker:${DOCKER_INFRA_VERSION}
487+
@make setup-clusterctl-config
488+
CLUSTER_TOPOLOGY=true clusterctl init --core cluster-api:${CLUSTERCTL_VERSION} --bootstrap kubeadm:${KUBEADM_VERSION},rke2:${RKE2_VERSION},k3s:${K3s_VERSION} --control-plane kubeadm:${KUBEADM_VERSION},rke2:${RKE2_VERSION},k3s:${K3s_VERSION} --infrastructure docker:${DOCKER_INFRA_VERSION}
487489

488490
.PHONY: kind-expose-cm
489491
kind-expose-cm: ## Expose the cluster manager service to the host
@@ -562,6 +564,19 @@ uninstall-cert-manager: ## Uninstall cert-manager using Helm.
562564
helm uninstall cert-manager --namespace cert-manager
563565
kubectl delete namespace cert-manager
564566

567+
# Define variables for k3s provider URLs
568+
BOOTSTRAP_URL := https://github.com/k3s-io/cluster-api-k3s/releases/$(K3s_VERSION)/bootstrap-components.yaml
569+
CONTROL_PLANE_URL := https://github.com/k3s-io/cluster-api-k3s/releases/$(K3s_VERSION)/control-plane-components.yaml
570+
571+
.PHONY: setup-clusterctl-config
572+
setup-clusterctl-config: ## Create clusterctl.yaml config for k3s providers
573+
@if [ -d "$$HOME/.config/cluster-api" ] && [ ! -w "$$HOME/.config/cluster-api" ]; then \
574+
echo "Error: Directory $$HOME/.config/cluster-api exists but is not writable."; \
575+
exit 1; \
576+
fi
577+
@mkdir -p "$$HOME/.config/cluster-api" || { echo "Error: Failed to create directory $$HOME/.config/cluster-api."; exit 1; }
578+
@printf "providers:\n - name: \"k3s\"\n url: \"$(BOOTSTRAP_URL)\"\n type: \"BootstrapProvider\"\n - name: \"k3s\"\n url: \"$(CONTROL_PLANE_URL)\"\n type: \"ControlPlaneProvider\"" > "$$HOME/.config/cluster-api/clusterctl.yaml"
579+
565580
.PHONY: clusterctl
566581
clusterctl: ## Download clusterctl binary
567582
ifndef CLUSTERCTL

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.1.3
1+
2.1.4-dev

‎cmd/template-controller/main.go

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ import (
3131
// +kubebuilder:scaffold:imports
3232

3333
// Imports for CAPI resources
34+
kthreesbootstrapv1beta2 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2"
35+
kthreescpv1beta2 "github.com/k3s-io/cluster-api-k3s/controlplane/api/v1beta2"
3436
intelv1alpha1 "github.com/open-edge-platform/cluster-api-provider-intel/api/v1alpha1"
3537
rke2bootstrapv1beta1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1beta1"
3638
rke2cpv1beta1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1"
@@ -51,31 +53,30 @@ func init() {
5153
utilruntime.Must(clusterv1alpha1.AddToScheme(scheme))
5254
// +kubebuilder:scaffold:scheme
5355

54-
// ---- DOCKER INFRASTRUCTURE PROVIDER ----
55-
// Add scheme for Docker infrastructure provider
56-
utilruntime.Must(dockerv1beta1.AddToScheme(scheme))
57-
58-
// ---- INTEL INFRASTRUCTURE PROVIDER ----
59-
// Add scheme for Intel infrastructure provider
60-
utilruntime.Must(intelv1alpha1.AddToScheme(scheme))
61-
62-
// ---- KUBEADM CONTROL PLANE PROVIDER ----
63-
// Add scheme for Kubeadm bootstrap provider
64-
utilruntime.Must(kubeadmbootstrapv1beta1.AddToScheme(scheme))
65-
66-
// Add scheme for Kubeadm control plane provider
67-
utilruntime.Must(kubeadmcp.AddToScheme(scheme))
68-
69-
// ---- RKE2 CONTROL PLANE PROVIDER ----
70-
// Add scheme for RKE2 bootstrap provider
71-
utilruntime.Must(rke2bootstrapv1beta1.AddToScheme(scheme))
72-
73-
// Add scheme for RKE2 control plane provider
74-
utilruntime.Must(rke2cpv1beta1.AddToScheme(scheme))
56+
capiSchemeAdders := []func(*runtime.Scheme) error{
57+
// Docker infrastructure provider
58+
dockerv1beta1.AddToScheme,
59+
// Intel infrastructure provider
60+
intelv1alpha1.AddToScheme,
61+
// Kubeadm bootstrap provider
62+
kubeadmbootstrapv1beta1.AddToScheme,
63+
// Kubeadm control plane provider
64+
kubeadmcp.AddToScheme,
65+
// RKE2 bootstrap provider
66+
rke2bootstrapv1beta1.AddToScheme,
67+
// RKE2 control plane provider
68+
rke2cpv1beta1.AddToScheme,
69+
// K3s bootstrap provider
70+
kthreesbootstrapv1beta2.AddToScheme,
71+
// K3s control plane provider
72+
kthreescpv1beta2.AddToScheme,
73+
// Cluster API core resources
74+
capi.AddToScheme,
75+
}
7576

76-
// ---- CAPI ----
77-
// Add scheme for Cluster API core resources
78-
utilruntime.Must(capi.AddToScheme(scheme))
77+
for _, adder := range capiSchemeAdders {
78+
utilruntime.Must(adder(scheme))
79+
}
7980
}
8081

8182
// version injected at build time

‎config/rbac/role.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ rules:
3030
resources:
3131
- kubeadmcontrolplanetemplates
3232
- rke2controlplanetemplates
33+
- kthreescontrolplanes
34+
- kthreescontrolplanetemplates
3335
verbs:
3436
- create
3537
- delete

‎default-cluster-templates/baseline-k3s.json

Lines changed: 28 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"infraprovidertype": "intel",
88
"clusterconfiguration": {
99
"kind": "KThreesControlPlaneTemplate",
10-
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta1",
10+
"apiVersion": "controlplane.cluster.x-k8s.io/v1beta2",
1111
"metadata": {
1212
"labels": {
1313
"cpumanager": "true"
@@ -16,72 +16,42 @@
1616
"spec": {
1717
"template": {
1818
"spec": {
19-
"files": [
20-
{
21-
"path": "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl",
22-
"content": "version = 2\n\n[plugins.\\\"io.containerd.internal.v1.opt\\\"]\n path = \\\"/var/lib/rancher/k3s/agent/containerd\\\"\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\"]\n stream_server_address = \\\"127.0.0.1\\\"\n stream_server_port = \\\"10010\\\"\n enable_selinux = false\n enable_unprivileged_ports = true\n enable_unprivileged_icmp = true\n sandbox_image = \\\"index.docker.io/rancher/mirrored-pause:3.6\\\"\n disable_apparmor = true\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".containerd]\n snapshotter = \\\"overlayfs\\\"\n disable_snapshot_annotations = true\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".containerd.runtimes.runc]\n runtime_type = \\\"io.containerd.runc.v2\\\"\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".containerd.runtimes.runc.options]\n SystemdCgroup = true\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".registry]\n config_path = \\\"/var/lib/rancher/k3s/agent/etc/containerd/certs.d\\\"\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".containerd.runtimes.kata-qemu]\n runtime_type = \\\"io.containerd.kata-qemu.v2\\\"\n runtime_path = \\\"/opt/kata/bin/containerd-shim-kata-v2\\\"\n privileged_without_host_devices = true\n pod_annotations = [\\\"io.katacontainers.*\\\"]\n\n[plugins.\\\"io.containerd.grpc.v1.cri\\\".containerd.runtimes.kata-qemu.options]\n ConfigPath = \\\"/opt/kata/share/defaults/kata-containers/configuration-qemu.toml\\\""
23-
}
24-
],
25-
"agentConfig": {
26-
"kubelet": {
27-
"extraArgs": [
19+
"kthreesConfigSpec": {
20+
"files": [
21+
{}
22+
],
23+
"agentConfig": {
24+
"kubeletArgs": [
2825
"--topology-manager-policy=best-effort",
2926
"--cpu-manager-policy=static",
3027
"--reserved-cpus=1",
3128
"--max-pods=250",
3229
"--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
3330
]
34-
}
35-
},
36-
"preK3sCommands": [
37-
"mkdir -p /etc/systemd/system/k3s-server.service.d",
38-
"echo '[Service]\nEnvironmentFile=/etc/environment' > /etc/systemd/system/k3s-server.service.d/override.conf",
39-
"make -p /var/lib/kubelet/static-pods",
40-
"sed -i '/kubelet-arg:/a\\- pod-manifest-path=/var/lib/kubelet/static-pods' /etc/rancher/k3s/config.yaml"
41-
],
42-
"postK3sCommands": [
43-
"kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/calico.yaml"
44-
],
45-
"privateRegistriesConfig": {},
46-
"serverConfig": {
47-
"cni": "calico",
48-
"cniMultusEnable": true,
49-
"disableComponents": {
50-
"kubernetesComponents": [
51-
"cloudController"
52-
]
5331
},
54-
"kubeApiServer": {
55-
"extraArgs": [
32+
"preK3sCommands": [
33+
"mkdir -p /etc/systemd/system/k3s-server.service.d",
34+
"echo '[Service]\nEnvironmentFile=/etc/environment' > /etc/systemd/system/k3s-server.service.d/override.conf",
35+
"mkdir -p /var/lib/kubelet/static-pods",
36+
"sed -i '/kubelet-arg:/a\\- pod-manifest-path=/var/lib/kubelet/static-pods' /etc/rancher/k3s/config.yaml",
37+
"sudo cp /var/lib/rancher/k3s/agent/pod-manifests/connect-agent.yaml /var/lib/kubelet/static-pods/connect-agent.yaml"
38+
],
39+
"postK3sCommands": [
40+
"kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.0/manifests/operator-crds.yaml",
41+
"kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.0/manifests/tigera-operator.yaml",
42+
"kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.0/manifests/custom-resources.yaml"
43+
],
44+
"serverConfig": {
45+
"kubeApiServerArg": [
5646
"--feature-gates=PortForwardWebsockets=true",
5747
"--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
58-
]
59-
},
60-
"kubeControllerManager": {
61-
"extraArgs": null
62-
},
63-
"kubeScheduler": {
64-
"extraArgs": null
48+
],
49+
"kubeControllerManagerArgs": [],
50+
"kubeSchedulerArgs": [],
51+
"clusterDomain": "cluster.edge",
52+
"disableCloudController": false
6553
},
66-
"etcd": {
67-
"exposeMetrics": false,
68-
"backupConfig": {
69-
"retention": "5",
70-
"scheduleCron": "0 */5 * * *"
71-
},
72-
"customConfig": {
73-
"extraArgs": [
74-
"cipher-suites=[TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]"
75-
]
76-
}
77-
}
78-
},
79-
"nodeDrainTimeout": "2m",
80-
"rolloutStrategy": {
81-
"type": "RollingUpdate",
82-
"rollingUpdate": {
83-
"maxSurge": 1
84-
}
54+
"version": "v1.32.4+k3s1"
8555
}
8656
}
8757
}
@@ -102,4 +72,4 @@
10272
"cluster-labels": {
10373
"default-extension": "baseline"
10474
}
105-
}
75+
}

‎deployment/charts/cluster-manager/Chart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ type: application
1616
# This is the chart version. This version number should be incremented each time you make changes
1717
# to the chart and its templates, including the app version.
1818
# Versions are expected to follow Semantic Versioning (https://semver.org/)
19-
version: 2.1.3
20-
appVersion: 2.1.3
19+
version: 2.1.4-dev
20+
appVersion: 2.1.4-dev
2121
annotations: {}

‎deployment/charts/cluster-manager/templates/rbac.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ rules:
3434
resources: ["machines"]
3535
verbs: ["get", "list", "watch"]
3636
- apiGroups: ["controlplane.cluster.x-k8s.io"]
37-
resources: ["kubeadmcontrolplanetemplates", "rke2controlplanetemplates"]
37+
resources: ["kubeadmcontrolplanetemplates", "rke2controlplanetemplates","kthreescontrolplanetemplates","kthreescontrolplanes"]
3838
verbs: ["create", "delete", "get", "list", "watch"]
3939
- apiGroups: ["infrastructure.cluster.x-k8s.io"]
4040
resources: ["dockerclustertemplates", "dockermachinetemplates", "dockermachines", "intelclustertemplates", "intelmachinebindings", "intelmachinetemplates", "intelmachines"]

‎deployment/charts/cluster-template-crd/Chart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ apiVersion: v2
66
name: cluster-template-crd
77
description: A Helm chart for the ClusterTemplate CRD
88
type: application
9-
version: 2.1.3
10-
appVersion: 2.1.3
9+
version: 2.1.4-dev
10+
appVersion: 2.1.4-dev
1111
annotations: {}

0 commit comments

Comments
 (0)