Control plane in k3d

.github/workflows/integration.yaml

@@ -95,6 +95,7 @@ jobs:
         flavors:
           - name: sonic
           - name: gardener
+          - name: k3s
 
     steps:
     - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211
@@ -109,6 +110,10 @@ jobs:
         # set if required:
         # DESIRED_VERSION: v0.59.0
 
+    - name: Setup k3d
+      run: |
+        curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
     - name: Checkout
       uses: actions/checkout@v4
 

Makefile

@@ -7,6 +7,7 @@
 YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4
 
 KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
+K3DCONFIG := $(or $(K3DCONFIG),control-plane/k3d.yaml)
 KUBECONFIG := $(shell pwd)/.kubeconfig
 
 METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
@@ -36,6 +37,12 @@ GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
 K8S_VERSION=1.32.5
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+else ifeq ($(MINI_LAB_FLAVOR),k3s)
+GARDENER_ENABLED=true
+# usually gardener restricts the maximum version for k8s:
+K8S_VERSION=1.32.5
+LAB_TOPOLOGY=mini-lab.sonic.yaml
+USE_K3D=true
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -45,6 +52,11 @@ ifneq ($(K8S_VERSION),)
 KIND_ARGS=--image kindest/node:v$(K8S_VERSION)
 endif
 
+K3D_ARGS=
+ifneq ($(K8S_VERSION),)
+K3D_ARGS=--image rancher/k3s:v$(K8S_VERSION)-k3s1
+endif
+
 ifeq ($(CI),true)
   DOCKER_COMPOSE_RUN_ARG=--no-TTY --rm
 else
@@ -94,12 +106,21 @@ create-proxy-registries:
 
 .PHONY: control-plane-bake
 control-plane-bake:
-	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
-	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
-		kind create cluster $(KIND_ARGS) \
-			--name metal-control-plane \
-			--config $(KINDCONFIG) \
-			--kubeconfig $(KUBECONFIG); fi
+	@if [ -n "$(USE_K3D)" ]; then \
+		if ! which k3d > /dev/null; then echo "k3d needs to be installed"; exit 1; fi; \
+		if ! k3d cluster list | grep metal-control-plane > /dev/null; then \
+			k3d cluster create metal-control-plane $(K3D_ARGS) \
+				--config $(K3DCONFIG); \
+		fi; \
+	else \
+		if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi; \
+		if ! kind get clusters | grep metal-control-plane > /dev/null; then \
+			kind create cluster $(KIND_ARGS) \
+				--name metal-control-plane \
+				--config $(KINDCONFIG) \
+				--kubeconfig $(KUBECONFIG); \
+		fi; \
+	fi;
 	$(MAKE) create-proxy-registries
 
 .PHONY: partition
@@ -142,6 +163,7 @@ cleanup: cleanup-control-plane cleanup-partition
 .PHONY: cleanup-control-plane
 cleanup-control-plane:
 	kind delete cluster --name metal-control-plane
+	k3d cluster delete metal-control-plane
 	docker compose down
 	rm -f $(KUBECONFIG)
 

control-plane/k3d.yaml

@@ -0,0 +1,55 @@
+apiVersion: k3d.io/v1alpha5
+kind: Simple
+servers: 1
+agents: 1
+kubeAPI:
+  hostIP: 0.0.0.0
+  hostPort: "6443"
+ports:
+  - port: 4443:4443
+    nodeFilters:
+      - server:*
+      - agents:*
+  - port: 8080:8080
+    nodeFilters:
+      - server:*
+      - agents:*
+  - port: 4150:4150
+    nodeFilters:
+      - server:*
+      - agents:*
+  - port: 50051:50051
+    nodeFilters:
+      - server:*
+      - agents:*
+options:
+  k3s:
+    extraArgs:
+      - arg: "--disable=traefik"
+        nodeFilters:
+          - server:*
+      - arg: "--disable=servicelb"
+        nodeFilters:
+          - server:*
+  k3d:
+    wait: true
+    timeout: 60s
+    disableLoadbalancer: false
+registries:
+  config: |
+    mirrors:
+      "docker.io":
+        endpoint:
+          - http://proxy-docker:5000
+      "gcr.io":
+        endpoint:
+          - http://proxy-gcr:5000
+      "ghcr.io":
+        endpoint:
+          - http://proxy-ghcr:5000
+      "quay.io":
+        endpoint:
+          - http://proxy-quay:5000
+      "registry.k8s.io":
+        endpoint:
+          - http://proxy-k8s:5000