feat(platform): support containerd as runtime (#1390)

api/platform/types.go

@@ -231,6 +231,13 @@ const (
 	GPUVirtual GPUType = "Virtual"
 )
 
+type ContainerRuntimeType = string
+
+const (
+	Containerd ContainerRuntimeType = "containerd"
+	Docker ContainerRuntimeType = "docker"
+)
+
 // ClusterPhase defines the phase of cluster constructor.
 type ClusterPhase string
 
@@ -388,6 +395,8 @@ type ClusterFeature struct {
 	// +optional
 	EnableCilium bool
 	// +optional
+	ContainerRuntime ContainerRuntimeType
+	// +optional
 	IPv6DualStack bool
 	// Upgrade control upgrade process.
 	// +optional

api/platform/v1/types.go

@@ -237,6 +237,13 @@ const (
 	GPUVirtual GPUType = "Virtual"
 )
 
+type ContainerRuntimeType = string
+
+const (
+	Containerd ContainerRuntimeType = "containerd"
+	Docker     ContainerRuntimeType = "docker"
+)
+
 // ClusterPhase defines the phase of cluster constructor.
 type ClusterPhase string
 
@@ -395,6 +402,8 @@ type ClusterFeature struct {
 	IPv6DualStack bool `json:"ipv6DualStack,omitempty" protobuf:"bytes,13,opt,name=ipv6DualStack"`
 	// +optional
 	EnableCilium bool `json:"enableCilium,omitempty" protobuf:"bytes,14,opt,name=enableCilium"`
+	// +optional
+	ContainerRuntime ContainerRuntimeType `json:"containerRuntime,omitempty" protobuf:"bytes,15,opt,name=containerRuntime"`
 	// Upgrade control upgrade process.
 	// +optional
 	Upgrade Upgrade `json:"upgrade,omitempty" protobuf:"bytes,22,opt,name=upgrade"`

build/docker/tools/provider-res/Dockerfile

@@ -26,14 +26,23 @@ COPY linux-arm64/cni-*.tar.gz          res/linux-arm64/
 COPY linux-amd64/conntrack-tools-*.tar.gz          res/linux-amd64/
 COPY linux-arm64/conntrack-tools-*.tar.gz          res/linux-arm64/
 
+COPY linux-amd64/nerdctl-*.tar.gz res/linux-amd64/
+COPY linux-arm64/nerdctl-*.tar.gz res/linux-arm64/
+
 COPY linux-amd64/docker-*.tar.gz       res/linux-amd64/
 COPY linux-arm64/docker-*.tar.gz       res/linux-arm64/
 
+COPY linux-amd64/containerd-*.tar.gz       res/linux-amd64/
+COPY linux-arm64/containerd-*.tar.gz       res/linux-arm64/
+
 COPY linux-amd64/kubernetes-*.tar.gz   res/linux-amd64/
 COPY linux-arm64/kubernetes-*.tar.gz   res/linux-arm64/
 
 COPY linux-amd64/NVIDIA-*.tar.gz       res/linux-amd64/
 
 COPY linux-amd64/nvidia-container-*.tar.gz res/linux-amd64/
 
+COPY containerd conf/containerd
+COPY kubeadm conf/kubeadm
+
 ENTRYPOINT ["sh"]

build/docker/tools/provider-res/Makefile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations under the License.
 
 IMAGE = provider-res
-VERSION = v1.20.4-1
+VERSION = v1.20.4-2
 REGISTRY_PREFIX ?= tkestack
 
 COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
@@ -34,6 +34,8 @@ include $(ROOT_DIR)/build/lib/docker-buildx.mk
 .PHONY: per.build
 per.build: download
 	@cp "$(ROOT_DIR)/build/docker/tools/$(IMAGE)/Dockerfile" "$(WORK_DIR)/Dockerfile"
+	@cp -r "$(ROOT_DIR)/pkg/platform/provider/baremetal/conf/containerd" "$(WORK_DIR)/containerd"
+	@cp -r "$(ROOT_DIR)/pkg/platform/provider/baremetal/conf/kubeadm" "$(WORK_DIR)/kubeadm"
 
 .PHONY: download
 download:
@@ -64,7 +66,7 @@ manifest.multiarch: per.build
 	#@rm -rf "$(WORK_DIR)"
 	@echo "===========> Pushing manifest $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) and then remove the local manifest list"
 	REGISTRY_PREFIX=$(REGISTRY_PREFIX) PLATFROMS="$(PLATFORMS)" IMAGE=$(IMAGE) VERSION=$(VERSION) \
-	  DOCKER_CLI_EXPERIMENTAL=enabled $(ROOT_DIR)/build/lib/create-manifest.sh 
+	  DOCKER_CLI_EXPERIMENTAL=enabled $(ROOT_DIR)/build/lib/create-manifest.sh
 
 ## help: Show this help info.
 .PHONY: help

build/docker/tools/provider-res/download.sh

@@ -56,6 +56,41 @@ function download::docker() {
   done
 }
 
+function download::containerd() {
+  if [ "${arch}" == "amd64" ]; then
+    containerd_arch=amd64
+    for version in ${CONTAINERD_VERSIONS}; do
+      wget -c "https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
+        -O "containerd-${platform}-${version}.tar.gz"
+    done
+  elif [ "${arch}" == "arm64" ]; then
+    containerd_arch=arm64
+    for version in ${CONTAINERD_VERSIONS}; do
+      wget -c "https://tke-release-1251707795.cos.ap-guangzhou.myqcloud.com/cri-containerd-cni-${version}-linux-${containerd_arch}.tar.gz" \
+        -O "containerd-${platform}-${version}.tar.gz"
+    done
+  else
+    echo "[ERROR] Fail to get containerd ${arch} on ${platform} platform."
+    exit 255
+  fi
+}
+
+function download::nerdctl() {
+  if [ "${arch}" == "amd64" ]; then
+    nerdctl_arch=x86_64
+  elif [ "${arch}" == "arm64" ]; then
+    nerdctl_arch=arm64
+  else
+    echo "[ERROR] Fail to get nerdctl ${arch} on ${platform} platform."
+    exit 255
+  fi
+
+  for version in ${NERDCTL_VERSIONS}; do
+    wget -c "https://github.com/containerd/nerdctl/releases/download/v${version}/nerdctl-${version}-linux-${arch}.tar.gz" \
+      -O "nerdctl-${platform}-${version}.tar.gz"
+  done
+}
+
 function download::kubernetes() {
   for version in ${K8S_VERSIONS}; do
     if [[ "${version}" =~ "tke" ]]; then
@@ -115,6 +150,8 @@ for os in ${OSS}; do
 
     download::cni_plugins
     download::docker
+    download::containerd
+    download::nerdctl
     download::kubernetes
     download::nvidia_driver
     download::nvidia_container_runtime

build/docker/tools/tke-installer/release.sh

@@ -23,7 +23,7 @@ set -o pipefail
 REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack}
 BUILDER=${BUILDER:-default}
 VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')}
-PROVIDER_RES_VERSION=v1.20.4-1
+PROVIDER_RES_VERSION=v1.20.4-2
 K8S_VERSION=${PROVIDER_RES_VERSION%-*}
 DOCKER_VERSION=19.03.14
 OSS=(linux)

cmd/setup-env/main.go

@@ -33,6 +33,8 @@ func main() {
 	env = append(env, fmt.Sprintf("OSS=%s", strings.Join(spec.OSs, " ")))
 	env = append(env, fmt.Sprintf("K8S_VERSIONS=%s", strings.Join(spec.K8sVersionsWithV, " ")))
 	env = append(env, fmt.Sprintf("DOCKER_VERSIONS=%s", strings.Join(spec.DockerVersions, " ")))
+	env = append(env, fmt.Sprintf("CONTAINERD_VERSIONS=%s", strings.Join(spec.ContainerdVersions, " ")))
+	env = append(env, fmt.Sprintf("NERDCTL_VERSIONS=%s", strings.Join(spec.NerdctlVersions, " ")))
 	env = append(env, fmt.Sprintf("CNI_PLUGINS_VERSIONS=%s", strings.Join(spec.CNIPluginsVersions, " ")))
 	env = append(env, fmt.Sprintf("NVIDIA_DRIVER_VERSIONS=%s", strings.Join(spec.NvidiaDriverVersions, " ")))
 	env = append(env, fmt.Sprintf("NVIDIA_CONTAINER_RUNTIME_VERSIONS=%s", strings.Join(spec.NvidiaContainerRuntimeVersions, " ")))

cmd/tke-installer/app/installer/images/images.go

@@ -76,7 +76,7 @@ var components = Components{
 	InfluxDB: containerregistry.Image{Name: "influxdb", Tag: "1.7.9"},
 	Thanos:   containerregistry.Image{Name: "thanos", Tag: "v0.15.0"},
 
-	ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.20.4-1"},
+	ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.20.4-2"},
 
 	TKEAuthAPI:               containerregistry.Image{Name: "tke-auth-api", Tag: Version},
 	TKEAuthController:        containerregistry.Image{Name: "tke-auth-controller", Tag: Version},

cmd/tke-installer/app/installer/installer.go

@@ -1577,7 +1577,7 @@ func (t *TKE) prepareImages(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
-		cmdString := fmt.Sprintf("docker pull %s", images.Get().TKEGateway.FullName())
+		cmdString := fmt.Sprintf("nerdctl --insecure-registry --namespace k8s.io pull %s", images.Get().TKEGateway.FullName())
 		_, err = machineSSH.CombinedOutput(cmdString)
 		if err != nil {
 			return errors.Wrap(err, machine.IP)

cmd/tke-installer/app/installer/manifests/tke-platform-controller/tke-platform-controller.yaml

@@ -30,12 +30,14 @@ spec:
         - command:
             - sh
             - -c
-            - cp -r /data/res /app/provider/baremetal/
+            - cp -r /data/res /app/provider/baremetal/ && cp -r /data/conf/* /app/provider/baremetal/conf/
           image: {{ .ProviderResImage }}
           name: provider-res
           volumeMounts:
             - mountPath: /app/provider/baremetal
               name: provider-volume
+            - mountPath: /app/provider/baremetal/conf/containerd
+              name: containerd-volume
       containers:
         - name: tke-platform-controller
           image: {{ .Image }}
@@ -48,6 +50,8 @@ spec:
               mountPath: /app/conf
             - name: provider-volume
               mountPath: /app/provider/baremetal
+            - name: containerd-volume
+              mountPath: /app/provider/baremetal/conf/containerd/
             - name: provider-config-volume
               mountPath: /app/provider/baremetal/conf/
             - name: docker-volume
@@ -91,6 +95,8 @@ spec:
             name: tke-platform-controller
         - name: provider-volume
           emptyDir: {}
+        - name: containerd-volume
+          emptyDir: {}
         - name: provider-config-volume
           configMap:
             name: provider-config
@@ -187,4 +193,4 @@ data:
       - context:
           cluster: tke
           user: admin-cert
-        name: tke
+        name: tke

pkg/platform/provider/baremetal/cluster/create.go

@@ -48,6 +48,7 @@ import (
 	"tkestack.io/tke/pkg/platform/provider/baremetal/images"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/addons/cniplugins"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/authzwebhook"
+	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/containerd"
 	csioperatorimage "tkestack.io/tke/pkg/platform/provider/baremetal/phases/csioperator/images"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/docker"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/galaxy"
@@ -502,6 +503,38 @@ func (p *Provider) EnsureNvidiaContainerRuntime(ctx context.Context, c *v1.Clust
 	return nil
 }
 
+func (p *Provider) EnsureContainerRuntime(ctx context.Context, c *v1.Cluster) error {
+	if c.Cluster.Spec.Features.ContainerRuntime == platformv1.Docker {
+		return p.EnsureDocker(ctx, c)
+	}
+	return p.EnsureContainerd(ctx, c)
+}
+
+func (p *Provider) EnsureContainerd(ctx context.Context, c *v1.Cluster) error {
+	insecureRegistries := []string{p.config.Registry.Domain}
+	if p.config.Registry.NeedSetHosts() && c.Spec.TenantID != "" {
+		insecureRegistries = append(insecureRegistries, c.Spec.TenantID+"."+p.config.Registry.Domain)
+	}
+	option := &containerd.Option{
+		InsecureRegistries: insecureRegistries,
+		SandboxImage:       images.Get().Pause.FullName(),
+	}
+	for _, machine := range c.Spec.Machines {
+		machineSSH, err := machine.SSH()
+		if err != nil {
+			return err
+		}
+
+		option.IsGPU = gpu.IsEnable(machine.Labels)
+		err = containerd.Install(machineSSH, option)
+		if err != nil {
+			return errors.Wrap(err, machine.IP)
+		}
+	}
+
+	return nil
+}
+
 func (p *Provider) EnsureDocker(ctx context.Context, c *v1.Cluster) error {
 	machines := map[bool][]platformv1.ClusterMachine{
 		true:  c.Spec.ScalingMachines,
@@ -537,13 +570,13 @@ func (p *Provider) EnsureKubernetesImages(ctx context.Context, c *v1.Cluster) er
 	machines := map[bool][]platformv1.ClusterMachine{
 		true:  c.Spec.ScalingMachines,
 		false: c.Spec.Machines}[len(c.Spec.ScalingMachines) > 0]
-	option := &image.Option{Version: c.Spec.Version, RegistryDomain: p.config.Registry.Domain}
+	option := &image.Option{Version: c.Spec.Version, RegistryDomain: p.config.Registry.Domain, KubeImages: images.KubecomponetNames}
 	for _, machine := range machines {
 		machineSSH, err := machine.SSH()
 		if err != nil {
 			return err
 		}
-		err = image.PullKubernetesImages(machineSSH, option)
+		err = image.PullKubernetesImages(c, machineSSH, option)
 		if err != nil {
 			return errors.Wrap(err, machine.IP)
 		}
@@ -581,7 +614,11 @@ func (p *Provider) EnsureKubeadm(ctx context.Context, c *v1.Cluster) error {
 			return err
 		}
 
-		err = kubeadm.Install(machineSSH, c.Spec.Version)
+		option := &kubeadm.Option{
+			RuntimeType: c.Spec.Features.ContainerRuntime,
+			Version:     c.Spec.Version,
+		}
+		err = kubeadm.Install(machineSSH, option)
 		if err != nil {
 			return errors.Wrap(err, machine.IP)
 		}

pkg/platform/provider/baremetal/cluster/kubeadm.go

@@ -75,6 +75,12 @@ func (p *Provider) getKubeadmJoinConfig(c *v1.Cluster, machineIP string) *kubead
 		}
 	}
 	nodeRegistration.KubeletExtraArgs = kubeletExtraArgs
+	// Specify cri runtime type
+	if c.Cluster.Spec.Features.ContainerRuntime == "docker" {
+		nodeRegistration.CRISocket = "/var/run/dockershim.sock"
+	} else {
+		nodeRegistration.CRISocket = "/var/run/containerd/containerd.sock"
+	}
 
 	return &kubeadmv1beta2.JoinConfiguration{
 		NodeRegistration: nodeRegistration,
@@ -121,7 +127,12 @@ func (p *Provider) getInitConfiguration(c *v1.Cluster) *kubeadmv1beta2.InitConfi
 		}
 	}
 	nodeRegistration.KubeletExtraArgs = kubeletExtraArgs
-
+	// Specify cri runtime type
+	if c.Cluster.Spec.Features.ContainerRuntime == "docker" {
+		nodeRegistration.CRISocket = "/var/run/dockershim.sock"
+	} else {
+		nodeRegistration.CRISocket = "/var/run/containerd/containerd.sock"
+	}
 	return &kubeadmv1beta2.InitConfiguration{
 		BootstrapTokens: []kubeadmv1beta2.BootstrapToken{
 			{

pkg/platform/provider/baremetal/cluster/provider.go

@@ -86,7 +86,7 @@ func NewProvider() (*Provider, error) {
 			// install packages
 			p.EnsureNvidiaDriver,
 			p.EnsureNvidiaContainerRuntime,
-			p.EnsureDocker,
+			p.EnsureContainerRuntime,
 			p.EnsureKubernetesImages,
 			p.EnsureKubelet,
 			p.EnsureCNIPlugins,

pkg/platform/provider/baremetal/cluster/update.go

@@ -74,7 +74,7 @@ func (p *Provider) EnsureRenewCerts(ctx context.Context, c *v1.Cluster) error {
 		}
 
 		logger.Info("RenewCerts doing")
-		err = kubeadm.RenewCerts(s)
+		err = kubeadm.RenewCerts(c, s)
 		if err != nil {
 			return errors.Wrap(err, machine.IP)
 		}
@@ -124,7 +124,7 @@ func (p *Provider) EnsureAPIServerCert(ctx context.Context, c *v1.Cluster) error
 		if err != nil {
 			return errors.Wrap(err, machine.IP)
 		}
-		err = kubeadm.RestartContainerByFilter(s, kubeadm.DockerFilterForControlPlane("kube-apiserver"))
+		err = kubeadm.RestartContainerByLabel(c, s, kubeadm.ContainerLabelOfControlPlane(c, "kube-apiserver"))
 		if err != nil {
 			return err
 		}