Merge pull request #1660 from fabriziopandini/support-multiple-cri

Upgrade Kind version & Add support for multiple CRI in kinder

kinder/ci/workflows/external-etcd.yaml

@@ -26,6 +26,7 @@ tasks:
     - --base-image={{ .vars.baseImage }}
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.kubernetesVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -40,6 +41,8 @@ tasks:
     - --control-plane-nodes=3
     - --worker-nodes=2
     - --external-etcd
+    - --automatic-copy-certs  # in v1.15 and after automatic copy certs can be configure in the kubeadm-config (which is generated at create time)
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 5m
 - name: init
@@ -77,7 +80,7 @@ tasks:
     - --loglevel=debug
 - name: e2e
   description: |
-    Runs e2e tests
+    Runs Kubernetes e2e test (conformance)
   cmd: kinder
   args:
     - test

kinder/ci/workflows/skew-1.15-on-1.14.yaml

@@ -17,6 +17,7 @@ tasks:
 # because:
 # - e2e-kubeadm is executed (nb. e2e-kubeadm was heavily refactored in v1.15 cycle)
 # - it uses the automatic copy certs feature, available in kubeadm since v1.14
+# - it configures automatic copy certs via the kubeadm-config, available in kubeadm since v1.15
 - name: pull-base-image
   description: |
     pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
@@ -36,6 +37,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.kubernetesVersion }}
     - --with-kubeadm={{ .vars.kubeadmVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -49,6 +51,9 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
+    - --automatic-copy-certs  # in v1.15 and after automatic copy certs can be configure in the kubeadm-config (which is generated at create time)
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -90,7 +95,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e
   description: |
-    Runs kubeadm e2e test
+    Runs Kubernetes e2e test (conformance)
   cmd: kinder
   args:
     - test

kinder/ci/workflows/skew-master-on-1.15.yaml

@@ -17,6 +17,7 @@ tasks:
 # because:
 # - e2e-kubeadm is executed (nb. e2e-kubeadm was heavily refactored in v1.15 cycle)
 # - it uses the automatic copy certs feature, available in kubeadm since v1.14
+# - it configures automatic copy certs via the kubeadm-config, available in kubeadm since v1.15
 - name: pull-base-image
   description: |
     pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
@@ -36,6 +37,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.kubernetesVersion }}
     - --with-kubeadm={{ .vars.kubeadmVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -49,6 +51,9 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
+    - --automatic-copy-certs  # in v1.15 and after automatic copy certs can be configure in the kubeadm-config (which is generated at create time)
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -90,7 +95,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e
   description: |
-    Runs kubeadm e2e test
+    Runs Kubernetes e2e test (conformance)
   cmd: kinder
   args:
     - test

kinder/ci/workflows/skew-x-on-y-tasks.yaml

@@ -36,6 +36,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.kubernetesVersion }}
     - --with-kubeadm={{ .vars.kubeadmVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -49,6 +50,8 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -79,7 +82,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e
   description: |
-    Runs kubeadm e2e test
+    Runs Kubernetes e2e test (conformance)
   cmd: kinder
   args:
     - test

kinder/ci/workflows/upgrade-1.14-1.15.yaml

@@ -37,6 +37,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.initVersion }}
     - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -50,6 +51,8 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -109,7 +112,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e-after
   description: |
-    Runs kubeadm e2e test on the cluster with Kubernetes "upgradeVersion"
+    Runs Kubernetes e2e test (conformance) on the cluster with Kubernetes "upgradeVersion"
   cmd: kinder
   args:
     - test

kinder/ci/workflows/upgrade-1.15-master.yaml

@@ -17,6 +17,7 @@ tasks:
 # because:
 # - e2e-kubeadm is executed after the upgrade (nb. e2e-kubeadm was heavily refactored in v1.15 cycle)
 # - it uses the automatic copy certs feature, available in kubeadm since v1.14
+# - it configures automatic copy certs via the kubeadm-config, available in kubeadm since v1.15
 - name: pull-base-image
   description: |
     pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
@@ -37,6 +38,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.initVersion }}
     - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -50,6 +52,9 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker            # this is temporary, until kinder will autodetect the CRI runtime installed into the node image
+    - --automatic-copy-certs  # in v1.15 and after automatic copy certs can be configure in the kubeadm-config (which is generated at create time)
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -109,7 +114,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e-after
   description: |
-    Runs kubeadm e2e test on the cluster with Kubernetes "upgradeVersion"
+    Runs Kubernetes e2e test (conformance) on the cluster with Kubernetes "upgradeVersion"
   cmd: kinder
   args:
     - test
@@ -152,4 +157,3 @@ tasks:
     - cluster
     - --name={{ .vars.clusterName }}
   force: true
-

kinder/ci/workflows/upgrade-tasks.yaml

@@ -36,6 +36,7 @@ tasks:
     - --image={{ .vars.image }}
     - --with-init-artifacts={{ .vars.initVersion }}
     - --with-upgrade-artifacts={{ .vars.upgradeVersion }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
     - --loglevel=debug
   timeout: 10m
 - name: create-cluster
@@ -49,6 +50,8 @@ tasks:
     - --image={{ .vars.image }}
     - --control-plane-nodes={{ .vars.controlPlaneNodes }}
     - --worker-nodes={{ .vars.workerNodes }}
+    - --cri=docker           # this is temporary, until kinder will autodetect the CRI runtime installed into the base image
+    - --loglevel=debug
   timeout: 5m
 - name: init
   description: |
@@ -97,7 +100,7 @@ tasks:
     - --name={{ .vars.clusterName }}
 - name: e2e-after
   description: |
-    Runs kubeadm e2e test on the cluster with Kubernetes "upgradeVersion"
+    Runs Kubernetes e2e test (conformance) on the cluster with Kubernetes "upgradeVersion"
   cmd: kinder
   args:
     - test

kinder/cmd/kinder/build/build.go

@@ -14,16 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package build implements the `build` command
 package build
 
 import (
 	"github.com/spf13/cobra"
 
-	"sigs.k8s.io/kind/cmd/kind/build/baseimage"
-	"sigs.k8s.io/kind/cmd/kind/build/nodeimage"
-
 	"k8s.io/kubeadm/kinder/cmd/kinder/build/nodevariant"
+	kindbaseimage "sigs.k8s.io/kind/cmd/kind/build/baseimage"
+	kindnodeimage "sigs.k8s.io/kind/cmd/kind/build/nodeimage"
 )
 
 // NewCommand returns a new cobra.Command for building
@@ -36,8 +34,8 @@ func NewCommand() *cobra.Command {
 		Long:  "Build the base node image (base-image) or the node image (node-image) or node image variants (node-variant)",
 	}
 	// add subcommands
-	cmd.AddCommand(baseimage.NewCommand())
-	cmd.AddCommand(nodeimage.NewCommand())
+	cmd.AddCommand(kindbaseimage.NewCommand())
+	cmd.AddCommand(kindnodeimage.NewCommand())
 	cmd.AddCommand(nodevariant.NewCommand())
 	return cmd
 }

kinder/cmd/kinder/build/build/TODO.md

@@ -0,0 +1 @@
+Add code for generating a kind(er) base images with docker container runtime installed internally

kinder/cmd/kinder/build/nodevariant/nodevariant.go

@@ -21,7 +21,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"k8s.io/kubeadm/kinder/pkg/build/alter"
-	"sigs.k8s.io/kind/pkg/build/node"
+	kindebuildnode "sigs.k8s.io/kind/pkg/build/node"
 )
 
 type flagpole struct {
@@ -33,6 +33,7 @@ type flagpole struct {
 	UpgradeArtifacts string
 	Kubeadm          string
 	Kubelet          string
+	CRI              string
 }
 
 // NewCommand returns a new cobra.Command for building the node image
@@ -50,13 +51,13 @@ func NewCommand() *cobra.Command {
 	}
 	cmd.Flags().StringVar(
 		&flags.Image, "image",
-		node.DefaultImage,
+		kindebuildnode.DefaultImage,
 		"name:tag of the resulting image to be built",
 	)
 	cmd.Flags().StringVar(
 		&flags.BaseImage, "base-image",
-		node.DefaultImage,
-		"name:tag of the base image to use for the build; this can be a kindest/base image or kindest/node image",
+		kindebuildnode.DefaultImage,
+		"name:tag of the source image; this can be a kindest/base image or kindest/node image",
 	)
 	cmd.Flags().StringVar(
 		&flags.InitArtifacts, "with-init-artifacts",
@@ -68,6 +69,12 @@ func NewCommand() *cobra.Command {
 		nil,
 		"version/build-label/path to images tar or folder with images tars to be added to the images",
 	)
+	//TODO: remove this as soon CRI autodetection is implemented
+	cmd.Flags().StringVar(
+		&flags.CRI, "cri",
+		"",
+		"specifies the cri installed inside the base image",
+	)
 	cmd.Flags().StringVar(
 		&flags.ImageNamePrefix, "image-name-prefix",
 		"",
@@ -92,10 +99,18 @@ func NewCommand() *cobra.Command {
 }
 
 func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
+	// check the cri flag
+	var err error
+	cri := flags.CRI
+	if cri == "" {
+		return errors.Wrap(err, "Please use the --cri flag to specify the container runtime installed inside the base image")
+	}
+
 	ctx, err := alter.NewContext(
 		// base build options
-		alter.WithImage(flags.Image),
 		alter.WithBaseImage(flags.BaseImage),
+		alter.WithCRI(cri),
+		alter.WithImage(flags.Image),
 		// bits to be added to the image
 		alter.WithInitArtifacts(flags.InitArtifacts),
 		alter.WithKubeadm(flags.Kubeadm),

kinder/cmd/kinder/cp/cp.go

@@ -14,15 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package cp implements the `cp` command
 package cp
 
 import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
-	kcluster "k8s.io/kubeadm/kinder/pkg/cluster"
-	"sigs.k8s.io/kind/pkg/cluster"
+	"k8s.io/kubeadm/kinder/pkg/cluster/manager"
+	"k8s.io/kubeadm/kinder/pkg/constants"
 )
 
 type flagpole struct {
@@ -52,31 +51,25 @@ func NewCommand() *cobra.Command {
 			return runE(flags, cmd, args)
 		},
 	}
-	cmd.Flags().StringVar(&flags.Name, "name", cluster.DefaultName, "cluster context name")
+	cmd.Flags().StringVar(
+		&flags.Name, "name",
+		constants.DefaultClusterName,
+		"cluster name",
+	)
 	return cmd
 }
 
 func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
-	// Check if the cluster name already exists
-	known, err := cluster.IsKnown(flags.Name)
+	// get a kinder cluster manager
+	o, err := manager.NewClusterManager(flags.Name)
 	if err != nil {
-		return err
-	}
-	if !known {
-		return errors.Errorf("a cluster with the name %q does not exists", flags.Name)
-	}
-
-	// create a cluster context from current nodes
-	ctx := cluster.NewContext(flags.Name)
-
-	kcfg, err := kcluster.NewKContext(ctx)
-	if err != nil {
-		return errors.Wrap(err, "failed to create cluster context")
+		return errors.Wrapf(err, "failed to create create a kinder cluster manager for %s", flags.Name)
 	}
 
-	err = kcfg.Copy(args[0], args[1])
+	// execute the copy action on selected target nodes
+	err = o.CopyFile(args[0], args[1])
 	if err != nil {
-		return errors.Wrap(err, "failed to exec command on cluster nodes")
+		return errors.Wrap(err, "failed to copy files")
 	}
 
 	return nil