Add --cni flag (replaces --enable-default-cni), fix --network-plugin handling

.github/workflows/master.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Download Dependencies
         run: go mod download
       - name: Build Binaries
-        run: |          
+        run: |
           make cross
           make e2e-cross
           cp -r test/integration/testdata ./out
@@ -541,7 +541,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 27 ];then echo "*** Failed to pass at least 27 ! ***";exit 2;fi
+          if [ "$numPass" -lt 26 ];then echo "*** Failed to pass at least 26 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   addons_certs_docker_ubuntu:
     runs-on: ubuntu-18.04
@@ -1030,17 +1030,17 @@ jobs:
     if: always()
     needs:
       [
-          functional_docker_ubuntu,
-          functional_virtualbox_macos,
-          functional_docker_windows,
-          functional_hyperv_windows,
-          functional_baremetal_ubuntu18_04,
-          addons_certs_docker_ubuntu,
-          addons_certs_virtualbox_macos,
-          multinode_docker_ubuntu,
-          multinode_virtualbox_macos,
-          pause_preload_dockerflags_docker_ubuntu,
-          pause_preload_dockerflags_virtualbox_macos,
+        functional_docker_ubuntu,
+        functional_virtualbox_macos,
+        functional_docker_windows,
+        functional_hyperv_windows,
+        functional_baremetal_ubuntu18_04,
+        addons_certs_docker_ubuntu,
+        addons_certs_virtualbox_macos,
+        multinode_docker_ubuntu,
+        multinode_virtualbox_macos,
+        pause_preload_dockerflags_docker_ubuntu,
+        pause_preload_dockerflags_virtualbox_macos,
       ]
     runs-on: ubuntu-18.04
     steps:

.github/workflows/pr.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Download Dependencies
         run: go mod download
       - name: Build Binaries
-        run: |          
+        run: |
           make cross
           make e2e-cross
           cp -r test/integration/testdata ./out
@@ -539,7 +539,7 @@ jobs:
           echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
-          if [ "$numPass" -lt 27 ];then echo "*** Failed to pass at least 27 ! ***";exit 2;fi
+          if [ "$numPass" -lt 26 ];then echo "*** Failed to pass at least 26 ! ***";exit 2;fi
           if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
   addons_certs_docker_ubuntu:
     runs-on: ubuntu-18.04
@@ -1028,17 +1028,17 @@ jobs:
     if: always()
     needs:
       [
-          functional_docker_ubuntu,
-          functional_virtualbox_macos,
-          functional_docker_windows,
-          functional_hyperv_windows,
-          functional_baremetal_ubuntu18_04,
-          addons_certs_docker_ubuntu,
-          addons_certs_virtualbox_macos,
-          multinode_docker_ubuntu,
-          multinode_virtualbox_macos,
-          pause_preload_dockerflags_docker_ubuntu,
-          pause_preload_dockerflags_virtualbox_macos,
+        functional_docker_ubuntu,
+        functional_virtualbox_macos,
+        functional_docker_windows,
+        functional_hyperv_windows,
+        functional_baremetal_ubuntu18_04,
+        addons_certs_docker_ubuntu,
+        addons_certs_virtualbox_macos,
+        multinode_docker_ubuntu,
+        multinode_virtualbox_macos,
+        pause_preload_dockerflags_docker_ubuntu,
+        pause_preload_dockerflags_virtualbox_macos,
       ]
     runs-on: ubuntu-18.04
     steps:

cmd/minikube/cmd/node_add.go

@@ -70,9 +70,7 @@ var nodeAddCmd = &cobra.Command{
 			}
 		}
 
-		// Add CNI config if it's not already there
-		// We need to run kubeadm.init here as well
-		if err := config.MultiNodeCNIConfig(cc); err != nil {
+		if err := config.SaveProfile(cc.Name, cc); err != nil {
 			exit.WithError("failed to save config", err)
 		}
 

cmd/minikube/cmd/start_flags.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -54,6 +55,7 @@ const (
 	criSocket               = "cri-socket"
 	networkPlugin           = "network-plugin"
 	enableDefaultCNI        = "enable-default-cni"
+	cniFlag                 = "cni"
 	hypervVirtualSwitch     = "hyperv-virtual-switch"
 	hypervUseExternalSwitch = "hyperv-use-external-switch"
 	hypervExternalAdapter   = "hyperv-external-adapter"
@@ -130,8 +132,9 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
 	startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
-	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
-	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
+	startCmd.Flags().String(networkPlugin, "", "Kubelet network plug-in to use (default: auto)")
+	startCmd.Flags().Bool(enableDefaultCNI, false, "DEPRECATED: Replaced by --cni=bridge")
+	startCmd.Flags().String(cniFlag, "", "CNI plug-in to use. Valid options: auto, bridge, flannel, kindnet, or path to a CNI manifest (default: auto)")
 	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllComponentsList, ",")))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
@@ -237,21 +240,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			exit.WithCodeT(exit.Config, "Generate unable to parse disk size '{{.diskSize}}': {{.error}}", out.V{"diskSize": viper.GetString(humanReadableDiskSize), "error": err})
 		}
 
-		r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})
-		if err != nil {
-			return cc, config.Node{}, errors.Wrap(err, "new runtime manager")
-		}
-
-		// Pick good default values for --network-plugin and --enable-default-cni based on runtime.
-		selectedEnableDefaultCNI := viper.GetBool(enableDefaultCNI)
-		selectedNetworkPlugin := viper.GetString(networkPlugin)
-		if r.DefaultCNI() && !cmd.Flags().Changed(networkPlugin) {
-			selectedNetworkPlugin = "cni"
-			if !cmd.Flags().Changed(enableDefaultCNI) {
-				selectedEnableDefaultCNI = true
-			}
-		}
-
 		repository := viper.GetString(imageRepository)
 		mirrorCountry := strings.ToLower(viper.GetString(imageMirrorCountry))
 		if strings.ToLower(repository) == "auto" || (mirrorCountry != "" && repository == "") {
@@ -275,6 +263,13 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 		}
 
+		// Backwards compatibility with --enable-default-cni
+		chosenCNI := viper.GetString(cniFlag)
+		if viper.GetBool(enableDefaultCNI) && !cmd.Flags().Changed(cniFlag) {
+			glog.Errorf("Found deprecated --enable-default-cni flag, setting --cni=bridge")
+			chosenCNI = "bridge"
+		}
+
 		cc = config.ClusterConfig{
 			Name:                    ClusterFlagValue(),
 			KeepContext:             viper.GetBool(keepContext),
@@ -318,16 +313,26 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 				FeatureGates:           viper.GetString(featureGates),
 				ContainerRuntime:       viper.GetString(containerRuntime),
 				CRISocket:              viper.GetString(criSocket),
-				NetworkPlugin:          selectedNetworkPlugin,
+				NetworkPlugin:          viper.GetString(networkPlugin),
 				ServiceCIDR:            viper.GetString(serviceCIDR),
 				ImageRepository:        repository,
 				ExtraOptions:           config.ExtraOptions,
 				ShouldLoadCachedImages: viper.GetBool(cacheImages),
-				EnableDefaultCNI:       selectedEnableDefaultCNI,
+				CNI:                    chosenCNI,
 				NodePort:               viper.GetInt(apiServerPort),
 			},
 		}
 		cc.VerifyComponents = interpretWaitFlag(*cmd)
+
+		cnm, err := cni.New(cc)
+		if err != nil {
+			return cc, config.Node{}, errors.Wrap(err, "cni")
+		}
+
+		if _, ok := cnm.(cni.Disabled); !ok {
+			glog.Infof("Found %q CNI - setting NetworkPlugin=cni", cnm)
+			cc.KubernetesConfig.NetworkPlugin = "cni"
+		}
 	}
 
 	r, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime})
@@ -354,6 +359,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 	validateFlags(cmd, existing.Driver)
 
 	cc := *existing
+
 	if cmd.Flags().Changed(containerRuntime) {
 		cc.KubernetesConfig.ContainerRuntime = viper.GetString(containerRuntime)
 	}
@@ -514,10 +520,6 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KubernetesConfig.CRISocket = viper.GetString(criSocket)
 	}
 
-	if cmd.Flags().Changed(criSocket) {
-		cc.KubernetesConfig.NetworkPlugin = viper.GetString(criSocket)
-	}
-
 	if cmd.Flags().Changed(networkPlugin) {
 		cc.KubernetesConfig.NetworkPlugin = viper.GetString(networkPlugin)
 	}
@@ -534,8 +536,15 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KubernetesConfig.ImageRepository = viper.GetString(imageRepository)
 	}
 
-	if cmd.Flags().Changed(enableDefaultCNI) {
-		cc.KubernetesConfig.EnableDefaultCNI = viper.GetBool(enableDefaultCNI)
+	if cmd.Flags().Changed(enableDefaultCNI) && !cmd.Flags().Changed(cniFlag) {
+		if viper.GetBool(enableDefaultCNI) {
+			glog.Errorf("Found deprecated --enable-default-cni flag, setting --cni=bridge")
+			cc.KubernetesConfig.CNI = "bridge"
+		}
+	}
+
+	if cmd.Flags().Changed(cniFlag) {
+		cc.KubernetesConfig.CNI = viper.GetString(cniFlag)
 	}
 
 	if cmd.Flags().Changed(waitComponents) {

hack/preload-images/generate.go

@@ -65,8 +65,9 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 	if err != nil {
 		return errors.Wrap(err, "kubeadm images")
 	}
+
 	if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428
-		imgs = append(imgs, kic.OverlayImage)
+		imgs = append(imgs, images.KindNet(""))
 	}
 
 	runner := command.NewKICRunner(profile, driver.OCIBinary)

pkg/drivers/kic/types.go

@@ -27,9 +27,6 @@ const (
 	Version = "v0.0.10"
 	// SHA of the kic base image
 	baseImageSHA = "f58e0c4662bac8a9b5dda7984b185bad8502ade5d9fa364bf2755d636ab51438"
-	// OverlayImage is the cni plugin used for overlay image, created by kind.
-	// CNI plugin image used for kic drivers created by kind.
-	OverlayImage = "kindest/kindnetd:0.5.4"
 )
 
 var (

pkg/minikube/bootstrapper/bsutil/files.go

@@ -18,17 +18,19 @@ limitations under the License.
 package bsutil
 
 import (
+	"os/exec"
 	"path"
 
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // KubeadmYamlPath is the path to the kubeadm configuration
 var KubeadmYamlPath = path.Join(vmpath.GuestEphemeralDir, "kubeadm.yaml")
 
 const (
-	//DefaultCNIConfigPath is the configuration file for CNI networks
-	DefaultCNIConfigPath = "/etc/cni/net.d/1-k8s.conf"
 	// KubeletServiceFile is the file for the systemd kubelet.service
 	KubeletServiceFile = "/lib/systemd/system/kubelet.service"
 	// KubeletSystemdConfFile is config for the systemd kubelet.service
@@ -38,3 +40,22 @@ const (
 	// KubeletInitPath is where Sys-V style init script is installed
 	KubeletInitPath = "/etc/init.d/kubelet"
 )
+
+// CopyFiles combines mkdir requests into a single call to reduce load
+func CopyFiles(runner command.Runner, files []assets.CopyableFile) error {
+	dirs := []string{}
+	for _, f := range files {
+		dirs = append(dirs, f.GetTargetDir())
+	}
+	args := append([]string{"mkdir", "-p"}, dirs...)
+	if _, err := runner.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
+
+	for _, f := range files {
+		if err := runner.Copy(f); err != nil {
+			return errors.Wrapf(err, "copy")
+		}
+	}
+	return nil
+}

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -26,6 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -65,6 +66,18 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
 	}
 
+	cnm, err := cni.New(cc)
+	if err != nil {
+		return nil, errors.Wrap(err, "cni")
+	}
+
+	podCIDR := cnm.CIDR()
+	overrideCIDR := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm)
+	if overrideCIDR != "" {
+		podCIDR = overrideCIDR
+	}
+	glog.Infof("Using pod CIDR: %s", podCIDR)
+
 	opts := struct {
 		CertDir             string
 		ServiceCIDR         string
@@ -88,7 +101,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 	}{
 		CertDir:           vmpath.GuestKubernetesCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
-		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
+		PodSubnet:         podCIDR,
 		AdvertiseAddress:  n.IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -53,7 +54,12 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 	}
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
+
+		if k8s.NetworkPlugin == "kubenet" {
+			extraOpts["pod-cidr"] = cni.DefaultPodCIDR
+		}
 	}
+
 	if _, ok := extraOpts["node-ip"]; !ok {
 		extraOpts["node-ip"] = nc.IP
 	}

pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go

@@ -32,7 +32,7 @@ import (
 
 // WaitForNodeReady waits till kube client reports node status as "ready"
 func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error {
-	glog.Info("waiting for node status to be ready ...")
+	glog.Infof("waiting %s for node status to be ready ...", timeout)
 	start := time.Now()
 	defer func() {
 		glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start))

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml

@@ -35,7 +35,7 @@ schedulerExtraArgs:
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "10.244.0.0/16"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml

@@ -35,7 +35,7 @@ schedulerExtraArgs:
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "10.244.0.0/16"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml

@@ -43,7 +43,7 @@ schedulerExtraArgs:
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "10.244.0.0/16"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1