Add --cni flag, fix --network-plugin=kubenet

cmd/minikube/cmd/node_add.go

@@ -70,9 +70,7 @@ var nodeAddCmd = &cobra.Command{
 			}
 		}
 
-		// Add CNI config if it's not already there
-		// We need to run kubeadm.init here as well
-		if err := config.MultiNodeCNIConfig(cc); err != nil {
+		if err := config.SaveProfile(cc.Name, cc); err != nil {
 			exit.WithError("failed to save config", err)
 		}
 

cmd/minikube/cmd/start_flags.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -54,6 +55,7 @@ const (
 	criSocket               = "cri-socket"
 	networkPlugin           = "network-plugin"
 	enableDefaultCNI        = "enable-default-cni"
+	cniFlag                 = "cni"
 	hypervVirtualSwitch     = "hyperv-virtual-switch"
 	hypervUseExternalSwitch = "hyperv-use-external-switch"
 	hypervExternalAdapter   = "hyperv-external-adapter"
@@ -130,8 +132,9 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
 	startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
-	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
-	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
+	startCmd.Flags().String(networkPlugin, "", "Kubelet network plug-in to use (default: auto)")
+	startCmd.Flags().Bool(enableDefaultCNI, false, "DEPRECATED: Replaced by --cni=custom")
+	startCmd.Flags().String(cniFlag, "", "CNI plug-in to use. Valid options: auto, calico, custom, flannel, kindnet (default: auto)")
 	startCmd.Flags().StringSlice(waitComponents, kverify.DefaultWaitList, fmt.Sprintf("comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to %q, available options: %q . other acceptable values are 'all' or 'none', 'true' and 'false'", strings.Join(kverify.DefaultWaitList, ","), strings.Join(kverify.AllComponentsList, ",")))
 	startCmd.Flags().Duration(waitTimeout, 6*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
@@ -237,21 +240,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			exit.WithCodeT(exit.Config, "Generate unable to parse disk size '{{.diskSize}}': {{.error}}", out.V{"diskSize": viper.GetString(humanReadableDiskSize), "error": err})
 		}
 
-		r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})
-		if err != nil {
-			return cc, config.Node{}, errors.Wrap(err, "new runtime manager")
-		}
-
-		// Pick good default values for --network-plugin and --enable-default-cni based on runtime.
-		selectedEnableDefaultCNI := viper.GetBool(enableDefaultCNI)
-		selectedNetworkPlugin := viper.GetString(networkPlugin)
-		if r.DefaultCNI() && !cmd.Flags().Changed(networkPlugin) {
-			selectedNetworkPlugin = "cni"
-			if !cmd.Flags().Changed(enableDefaultCNI) {
-				selectedEnableDefaultCNI = true
-			}
-		}
-
 		repository := viper.GetString(imageRepository)
 		mirrorCountry := strings.ToLower(viper.GetString(imageMirrorCountry))
 		if strings.ToLower(repository) == "auto" || (mirrorCountry != "" && repository == "") {
@@ -275,6 +263,13 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			out.T(out.SuccessType, "Using image repository {{.name}}", out.V{"name": repository})
 		}
 
+		// Backwards compatibility with --enable-default-cni
+		chosenCNI := viper.GetString(cniFlag)
+		if viper.GetBool(enableDefaultCNI) && !cmd.Flags().Changed(cniFlag) {
+			glog.Errorf("Found deprecated --enable-default-cni flag, setting --cni=bridge")
+			chosenCNI = "bridge"
+		}
+
 		cc = config.ClusterConfig{
 			Name:                    ClusterFlagValue(),
 			KeepContext:             viper.GetBool(keepContext),
@@ -318,16 +313,26 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 				FeatureGates:           viper.GetString(featureGates),
 				ContainerRuntime:       viper.GetString(containerRuntime),
 				CRISocket:              viper.GetString(criSocket),
-				NetworkPlugin:          selectedNetworkPlugin,
+				NetworkPlugin:          viper.GetString(networkPlugin),
 				ServiceCIDR:            viper.GetString(serviceCIDR),
 				ImageRepository:        repository,
 				ExtraOptions:           config.ExtraOptions,
 				ShouldLoadCachedImages: viper.GetBool(cacheImages),
-				EnableDefaultCNI:       selectedEnableDefaultCNI,
+				CNI:                    chosenCNI,
 				NodePort:               viper.GetInt(apiServerPort),
 			},
 		}
 		cc.VerifyComponents = interpretWaitFlag(*cmd)
+
+		cnm, err := cni.New(cc)
+		if err != nil {
+			return cc, config.Node{}, errors.Wrap(err, "cni")
+		}
+
+		if _, ok := cnm.(cni.Disabled); !ok {
+			glog.Infof("Found %q CNI - setting NetworkPlugin=cni", cnm)
+			cc.KubernetesConfig.NetworkPlugin = "cni"
+		}
 	}
 
 	r, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime})
@@ -354,6 +359,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 	validateFlags(cmd, existing.Driver)
 
 	cc := *existing
+
 	if cmd.Flags().Changed(containerRuntime) {
 		cc.KubernetesConfig.ContainerRuntime = viper.GetString(containerRuntime)
 	}
@@ -514,10 +520,6 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KubernetesConfig.CRISocket = viper.GetString(criSocket)
 	}
 
-	if cmd.Flags().Changed(criSocket) {
-		cc.KubernetesConfig.NetworkPlugin = viper.GetString(criSocket)
-	}
-
 	if cmd.Flags().Changed(networkPlugin) {
 		cc.KubernetesConfig.NetworkPlugin = viper.GetString(networkPlugin)
 	}
@@ -534,8 +536,15 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KubernetesConfig.ImageRepository = viper.GetString(imageRepository)
 	}
 
-	if cmd.Flags().Changed(enableDefaultCNI) {
-		cc.KubernetesConfig.EnableDefaultCNI = viper.GetBool(enableDefaultCNI)
+	if cmd.Flags().Changed(enableDefaultCNI) && !cmd.Flags().Changed(cniFlag) {
+		if viper.GetBool(enableDefaultCNI) {
+			glog.Errorf("Found deprecated --enable-default-cni flag, setting --cni=bridge")
+			cc.KubernetesConfig.CNI = "bridge"
+		}
+	}
+
+	if cmd.Flags().Changed(cniFlag) {
+		cc.KubernetesConfig.CNI = viper.GetString(cniFlag)
 	}
 
 	if cmd.Flags().Changed(waitComponents) {

hack/jenkins/cloudshell_integration_tests_none.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the none Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+
+set -e
+
+OS_ARCH="linux-amd64"
+VM_DRIVER="none"
+JOB_NAME="none_cloudshell"
+
+
+
+SUDO_PREFIX="sudo -E "
+export KUBECONFIG="/root/.kube/config"
+
+gcloud alpha cloud-shell ssh --boosted "uptime"
+gcloud alpha cloud-shell scp --boosted ""
+
+mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+source ./common.sh

hack/jenkins/linux_conformance_tests_kvm.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+set -e
+
+OS_ARCH="linux-amd64"
+VM_DRIVER="kvm2"
+JOB_NAME="KVM_Linux"
+EXPECTED_DEFAULT_DRIVER="kvm2"
+
+# We pick kvm as our gvisor testbed because it is fast & reliable
+EXTRA_TEST_ARGS="-gvisor"
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+source ./common.sh

hack/preload-images/generate.go

@@ -65,8 +65,9 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 	if err != nil {
 		return errors.Wrap(err, "kubeadm images")
 	}
+
 	if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428
-		imgs = append(imgs, kic.OverlayImage)
+		imgs = append(imgs, images.KindNet(""))
 	}
 
 	runner := command.NewKICRunner(profile, driver.OCIBinary)

pkg/drivers/kic/types.go

@@ -27,9 +27,6 @@ const (
 	Version = "v0.0.10"
 	// SHA of the kic base image
 	baseImageSHA = "f58e0c4662bac8a9b5dda7984b185bad8502ade5d9fa364bf2755d636ab51438"
-	// OverlayImage is the cni plugin used for overlay image, created by kind.
-	// CNI plugin image used for kic drivers created by kind.
-	OverlayImage = "kindest/kindnetd:0.5.4"
 )
 
 var (

pkg/minikube/bootstrapper/bsutil/files.go

@@ -18,17 +18,19 @@ limitations under the License.
 package bsutil
 
 import (
+	"os/exec"
 	"path"
 
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // KubeadmYamlPath is the path to the kubeadm configuration
 var KubeadmYamlPath = path.Join(vmpath.GuestEphemeralDir, "kubeadm.yaml")
 
 const (
-	//DefaultCNIConfigPath is the configuration file for CNI networks
-	DefaultCNIConfigPath = "/etc/cni/net.d/1-k8s.conf"
 	// KubeletServiceFile is the file for the systemd kubelet.service
 	KubeletServiceFile = "/lib/systemd/system/kubelet.service"
 	// KubeletSystemdConfFile is config for the systemd kubelet.service
@@ -38,3 +40,22 @@ const (
 	// KubeletInitPath is where Sys-V style init script is installed
 	KubeletInitPath = "/etc/init.d/kubelet"
 )
+
+// CopyFiles combines mkdir requests into a single call to reduce load
+func CopyFiles(runner command.Runner, files []assets.CopyableFile) error {
+	dirs := []string{}
+	for _, f := range files {
+		dirs = append(dirs, f.GetTargetDir())
+	}
+	args := append([]string{"mkdir", "-p"}, dirs...)
+	if _, err := runner.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
+
+	for _, f := range files {
+		if err := runner.Copy(f); err != nil {
+			return errors.Wrapf(err, "copy")
+		}
+	}
+	return nil
+}

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -26,6 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -65,6 +66,18 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
 	}
 
+	cnm, err := cni.New(cc)
+	if err != nil {
+		return nil, errors.Wrap(err, "cni")
+	}
+
+	podCIDR := cnm.CIDR()
+	overrideCIDR := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm)
+	if overrideCIDR != "" {
+		podCIDR = overrideCIDR
+	}
+	glog.Infof("Using pod CIDR: %s", podCIDR)
+
 	opts := struct {
 		CertDir             string
 		ServiceCIDR         string
@@ -87,7 +100,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 	}{
 		CertDir:           vmpath.GuestKubernetesCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
-		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
+		PodSubnet:         podCIDR,
 		AdvertiseAddress:  n.IP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -53,7 +54,12 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 	}
 	if k8s.NetworkPlugin != "" {
 		extraOpts["network-plugin"] = k8s.NetworkPlugin
+
+		if k8s.NetworkPlugin == "kubenet" {
+			extraOpts["pod-cidr"] = cni.DefaultPodCIDR
+		}
 	}
+
 	if _, ok := extraOpts["node-ip"]; !ok {
 		extraOpts["node-ip"] = nc.IP
 	}

pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go

@@ -32,7 +32,7 @@ import (
 
 // WaitForNodeReady waits till kube client reports node status as "ready"
 func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error {
-	glog.Info("waiting for node status to be ready ...")
+	glog.Infof("waiting %s for node status to be ready ...", timeout)
 	start := time.Now()
 	defer func() {
 		glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start))