Merge pull request #4 from alonyb/patch-1

documentation

hack/images/kicbase.Dockerfile

@@ -2,7 +2,7 @@ ARG COMMIT_SHA
 # using base image created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/Dockerfile
 # which is an ubuntu 19.10 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
-FROM kindest/base:v20200122-2dfe64b2 as base
+FROM kindest/base:v20200317-92225082 as base
 USER root
 # specify version of everything explicitly using 'apt-cache policy'
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -19,9 +19,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \    
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
     apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends cri-o-1.17=1.17.0-3
+    apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
 # install podman
-RUN apt-get install -y --no-install-recommends podman=1.8.2~1
+RUN apt-get install -y --no-install-recommends podman=1.8.2~144
 # disable non-docker runtimes by default
 RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
 # enable docker which is default

pkg/drivers/kic/kic.go

@@ -361,13 +361,20 @@ func (d *Driver) Stop() error {
 		}
 		if len(containers) > 0 {
 			if err := runtime.StopContainers(containers); err != nil {
-				glog.Errorf("unable to stop containers : %v", err)
+				glog.Infof("unable to stop containers : %v", err)
+			}
+			if err := runtime.KillContainers(containers); err != nil {
+				glog.Errorf("unable to kill containers : %v", err)
 			}
 		}
 		glog.Infof("successfully stopped kubernetes!")
 
 	}
 
+	if err := killAPIServerProc(d.exec); err != nil {
+		glog.Warningf("couldn't stop kube-apiserver proc: %v", err)
+	}
+
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName)
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "stopping %s", d.MachineName)
@@ -379,3 +386,20 @@ func (d *Driver) Stop() error {
 func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support RunSSHCommandFromDriver commands")
 }
+
+// killAPIServerProc will kill an api server proc if it exists
+// to ensure this never happens https://github.com/kubernetes/minikube/issues/7521
+func killAPIServerProc(runner command.Runner) error {
+	// first check if it exists
+	rr, err := runner.RunCmd(exec.Command("pgrep", "kube-apiserver"))
+	if err == nil { // this means we might have a running kube-apiserver
+		pid, err := strconv.Atoi(rr.Stdout.String())
+		if err == nil { // this means we have a valid pid
+			glog.Warningf("Found a kube-apiserver running with pid %d, will try to kill the proc", pid)
+			if _, err = runner.RunCmd(exec.Command("pkill", "-9", string(pid))); err != nil {
+				return errors.Wrap(err, "kill")
+			}
+		}
+	}
+	return nil
+}

pkg/drivers/kic/types.go

@@ -30,13 +30,12 @@ const (
 	DefaultPodCIDR = "10.244.0.0/16"
 
 	// Version is the current version of kic
-	Version = "v0.0.8"
+	Version = "v0.0.9"
 	// SHA of the kic base image
-	baseImageSHA = "2f3380ebf1bb0c75b0b47160fd4e61b7b8fef0f1f32f9def108d3eada50a7a81"
-
+	baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
-	OverlayImage = "kindest/kindnetd:0.5.3"
+	OverlayImage = "kindest/kindnetd:0.5.4"
 )
 
 var (

pkg/minikube/assets/vm_assets.go

@@ -93,30 +93,37 @@ func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset
 // NewFileAsset creates a new FileAsset
 func NewFileAsset(src, targetDir, targetName, permissions string) (*FileAsset, error) {
 	glog.V(4).Infof("NewFileAsset: %s -> %s", src, path.Join(targetDir, targetName))
+
 	f, err := os.Open(src)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error opening file asset: %s", src)
+		return nil, errors.Wrap(err, "open")
 	}
+
 	info, err := os.Stat(src)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error getting info for %s", src)
+		return nil, errors.Wrapf(err, "stat")
 	}
-	r := io.NewSectionReader(f, 0, info.Size())
+
+	if info.Size() == 0 {
+		glog.Warningf("NewFileAsset: %s is an empty file!", src)
+	}
+
 	return &FileAsset{
 		BaseAsset: BaseAsset{
 			SourcePath:  src,
 			TargetDir:   targetDir,
 			TargetName:  targetName,
 			Permissions: permissions,
 		},
-		reader: r,
+		reader: io.NewSectionReader(f, 0, info.Size()),
 	}, nil
 }
 
 // GetLength returns the file length, or 0 (on error)
 func (f *FileAsset) GetLength() (flen int) {
 	fi, err := os.Stat(f.SourcePath)
 	if err != nil {
+		glog.Errorf("stat(%q) failed: %v", f.SourcePath, err)
 		return 0
 	}
 	return int(fi.Size())
@@ -126,6 +133,7 @@ func (f *FileAsset) GetLength() (flen int) {
 func (f *FileAsset) GetModTime() (time.Time, error) {
 	fi, err := os.Stat(f.SourcePath)
 	if err != nil {
+		glog.Errorf("stat(%q) failed: %v", f.SourcePath, err)
 		return time.Time{}, err
 	}
 	return fi.ModTime(), nil

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -60,8 +60,8 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.
 				return errors.Wrapf(err, "downloading %s", name)
 			}
 
-			if name == "kubelet" {
-				if err := sm.ForceStop("kubelet"); err != nil {
+			if name == "kubelet" && sm.Active(name) {
+				if err := sm.ForceStop(name); err != nil {
 					glog.Errorf("unable to stop kubelet: %v", err)
 				}
 			}

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -212,15 +212,18 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error {
 		return errors.Wrap(err, "run")
 	}
 
-	// this is required for containerd and cri-o runtime. till we close https://github.com/kubernetes/minikube/issues/7428
-	if driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker" {
-		if err := k.applyKicOverlay(cfg); err != nil {
-			return errors.Wrap(err, "apply kic overlay")
-		}
-	}
-
 	var wg sync.WaitGroup
-	wg.Add(3)
+	wg.Add(4)
+
+	go func() {
+		// the overlay is required for containerd and cri-o runtime: see #7428
+		if driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker" {
+			if err := k.applyKicOverlay(cfg); err != nil {
+				glog.Errorf("failed to apply kic overlay: %v", err)
+			}
+		}
+		wg.Done()
+	}()
 
 	go func() {
 		if err := k.applyNodeLabels(cfg); err != nil {
@@ -242,6 +245,7 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error {
 		}
 		wg.Done()
 	}()
+
 	wg.Wait()
 	return nil
 }
@@ -762,20 +766,30 @@ func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error {
 
 // applyKicOverlay applies the CNI plugin needed to make kic work
 func (k *Bootstrapper) applyKicOverlay(cfg config.ClusterConfig) error {
-	// Allow no more than 5 seconds for apply kic overlay
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
+
 	cmd := exec.CommandContext(ctx, "sudo",
 		path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), "create", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")),
 		"-f", "-")
+
 	b := bytes.Buffer{}
 	if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil {
 		return err
 	}
+
 	cmd.Stdin = bytes.NewReader(b.Bytes())
 	if rr, err := k.c.RunCmd(cmd); err != nil {
 		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
 	}
+
+	// Inform cri-o that the CNI has changed
+	if cfg.KubernetesConfig.ContainerRuntime == "crio" {
+		if err := sysinit.New(k.c).Restart("crio"); err != nil {
+			return errors.Wrap(err, "restart crio")
+		}
+	}
+
 	return nil
 }
 

pkg/minikube/cruntime/cri.go

@@ -195,7 +195,7 @@ func stopCRIContainers(cr CommandRunner, ids []string) error {
 	glog.Infof("Stopping containers: %s", ids)
 
 	crictl := getCrictlPath(cr)
-	args := append([]string{crictl, "rm"}, ids...)
+	args := append([]string{crictl, "stop"}, ids...)
 	c := exec.Command("sudo", args...)
 	if _, err := cr.RunCmd(c); err != nil {
 		return errors.Wrap(err, "crictl")

pkg/minikube/cruntime/cruntime.go

@@ -166,6 +166,7 @@ func ContainerStatusCommand() string {
 
 // disableOthers disables all other runtimes except for me.
 func disableOthers(me Manager, cr CommandRunner) error {
+
 	// valid values returned by manager.Name()
 	runtimes := []string{"containerd", "crio", "docker"}
 	for _, name := range runtimes {
@@ -178,13 +179,22 @@ func disableOthers(me Manager, cr CommandRunner) error {
 		if r.Name() == me.Name() {
 			continue
 		}
+
+		// Don't disable containerd if we are bound to it
+		if me.Name() == "Docker" && r.Name() == "containerd" && dockerBoundToContainerd(cr) {
+			glog.Infof("skipping containerd shutdown because we are bound to it")
+			continue
+		}
+
 		// runtime is already disabled, nothing to do.
 		if !r.Active() {
 			continue
 		}
+
 		if err = r.Disable(); err != nil {
 			glog.Warningf("disable failed: %v", err)
 		}
+
 		// Validate that the runtime really is offline - and that Active & Disable are properly written.
 		if r.Active() {
 			return fmt.Errorf("%s is still active", r.Name())

pkg/minikube/cruntime/docker.go

@@ -364,3 +364,18 @@ func DockerImagesPreloaded(runner command.Runner, images []string) bool {
 	}
 	return true
 }
+
+func dockerBoundToContainerd(runner command.Runner) bool {
+	// NOTE: assumes systemd
+	rr, err := runner.RunCmd(exec.Command("sudo", "systemctl", "cat", "docker.service"))
+	if err != nil {
+		glog.Warningf("unable to check if docker is bound to containerd")
+		return false
+	}
+
+	if strings.Contains(rr.Stdout.String(), "\nBindsTo=containerd") {
+		return true
+	}
+
+	return false
+}

pkg/minikube/node/cache.go

@@ -21,6 +21,7 @@ import (
 	"runtime"
 
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
@@ -144,7 +145,7 @@ func saveImagesToTarFromConfig() error {
 func CacheAndLoadImagesInConfig() error {
 	images, err := imagesInConfigFile()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "images")
 	}
 	if len(images) == 0 {
 		return nil
@@ -155,7 +156,7 @@ func CacheAndLoadImagesInConfig() error {
 func imagesInConfigFile() ([]string, error) {
 	configFile, err := config.ReadConfig(localpath.ConfigFile())
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "read")
 	}
 	if values, ok := configFile[cacheImageConfigKey]; ok {
 		var images []string

pkg/minikube/node/start.go

@@ -140,7 +140,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 	wg.Add(1)
 	go func() {
 		if err := CacheAndLoadImagesInConfig(); err != nil {
-			out.FailureT("Unable to load cached images from config file: {{error}}", out.V{"error": err})
+			out.FailureT("Unable to push cached images: {{error}}", out.V{"error": err})
 		}
 		wg.Done()
 	}()

pkg/minikube/registry/drvs/docker/docker.go

@@ -70,30 +70,40 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 }
 
 func status() registry.State {
+	docURL := "https://minikube.sigs.k8s.io/docs/drivers/docker/"
 	_, err := exec.LookPath(oci.Docker)
 	if err != nil {
-		return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Install Docker.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker/#install-docker"}
+		return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Install Docker", Doc: docURL}
 	}
 
-	// Allow no more than 3 seconds for docker info
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
 	defer cancel()
 
-	err = exec.CommandContext(ctx, oci.Docker, "info").Run()
+	// Quickly returns an error code if server is not running
+	cmd := exec.CommandContext(ctx, oci.Docker, "version", "--format", "{{.Server.Version}}")
+	_, err = cmd.Output()
+	if err == nil {
+		return registry.State{Installed: true, Healthy: true}
+	}
+
+	glog.Warningf("docker returned error: %v", err)
 
+	// Basic timeout
 	if ctx.Err() == context.DeadlineExceeded {
-		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Docker responds too slow. Restart the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"}
+		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Restart the Docker service", Doc: docURL}
 	}
-	if err != nil {
-		glog.Infof("docker info returned error: %v", err)
-		if strings.Contains(err.Error(), "Cannot connect to the Docker daemon") {
-			return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Start the Docker Service.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"}
+
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		stderr := strings.TrimSpace(string(exitErr.Stderr))
+		newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
+
+		if strings.Contains(stderr, "Cannot connect") || strings.Contains(stderr, "refused") || strings.Contains(stderr, "Is the docker daemon running") {
+			return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: "Start the Docker service", Doc: docURL}
 		}
-		// if we get here, something is really wrong on their docker.
-		// our best suggestion would be re-install latest docker.
-		return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Re-install the latest version of Docker.", Doc: "https://minikube.sigs.k8s.io/docs/drivers/docker"}
 
+		// We don't have good advice, but at least we can provide a good error message
+		return registry.State{Error: newErr, Installed: true, Healthy: false, Doc: docURL}
 	}
 
-	return registry.State{Installed: true, Healthy: true}
+	return registry.State{Error: err, Installed: true, Healthy: false, Doc: docURL}
 }