From 557339acc29b3b3891d07fbc815e7d54153d1614 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 14 Nov 2019 15:00:56 -0800 Subject: [PATCH 1/5] Recover from aborted kubeadm runs by checking for file existence --- cmd/minikube/cmd/start.go | 11 --------- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 25 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index c6162911dcd9..1be0db1f1205 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -1210,9 +1210,6 @@ func configureRuntimes(runner cruntime.CommandRunner, drvName string, k8s cfg.Ku // bootstrapCluster starts Kubernetes using the chosen bootstrapper func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner command.Runner, kc cfg.KubernetesConfig, preexisting bool, isUpgrade bool) { - // hum. bootstrapper.Bootstrapper should probably have a Name function. - bsName := viper.GetString(cmdcfg.Bootstrapper) - if isUpgrade || !preexisting { out.T(out.Pulling, "Pulling images ...") if err := bs.PullImages(kc); err != nil { @@ -1220,14 +1217,6 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner c } } - if preexisting { - out.T(out.Restarting, "Relaunching Kubernetes using {{.bootstrapper}} ... ", out.V{"bootstrapper": bsName}) - if err := bs.RestartCluster(kc); err != nil { - exit.WithLogEntries("Error restarting cluster", err, logs.FindProblems(r, bs, runner)) - } - return - } - out.T(out.Launch, "Launching Kubernetes ... ") if err := bs.StartCluster(kc); err != nil { exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(r, bs, runner)) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 08f1d857c248..e30d8d19b37f 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -94,6 +94,17 @@ var KubeadmExtraArgsWhitelist = map[int][]string{ }, } +// remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. +var expectedArtifacts = []string{ + "/etc/kubernetes/scheduler.conf", + "/etc/kubernetes/kubelet.conf", + "/var/lib/kubelet/kubeadm-flags.env", + "/var/lib/kubelet/config.yaml", + "/var/lib/minikube", + "/etc/kubernetes/manifests/kube-apiserver.yaml", + "/etc/kubernetes/manifests/etcd.yaml", +} + // yamlConfigPath is the path to the kubeadm configuration var yamlConfigPath = path.Join(vmpath.GuestEphemeralDir, "kubeadm.yaml") @@ -222,8 +233,20 @@ func (k *Bootstrapper) createCompatSymlinks() error { return nil } +func (k *Bootstrapper) existingConfig() error { + _, err := k.c.RunCmd(exec.Command("ls", expectedArtifacts...)) + return err +} + // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { + err := k.existingConfig() + if err == nil { + return k.RestartCluster(k8s) + } else { + glog.Infof("existence check: %v", err) + } + start := time.Now() glog.Infof("StartCluster: %+v", k8s) defer func() { @@ -281,6 +304,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { if err := k.adjustResourceLimits(); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) } + return nil } @@ -454,6 +478,7 @@ func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time. // RestartCluster restarts the Kubernetes cluster configured by kubeadm func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { glog.Infof("RestartCluster start") + start := time.Now() defer func() { glog.Infof("RestartCluster took %s", time.Since(start)) From a6ece0cbe8bc25e05496096aaa09f34cac38f32d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 14 Nov 2019 15:06:05 -0800 Subject: [PATCH 2/5] Make restartCluster private in preference to idempotent StartCluster --- pkg/minikube/bootstrapper/bootstrapper.go | 1 - pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 15 +++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index f54c0b4f41b5..119466da0a1e 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -39,7 +39,6 @@ type Bootstrapper interface { PullImages(config.KubernetesConfig) error StartCluster(config.KubernetesConfig) error UpdateCluster(config.KubernetesConfig) error - RestartCluster(config.KubernetesConfig) error DeleteCluster(config.KubernetesConfig) error WaitForCluster(config.KubernetesConfig, time.Duration) error // LogCommands returns a map of log type to a command which will display that log. diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e30d8d19b37f..267275b5a9cd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -242,10 +242,9 @@ func (k *Bootstrapper) existingConfig() error { func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { err := k.existingConfig() if err == nil { - return k.RestartCluster(k8s) - } else { - glog.Infof("existence check: %v", err) + return k.restartCluster(k8s) } + glog.Infof("existence check: %v", err) start := time.Now() glog.Infof("StartCluster: %+v", k8s) @@ -475,13 +474,13 @@ func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time. return k.waitForSystemPods(start, k8s, timeout) } -// RestartCluster restarts the Kubernetes cluster configured by kubeadm -func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { - glog.Infof("RestartCluster start") +// restartCluster restarts the Kubernetes cluster configured by kubeadm +func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { + glog.Infof("restartCluster start") start := time.Now() defer func() { - glog.Infof("RestartCluster took %s", time.Since(start)) + glog.Infof("restartCluster took %s", time.Since(start)) }() version, err := parseKubernetesVersion(k8s.KubernetesVersion) @@ -554,7 +553,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { return nil } -// PullImages downloads images that will be used by RestartCluster +// PullImages downloads images that will be used by Kubernetes func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { version, err := parseKubernetesVersion(k8s.KubernetesVersion) if err != nil { From a5865138313fc49a7678e1727c3355e7215df234 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 14 Nov 2019 19:11:57 -0800 Subject: [PATCH 3/5] Make sure etcd existence is recoverable too --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 267275b5a9cd..daff76064c98 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -266,6 +266,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { ignore := []string{ fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(path.Join(vmpath.GuestPersistentDir, "etcd"), "/", "-", -1)), "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", From 39c0c8be13654e04759fa88ba2df58e147dc85a0 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 14 Nov 2019 20:17:31 -0800 Subject: [PATCH 4/5] Use etcdDataDir() instead --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index daff76064c98..c0c75dfd946a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -266,7 +266,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { ignore := []string{ fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), - fmt.Sprintf("DirAvailable-%s", strings.Replace(path.Join(vmpath.GuestPersistentDir, "etcd"), "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", etcdDataDir(), "/", "-", -1)), "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml", From 2223c89555523eeb2c5596ee96c214e23e1fbbc7 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 15 Nov 2019 07:02:44 -0800 Subject: [PATCH 5/5] Check for etcd, but not /etc --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index c0c75dfd946a..5302e21cbe08 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -95,14 +95,11 @@ var KubeadmExtraArgsWhitelist = map[int][]string{ } // remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. +// NOTE: /etc is not persistent across restarts, so don't bother checking there var expectedArtifacts = []string{ - "/etc/kubernetes/scheduler.conf", - "/etc/kubernetes/kubelet.conf", "/var/lib/kubelet/kubeadm-flags.env", "/var/lib/kubelet/config.yaml", - "/var/lib/minikube", - "/etc/kubernetes/manifests/kube-apiserver.yaml", - "/etc/kubernetes/manifests/etcd.yaml", + etcdDataDir(), } // yamlConfigPath is the path to the kubeadm configuration @@ -234,7 +231,8 @@ func (k *Bootstrapper) createCompatSymlinks() error { } func (k *Bootstrapper) existingConfig() error { - _, err := k.c.RunCmd(exec.Command("ls", expectedArtifacts...)) + args := append([]string{"ls"}, expectedArtifacts...) + _, err := k.c.RunCmd(exec.Command("sudo", args...)) return err } @@ -266,7 +264,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { ignore := []string{ fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), - fmt.Sprintf("DirAvailable-%s", etcdDataDir(), "/", "-", -1)), + fmt.Sprintf("DirAvailable-%s", strings.Replace(etcdDataDir(), "/", "-", -1)), "FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml", "FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml", "FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml",