Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix stale hyperkit.pid making minikube start hang #3593

Merged
merged 5 commits into from
Jan 30, 2019
Merged
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 70 additions & 14 deletions pkg/drivers/hyperkit/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,15 @@ import (
"syscall"
"time"

"github.com/mitchellh/go-ps"

"io/ioutil"

"github.com/docker/machine/libmachine/drivers"
"github.com/docker/machine/libmachine/log"
"github.com/docker/machine/libmachine/state"
nfsexports "github.com/johanneswuerbach/nfsexports"
hyperkit "github.com/moby/hyperkit/go"
"github.com/johanneswuerbach/nfsexports"
"github.com/moby/hyperkit/go"
"github.com/pkg/errors"
pkgdrivers "k8s.io/minikube/pkg/drivers"
"k8s.io/minikube/pkg/minikube/constants"
Expand Down Expand Up @@ -169,12 +173,14 @@ func (d *Driver) Restart() error {

// Start a host
func (d *Driver) Start() error {
h, err := hyperkit.New("", d.VpnKitSock, filepath.Join(d.StorePath, "machines", d.MachineName))
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
if err := d.recoverFromUncleanShutdown(); err != nil {
return err
}
h, err := hyperkit.New("", d.VpnKitSock, stateDir)
if err != nil {
return errors.Wrap(err, "new-ing Hyperkit")
}

// TODO: handle the rest of our settings.
h.Kernel = d.ResolveStorePath("bzimage")
h.Initrd = d.ResolveStorePath("initrd")
h.VMNet = true
Expand All @@ -183,13 +189,20 @@ func (d *Driver) Start() error {
h.CPUs = d.CPU
h.Memory = d.Memory
h.UUID = d.UUID

if vsockPorts, err := d.extractVSockPorts(); err != nil {
return err
} else if len(vsockPorts) >= 1 {
h.VSock = true
h.VSockPorts = vsockPorts
}
h.Disks = []hyperkit.DiskConfig{
{
Path: pkgdrivers.GetDiskPath(d.BaseDriver),
Size: d.DiskSize,
Driver: "virtio-blk",
},
}
//}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove comment


log.Infof("Using UUID %s", h.UUID)
mac, err := GetMACAddressFromUUID(h.UUID)
Expand All @@ -200,13 +213,7 @@ func (d *Driver) Start() error {
// Need to strip 0's
mac = trimMacAddress(mac)
log.Infof("Generated MAC %s", mac)
h.Disks = []hyperkit.DiskConfig{
{
Path: pkgdrivers.GetDiskPath(d.BaseDriver),
Size: d.DiskSize,
Driver: "virtio-blk",
},
}

log.Infof("Starting with cmdline: %s", d.Cmdline)
if err := h.Start(d.Cmdline); err != nil {
return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline)
Expand Down Expand Up @@ -240,6 +247,56 @@ func (d *Driver) Start() error {
return nil
}

//recoverFromUncleanShutdown searches for an existing hyperkit.pid file in
//the machine directory. If it can't find it, a clean shutdown is assumed.
//If it finds the pid file, it checks for a running hyperkit process with that pid
//as the existence of a file might not indicate an unclean shutdown but an actual running
//hyperkit server. This is an error situation - we shouldn't start minikube as there is likely
//an instance running already. If the PID in the pidfile does not belong to a running hyperkit
//process, we can safely delete it, and there is a good chance the machine will recover when restarted.
func (d *Driver) recoverFromUncleanShutdown() error {
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
pidFile := filepath.Join(stateDir, pidFileName)

_, err := os.Stat(pidFile)

if os.IsNotExist(err) {
log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
return nil
}

if err != nil {
return errors.Wrap(err, "checking hyperkit pid file existence")
}

log.Warnf("minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: %s", pidFile)

content, err := ioutil.ReadFile(pidFile)
balopat marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return errors.Wrapf(err, "reading pidfile %s", pidFile)
}
pid, err := strconv.Atoi(string(content))
if err != nil {
return errors.Wrapf(err, "parsing pidfile %s", pidFile)
}

p, err := ps.FindProcess(pid)
if err != nil {
return errors.Wrapf(err, "trying to find process for PID %s", pid)
}

if p != nil && !strings.Contains(p.Executable(), "hyperkit") {
return fmt.Errorf("something is not right...please stop all minikube instances, seemingly a hyperkit server is already running with pid %d, executable: %s", pid, p.Executable())
}

log.Infof("No running hyperkit process found with PID %d, removing %s...", pid, pidFile)
if err := os.Remove(pidFile); err != nil {
return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile))
}

return nil
}

// Stop a host gracefully
func (d *Driver) Stop() error {
d.cleanupNfsExports()
Expand Down Expand Up @@ -350,7 +407,6 @@ func (d *Driver) sendSignal(s os.Signal) error {

func (d *Driver) getPid() int {
pidPath := d.ResolveStorePath(machineFileName)

f, err := os.Open(pidPath)
if err != nil {
log.Warnf("Error reading pid file: %v", err)
Expand Down