Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix stale hyperkit.pid making minikube start hang #3593

Merged
merged 5 commits into from
Jan 30, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix stale hyperkit.pid making minikube start hang
  • Loading branch information
balopat committed Jan 26, 2019
commit 9b97a85a8fd82f1c7e1d4b9a8cacc97a80d15ce8
74 changes: 60 additions & 14 deletions pkg/drivers/hyperkit/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,13 @@ import (
"github.com/docker/machine/libmachine/drivers"
"github.com/docker/machine/libmachine/log"
"github.com/docker/machine/libmachine/state"
nfsexports "github.com/johanneswuerbach/nfsexports"
hyperkit "github.com/moby/hyperkit/go"
"github.com/johanneswuerbach/nfsexports"
"github.com/moby/hyperkit/go"
"github.com/pkg/errors"
pkgdrivers "k8s.io/minikube/pkg/drivers"
"k8s.io/minikube/pkg/minikube/constants"
commonutil "k8s.io/minikube/pkg/util"
"io/ioutil"
)

const (
Expand Down Expand Up @@ -126,6 +127,7 @@ func (d *Driver) GetURL() (string, error) {
// GetState returns the state that the host is in (running, stopped, etc)
func (d *Driver) GetState() (state.State, error) {
pid := d.getPid()
log.Infof("Found hyperkit pid: %d", pid)
if pid == 0 {
return state.Stopped, nil
}
Expand Down Expand Up @@ -169,12 +171,14 @@ func (d *Driver) Restart() error {

// Start a host
func (d *Driver) Start() error {
h, err := hyperkit.New("", d.VpnKitSock, filepath.Join(d.StorePath, "machines", d.MachineName))
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
if err := d.checkForPidFile(); err != nil {
return err
}
h, err := hyperkit.New("", d.VpnKitSock, stateDir)
if err != nil {
return errors.Wrap(err, "new-ing Hyperkit")
}

// TODO: handle the rest of our settings.
h.Kernel = d.ResolveStorePath("bzimage")
h.Initrd = d.ResolveStorePath("initrd")
h.VMNet = true
Expand All @@ -183,13 +187,20 @@ func (d *Driver) Start() error {
h.CPUs = d.CPU
h.Memory = d.Memory
h.UUID = d.UUID

if vsockPorts, err := d.extractVSockPorts(); err != nil {
return err
} else if len(vsockPorts) >= 1 {
h.VSock = true
h.VSockPorts = vsockPorts
}
h.Disks = []hyperkit.DiskConfig{
{
Path: pkgdrivers.GetDiskPath(d.BaseDriver),
Size: d.DiskSize,
Driver: "virtio-blk",
},
}
//}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove comment


log.Infof("Using UUID %s", h.UUID)
mac, err := GetMACAddressFromUUID(h.UUID)
Expand All @@ -200,13 +211,7 @@ func (d *Driver) Start() error {
// Need to strip 0's
mac = trimMacAddress(mac)
log.Infof("Generated MAC %s", mac)
h.Disks = []hyperkit.DiskConfig{
{
Path: pkgdrivers.GetDiskPath(d.BaseDriver),
Size: d.DiskSize,
Driver: "virtio-blk",
},
}

log.Infof("Starting with cmdline: %s", d.Cmdline)
if err := h.Start(d.Cmdline); err != nil {
return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline)
Expand Down Expand Up @@ -240,6 +245,48 @@ func (d *Driver) Start() error {
return nil
}

func (d *Driver) checkForPidFile() error {
balopat marked this conversation as resolved.
Show resolved Hide resolved
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
pidFile := filepath.Join(stateDir, pidFileName)

if _, err := os.Stat(pidFile); err != nil {
balopat marked this conversation as resolved.
Show resolved Hide resolved
log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
return nil
}

log.Warnf("hyperkit pid file exists: %s", pidFile)
balopat marked this conversation as resolved.
Show resolved Hide resolved
// we have a pid file, hyperkit might be running!
// Make sure the pid written by hyperkit is the same as in the json
balopat marked this conversation as resolved.
Show resolved Hide resolved
content, err := ioutil.ReadFile(pidFile)
balopat marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return errors.Wrapf(err, "reading pidfile %s", pidFile)
}
pid, err := strconv.Atoi(string(content[:]))
if err != nil {
return errors.Wrapf(err, "parsing pidfile %s", pidFile)
}

p, _ := os.FindProcess(pid) //noop on linux
balopat marked this conversation as resolved.
Show resolved Hide resolved
// Sending a signal of 0 can be used to check the existence of a process.
if err := p.Signal(syscall.Signal(0)); err == nil {
return fmt.Errorf("something is not right...please stop all minikube instances, seemingly a hyperkit server is already running with pid %d", pid)
balopat marked this conversation as resolved.
Show resolved Hide resolved
}

machinePID := d.getPid()

if machinePID != pid {
return fmt.Errorf("something is not right...the PID reported in %s (%d) is different from machine PID (%d from %s)", pidFile, pid, machinePID, machineFileName)
balopat marked this conversation as resolved.
Show resolved Hide resolved
}

log.Warnf("No running process found with PID %d, removing %s...", machinePID, pidFile)
if err := os.Remove(pidFile); err != nil {
return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile))
}
log.Infof("Removed PID file.")
balopat marked this conversation as resolved.
Show resolved Hide resolved

return nil
}

// Stop a host gracefully
func (d *Driver) Stop() error {
d.cleanupNfsExports()
Expand Down Expand Up @@ -350,7 +397,6 @@ func (d *Driver) sendSignal(s os.Signal) error {

func (d *Driver) getPid() int {
pidPath := d.ResolveStorePath(machineFileName)

f, err := os.Open(pidPath)
if err != nil {
log.Warnf("Error reading pid file: %v", err)
Expand Down