Skip to content

Commit

Permalink
kvm2 driver: add static ip
Browse files Browse the repository at this point in the history
  • Loading branch information
@prezha
prezha committed Mar 27, 2021
1 parent fcc6283 commit 06d4313
Show file tree
Hide file tree
Showing 12 changed files with 284 additions and 268 deletions.
23 changes: 4 additions & 19 deletions cmd/minikube/cmd/start_flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ const (
hypervUseExternalSwitch = "hyperv-use-external-switch"
hypervExternalAdapter = "hyperv-external-adapter"
kvmNetwork = "kvm-network"
kvmPrivateNetwork = "kvm-private-network"
kvmQemuURI = "kvm-qemu-uri"
kvmGPU = "kvm-gpu"
kvmHidden = "kvm-hidden"
Expand Down Expand Up @@ -162,7 +161,7 @@ func initMinikubeFlags() {
startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Defaults to false.")
startCmd.Flags().StringP(network, "", "", "network to run minikube with. Only available with the docker/podman drivers. If left empty, minikube will create a new network.")
startCmd.Flags().StringP(network, "", "", "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.")
startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
}
Expand Down Expand Up @@ -193,7 +192,6 @@ func initDriverFlags() {

// kvm2
startCmd.Flags().String(kvmNetwork, "default", "The KVM default network name. (kvm2 driver only)")
startCmd.Flags().String(kvmPrivateNetwork, "", "The KVM private network name. (kvm2 driver only) (default: 'mk-<cluster_name>')")
startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)")
startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube")
startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)")
Expand Down Expand Up @@ -313,8 +311,8 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
out.WarningT("With --network-plugin=cni, you will need to provide your own CNI. See --cni flag as a user-friendly alternative")
}

if !driver.IsKIC(drvName) && viper.GetString(network) != "" {
out.WarningT("--network flag is only valid with the docker/podman drivers, it will be ignored")
if !(driver.IsKIC(drvName) || driver.IsKVM(drvName)) && viper.GetString(network) != "" {
out.WarningT("--network flag is only valid with the docker/podman and KVM drivers, it will be ignored")
}

checkNumaCount(k8sVersion)
Expand Down Expand Up @@ -344,7 +342,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch),
HypervExternalAdapter: viper.GetString(hypervExternalAdapter),
KVMNetwork: viper.GetString(kvmNetwork),
KVMPrivateNetwork: viper.GetString(kvmPrivateNetwork),
KVMQemuURI: viper.GetString(kvmQemuURI),
KVMGPU: viper.GetBool(kvmGPU),
KVMHidden: viper.GetBool(kvmHidden),
Expand Down Expand Up @@ -383,10 +380,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
},
MultiNodeRequested: viper.GetInt(nodes) > 1,
}
// if KVMPrivateNetwork is not user-defined, defaults to "mk-<cluster_name>"
if cc.KVMPrivateNetwork == "" {
cc.KVMPrivateNetwork = fmt.Sprintf("mk-%s", cc.KubernetesConfig.ClusterName)
}
cc.VerifyComponents = interpretWaitFlag(*cmd)
if viper.GetBool(createMount) && driver.IsKIC(drvName) {
cc.ContainerVolumeMounts = []string{viper.GetString(mountString)}
Expand Down Expand Up @@ -561,15 +554,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
}

if cmd.Flags().Changed(kvmNetwork) {
if cc.KVMNetwork != viper.GetString(kvmNetwork) {
out.WarningT("You cannot change the KVM Default Network name for an exiting minikube cluster. Please first delete the cluster.")
}
}

if cmd.Flags().Changed(kvmPrivateNetwork) {
if cc.KVMPrivateNetwork != viper.GetString(kvmPrivateNetwork) {
out.WarningT("You cannot change the KVM Private Network name for an exiting minikube cluster. Please first delete the cluster.")
}
cc.KVMNetwork = viper.GetString(kvmNetwork)
}

if cmd.Flags().Changed(kvmQemuURI) {
Expand Down
51 changes: 12 additions & 39 deletions pkg/drivers/kvm/domain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@ package kvm

import (
"bytes"
"crypto/rand"
"fmt"
"net"
"text/template"

libvirt "github.com/libvirt/libvirt-go"
Expand Down Expand Up @@ -68,12 +66,10 @@ const domainTmpl = `
</disk>
<interface type='network'>
<source network='{{.Network}}'/>
<mac address='{{.MAC}}'/>
<model type='virtio'/>
</interface>
<interface type='network'>
<source network='{{.PrivateNetwork}}'/>
<mac address='{{.PrivateMAC}}'/>
<model type='virtio'/>
</interface>
<serial type='pty'>
Expand All @@ -92,25 +88,6 @@ const domainTmpl = `
</domain>
`

func randomMAC() (net.HardwareAddr, error) {
buf := make([]byte, 6)
_, err := rand.Read(buf)
if err != nil {
return nil, err
}
// We unset the first and second least significant bits (LSB) of the MAC
//
// The LSB of the first octet
// 0 for unicast
// 1 for multicast
//
// The second LSB of the first octet
// 0 for universally administered addresses
// 1 for locally administered addresses
buf[0] &= 0xfc
return buf, nil
}

func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) {
conn, err := getConnection(d.ConnectionURI)
if err != nil {
Expand Down Expand Up @@ -146,22 +123,6 @@ func closeDomain(dom *libvirt.Domain, conn *libvirt.Connect) error {
}

func (d *Driver) createDomain() (*libvirt.Domain, error) {
// create random MAC addresses first for our NICs
if d.MAC == "" {
mac, err := randomMAC()
if err != nil {
return nil, errors.Wrap(err, "generating mac address")
}
d.MAC = mac.String()
}

if d.PrivateMAC == "" {
mac, err := randomMAC()
if err != nil {
return nil, errors.Wrap(err, "generating mac address")
}
d.PrivateMAC = mac.String()
}
// create the XML for the domain using our domainTmpl template
tmpl := template.Must(template.New("domain").Parse(domainTmpl))
var domainXML bytes.Buffer
Expand All @@ -180,5 +141,17 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) {
return nil, errors.Wrapf(err, "error defining domain xml: %s", domainXML.String())
}

// save MAC address
dmac, err := macFromXML(conn, d.MachineName, d.Network)
if err != nil {
return nil, fmt.Errorf("failed saving MAC address: %w", err)
}
d.MAC = dmac
pmac, err := macFromXML(conn, d.MachineName, d.PrivateNetwork)
if err != nil {
return nil, fmt.Errorf("failed saving MAC address: %w", err)
}
d.PrivateMAC = pmac

return dom, nil
}
66 changes: 43 additions & 23 deletions pkg/drivers/kvm/kvm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
libvirt "github.com/libvirt/libvirt-go"
"github.com/pkg/errors"
pkgdrivers "k8s.io/minikube/pkg/drivers"
"k8s.io/minikube/pkg/util/retry"
)

// Driver is the machine driver for KVM
Expand Down Expand Up @@ -209,12 +210,14 @@ func (d *Driver) GetIP() (string, error) {
if s != state.Running {
return "", errors.New("host is not running")
}
ip, err := d.lookupIP()

conn, err := getConnection(d.ConnectionURI)
if err != nil {
return "", errors.Wrap(err, "getting IP")
return "", errors.Wrap(err, "getting libvirt connection")
}
defer conn.Close()

return ip, nil
return ipFromXML(conn, d.MachineName, d.PrivateNetwork)
}

// GetSSHHostname returns hostname for use with ssh
Expand Down Expand Up @@ -272,32 +275,43 @@ func (d *Driver) Start() (err error) {
}

log.Info("Waiting to get IP...")
for i := 0; i <= 40; i++ {
ip, err := d.GetIP()
if err := d.waitForStaticIP(conn); err != nil {
return errors.Wrap(err, "IP not available after waiting")
}

log.Info("Waiting for SSH to be available...")
if err := drivers.WaitForSSH(d); err != nil {
return errors.Wrap(err, "SSH not available after waiting")
}

return nil
}

// waitForStaticIP waits for IP address of domain that has been created & starting and then makes that IP static.
func (d *Driver) waitForStaticIP(conn *libvirt.Connect) error {
query := func() error {
sip, err := ipFromAPI(conn, d.MachineName, d.PrivateNetwork)
if err != nil {
return errors.Wrap(err, "getting ip during machine start")
return fmt.Errorf("failed getting IP during machine start, will retry: %w", err)
}
if ip == "" {
log.Debugf("Waiting for machine to come up %d/%d", i, 40)
time.Sleep(3 * time.Second)
continue
if sip == "" {
return fmt.Errorf("waiting for machine to come up")
}

if ip != "" {
log.Infof("Found IP for machine: %s", ip)
d.IPAddress = ip
break
}
}
log.Infof("Found IP for machine: %s", sip)
d.IPAddress = sip

if d.IPAddress == "" {
return errors.New("machine didn't return an IP after 120 seconds")
return nil
}
if err := retry.Local(query, 1*time.Minute); err != nil {
return fmt.Errorf("machine %s didn't return IP after 1 minute", d.MachineName)
}

log.Info("Waiting for SSH to be available...")
if err := drivers.WaitForSSH(d); err != nil {
d.IPAddress = ""
return errors.Wrap(err, "SSH not available after waiting")
log.Info("Reserving static IP address...")
if err := addStaticIP(conn, d.PrivateNetwork, d.MachineName, d.PrivateMAC, d.IPAddress); err != nil {
log.Warnf("Failed reserving static IP %s for host %s, will continue anyway: %v", d.IPAddress, d.MachineName, err)
} else {
log.Infof("Reserved static IP address: %s", d.IPAddress)
}

return nil
Expand Down Expand Up @@ -385,7 +399,6 @@ func ensureDirPermissions(store string) error {

// Stop a host gracefully
func (d *Driver) Stop() (err error) {
d.IPAddress = ""
s, err := d.GetState()
if err != nil {
return errors.Wrap(err, "getting state of VM")
Expand Down Expand Up @@ -458,6 +471,13 @@ func (d *Driver) Remove() error {
return errors.Wrap(err, "undefine domain")
}

log.Info("Removing static IP address...")
if err := delStaticIP(conn, d.PrivateNetwork, "", "", d.IPAddress); err != nil {
log.Warnf("failed removing static IP %s for host %s, will continue anyway: %v", d.IPAddress, d.MachineName, err)
} else {
log.Info("Removed static IP address")
}

return nil
}

Expand Down
Loading

0 comments on commit 06d4313

Please sign in to comment.