add dedicated network for docker driver

cmd/minikube/cmd/delete.go

@@ -260,6 +260,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 		glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
 	}
 
+	errs = oci.DeleteKICNetworks()
+	if errs != nil {
+		glog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs)
+	}
+
 	if bin == oci.Podman {
 		// podman prune does not support --filter
 		return

cmd/minikube/cmd/mount.go

@@ -110,7 +110,7 @@ var mountCmd = &cobra.Command{
 		var ip net.IP
 		var err error
 		if mountIP == "" {
-			ip, err = cluster.HostIP(co.CP.Host)
+			ip, err = cluster.HostIP(co.CP.Host, co.Config.Name)
 			if err != nil {
 				exit.Error(reason.IfHostIP, "Error getting the host IP address to use from within the VM", err)
 			}

hack/preload-images/generate.go

@@ -39,6 +39,7 @@ import (
 
 func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error {
 	driver := kic.NewDriver(kic.Config{
+		ClusterName:       profile,
 		KubernetesVersion: kubernetesVersion,
 		ContainerRuntime:  containerRuntime,
 		OCIBinary:         oci.Docker,

pkg/drivers/kic/kic.go

@@ -37,6 +37,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -81,6 +83,17 @@ func (d *Driver) Create() error {
 		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
+	if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
+		out.WarningT("Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}", out.V{"error": err})
+	} else {
+		params.Network = d.NodeConfig.ClusterName
+		ip := gateway.To4()
+		// calculate the container IP based on guessing the machine index
+		ip[3] += byte(driver.IndexFromMachineName(d.NodeConfig.MachineName))
+		glog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName)
+		params.IP = ip.String()
+	}
+
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: oci.DefaultBindIPV4,
@@ -289,6 +302,10 @@ func (d *Driver) Remove() error {
 	if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" {
 		return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
 	}
+
+	if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
+		glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
+	}
 	return nil
 }
 

pkg/drivers/kic/oci/errors.go

@@ -39,12 +39,27 @@ var ErrWindowsContainers = &FailFastError{errors.New("docker container type is w
 // ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container
 var ErrCPUCountLimit = &FailFastError{errors.New("not enough CPUs is available for container")}
 
+// ErrIPinUse is thrown when the container been given an IP used by another container
+var ErrIPinUse = &FailFastError{errors.New("can't create with that IP, address already in use")}
+
 // ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore.
 var ErrExitedUnexpectedly = errors.New("container exited unexpectedly")
 
 // ErrDaemonInfo is thrown when docker/podman info is failing or not responding
 var ErrDaemonInfo = errors.New("daemon info not responding")
 
+// ErrNetworkSubnetTaken is thrown when a subnet is taken by another network
+var ErrNetworkSubnetTaken = errors.New("subnet is taken")
+
+// ErrNetworkNotFound is when given network was not found
+var ErrNetworkNotFound = errors.New("kic network not found")
+
+// ErrNetworkGatewayTaken is when given network gatway is taken
+var ErrNetworkGatewayTaken = errors.New("network gateway is taken")
+
+// ErrNetworkInUse is when trying to delete a network which is attached to another container
+var ErrNetworkInUse = errors.New("unable to delete a network that is attached to a running container")
+
 // LogContainerDebug will print relevant docker/podman infos after a container fails
 func LogContainerDebug(ociBin string, name string) string {
 	rr, err := containerInspect(ociBin, name)

pkg/drivers/kic/oci/network.go

@@ -31,17 +31,26 @@ import (
 
 // RoutableHostIPFromInside returns the ip/dns of the host that container lives on
 // is routable from inside the container
-func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) {
+func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) {
 	if ociBin == Docker {
 		if runtime.GOOS == "linux" {
-			return dockerGatewayIP(containerName)
+			_, gateway, err := dockerNetworkInspect(clusterName)
+			if err != nil {
+				if errors.Is(err, ErrNetworkNotFound) {
+					glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube <v1.14, will try to get the IP using container gatway", containerName)
+
+					return containerGatewayIP(Docker, containerName)
+				}
+				return gateway, errors.Wrap(err, "network inspect")
+			}
+			return gateway, nil
 		}
 		// for windows and mac, the gateway ip is not routable so we use dns trick.
 		return digDNS(ociBin, containerName, "host.docker.internal")
 	}
-
+	// podman
 	if runtime.GOOS == "linux" {
-		return containerGatewayIP(ociBin, containerName)
+		return containerGatewayIP(Podman, containerName)
 	}
 
 	return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
@@ -59,56 +68,8 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 	return ip, nil
 }
 
-// profileInContainers checks whether the profile is within the containers list
-func profileInContainers(profile string, containers []string) bool {
-	for _, container := range containers {
-		if container == profile {
-			return true
-		}
-	}
-	return false
-}
-
-// dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine
-// gets the ip from user's host docker
-func dockerGatewayIP(profile string) (net.IP, error) {
-	var bridgeID string
-	rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}"))
-	if err != nil {
-		return nil, errors.Wrapf(err, "get network bridge")
-	}
-	networksOutput := strings.TrimSpace(rr.Stdout.String())
-	networksSlice := strings.Fields(networksOutput)
-	// Look for the minikube container within each docker network
-	for _, net := range networksSlice {
-		// get all containers in the network
-		rs, err := runCmd(exec.Command(Docker, "network", "inspect", net, "-f", "{{range $k, $v := .Containers}}{{$v.Name}} {{end}}"))
-		if err != nil {
-			return nil, errors.Wrapf(err, "get containers in network")
-		}
-		containersSlice := strings.Fields(rs.Stdout.String())
-		if profileInContainers(profile, containersSlice) {
-			bridgeID = net
-			break
-		}
-	}
-
-	if bridgeID == "" {
-		return nil, errors.Errorf("unable to determine bridge network id from %q", networksOutput)
-	}
-	rr, err = runCmd(exec.Command(Docker, "network", "inspect",
-		"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID))
-	if err != nil {
-		return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID)
-	}
-
-	ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
-	glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
-	return ip, nil
-}
-
 // containerGatewayIP gets the default gateway ip for the container
-func containerGatewayIP(ociBin, containerName string) (net.IP, error) {
+func containerGatewayIP(ociBin string, containerName string) (net.IP, error) {
 	rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
 	if err != nil {
 		return nil, errors.Wrapf(err, "inspect gateway")

pkg/drivers/kic/oci/network_create.go

@@ -0,0 +1,191 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+// firstSubnetAddr subnet to be used on first kic cluster
+// it is one octet more than the one used by KVM to avoid possible conflict
+const firstSubnetAddr = "192.168.49.0"
+
+// big enough for a cluster of 254 nodes
+const defaultSubnetMask = 24
+
+// CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
+func CreateNetwork(ociBin string, name string) (net.IP, error) {
+	if ociBin != Docker {
+		return nil, fmt.Errorf("%s network not implemented yet", ociBin)
+	}
+	return createDockerNetwork(name)
+}
+
+func createDockerNetwork(clusterName string) (net.IP, error) {
+	// check if the network already exists
+	subnet, gateway, err := dockerNetworkInspect(clusterName)
+	if err == nil {
+		glog.Infof("Found existing network with subnet %s and gateway %s.", subnet, gateway)
+		return gateway, nil
+	}
+
+	attempts := 0
+	subnetAddr := firstSubnetAddr
+	// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
+	// will be like 192.168.49.0/24 ,...,192.168.239.0/24
+	for attempts < 20 {
+		gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
+		if err == nil {
+			return gateway, nil
+		}
+
+		// don't retry if error is not adddress is taken
+		if !(errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken)) {
+			glog.Errorf("error while trying to create network %v", err)
+			return nil, errors.Wrap(err, "un-retryable")
+		}
+		attempts++
+		// Find an open subnet by incrementing the 3rd octet by 10 for each try
+		// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
+		// at most it will add up to 169 which is still less than max allowed 255
+		// this is large enough to try more and not too small to not try enough
+		// can be tuned in the next iterations
+		newSubnet := net.ParseIP(subnetAddr).To4()
+		newSubnet[2] += byte(9 + attempts)
+		subnetAddr = newSubnet.String()
+	}
+	return gateway, fmt.Errorf("failed to create network after 20 attempts")
+}
+
+func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net.IP, error) {
+	gateway := net.ParseIP(subnetAddr)
+	gateway.To4()[3]++ // first ip for gateway
+	glog.Infof("attempt to create network %s/%d with subnet: %s and gateway %s...", subnetAddr, subnetMask, name, gateway)
+	// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
+	rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetMask)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
+	if err != nil {
+		// Pool overlaps with other one on this address space
+		if strings.Contains(rr.Output(), "Pool overlaps") {
+			return nil, ErrNetworkSubnetTaken
+		}
+		if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") {
+			return nil, ErrNetworkGatewayTaken
+		}
+		return nil, errors.Wrapf(err, "create network %s", fmt.Sprintf("%s %s/%d", name, subnetAddr, subnetMask))
+	}
+	return gateway, nil
+}
+
+// returns subnet and gate if exists
+func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
+	rr, err := runCmd(exec.Command(Docker, "network", "inspect", name, "--format", "{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}"))
+	if err != nil {
+		if strings.Contains(rr.Output(), "No such network") {
+			return nil, nil, ErrNetworkNotFound
+		}
+		return nil, nil, err
+	}
+	// results looks like 172.17.0.0/16,172.17.0.1
+	ips := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",")
+	if len(ips) == 0 {
+		return nil, nil, fmt.Errorf("empty IP list parsed from: %q", rr.Output())
+	}
+
+	_, subnet, err := net.ParseCIDR(ips[0])
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "parse subnet for %s", name)
+	}
+	var gateway net.IP
+	if len(ips) > 0 {
+		gateway = net.ParseIP(ips[1])
+	}
+	return subnet, gateway, nil
+}
+
+// RemoveNetwork removes a network
+func RemoveNetwork(name string) error {
+	if !networkExists(name) {
+		return nil
+	}
+	rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
+	if err != nil {
+		if strings.Contains(rr.Output(), "No such network") {
+			return ErrNetworkNotFound
+		}
+		// Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints
+		if strings.Contains(rr.Output(), "has active endpoints") {
+			return ErrNetworkInUse
+		}
+	}
+
+	return err
+}
+
+func networkExists(name string) bool {
+	_, _, err := dockerNetworkInspect(name)
+	if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error
+		glog.Warningf("Error inspecting docker network %s: %v", name, err)
+	}
+	return err == nil
+}
+
+// networkNamesByLabel returns all network names created by a label
+func networkNamesByLabel(ociBin string, label string) ([]string, error) {
+	if ociBin != Docker {
+		return nil, fmt.Errorf("%s not supported", ociBin)
+	}
+
+	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}'
+	rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
+	if err != nil {
+		return nil, err
+	}
+	var lines []string
+	scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
+	for scanner.Scan() {
+		lines = append(lines, strings.TrimSpace(scanner.Text()))
+	}
+
+	return lines, nil
+}
+
+// DeleteKICNetworks deletes all networks created by kic
+func DeleteKICNetworks() []error {
+	var errs []error
+	ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true")
+	if err != nil {
+		return []error{errors.Wrap(err, "list all volume")}
+	}
+	for _, n := range ns {
+		err := RemoveNetwork(n)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}

pkg/drivers/kic/oci/oci.go

@@ -169,6 +169,11 @@ func CreateContainerNode(p CreateParams) error {
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
+		// to provide a static IP for docker
+		if p.Network != "" && p.IP != "" {
+			runArgs = append(runArgs, "--network", p.Network)
+			runArgs = append(runArgs, "--ip", p.IP)
+		}
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
@@ -285,6 +290,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
 		if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available
 			return ErrCPUCountLimit
 		}
+		// example: docker: Error response from daemon: Address already in use.
+		if strings.Contains(rr.Output(), "Address already in use") {
+			return ErrIPinUse
+		}
 		return err
 	}