From ae9f4b20c7fcb7a89fc65e6eab0fb716f4d1a4ed Mon Sep 17 00:00:00 2001 From: Balint Pato Date: Thu, 18 Oct 2018 11:01:47 -0700 Subject: [PATCH] minikube tunnel (#3015) This commit introduces a new command, `minikube tunnel`, a LoadBalancer emulator functionality, that must be run with root permissions. This command: * Establishes networking routes from the host into the VM for all IP ranges used by Kubernetes. * Enables a cluster controller that allocates IPs to services external `LoadBalancer` IPs. * Cleans up routes and IPs when stopped (Ctrl+C), when `minikube` stops, and when `minikube tunnel` is ran with the `--cleanup` flag --- cmd/minikube/cmd/config/util.go | 2 +- cmd/minikube/cmd/env.go | 2 +- cmd/minikube/cmd/env_test.go | 14 +- cmd/minikube/cmd/ssh.go | 3 +- cmd/minikube/cmd/start.go | 22 +- cmd/minikube/cmd/status.go | 2 +- cmd/minikube/cmd/tunnel.go | 83 ++++ cmd/minikube/cmd/update-context.go | 7 +- docs/networking.md | 50 ++ docs/tunnel.md | 144 ++++++ .../unix_white_box_integration_tests.sh | 20 + pkg/minikube/cluster/cluster.go | 31 +- pkg/minikube/cluster/cluster_test.go | 2 +- pkg/minikube/config/config.go | 36 ++ pkg/minikube/constants/constants.go | 4 + pkg/minikube/service/service.go | 37 +- pkg/minikube/service/service_test.go | 74 +-- pkg/minikube/tests/api_mock.go | 49 +- pkg/minikube/tests/driver_mock.go | 11 +- pkg/minikube/tests/fake_store.go | 72 +++ pkg/minikube/tunnel/cluster_inspector.go | 101 ++++ pkg/minikube/tunnel/cluster_inspector_test.go | 143 ++++++ pkg/minikube/tunnel/loadbalancer_patcher.go | 162 ++++++ .../tunnel/loadbalancer_patcher_test.go | 347 +++++++++++++ pkg/minikube/tunnel/process.go | 55 ++ pkg/minikube/tunnel/registry.go | 195 ++++++++ pkg/minikube/tunnel/registry_test.go | 268 ++++++++++ pkg/minikube/tunnel/reporter.go | 92 ++++ pkg/minikube/tunnel/reporter_test.go | 131 +++++ pkg/minikube/tunnel/route.go | 112 +++++ pkg/minikube/tunnel/route_darwin.go | 168 +++++++ pkg/minikube/tunnel/route_darwin_test.go | 157 ++++++ pkg/minikube/tunnel/route_linux.go | 136 +++++ pkg/minikube/tunnel/route_linux_test.go | 135 +++++ pkg/minikube/tunnel/route_test.go | 139 ++++++ pkg/minikube/tunnel/route_windows.go | 142 ++++++ pkg/minikube/tunnel/route_windows_test.go | 173 +++++++ pkg/minikube/tunnel/test_doubles.go | 91 ++++ pkg/minikube/tunnel/tunnel.go | 177 +++++++ pkg/minikube/tunnel/tunnel_manager.go | 144 ++++++ pkg/minikube/tunnel/tunnel_manager_test.go | 282 +++++++++++ pkg/minikube/tunnel/tunnel_test.go | 470 ++++++++++++++++++ pkg/minikube/tunnel/types.go | 102 ++++ test.sh | 2 +- test/integration/functional_test.go | 1 + test/integration/testdata/testsvc.yaml | 31 ++ test/integration/tunnel_test.go | 109 ++++ 47 files changed, 4584 insertions(+), 146 deletions(-) create mode 100644 cmd/minikube/cmd/tunnel.go create mode 100644 docs/tunnel.md create mode 100755 hack/jenkins/unix_white_box_integration_tests.sh create mode 100644 pkg/minikube/tests/fake_store.go create mode 100644 pkg/minikube/tunnel/cluster_inspector.go create mode 100644 pkg/minikube/tunnel/cluster_inspector_test.go create mode 100644 pkg/minikube/tunnel/loadbalancer_patcher.go create mode 100644 pkg/minikube/tunnel/loadbalancer_patcher_test.go create mode 100644 pkg/minikube/tunnel/process.go create mode 100644 pkg/minikube/tunnel/registry.go create mode 100644 pkg/minikube/tunnel/registry_test.go create mode 100644 pkg/minikube/tunnel/reporter.go create mode 100644 pkg/minikube/tunnel/reporter_test.go create mode 100644 pkg/minikube/tunnel/route.go create mode 100644 pkg/minikube/tunnel/route_darwin.go create mode 100644 pkg/minikube/tunnel/route_darwin_test.go create mode 100644 pkg/minikube/tunnel/route_linux.go create mode 100644 pkg/minikube/tunnel/route_linux_test.go create mode 100644 pkg/minikube/tunnel/route_test.go create mode 100644 pkg/minikube/tunnel/route_windows.go create mode 100644 pkg/minikube/tunnel/route_windows_test.go create mode 100644 pkg/minikube/tunnel/test_doubles.go create mode 100644 pkg/minikube/tunnel/tunnel.go create mode 100644 pkg/minikube/tunnel/tunnel_manager.go create mode 100644 pkg/minikube/tunnel/tunnel_manager_test.go create mode 100644 pkg/minikube/tunnel/tunnel_test.go create mode 100644 pkg/minikube/tunnel/types.go create mode 100644 test/integration/testdata/testsvc.yaml create mode 100644 test/integration/tunnel_test.go diff --git a/cmd/minikube/cmd/config/util.go b/cmd/minikube/cmd/config/util.go index 44ca8ea2fc10..06e93c698d27 100644 --- a/cmd/minikube/cmd/config/util.go +++ b/cmd/minikube/cmd/config/util.go @@ -114,7 +114,7 @@ func EnableOrDisableAddon(name string, val string) error { if err != nil { return err } - host, err := cluster.CheckIfApiExistsAndLoad(api) + host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName()) if err != nil { return errors.Wrap(err, "getting host") } diff --git a/cmd/minikube/cmd/env.go b/cmd/minikube/cmd/env.go index d5a0cf4ad8cc..82cabdb8b2e1 100644 --- a/cmd/minikube/cmd/env.go +++ b/cmd/minikube/cmd/env.go @@ -306,7 +306,7 @@ var dockerEnvCmd = &cobra.Command{ os.Exit(1) } defer api.Close() - host, err := cluster.CheckIfApiExistsAndLoad(api) + host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName()) if err != nil { fmt.Fprintf(os.Stderr, "Error getting host: %s\n", err) os.Exit(1) diff --git a/cmd/minikube/cmd/env_test.go b/cmd/minikube/cmd/env_test.go index 338183ae4859..ffc01d75e073 100644 --- a/cmd/minikube/cmd/env_test.go +++ b/cmd/minikube/cmd/env_test.go @@ -45,10 +45,12 @@ func (f FakeNoProxyGetter) GetNoProxyVar() (string, string) { } var defaultAPI = &tests.MockAPI{ - Hosts: map[string]*host.Host{ - config.GetMachineName(): { - Name: config.GetMachineName(), - Driver: &tests.MockDriver{}, + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + config.GetMachineName(): { + Name: config.GetMachineName(), + Driver: &tests.MockDriver{}, + }, }, }, } @@ -81,7 +83,9 @@ func TestShellCfgSet(t *testing.T) { { description: "no host specified", api: &tests.MockAPI{ - Hosts: make(map[string]*host.Host), + FakeStore: tests.FakeStore{ + Hosts: make(map[string]*host.Host), + }, }, shell: "bash", expectedShellCfg: nil, diff --git a/cmd/minikube/cmd/ssh.go b/cmd/minikube/cmd/ssh.go index 8a26f537f89e..c664e0ae0dc8 100644 --- a/cmd/minikube/cmd/ssh.go +++ b/cmd/minikube/cmd/ssh.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/minikube/pkg/minikube/cluster" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/machine" ) @@ -39,7 +40,7 @@ var sshCmd = &cobra.Command{ os.Exit(1) } defer api.Close() - host, err := cluster.CheckIfApiExistsAndLoad(api) + host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName()) if err != nil { fmt.Fprintf(os.Stderr, "Error getting host: %s\n", err) os.Exit(1) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 3b9d136e3316..ed780bfe9381 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -187,7 +187,7 @@ func runStart(cmd *cobra.Command, args []string) { selectedKubernetesVersion = constants.DefaultKubernetesVersion } // Load profile cluster config from file - cc, err := loadConfigFromFile(viper.GetString(cfg.MachineProfile)) + cc, err := cfg.Load() if err != nil && !os.IsNotExist(err) { glog.Errorln("Error loading profile config: ", err) } @@ -463,23 +463,3 @@ func saveConfigToFile(data []byte, file string) error { } return nil } - -func loadConfigFromFile(profile string) (cfg.Config, error) { - var cc cfg.Config - - profileConfigFile := constants.GetProfileFile(profile) - - if _, err := os.Stat(profileConfigFile); os.IsNotExist(err) { - return cc, err - } - - data, err := ioutil.ReadFile(profileConfigFile) - if err != nil { - return cc, err - } - - if err := json.Unmarshal(data, &cc); err != nil { - return cc, err - } - return cc, nil -} diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 6c74d7031e4a..856ac81ef1d5 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -88,7 +88,7 @@ var statusCmd = &cobra.Command{ returnCode |= clusterNotRunningStatusFlag } - ip, err := cluster.GetHostDriverIP(api) + ip, err := cluster.GetHostDriverIP(api, config.GetMachineName()) if err != nil { glog.Errorln("Error host driver ip status:", err) cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode) diff --git a/cmd/minikube/cmd/tunnel.go b/cmd/minikube/cmd/tunnel.go new file mode 100644 index 000000000000..f49fa7cac988 --- /dev/null +++ b/cmd/minikube/cmd/tunnel.go @@ -0,0 +1,83 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "context" + "os" + "os/signal" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/service" + "k8s.io/minikube/pkg/minikube/tunnel" +) + +var cleanup bool + +// tunnelCmd represents the tunnel command +var tunnelCmd = &cobra.Command{ + Use: "tunnel", + Short: "tunnel makes services of type LoadBalancer accessible on localhost", + Long: `tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + RootCmd.PersistentPreRun(cmd, args) + }, + Run: func(cmd *cobra.Command, args []string) { + manager := tunnel.NewManager() + + if cleanup { + glog.Info("Checking for tunnels to cleanup...") + if err := manager.CleanupNotRunningTunnels(); err != nil { + glog.Errorf("error cleaning up: %s", err) + } + return + } + + glog.Infof("Creating docker machine client...") + api, err := machine.NewAPIClient() + if err != nil { + glog.Fatalf("error creating dockermachine client: %s", err) + } + glog.Infof("Creating k8s client...") + clientset, err := service.K8s.GetClientset() + if err != nil { + glog.Fatalf("error creating K8S clientset: %s", err) + } + + ctrlC := make(chan os.Signal, 1) + signal.Notify(ctrlC, os.Interrupt) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-ctrlC + cancel() + }() + + done, err := manager.StartTunnel(ctx, config.GetMachineName(), api, config.DefaultLoader, clientset.CoreV1()) + if err != nil { + glog.Fatalf("error starting tunnel: %s", err) + } + <-done + }, +} + +func init() { + tunnelCmd.Flags().BoolVarP(&cleanup, "cleanup", "c", false, "call with cleanup=true to remove old tunnels") + RootCmd.AddCommand(tunnelCmd) +} diff --git a/cmd/minikube/cmd/update-context.go b/cmd/minikube/cmd/update-context.go index 3f1976dcf6cb..3eddba38cbd6 100644 --- a/cmd/minikube/cmd/update-context.go +++ b/cmd/minikube/cmd/update-context.go @@ -43,17 +43,18 @@ var updateContextCmd = &cobra.Command{ os.Exit(1) } defer api.Close() - ip, err := cluster.GetHostDriverIP(api) + machineName := config.GetMachineName() + ip, err := cluster.GetHostDriverIP(api, machineName) if err != nil { glog.Errorln("Error host driver ip status:", err) cmdUtil.MaybeReportErrorAndExit(err) } - kstatus, err := kcfg.UpdateKubeconfigIP(ip, constants.KubeconfigPath, config.GetMachineName()) + ok, err := kcfg.UpdateKubeconfigIP(ip, constants.KubeconfigPath, machineName) if err != nil { glog.Errorln("Error kubeconfig status:", err) cmdUtil.MaybeReportErrorAndExit(err) } - if kstatus { + if ok { fmt.Println("Reconfigured kubeconfig IP, now pointing at " + ip.String()) } else { fmt.Println("Kubeconfig IP correctly configured, pointing at " + ip.String()) diff --git a/docs/networking.md b/docs/networking.md index c6f0f9c805cb..5efff8bf5fc8 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -10,3 +10,53 @@ To determine the NodePort for your service, you can use a `kubectl` command like We also have a shortcut for fetching the minikube IP and a service's `NodePort`: `minikube service --url $SERVICE` + +### LoadBalancer emulation (`minikube tunnel`) + +Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. + +````shell +minikube tunnel +```` + +Will output: + +``` +out/minikube tunnel +Password: ***** +Status: + machine: minikube + pid: 59088 + route: 10.96.0.0/12 -> 192.168.99.101 + minikube: Running + services: [] + errors: + minikube: no errors + router: no errors + loadbalancer emulator: no errors + + +```` + +Tunnel might ask you for password for creating and deleting network routes. + +# Cleaning up orphaned routes + +If the `minikube tunnel` shuts down in an unclean way, it might leave a network route around. +This case the ~/.minikube/tunnels.json file will contain an entry for that tunnel. +To cleanup orphaned routes, run: +```` +minikube tunnel --cleanup +```` + +# (Advanced) Running tunnel as root to avoid entering password multiple times + +`minikube tunnel` runs as a separate daemon, creates a network route on the host to the service CIDR of the cluster using the cluster's IP address as a gateway. +Adding a route requires root privileges for the user, and thus there are differences in how to run `minikube tunnel` depending on the OS. + +Recommended way to use on Linux with KVM2 driver and MacOSX with Hyperkit driver: + +`sudo -E minikube tunnel` + +Using VirtualBox on Windows, Mac and Linux _both_ `minikube start` and `minikube tunnel` needs to be started from the same Administrator user session otherwise [VBoxManage can't recognize the created VM](https://forums.virtualbox.org/viewtopic.php?f=6&t=81551). + diff --git a/docs/tunnel.md b/docs/tunnel.md new file mode 100644 index 000000000000..87e5fd799e31 --- /dev/null +++ b/docs/tunnel.md @@ -0,0 +1,144 @@ +# Minikube Tunnel Design Doc + +## Background + +Minikube today only exposes a single IP address for all cluster and VM communication. +This effectively requires users to connect to any running Pods, Services or LoadBalancers over ClusterIPs, which can require modifications to workflows when compared to developing against a production cluster. + +A main goal of Minikube is to minimize the differences required in code and configuration between development and production, so this is not ideal. +If all cluster IP addresses and Load Balancers were made available on the minikube host machine, these modifications would not be necessary and users would get the "magic" experience of developing from inside a cluster. + +Tools like telepresence.io, sshuttle, and the OpenVPN chart provide similar capabilities already. + +Also, Steve Sloka has provided a very detailed guide on how to setup a similar configuration [manually](https://stevesloka.com/2017/06/12/access-minikube-service-from-linux-host/). + +Elson Rodriguez has provided a similar guide, including a Minikube [external LB controller](https://github.com/elsonrodriguez/minikube-lb-patch). + +## Example usage + +```shell +$ minikube tunnel +Starting minikube tunnel process. Press Ctrl+C to exit. +All cluster IPs and load balancers are now available from your host machine. +``` + +## Overview + +We will introduce a new command, `minikube tunnel`, that must be run with root permissions. +This command will: + +* Establish networking routes from the host into the VM for all IP ranges used by Kubernetes. +* Enable a cluster controller that allocates IPs to services external `LoadBalancer` IPs. +* Clean up routes and IPs when stopped, or when `minikube` stops. + +Additionally, we will introduce a Minikube LoadBalancer controller that manages a CIDR of IPs and assigns them to services of type `LoadBalancer`. +These IPs will also be made available on the host machine. + +## Network Routes + +Minikube drivers usually establish "host-only" IP addresses (192.168.1.1, for example) that route into the running VM +from the host. + +The new `minikube tunnel` command will create a static routing table entry that maps the CIDRs used by Pods, Services and LoadBalancers to the host-only IP, obtainable via the `minikube ip` command. + +The commands below detail adding routes for the entire `/8` block, we should probably add individual entries for each CIDR we manage instead. + +### Linux + +Route entries for the entire 10.* block can be added via: + +```shell +sudo ip route add 10.0.0.0/8 via $(minikube ip) +``` + +and deleted via: + +```shell +sudo ip route delete 10.0.0.0/8 +``` + +The routing table can be queried with `netstat -nr -f inet` + +### OSX + +Route entries can be added via: + +```shell +sudo route -n add 10.0.0.0/8 $(minikube ip) +``` + +and deleted via: + +```shell +sudo route -n delete 10.0.0.0/8 + +``` + +The routing table can be queried with `netstat -nr -f inet` + +### Windows + +Route entries can be added via: + +```shell +route ADD 10.0.0.0 MASK 255.0.0.0 +``` + +and deleted via: + +```shell +route DELETE 10.0.0.0 +``` + +The routing table can be queried with `route print -4` + +### Handling unclean shutdowns + +Unclean shutdowns of the tunnel process can result in partially executed cleanup process, leaving network routes in the routing table. +We will keep track of the routes created by each tunnel in a centralized location in the main minikube config directory. +This list serves as a registry for tunnels containing information about +- machine profile +- process ID +- and the route that was created + +The cleanup command cleans the routes from both the routing table and the registry for tunnels that are not running: + +``` +minikube tunnel --cleanup +``` + +Updating the tunnel registry and the routing table is an atomic transaction: + +- create route in the routing table + create registry entry if both are successful, otherwise rollback +- delete route in the routing table + remove registry entry if both are successful, otherwise rollback + +*Note*: because we don't support currently real multi cluster setup (due to overlapping CIDRs), the handling of running/not-running processes is not strictly required however it is forward looking. + +### Handling routing table conflicts + +A routing table conflict happens when a destination CIDR of the route required by the tunnel overlaps with an existing route. +Minikube tunnel will warn the user if this happens and should not create the rule. +There should not be any automated removal of conflicting routes. + +*Note*: If the user removes the minikube config directory, this might leave conflicting rules in the network routing table that will have to be cleaned up manually. + + +## Load Balancer Controller + +In addition to making IPs routable, minikube tunnel will assign an external IP (the ClusterIP) to all services of type `LoadBalancer`. + +The logic of this controller will be, roughly: + +```python +for service in services: + if service.type == "LoadBalancer" and len(service.ingress) == 0: + add_ip_to_service(ClusterIP, service) +sleep +``` + +Note that the Minikube ClusterIP can change over time (during system reboots) and this loop should also handle reconcilliation of those changes. + +## Handling multiple clusters + +Multiple clusters are currently not supported due to our inability to specify ServiceCIDR. +This causes conflicting routes having the same destination CIDR. diff --git a/hack/jenkins/unix_white_box_integration_tests.sh b/hack/jenkins/unix_white_box_integration_tests.sh new file mode 100755 index 000000000000..b514e8a34e50 --- /dev/null +++ b/hack/jenkins/unix_white_box_integration_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +for pkg in $(go list ./pkg/minikube/...); do + #we are assuming that the runner has NOPASSWD sudoer + go test -v $pkg -tags integration +done diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index f37ccd2f91da..6091954a8c23 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -69,10 +69,10 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error) glog.Infoln("Machine does not exist... provisioning new machine") glog.Infof("Provisioning machine with config: %+v", config) return createHost(api, config) - } else { - glog.Infoln("Skipping create...Using existing machine configuration") } + glog.Infoln("Skipping create...Using existing machine configuration") + h, err := api.Load(cfg.GetMachineName()) if err != nil { return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.") @@ -152,8 +152,8 @@ func GetHostStatus(api libmachine.API) (string, error) { } // GetHostDriverIP gets the ip address of the current minikube cluster -func GetHostDriverIP(api libmachine.API) (net.IP, error) { - host, err := CheckIfApiExistsAndLoad(api) +func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) { + host, err := CheckIfHostExistsAndLoad(api, machineName) if err != nil { return nil, err } @@ -164,7 +164,7 @@ func GetHostDriverIP(api libmachine.API) (net.IP, error) { } ip := net.ParseIP(ipStr) if ip == nil { - return nil, errors.Wrap(err, "Error parsing IP") + return nil, fmt.Errorf("error parsing IP: %s", ipStr) } return ip, nil } @@ -251,7 +251,7 @@ func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error // GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm func GetHostDockerEnv(api libmachine.API) (map[string]string, error) { - host, err := CheckIfApiExistsAndLoad(api) + host, err := CheckIfHostExistsAndLoad(api, cfg.GetMachineName()) if err != nil { return nil, errors.Wrap(err, "Error checking that api exists and loading it") } @@ -273,7 +273,7 @@ func GetHostDockerEnv(api libmachine.API) (map[string]string, error) { // MountHost runs the mount command from the 9p client on the VM to the 9p server on the host func MountHost(api libmachine.API, ip net.IP, path, port, mountVersion string, uid, gid, msize int) error { - host, err := CheckIfApiExistsAndLoad(api) + host, err := CheckIfHostExistsAndLoad(api, cfg.GetMachineName()) if err != nil { return errors.Wrap(err, "Error checking that api exists and loading it") } @@ -344,24 +344,25 @@ func getIPForInterface(name string) (net.IP, error) { return nil, errors.Errorf("Error finding IPV4 address for %s", name) } -func CheckIfApiExistsAndLoad(api libmachine.API) (*host.Host, error) { - exists, err := api.Exists(cfg.GetMachineName()) +func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) { + exists, err := api.Exists(machineName) if err != nil { - return nil, errors.Wrapf(err, "Error checking that api exists for: %s", cfg.GetMachineName()) + return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName) } if !exists { - return nil, errors.Errorf("Machine does not exist for api.Exists(%s)", cfg.GetMachineName()) + return nil, errors.Errorf("Machine does not exist for api.Exists(%s)", machineName) } - host, err := api.Load(cfg.GetMachineName()) + host, err := api.Load(machineName) if err != nil { - return nil, errors.Wrapf(err, "Error loading api for: %s", cfg.GetMachineName()) + return nil, errors.Wrapf(err, "Error loading store for: %s", machineName) } return host, nil } func CreateSSHShell(api libmachine.API, args []string) error { - host, err := CheckIfApiExistsAndLoad(api) + machineName := cfg.GetMachineName() + host, err := CheckIfHostExistsAndLoad(api, machineName) if err != nil { return errors.Wrap(err, "Error checking if api exist and loading it") } @@ -372,7 +373,7 @@ func CreateSSHShell(api libmachine.API, args []string) error { } if currentState != state.Running { - return errors.Errorf("Error: Cannot run ssh command: Host %q is not running", cfg.GetMachineName()) + return errors.Errorf("Error: Cannot run ssh command: Host %q is not running", machineName) } client, err := host.CreateSSHClient() diff --git a/pkg/minikube/cluster/cluster_test.go b/pkg/minikube/cluster/cluster_test.go index 26e8b2ce9aba..64c35ee01857 100644 --- a/pkg/minikube/cluster/cluster_test.go +++ b/pkg/minikube/cluster/cluster_test.go @@ -271,7 +271,7 @@ func TestDeleteHostMultipleErrors(t *testing.T) { t.Fatal("Expected error deleting host, didn't get one.") } - expectedErrors := []string{"Error removing " + config.GetMachineName(), "Error deleting machine"} + expectedErrors := []string{"error removing " + config.GetMachineName(), "error deleting machine"} for _, expectedError := range expectedErrors { if !strings.Contains(err.Error(), expectedError) { t.Fatalf("Error %s expected to contain: %s.", err, expectedError) diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 8a53d54b2b28..ad335fae5549 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -23,6 +23,8 @@ import ( "io" "os" + "io/ioutil" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/constants" ) @@ -88,3 +90,37 @@ func GetMachineName() string { } return viper.GetString(MachineProfile) } + +// Load loads the kubernetes and machine config for the current machine +func Load() (Config, error) { + return DefaultLoader.LoadConfigFromFile(GetMachineName()) +} + +// Loader loads the kubernetes and machine config based on the machine profile name +type Loader interface { + LoadConfigFromFile(profile string) (Config, error) +} + +type simpleConfigLoader struct{} + +var DefaultLoader Loader = &simpleConfigLoader{} + +func (c *simpleConfigLoader) LoadConfigFromFile(profile string) (Config, error) { + var cc Config + + path := constants.GetProfileFile(profile) + + if _, err := os.Stat(path); os.IsNotExist(err) { + return cc, err + } + + data, err := ioutil.ReadFile(path) + if err != nil { + return cc, err + } + + if err := json.Unmarshal(data, &cc); err != nil { + return cc, err + } + return cc, nil +} diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index f367e694c1e5..4b01fbaa1ba8 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -87,6 +87,10 @@ const DefaultStorageClassProvisioner = "standard" // Used to modify the cache field in the config file const Cache = "cache" +func TunnelRegistryPath() string { + return filepath.Join(GetMinipath(), "tunnels.json") +} + // MakeMiniPath is a utility to calculate a relative path to our directory. func MakeMiniPath(fileName ...string) string { args := []string{GetMinipath()} diff --git a/pkg/minikube/service/service.go b/pkg/minikube/service/service.go index 6b4cfeeb5fac..40424255e92c 100644 --- a/pkg/minikube/service/service.go +++ b/pkg/minikube/service/service.go @@ -80,6 +80,7 @@ func (*K8sClientGetter) GetClientset() (*kubernetes.Clientset, error) { if err != nil { return nil, fmt.Errorf("Error creating kubeConfig: %s", err) } + clientConfig.Timeout = 1 * time.Second client, err := kubernetes.NewForConfig(clientConfig) if err != nil { return nil, errors.Wrap(err, "Error creating new client from kubeConfig.ClientConfig()") @@ -88,18 +89,18 @@ func (*K8sClientGetter) GetClientset() (*kubernetes.Clientset, error) { return client, nil } -type ServiceURL struct { +type URL struct { Namespace string Name string URLs []string } -type ServiceURLs []ServiceURL +type URLs []URL // Returns all the node port URLs for every service in a particular namespace // Accepts a template for formatting -func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) (ServiceURLs, error) { - host, err := cluster.CheckIfApiExistsAndLoad(api) +func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) (URLs, error) { + host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName()) if err != nil { return nil, err } @@ -121,13 +122,13 @@ func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) return nil, err } - var serviceURLs []ServiceURL + var serviceURLs []URL for _, svc := range svcs.Items { urls, err := printURLsForService(client, ip, svc.Name, svc.Namespace, t) if err != nil { return nil, err } - serviceURLs = append(serviceURLs, ServiceURL{Namespace: svc.Namespace, Name: svc.Name, URLs: urls}) + serviceURLs = append(serviceURLs, URL{Namespace: svc.Namespace, Name: svc.Name, URLs: urls}) } return serviceURLs, nil @@ -136,7 +137,7 @@ func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) // Returns all the node ports for a service in a namespace // with optional formatting func GetServiceURLsForService(api libmachine.API, namespace, service string, t *template.Template) ([]string, error) { - host, err := cluster.CheckIfApiExistsAndLoad(api) + host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName()) if err != nil { return nil, errors.Wrap(err, "Error checking if api exist and loading it") } @@ -211,19 +212,19 @@ func CheckService(namespace string, service string) error { return nil } -func OptionallyHttpsFormattedUrlString(bareUrlString string, https bool) (string, bool) { - httpsFormattedString := bareUrlString - isHttpSchemedURL := false +func OptionallyHTTPSFormattedURLString(bareURLString string, https bool) (string, bool) { + httpsFormattedString := bareURLString + isHTTPSchemedURL := false - if u, parseErr := url.Parse(bareUrlString); parseErr == nil { - isHttpSchemedURL = u.Scheme == "http" + if u, parseErr := url.Parse(bareURLString); parseErr == nil { + isHTTPSchemedURL = u.Scheme == "http" } - if isHttpSchemedURL && https { - httpsFormattedString = strings.Replace(bareUrlString, "http", "https", 1) + if isHTTPSchemedURL && https { + httpsFormattedString = strings.Replace(bareURLString, "http", "https", 1) } - return httpsFormattedString, isHttpSchemedURL + return httpsFormattedString, isHTTPSchemedURL } func WaitAndMaybeOpenService(api libmachine.API, namespace string, service string, urlTemplate *template.Template, urlMode bool, https bool, @@ -236,10 +237,10 @@ func WaitAndMaybeOpenService(api libmachine.API, namespace string, service strin if err != nil { return errors.Wrap(err, "Check that minikube is running and that you have specified the correct namespace") } - for _, bareUrlString := range urls { - urlString, isHttpSchemedURL := OptionallyHttpsFormattedUrlString(bareUrlString, https) + for _, bareURLString := range urls { + urlString, isHTTPSchemedURL := OptionallyHTTPSFormattedURLString(bareURLString, https) - if urlMode || !isHttpSchemedURL { + if urlMode || !isHTTPSchemedURL { fmt.Fprintln(os.Stdout, urlString) } else { fmt.Fprintln(os.Stderr, "Opening kubernetes service "+namespace+"/"+service+" in default browser...") diff --git a/pkg/minikube/service/service_test.go b/pkg/minikube/service/service_test.go index 7268e49eaaed..fd2b1040379e 100644 --- a/pkg/minikube/service/service_test.go +++ b/pkg/minikube/service/service_test.go @@ -252,38 +252,38 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) { var tests = []struct { description string - bareUrlString string + bareURLString string https bool - expectedHttpsFormattedUrlString string - expectedIsHttpSchemedURL bool + expectedHTTPSFormattedURLString string + expectedIsHTTPSchemedURL bool }{ { description: "no https for http schemed with no https option", - bareUrlString: "http://192.168.99.100:30563", + bareURLString: "http://192.168.99.100:30563", https: false, - expectedHttpsFormattedUrlString: "http://192.168.99.100:30563", - expectedIsHttpSchemedURL: true, + expectedHTTPSFormattedURLString: "http://192.168.99.100:30563", + expectedIsHTTPSchemedURL: true, }, { description: "no https for non-http schemed with no https option", - bareUrlString: "xyz.http.myservice:30563", + bareURLString: "xyz.http.myservice:30563", https: false, - expectedHttpsFormattedUrlString: "xyz.http.myservice:30563", - expectedIsHttpSchemedURL: false, + expectedHTTPSFormattedURLString: "xyz.http.myservice:30563", + expectedIsHTTPSchemedURL: false, }, { description: "https for http schemed with https option", - bareUrlString: "http://192.168.99.100:30563", + bareURLString: "http://192.168.99.100:30563", https: true, - expectedHttpsFormattedUrlString: "https://192.168.99.100:30563", - expectedIsHttpSchemedURL: true, + expectedHTTPSFormattedURLString: "https://192.168.99.100:30563", + expectedIsHTTPSchemedURL: true, }, { description: "no https for non-http schemed with https option and http substring", - bareUrlString: "xyz.http.myservice:30563", + bareURLString: "xyz.http.myservice:30563", https: true, - expectedHttpsFormattedUrlString: "xyz.http.myservice:30563", - expectedIsHttpSchemedURL: false, + expectedHTTPSFormattedURLString: "xyz.http.myservice:30563", + expectedIsHTTPSchemedURL: false, }, } @@ -291,15 +291,15 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) { test := test t.Run(test.description, func(t *testing.T) { t.Parallel() - httpsFormattedUrlString, isHttpSchemedURL := OptionallyHttpsFormattedUrlString(test.bareUrlString, test.https) + httpsFormattedURLString, isHTTPSchemedURL := OptionallyHTTPSFormattedURLString(test.bareURLString, test.https) - if httpsFormattedUrlString != test.expectedHttpsFormattedUrlString { - t.Errorf("\nhttpsFormattedUrlString, Expected %v \nActual: %v \n\n", test.expectedHttpsFormattedUrlString, httpsFormattedUrlString) + if httpsFormattedURLString != test.expectedHTTPSFormattedURLString { + t.Errorf("\nhttpsFormattedURLString, Expected %v \nActual: %v \n\n", test.expectedHTTPSFormattedURLString, httpsFormattedURLString) } - if isHttpSchemedURL != test.expectedIsHttpSchemedURL { - t.Errorf("\nisHttpSchemedURL, Expected %v \nActual: %v \n\n", - test.expectedHttpsFormattedUrlString, httpsFormattedUrlString) + if isHTTPSchemedURL != test.expectedIsHTTPSchemedURL { + t.Errorf("\nisHTTPSchemedURL, Expected %v \nActual: %v \n\n", + test.expectedHTTPSFormattedURLString, httpsFormattedURLString) } }) } @@ -307,10 +307,12 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) { func TestGetServiceURLs(t *testing.T) { defaultAPI := &tests.MockAPI{ - Hosts: map[string]*host.Host{ - config.GetMachineName(): { - Name: config.GetMachineName(), - Driver: &tests.MockDriver{}, + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + config.GetMachineName(): { + Name: config.GetMachineName(), + Driver: &tests.MockDriver{}, + }, }, }, } @@ -320,13 +322,15 @@ func TestGetServiceURLs(t *testing.T) { description string api libmachine.API namespace string - expected ServiceURLs + expected URLs err bool }{ { description: "no host", api: &tests.MockAPI{ - Hosts: make(map[string]*host.Host), + FakeStore: tests.FakeStore{ + Hosts: make(map[string]*host.Host), + }, }, err: true, }, @@ -334,7 +338,7 @@ func TestGetServiceURLs(t *testing.T) { description: "correctly return serviceURLs", namespace: "default", api: defaultAPI, - expected: []ServiceURL{ + expected: []URL{ { Namespace: "default", Name: "mock-dashboard", @@ -374,10 +378,12 @@ func TestGetServiceURLs(t *testing.T) { func TestGetServiceURLsForService(t *testing.T) { defaultAPI := &tests.MockAPI{ - Hosts: map[string]*host.Host{ - config.GetMachineName(): { - Name: config.GetMachineName(), - Driver: &tests.MockDriver{}, + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + config.GetMachineName(): { + Name: config.GetMachineName(), + Driver: &tests.MockDriver{}, + }, }, }, } @@ -394,7 +400,9 @@ func TestGetServiceURLsForService(t *testing.T) { { description: "no host", api: &tests.MockAPI{ - Hosts: make(map[string]*host.Host), + FakeStore: tests.FakeStore{ + Hosts: make(map[string]*host.Host), + }, }, err: true, }, diff --git a/pkg/minikube/tests/api_mock.go b/pkg/minikube/tests/api_mock.go index a4d9a7ea6d25..1dc69609ef6c 100644 --- a/pkg/minikube/tests/api_mock.go +++ b/pkg/minikube/tests/api_mock.go @@ -20,17 +20,14 @@ import ( "encoding/json" "fmt" - "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/auth" "github.com/docker/machine/libmachine/host" - "github.com/docker/machine/libmachine/mcnerror" - "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" ) // MockAPI is a struct used to mock out libmachine.API type MockAPI struct { - Hosts map[string]*host.Host + FakeStore CreateError bool RemoveError bool SaveCalled bool @@ -38,7 +35,9 @@ type MockAPI struct { func NewMockAPI() *MockAPI { m := MockAPI{ - Hosts: make(map[string]*host.Host), + FakeStore: FakeStore{ + Hosts: make(map[string]*host.Host), + }, } return &m } @@ -52,7 +51,7 @@ func (api *MockAPI) Close() error { func (api *MockAPI) NewHost(driverName string, rawDriver []byte) (*host.Host, error) { var driver MockDriver if err := json.Unmarshal(rawDriver, &driver); err != nil { - return nil, errors.Wrap(err, "Error unmarshalling json") + return nil, errors.Wrap(err, "error unmarshalling json") } h := &host.Host{ DriverName: driverName, @@ -67,38 +66,20 @@ func (api *MockAPI) NewHost(driverName string, rawDriver []byte) (*host.Host, er // Create creates the actual host. func (api *MockAPI) Create(h *host.Host) error { if api.CreateError { - return fmt.Errorf("Error creating host.") + return errors.New("error creating host") } return h.Driver.Create() } -// Exists determines if the host already exists. -func (api *MockAPI) Exists(name string) (bool, error) { - _, ok := api.Hosts[name] - return ok, nil -} - // List the existing hosts. func (api *MockAPI) List() ([]string, error) { return []string{}, nil } -// Load loads a host from disk. -func (api *MockAPI) Load(name string) (*host.Host, error) { - h, ok := api.Hosts[name] - if !ok { - return nil, mcnerror.ErrHostDoesNotExist{ - Name: name, - } - - } - return h, nil -} - // Remove a host. func (api *MockAPI) Remove(name string) error { if api.RemoveError { - return fmt.Errorf("Error removing %s", name) + return fmt.Errorf("error removing %s", name) } delete(api.Hosts, name) @@ -107,25 +88,11 @@ func (api *MockAPI) Remove(name string) error { // Save saves a host to disk. func (api *MockAPI) Save(host *host.Host) error { - api.Hosts[host.Name] = host api.SaveCalled = true - return nil + return api.FakeStore.Save(host) } // GetMachinesDir returns the directory to store machines in. func (api MockAPI) GetMachinesDir() string { return "" } - -// State returns the state of a host. -func State(api libmachine.API, name string) state.State { - host, _ := api.Load(name) - machineState, _ := host.Driver.GetState() - return machineState -} - -// Exists tells whether a named host exists. -func Exists(api libmachine.API, name string) bool { - exists, _ := api.Exists(name) - return exists -} diff --git a/pkg/minikube/tests/driver_mock.go b/pkg/minikube/tests/driver_mock.go index f03984f81bf6..6ffe72bfdaf8 100644 --- a/pkg/minikube/tests/driver_mock.go +++ b/pkg/minikube/tests/driver_mock.go @@ -17,11 +17,10 @@ limitations under the License. package tests import ( - "fmt" - "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/mcnflag" "github.com/docker/machine/libmachine/state" + "github.com/pkg/errors" ) // MockDriver is a struct used to mock out libmachine.Driver @@ -31,6 +30,7 @@ type MockDriver struct { RemoveError bool HostError bool Port int + IP string } // Create creates a MockDriver instance @@ -40,6 +40,9 @@ func (driver *MockDriver) Create() error { } func (driver *MockDriver) GetIP() (string, error) { + if driver.IP != "" { + return driver.IP, nil + } if driver.BaseDriver.IPAddress != "" { return driver.BaseDriver.IPAddress, nil } @@ -58,7 +61,7 @@ func (driver *MockDriver) GetSSHPort() (int, error) { // GetSSHHostname returns the hostname for SSH func (driver *MockDriver) GetSSHHostname() (string, error) { if driver.HostError { - return "", fmt.Errorf("Error getting host!") + return "", errors.New("error getting host") } return "localhost", nil } @@ -87,7 +90,7 @@ func (driver *MockDriver) Kill() error { // Remove removes the machine func (driver *MockDriver) Remove() error { if driver.RemoveError { - return fmt.Errorf("Error deleting machine.") + return errors.New("error deleting machine") } return nil } diff --git a/pkg/minikube/tests/fake_store.go b/pkg/minikube/tests/fake_store.go new file mode 100644 index 000000000000..f8226da787bb --- /dev/null +++ b/pkg/minikube/tests/fake_store.go @@ -0,0 +1,72 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "github.com/docker/machine/libmachine/host" + "github.com/docker/machine/libmachine/mcnerror" +) + +//implements persist.Store from libmachine +type FakeStore struct { + Hosts map[string]*host.Host +} + +// Exists determines if the host already exists. +func (s *FakeStore) Exists(name string) (bool, error) { + _, ok := s.Hosts[name] + return ok, nil +} + +func (s *FakeStore) List() ([]string, error) { + hostNames := []string{} + for h := range s.Hosts { + hostNames = append(hostNames, h) + } + return hostNames, nil +} + +// Load loads a host from disk. +func (s *FakeStore) Load(name string) (*host.Host, error) { + h, ok := s.Hosts[name] + if !ok { + return nil, mcnerror.ErrHostDoesNotExist{ + Name: name, + } + + } + return h, nil +} + +// Remove removes a machine from the store +func (s *FakeStore) Remove(name string) error { + _, ok := s.Hosts[name] + if !ok { + return mcnerror.ErrHostDoesNotExist{ + Name: name, + } + + } + delete(s.Hosts, name) + return nil +} + +// Save persists a machine in the store +func (s *FakeStore) Save(host *host.Host) error { + s.Hosts[host.Name] = host + return nil +} diff --git a/pkg/minikube/tunnel/cluster_inspector.go b/pkg/minikube/tunnel/cluster_inspector.go new file mode 100644 index 000000000000..39d3b5ec9b3f --- /dev/null +++ b/pkg/minikube/tunnel/cluster_inspector.go @@ -0,0 +1,101 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "net" + + "github.com/docker/machine/libmachine" + "github.com/docker/machine/libmachine/host" + "github.com/docker/machine/libmachine/state" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/cluster" + "k8s.io/minikube/pkg/minikube/config" +) + +type clusterInspector struct { + machineAPI libmachine.API + configLoader config.Loader + machineName string +} + +func (m *clusterInspector) getStateAndHost() (HostState, *host.Host, error) { + + h, err := cluster.CheckIfHostExistsAndLoad(m.machineAPI, m.machineName) + + if err != nil { + err = errors.Wrapf(err, "error loading docker-machine host for: %s", m.machineName) + return Unknown, nil, err + } + + var s state.State + s, err = h.Driver.GetState() + if err != nil { + err = errors.Wrapf(err, "error getting host status for %s", m.machineName) + return Unknown, nil, err + } + + if s == state.Running { + return Running, h, nil + } + + return Stopped, h, nil +} + +func (m *clusterInspector) getStateAndRoute() (HostState, *Route, error) { + hostState, h, err := m.getStateAndHost() + defer m.machineAPI.Close() + if err != nil { + return hostState, nil, err + } + var c config.Config + c, err = m.configLoader.LoadConfigFromFile(m.machineName) + if err != nil { + err = errors.Wrapf(err, "error loading config for %s", m.machineName) + return hostState, nil, err + } + + var route *Route + route, err = getRoute(h, c) + if err != nil { + err = errors.Wrapf(err, "error getting Route info for %s", m.machineName) + return hostState, nil, err + } + return hostState, route, nil +} + +func getRoute(host *host.Host, clusterConfig config.Config) (*Route, error) { + hostDriverIP, err := host.Driver.GetIP() + if err != nil { + return nil, errors.Wrapf(err, "error getting host IP for %s", host.Name) + } + + _, ipNet, err := net.ParseCIDR(clusterConfig.KubernetesConfig.ServiceCIDR) + if err != nil { + return nil, fmt.Errorf("error parsing service CIDR: %s", err) + } + ip := net.ParseIP(hostDriverIP) + if ip == nil { + return nil, fmt.Errorf("invalid IP for host %s", hostDriverIP) + } + + return &Route{ + Gateway: ip, + DestCIDR: ipNet, + }, nil +} diff --git a/pkg/minikube/tunnel/cluster_inspector_test.go b/pkg/minikube/tunnel/cluster_inspector_test.go new file mode 100644 index 000000000000..2b5179b9ecb5 --- /dev/null +++ b/pkg/minikube/tunnel/cluster_inspector_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "testing" + + "net" + "reflect" + "strings" + + "github.com/docker/machine/libmachine/host" + "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/tests" +) + +func TestAPIError(t *testing.T) { + machineName := "nonexistentmachine" + + machineAPI := tests.NewMockAPI() + configLoader := &stubConfigLoader{} + inspector := &clusterInspector{ + machineAPI, configLoader, machineName, + } + + s, r, err := inspector.getStateAndRoute() + + if err == nil || !strings.Contains(err.Error(), "Machine does not exist") { + t.Errorf("cluster inspector should propagate errors from API, getStateAndRoute() returned \"%v, %v\", %v", s, r, err) + } +} + +func TestMinikubeCheckReturnsHostInformation(t *testing.T) { + machineAPI := &tests.MockAPI{ + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + "testmachine": { + Driver: &tests.MockDriver{ + CurrentState: state.Running, + IP: "1.2.3.4", + }, + }, + }, + }, + } + + configLoader := &stubConfigLoader{ + c: config.Config{ + KubernetesConfig: config.KubernetesConfig{ + ServiceCIDR: "96.0.0.0/12", + }, + }, + } + inspector := &clusterInspector{ + machineAPI, configLoader, "testmachine", + } + + s, r, err := inspector.getStateAndRoute() + + if err != nil { + t.Errorf("`error` is not nil") + } + + ip := net.ParseIP("1.2.3.4") + _, ipNet, _ := net.ParseCIDR("96.0.0.0/12") + + expectedRoute := &Route{ + Gateway: ip, + DestCIDR: ipNet, + } + + if s != Running { + t.Errorf("expected running, got %s", s) + } + if !reflect.DeepEqual(r, expectedRoute) { + t.Errorf("expected %v, got %v", expectedRoute, r) + } +} + +func TestUnparseableCIDR(t *testing.T) { + cfg := config.Config{ + KubernetesConfig: config.KubernetesConfig{ + ServiceCIDR: "bad.cidr.0.0/12", + }} + h := &host.Host{ + Driver: &tests.MockDriver{ + IP: "192.168.1.1", + }, + } + + _, err := getRoute(h, cfg) + + if err == nil { + t.Errorf("expected non nil error, instead got %s", err) + } +} + +func TestRouteIPDetection(t *testing.T) { + expectedTargetCIDR := "10.96.0.0/12" + + cfg := config.Config{ + KubernetesConfig: config.KubernetesConfig{ + ServiceCIDR: expectedTargetCIDR, + }, + } + + expectedGatewayIP := "192.168.1.1" + h := &host.Host{ + Driver: &tests.MockDriver{ + IP: expectedGatewayIP, + }, + } + + routerConfig, err := getRoute(h, cfg) + + if err != nil { + t.Errorf("expected no errors but got: %s", err) + } + + if routerConfig.DestCIDR.String() != expectedTargetCIDR { + t.Errorf("addTargetCIDR doesn't match, expected '%s', got '%s'", expectedTargetCIDR, routerConfig.DestCIDR) + } + + if routerConfig.Gateway.String() != expectedGatewayIP { + t.Errorf("add gateway IP doesn't match, expected '%s', got '%s'", expectedGatewayIP, routerConfig.Gateway) + } + +} diff --git a/pkg/minikube/tunnel/loadbalancer_patcher.go b/pkg/minikube/tunnel/loadbalancer_patcher.go new file mode 100644 index 000000000000..d7659eafaa6a --- /dev/null +++ b/pkg/minikube/tunnel/loadbalancer_patcher.go @@ -0,0 +1,162 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + + "github.com/golang/glog" + core_v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s_types "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +//requestSender is an interface exposed for testing what requests are sent through the k8s REST client +type requestSender interface { + send(request *rest.Request) ([]byte, error) +} + +//patchConverter is an interface exposed for testing what patches are sent through the k8s REST client +type patchConverter interface { + convert(restClient rest.Interface, patch *Patch) *rest.Request +} + +//loadBalancerEmulator is the main struct for emulating the loadbalancer behavior. it sets the ingress to the cluster IP +type loadBalancerEmulator struct { + coreV1Client v1.CoreV1Interface + requestSender requestSender + patchConverter patchConverter +} + +func (l *loadBalancerEmulator) PatchServices() ([]string, error) { + return l.applyOnLBServices(func(restClient rest.Interface, svc core_v1.Service) ([]byte, error) { + return l.updateService(restClient, svc) + }) +} + +func (l *loadBalancerEmulator) Cleanup() ([]string, error) { + return l.applyOnLBServices(func(restClient rest.Interface, svc core_v1.Service) ([]byte, error) { + return l.cleanupService(restClient, svc) + }) +} + +func (l *loadBalancerEmulator) applyOnLBServices(action func(restClient rest.Interface, svc core_v1.Service) ([]byte, error)) ([]string, error) { + services := l.coreV1Client.Services("") + serviceList, err := services.List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + restClient := l.coreV1Client.RESTClient() + + var managedServices []string + + for _, svc := range serviceList.Items { + if svc.Spec.Type != "LoadBalancer" { + glog.V(3).Infof("%s is not type LoadBalancer, skipping.", svc.Name) + continue + } + glog.Infof("%s is type LoadBalancer.", svc.Name) + managedServices = append(managedServices, svc.Name) + result, err := action(restClient, svc) + if err != nil { + glog.Errorf("%s", result) + glog.Errorf("error patching service %s/%s: %s", svc.Namespace, svc.Name, err) + continue + } + + } + return managedServices, nil +} +func (l *loadBalancerEmulator) updateService(restClient rest.Interface, svc core_v1.Service) ([]byte, error) { + clusterIP := svc.Spec.ClusterIP + ingresses := svc.Status.LoadBalancer.Ingress + if len(ingresses) == 1 && ingresses[0].IP == clusterIP { + return nil, nil + } + glog.V(3).Infof("[%s] setting ClusterIP as the LoadBalancer Ingress", svc.Name) + jsonPatch := fmt.Sprintf(`[{"op": "add", "path": "/status/loadBalancer/ingress", "value": [ { "ip": "%s" } ] }]`, clusterIP) + patch := &Patch{ + Type: k8s_types.JSONPatchType, + ResourceName: svc.Name, + NameSpaceSet: true, + NameSpace: svc.Namespace, + Subresource: "status", + Resource: "services", + BodyContent: jsonPatch, + } + request := l.patchConverter.convert(restClient, patch) + result, err := l.requestSender.send(request) + if err != nil { + glog.Infof("Patched %s with IP %s", svc.Name, clusterIP) + } else { + glog.Errorf("error patching %s with IP %s: %s", svc.Name, clusterIP, err) + } + return result, err +} + +func (l *loadBalancerEmulator) cleanupService(restClient rest.Interface, svc core_v1.Service) ([]byte, error) { + ingresses := svc.Status.LoadBalancer.Ingress + if len(ingresses) == 0 { + return nil, nil + } + glog.V(3).Infof("[%s] cleanup: unset load balancer ingress", svc.Name) + jsonPatch := `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]` + patch := &Patch{ + Type: k8s_types.JSONPatchType, + ResourceName: svc.Name, + NameSpaceSet: true, + NameSpace: svc.Namespace, + Subresource: "status", + Resource: "services", + BodyContent: jsonPatch, + } + request := l.patchConverter.convert(restClient, patch) + result, err := l.requestSender.send(request) + glog.Infof("Removed load balancer ingress from %s.", svc.Name) + return result, err + +} + +func newLoadBalancerEmulator(corev1Client v1.CoreV1Interface) loadBalancerEmulator { + return loadBalancerEmulator{ + coreV1Client: corev1Client, + requestSender: &defaultRequestSender{}, + patchConverter: &defaultPatchConverter{}, + } +} + +type defaultPatchConverter struct{} + +func (c *defaultPatchConverter) convert(restClient rest.Interface, patch *Patch) *rest.Request { + request := restClient.Patch(patch.Type) + request.Name(patch.ResourceName) + request.Resource(patch.Resource) + request.SubResource(patch.Subresource) + if patch.NameSpaceSet { + request.Namespace(patch.NameSpace) + } + request.Body([]byte(patch.BodyContent)) + return request +} + +type defaultRequestSender struct{} + +func (r *defaultRequestSender) send(request *rest.Request) ([]byte, error) { + return request.Do().Raw() +} diff --git a/pkg/minikube/tunnel/loadbalancer_patcher_test.go b/pkg/minikube/tunnel/loadbalancer_patcher_test.go new file mode 100644 index 000000000000..0d97a21d11f0 --- /dev/null +++ b/pkg/minikube/tunnel/loadbalancer_patcher_test.go @@ -0,0 +1,347 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "testing" + + "reflect" + + apiV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/kubernetes/typed/core/v1/fake" + "k8s.io/client-go/rest" +) + +type stubCoreClient struct { + fake.FakeCoreV1 + servicesList *apiV1.ServiceList + restClient *rest.RESTClient +} + +func (c *stubCoreClient) Services(namespace string) v1.ServiceInterface { + return &stubServices{ + fake.FakeServices{Fake: &c.FakeCoreV1}, + c.servicesList, + } +} + +func (c *stubCoreClient) RESTClient() rest.Interface { + return c.restClient +} + +type stubServices struct { + fake.FakeServices + servicesList *apiV1.ServiceList +} + +func (s *stubServices) List(opts metaV1.ListOptions) (*apiV1.ServiceList, error) { + return s.servicesList, nil +} + +func newStubCoreClient(servicesList *apiV1.ServiceList) *stubCoreClient { + if servicesList == nil { + servicesList = &apiV1.ServiceList{ + Items: []apiV1.Service{}} + } + return &stubCoreClient{ + servicesList: servicesList, + restClient: nil, + } +} + +type countingRequestSender struct { + requests int +} + +func (s *countingRequestSender) send(request *rest.Request) (result []byte, err error) { + s.requests++ + return nil, nil +} + +type recordingPatchConverter struct { + patches []*Patch +} + +func (r *recordingPatchConverter) convert(restClient rest.Interface, patch *Patch) *rest.Request { + r.patches = append(r.patches, patch) + return nil +} + +func TestEmptyListOfServicesDoesNothing(t *testing.T) { + client := newStubCoreClient(&apiV1.ServiceList{ + Items: []apiV1.Service{}}) + + patcher := newLoadBalancerEmulator(client) + + serviceNames, err := patcher.PatchServices() + + if len(serviceNames) > 0 || err != nil { + t.Errorf("Expected: [], nil\n Got: %v, %s", serviceNames, err) + } + +} + +func TestServicesWithNoLoadbalancerType(t *testing.T) { + client := newStubCoreClient(&apiV1.ServiceList{ + Items: []apiV1.Service{ + { + Spec: apiV1.ServiceSpec{ + Type: "ClusterIP", + }, + }, + { + Spec: apiV1.ServiceSpec{ + Type: "NodeIP", + }, + }, + }, + }) + + patcher := newLoadBalancerEmulator(client) + + serviceNames, err := patcher.PatchServices() + + if len(serviceNames) > 0 || err != nil { + t.Errorf("Expected: [], nil\n Got: %v, %s", serviceNames, err) + } + +} + +func TestServicesWithLoadbalancerType(t *testing.T) { + client := newStubCoreClient(&apiV1.ServiceList{ + Items: []apiV1.Service{ + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc1-up-to-date", + Namespace: "ns1", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.3", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{ + { + IP: "10.96.0.3", + }, + }, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc2-out-of-date", + Namespace: "ns2", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.4", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{ + { + IP: "10.96.0.5", + }, + }, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc3-empty-ingress", + Namespace: "ns3", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.2", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{}, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc4-not-lb", + }, + Spec: apiV1.ServiceSpec{ + Type: "NodeIP", + }, + }, + }, + }) + + expectedPatches := []*Patch{ + { + Type: "application/json-patch+json", + NameSpace: "ns2", + NameSpaceSet: true, + Resource: "services", + Subresource: "status", + ResourceName: "svc2-out-of-date", + BodyContent: `[{"op": "add", "path": "/status/loadBalancer/ingress", "value": [ { "ip": "10.96.0.4" } ] }]`, + }, + { + Type: "application/json-patch+json", + NameSpace: "ns3", + NameSpaceSet: true, + Resource: "services", + Subresource: "status", + ResourceName: "svc3-empty-ingress", + BodyContent: `[{"op": "add", "path": "/status/loadBalancer/ingress", "value": [ { "ip": "10.96.0.2" } ] }]`, + }, + } + + requestSender := &countingRequestSender{} + patchConverter := &recordingPatchConverter{} + + patcher := newLoadBalancerEmulator(client) + patcher.requestSender = requestSender + patcher.patchConverter = patchConverter + + serviceNames, err := patcher.PatchServices() + + expectedServices := []string{"svc1-up-to-date", "svc2-out-of-date", "svc3-empty-ingress"} + + if !reflect.DeepEqual(serviceNames, expectedServices) || err != nil { + t.Errorf("error.\nExpected: %s, \nGot: %v, %v", expectedServices, serviceNames, err) + } + + if !reflect.DeepEqual(patchConverter.patches, expectedPatches) { + t.Errorf("error in patches.\nExpected: %v, \nGot: %v", expectedPatches, patchConverter.patches) + } + + if requestSender.requests != 2 { + t.Errorf("error in number of requests sent.\nExpected: %v, \nGot: %v", 2, requestSender.requests) + } + +} + +func TestCleanupPatchedIPs(t *testing.T) { + expectedPatches := []*Patch{ + { + Type: "application/json-patch+json", + NameSpace: "ns1", + NameSpaceSet: true, + Resource: "services", + Subresource: "status", + ResourceName: "svc1-up-to-date", + BodyContent: `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]`, + }, + + { + Type: "application/json-patch+json", + NameSpace: "ns2", + NameSpaceSet: true, + Resource: "services", + Subresource: "status", + ResourceName: "svc2-out-of-date", + BodyContent: `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]`, + }, + } + + client := newStubCoreClient(&apiV1.ServiceList{ + Items: []apiV1.Service{ + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc1-up-to-date", + Namespace: "ns1", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.3", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{ + { + IP: "10.96.0.3", + }, + }, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc2-out-of-date", + Namespace: "ns2", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.4", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{ + { + IP: "10.96.0.5", + }, + }, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc3-empty-ingress", + Namespace: "ns3", + }, + Spec: apiV1.ServiceSpec{ + Type: "LoadBalancer", + ClusterIP: "10.96.0.2", + }, + Status: apiV1.ServiceStatus{ + LoadBalancer: apiV1.LoadBalancerStatus{ + Ingress: []apiV1.LoadBalancerIngress{}, + }, + }, + }, + { + ObjectMeta: metaV1.ObjectMeta{ + Name: "svc4-not-lb", + }, + Spec: apiV1.ServiceSpec{ + Type: "NodeIP", + }, + }, + }, + }) + + requestSender := &countingRequestSender{} + patchConverter := &recordingPatchConverter{} + + patcher := newLoadBalancerEmulator(client) + patcher.requestSender = requestSender + patcher.patchConverter = patchConverter + + serviceNames, err := patcher.Cleanup() + expectedServices := []string{"svc1-up-to-date", "svc2-out-of-date", "svc3-empty-ingress"} + + if !reflect.DeepEqual(serviceNames, expectedServices) || err != nil { + t.Errorf("error.\nExpected: %s, \nGot: %v, %v", expectedServices, serviceNames, err) + } + if !reflect.DeepEqual(patchConverter.patches, expectedPatches) { + t.Errorf("error in patches.\nExpected: %v, \nGot: %v", expectedPatches, patchConverter.patches) + } + if requestSender.requests != 2 { + t.Errorf("error in number of requests sent.\nExpected: %v, \nGot: %v", 2, requestSender.requests) + } +} diff --git a/pkg/minikube/tunnel/process.go b/pkg/minikube/tunnel/process.go new file mode 100644 index 000000000000..b240c6623864 --- /dev/null +++ b/pkg/minikube/tunnel/process.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "os" + "runtime" + "syscall" +) + +var checkIfRunning func(pid int) (bool, error) +var getPid func() int + +func init() { + checkIfRunning = osCheckIfRunning + getPid = osGetPid +} + +func osGetPid() int { + return os.Getpid() +} + +//TODO(balintp): this is vulnerable to pid reuse we should include process name in the check +func osCheckIfRunning(pid int) (bool, error) { + p, err := os.FindProcess(pid) + if runtime.GOOS == "windows" { + return err == nil, nil + } + //on unix systems further checking is required, as findProcess is noop + if err != nil { + return false, fmt.Errorf("error finding process %d: %s", pid, err) + } + if err := p.Signal(syscall.Signal(0)); err != nil { + return false, nil + } + if p == nil { + return false, nil + } + return true, nil +} diff --git a/pkg/minikube/tunnel/registry.go b/pkg/minikube/tunnel/registry.go new file mode 100644 index 000000000000..e4ccde963b93 --- /dev/null +++ b/pkg/minikube/tunnel/registry.go @@ -0,0 +1,195 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + + "github.com/golang/glog" + "github.com/pkg/errors" +) + +// There is one tunnel registry per user, shared across multiple vms. +// It can register, list and check for existing and running tunnels +type ID struct { + //Route is the key + Route *Route + //the rest is metadata + MachineName string + Pid int +} + +func (t *ID) Equal(other *ID) bool { + return t.Route.Equal(other.Route) && + t.MachineName == other.MachineName && + t.Pid == other.Pid +} + +func (t *ID) String() string { + return fmt.Sprintf("ID { Route: %v, machineName: %s, Pid: %d }", t.Route, t.MachineName, t.Pid) +} + +type persistentRegistry struct { + path string +} + +func (r *persistentRegistry) IsAlreadyDefinedAndRunning(tunnel *ID) (*ID, error) { + tunnels, err := r.List() + if err != nil { + return nil, fmt.Errorf("failed to list: %s", err) + } + + for _, t := range tunnels { + if t.Route.Equal(tunnel.Route) { + isRunning, err := checkIfRunning(t.Pid) + if err != nil { + return nil, fmt.Errorf("error checking whether conflicting tunnel (%v) is running: %s", t, err) + } + if isRunning { + return t, nil + } + } + } + return nil, nil +} + +func (r *persistentRegistry) Register(tunnel *ID) error { + glog.V(3).Infof("registering tunnel: %s", tunnel) + if tunnel.Route == nil { + return errors.New("tunnel.Route should not be nil") + } + + tunnels, err := r.List() + if err != nil { + return fmt.Errorf("failed to list: %s", err) + } + + alreadyExists := false + for i, t := range tunnels { + if t.Route.Equal(tunnel.Route) { + isRunning, err := checkIfRunning(t.Pid) + if err != nil { + return fmt.Errorf("error checking whether conflicting tunnel (%v) is running: %s", t, err) + } + if isRunning { + return errorTunnelAlreadyExists(t) + } + tunnels[i] = tunnel + alreadyExists = true + } + } + + if !alreadyExists { + tunnels = append(tunnels, tunnel) + } + + bytes, err := json.Marshal(tunnels) + if err != nil { + return fmt.Errorf("error marshalling json %s", err) + } + + glog.V(5).Infof("json marshalled: %v, %s\n", tunnels, bytes) + + f, err := os.OpenFile(r.path, os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + if os.IsNotExist(err) { + f, err = os.Create(r.path) + if err != nil { + return fmt.Errorf("error creating registry file (%s): %s", r.path, err) + } + } else { + return err + } + } + defer func() { + err := f.Close() + if err != nil { + fmt.Errorf("error closing registry file: %s", err) + } + }() + + n, err := f.Write(bytes) + if n < len(bytes) || err != nil { + return fmt.Errorf("error registering tunnel while writing tunnels file: %s", err) + } + + return nil +} + +func (r *persistentRegistry) Remove(route *Route) error { + glog.V(3).Infof("removing tunnel from registry: %s", route) + tunnels, err := r.List() + if err != nil { + return err + } + idx := -1 + for i := range tunnels { + if tunnels[i].Route.Equal(route) { + idx = i + break + } + } + if idx == -1 { + return fmt.Errorf("can't remove route: %s not found in tunnel registry", route) + } + tunnels = append(tunnels[:idx], tunnels[idx+1:]...) + glog.V(4).Infof("tunnels after remove: %s", tunnels) + f, err := os.OpenFile(r.path, os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("error removing tunnel %s", err) + } + defer func() { + err := f.Close() + if err != nil { + fmt.Errorf("error closing tunnel registry file: %s", err) + } + }() + + var bytes []byte + bytes, err = json.Marshal(tunnels) + if err != nil { + return fmt.Errorf("error removing tunnel %s", err) + } + n, err := f.Write(bytes) + if n < len(bytes) || err != nil { + return fmt.Errorf("error removing tunnel %s", err) + } + + return nil +} +func (r *persistentRegistry) List() ([]*ID, error) { + f, err := os.Open(r.path) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + return []*ID{}, nil + } + byteValue, _ := ioutil.ReadAll(f) + var tunnels []*ID + if len(byteValue) == 0 { + return tunnels, nil + } + if err = json.Unmarshal(byteValue, &tunnels); err != nil { + return nil, err + } + + return tunnels, nil +} diff --git a/pkg/minikube/tunnel/registry_test.go b/pkg/minikube/tunnel/registry_test.go new file mode 100644 index 000000000000..9240908ee25b --- /dev/null +++ b/pkg/minikube/tunnel/registry_test.go @@ -0,0 +1,268 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "io/ioutil" + "os" + "reflect" + "testing" +) + +func TestPersistentRegistryWithNoKey(t *testing.T) { + registry, cleanup := createTestRegistry(t) + defer cleanup() + + route := &ID{} + err := registry.Register(route) + + if err == nil { + t.Errorf("attempting to register ID without key should throw error") + } +} + +func TestPersistentRegistryNullableMetadata(t *testing.T) { + registry, cleanup := createTestRegistry(t) + defer cleanup() + + route := &ID{ + Route: unsafeParseRoute("1.2.3.4", "10.96.0.0/12"), + } + err := registry.Register(route) + + if err != nil { + t.Errorf("metadata should be nullable, expected no error, got %s", err) + } +} + +func TestListOnEmptyRegistry(t *testing.T) { + reg := &persistentRegistry{ + path: "nonexistent.txt", + } + + info, err := reg.List() + expectedInfo := []*ID{} + if !reflect.DeepEqual(info, expectedInfo) || err != nil { + t.Errorf("expected %s, nil error, got %s, %s", expectedInfo, info, err) + } +} + +func TestRemoveOnEmptyRegistry(t *testing.T) { + reg := &persistentRegistry{ + path: "nonexistent.txt", + } + + e := reg.Remove(unsafeParseRoute("1.2.3.4", "1.2.3.4/5")) + if e == nil { + t.Errorf("expected error, got %s", e) + } +} + +func TestRegisterOnEmptyRegistry(t *testing.T) { + reg := &persistentRegistry{ + path: "nonexistent.txt", + } + + err := reg.Register(&ID{Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5")}) + if err != nil { + t.Errorf("expected no error, got %s", err) + } + f, err := os.Open("nonexistent.txt") + if err != nil { + t.Errorf("expected file to exist, got: %s", err) + return + } + f.Close() + err = os.Remove("nonexistent.txt") + if err != nil { + t.Errorf("error removing nonexistent.txt: %s", err) + } +} + +func TestRemoveOnNonExistentTunnel(t *testing.T) { + file := tmpFile(t) + reg := &persistentRegistry{ + path: file, + } + defer os.Remove(file) + + err := reg.Register(&ID{Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5")}) + if err != nil { + t.Errorf("expected no error, got %s", err) + } + + err = reg.Remove(unsafeParseRoute("5.6.7.8", "1.2.3.4/5")) + if err == nil { + t.Errorf("expected error, got nil") + } +} + +func TestListAfterRegister(t *testing.T) { + file := tmpFile(t) + reg := &persistentRegistry{ + path: file, + } + defer os.Remove(file) + err := reg.Register(&ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: 1234, + }) + if err != nil { + t.Errorf("failed to register: expected no error, got %s", err) + } + + tunnelList, err := reg.List() + if err != nil { + t.Errorf("failed to list: expected no error, got %s", err) + } + + expectedList := []*ID{ + { + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: 1234, + }, + } + + if len(tunnelList) != 1 || !tunnelList[0].Equal(expectedList[0]) { + t.Errorf("\nexpected %+v,\ngot %+v", expectedList, tunnelList) + } +} +func TestRegisterRemoveList(t *testing.T) { + file := tmpFile(t) + reg := &persistentRegistry{ + path: file, + } + defer os.Remove(file) + + err := reg.Register(&ID{ + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 1234, + }) + if err != nil { + t.Errorf("failed to register: expected no error, got %s", err) + } + + err = reg.Remove(unsafeParseRoute("192.168.1.25", "10.96.0.0/12")) + + if err != nil { + t.Errorf("failed to remove: expected no error, got %s", err) + } + + tunnelList, err := reg.List() + if err != nil { + t.Errorf("failed to list: expected no error, got %s", err) + } + + expectedList := []*ID{} + + if len(tunnelList) != 0 { + t.Errorf("\nexpected %+v,\ngot %+v", expectedList, tunnelList) + } +} + +func TestDuplicateRouteError(t *testing.T) { + file := tmpFile(t) + reg := &persistentRegistry{ + path: file, + } + defer os.Remove(file) + + err := reg.Register(&ID{ + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: os.Getpid(), + }) + if err != nil { + t.Errorf("failed to register: expected no error, got %s", err) + } + err = reg.Register(&ID{ + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 5678, + }) + if err == nil { + t.Error("expected error on duplicate route, got nil") + } +} + +func TestTunnelTakeoverFromNonRunningProcess(t *testing.T) { + file := tmpFile(t) + reg := &persistentRegistry{ + path: file, + } + defer os.Remove(file) + + err := reg.Register(&ID{ + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 12341234, + }) + if err != nil { + t.Errorf("failed to register: expected no error, got %s", err) + } + err = reg.Register(&ID{ + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 5678, + }) + if err != nil { + t.Errorf("failed to register: expected no error, got %s", err) + } + + tunnelList, err := reg.List() + if err != nil { + t.Errorf("failed to list: expected no error, got %s", err) + } + + expectedList := []*ID{ + { + Route: unsafeParseRoute("192.168.1.25", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 5678, + }, + } + + if len(tunnelList) != 1 || !tunnelList[0].Equal(expectedList[0]) { + t.Errorf("\nexpected %+v,\ngot %+v", expectedList, tunnelList) + } +} + +func tmpFile(t *testing.T) string { + t.Helper() + f, err := ioutil.TempFile(os.TempDir(), "reg_") + f.Close() + if err != nil { + t.Errorf("failed to create temp file %s", err) + } + return f.Name() +} + +func createTestRegistry(t *testing.T) (reg *persistentRegistry, cleanup func()) { + f, err := ioutil.TempFile(os.TempDir(), "reg_") + f.Close() + if err != nil { + t.Errorf("failed to create temp file %s", err) + } + + registry := &persistentRegistry{ + path: f.Name(), + } + return registry, func() { os.Remove(f.Name()) } +} diff --git a/pkg/minikube/tunnel/reporter.go b/pkg/minikube/tunnel/reporter.go new file mode 100644 index 000000000000..6b64529b680d --- /dev/null +++ b/pkg/minikube/tunnel/reporter.go @@ -0,0 +1,92 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + + "io" + "strings" + + "github.com/golang/glog" +) + +//reporter that reports the status of a tunnel +type reporter interface { + Report(tunnelState *Status) +} + +type simpleReporter struct { + out io.Writer + lastState *Status +} + +const noErrors = "no errors" + +func (r *simpleReporter) Report(tunnelState *Status) { + if r.lastState == tunnelState { + return + } + r.lastState = tunnelState + minikubeState := tunnelState.MinikubeState.String() + + managedServices := fmt.Sprintf("[%s]", strings.Join(tunnelState.PatchedServices, ", ")) + + lbError := noErrors + if tunnelState.LoadBalancerEmulatorError != nil { + lbError = tunnelState.LoadBalancerEmulatorError.Error() + } + + minikubeError := noErrors + if tunnelState.MinikubeError != nil { + minikubeError = tunnelState.MinikubeError.Error() + } + + routerError := noErrors + if tunnelState.RouteError != nil { + routerError = tunnelState.RouteError.Error() + } + + errors := fmt.Sprintf(` errors: + minikube: %s + router: %s + loadbalancer emulator: %s +`, minikubeError, routerError, lbError) + + _, err := r.out.Write([]byte(fmt.Sprintf( + `Status: + machine: %s + pid: %d + route: %s + minikube: %s + services: %s +%s`, tunnelState.TunnelID.MachineName, + tunnelState.TunnelID.Pid, + tunnelState.TunnelID.Route, + minikubeState, + managedServices, + errors))) + if err != nil { + glog.Errorf("failed to report state %s", err) + } +} + +func newReporter(out io.Writer) reporter { + return &simpleReporter{ + out: out, + } +} diff --git a/pkg/minikube/tunnel/reporter_test.go b/pkg/minikube/tunnel/reporter_test.go new file mode 100644 index 000000000000..67f0b696e28c --- /dev/null +++ b/pkg/minikube/tunnel/reporter_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "errors" + "fmt" + + "testing" +) + +func TestReporter(t *testing.T) { + testCases := []struct { + name string + tunnelState *Status + expectedOutput string + }{ + { + name: "simple", + tunnelState: &Status{ + TunnelID: ID{ + Route: unsafeParseRoute("1.2.3.4", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 1234, + }, + MinikubeState: Running, + MinikubeError: nil, + + RouteError: nil, + + PatchedServices: []string{"svc1", "svc2"}, + LoadBalancerEmulatorError: nil, + }, + expectedOutput: `Status: + machine: testmachine + pid: 1234 + route: 10.96.0.0/12 -> 1.2.3.4 + minikube: Running + services: [svc1, svc2] + errors: + minikube: no errors + router: no errors + loadbalancer emulator: no errors +`, + }, + { + name: "errors", + tunnelState: &Status{ + TunnelID: ID{ + Route: unsafeParseRoute("1.2.3.4", "10.96.0.0/12"), + MachineName: "testmachine", + Pid: 1234, + }, + MinikubeState: Unknown, + MinikubeError: errors.New("minikubeerror"), + + RouteError: errors.New("routeerror"), + + PatchedServices: nil, + LoadBalancerEmulatorError: errors.New("lberror"), + }, + expectedOutput: `Status: + machine: testmachine + pid: 1234 + route: 10.96.0.0/12 -> 1.2.3.4 + minikube: Unknown + services: [] + errors: + minikube: minikubeerror + router: routeerror + loadbalancer emulator: lberror +`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + out := &recordingWriter{} + reporter := newReporter(out) + reporter.Report(tc.tunnelState) + if tc.expectedOutput != out.output { + t.Errorf(`%s [FAIL]. +Expected: "%s" +Got: "%s"`, tc.name, tc.expectedOutput, out.output) + } + }) + } + + //testing deduplication + out := &recordingWriter{} + reporter := newReporter(out) + reporter.Report(testCases[0].tunnelState) + reporter.Report(testCases[0].tunnelState) + reporter.Report(testCases[1].tunnelState) + reporter.Report(testCases[1].tunnelState) + reporter.Report(testCases[0].tunnelState) + + expectedOutput := fmt.Sprintf("%s%s%s", + testCases[0].expectedOutput, + testCases[1].expectedOutput, + testCases[0].expectedOutput) + + if out.output != expectedOutput { + t.Errorf(`Deduplication test [FAIL]. +Expected: "%s" +Got: "%s"`, expectedOutput, out.output) + } +} + +type recordingWriter struct { + output string +} + +func (w *recordingWriter) Write(p []byte) (n int, err error) { + w.output = fmt.Sprintf("%s%s", w.output, p) + return 0, nil +} diff --git a/pkg/minikube/tunnel/route.go b/pkg/minikube/tunnel/route.go new file mode 100644 index 000000000000..1b62873758c0 --- /dev/null +++ b/pkg/minikube/tunnel/route.go @@ -0,0 +1,112 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "strings" + + "github.com/golang/glog" +) + +//router manages the routing table on the host, implementations should cater for OS specific methods +type router interface { + //Inspect checks if the given route exists or not in the routing table + //conflict is defined as: same destination CIDR, different Gateway + //overlaps are defined as: routes that have overlapping but not exactly matching destination CIDR + Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) + + //EnsureRouteIsAdded is an idempotent way to add a route to the routing table + //it fails if there is a conflict + EnsureRouteIsAdded(route *Route) error + + //Cleanup is an idempotent way to remove a route from the routing table + //it fails if there is a conflict + Cleanup(route *Route) error +} + +type osRouter struct{} + +type routingTableLine struct { + route *Route + line string +} + +func isValidToAddOrDelete(router router, r *Route) (bool, error) { + exists, conflict, overlaps, err := router.Inspect(r) + if err != nil { + return false, err + } + + if len(overlaps) > 0 { + glog.Warningf("overlapping CIDR detected in routing table with minikube tunnel (CIDR: %s). It is advisable to remove these rules:\n%v", r.DestCIDR, strings.Join(overlaps, "\n")) + } + + if exists { + return true, nil + } + + if len(conflict) > 0 { + return false, fmt.Errorf("conflicting rule in routing table: %s", conflict) + } + + return false, nil +} + +//a partial representation of the routing table on the host +//tunnel only requires the destination CIDR, the gateway and the actual textual representation per line +type routingTable []routingTableLine + +func (t *routingTable) Check(route *Route) (exists bool, conflict string, overlaps []string) { + conflict = "" + exists = false + overlaps = []string{} + for _, tableLine := range *t { + if route.Equal(tableLine.route) { + exists = true + } else if route.DestCIDR.String() == tableLine.route.DestCIDR.String() && + route.Gateway.String() != tableLine.route.Gateway.String() { + conflict = tableLine.line + } else if route.DestCIDR.Contains(tableLine.route.DestCIDR.IP) || tableLine.route.DestCIDR.Contains(route.DestCIDR.IP) { + overlaps = append(overlaps, tableLine.line) + } + } + return +} + +func (t *routingTable) String() string { + result := fmt.Sprintf("table (%d routes)", len(*t)) + for _, l := range *t { + result = fmt.Sprintf("%s\n %s\t|%s", result, l.route.String(), l.line) + } + return result +} + +func (t *routingTable) Equal(other *routingTable) bool { + if other == nil || len(*t) != len(*other) { + return false + } + + for i := range *t { + routesEqual := (*t)[i].route.Equal((*other)[i].route) + linesEqual := (*t)[i].line == ((*other)[i].line) + if !(routesEqual && linesEqual) { + return false + } + } + return true +} diff --git a/pkg/minikube/tunnel/route_darwin.go b/pkg/minikube/tunnel/route_darwin.go new file mode 100644 index 000000000000..ad5a7db0e1d8 --- /dev/null +++ b/pkg/minikube/tunnel/route_darwin.go @@ -0,0 +1,168 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "net" + "os/exec" + "regexp" + "strings" + + "github.com/golang/glog" +) + +func (router *osRouter) EnsureRouteIsAdded(route *Route) error { + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if exists { + return nil + } + + serviceCIDR := route.DestCIDR.String() + gatewayIP := route.Gateway.String() + + glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP) + command := exec.Command("sudo", "route", "-n", "add", serviceCIDR, gatewayIP) + glog.Infof("About to run command: %s", command.Args) + stdInAndOut, err := command.CombinedOutput() + message := fmt.Sprintf("%s", stdInAndOut) + re := regexp.MustCompile(fmt.Sprintf("add net (.*): gateway %s\n", gatewayIP)) + if !re.MatchString(message) { + return fmt.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n"))) + } + glog.Infof("%s", stdInAndOut) + if err != nil { + return err + } + return nil +} + +func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) { + cmd := exec.Command("netstat", "-nr", "-f", "inet") + cmd.Env = append(cmd.Env, "LC_ALL=C") + stdInAndOut, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("error running '%v': %s", cmd, err) + return + } + + rt := router.parseTable(stdInAndOut) + + exists, conflict, overlaps = rt.Check(route) + + return +} + +func (router *osRouter) parseTable(table []byte) routingTable { + t := routingTable{} + skip := true + for _, line := range strings.Split(string(table), "\n") { + //header + if strings.HasPrefix(line, "Destination") { + skip = false + continue + } + //don't care about the 0.0.0.0 routes + if skip || strings.HasPrefix(line, "default") { + continue + } + fields := strings.Fields(line) + + if len(fields) <= 2 { + continue + } + dstCIDRString := router.padCIDR(fields[0]) + gatewayIPString := fields[1] + gatewayIP := net.ParseIP(gatewayIPString) + + _, ipNet, err := net.ParseCIDR(dstCIDRString) + if err != nil { + glog.V(4).Infof("skipping line: can't parse CIDR from routing table: %s", dstCIDRString) + } else if gatewayIP == nil { + glog.V(4).Infof("skipping line: can't parse IP from routing table: %s", gatewayIPString) + } else { + tableLine := routingTableLine{ + route: &Route{ + DestCIDR: ipNet, + Gateway: gatewayIP, + }, + line: line, + } + t = append(t, tableLine) + } + } + + return t +} + +func (router *osRouter) padCIDR(origCIDR string) string { + s := "" + dots := 0 + slash := false + for i, c := range origCIDR { + if c == '.' { + dots++ + } + if c == '/' { + for dots < 3 { + s += ".0" + dots++ + } + slash = true + } + if i == len(origCIDR)-1 { + s += string(c) + bits := 32 - 8*(3-dots) + for dots < 3 { + s += ".0" + dots++ + } + if !slash { + s += fmt.Sprintf("/%d", bits) + } + } else { + s += string(c) + } + } + return s +} + +func (router *osRouter) Cleanup(route *Route) error { + glog.V(3).Infof("Cleaning up %s\n", route) + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if !exists { + return nil + } + command := exec.Command("sudo", "route", "-n", "delete", route.DestCIDR.String()) + stdInAndOut, err := command.CombinedOutput() + if err != nil { + return err + } + message := fmt.Sprintf("%s", stdInAndOut) + glog.V(4).Infof("%s", message) + re := regexp.MustCompile("^delete net ([^:]*)$") + if !re.MatchString(message) { + return fmt.Errorf("error deleting route: %s, %d", message, len(strings.Split(message, "\n"))) + } + return nil +} diff --git a/pkg/minikube/tunnel/route_darwin_test.go b/pkg/minikube/tunnel/route_darwin_test.go new file mode 100644 index 000000000000..f144eda8fbc0 --- /dev/null +++ b/pkg/minikube/tunnel/route_darwin_test.go @@ -0,0 +1,157 @@ +// +build darwin,integration + +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "net" + "os/exec" + "testing" + + "fmt" + + "reflect" +) + +func TestDarwinRouteFailsOnConflictIntegrationTest(t *testing.T) { + cfg := &Route{ + Gateway: net.IPv4(127, 0, 0, 1), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + + addRoute(t, "10.96.0.0/12", "127.0.0.2") + err := (&osRouter{}).EnsureRouteIsAdded(cfg) + if err == nil { + t.Errorf("add should have error, but it is nil") + } +} + +func TestDarwinRouteIdempotentIntegrationTest(t *testing.T) { + cfg := &Route{ + Gateway: net.IPv4(127, 0, 0, 1), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + + cleanRoute(t, "10.96.0.0/12") + err := (&osRouter{}).EnsureRouteIsAdded(cfg) + if err != nil { + t.Errorf("add error: %s", err) + } + + err = (&osRouter{}).EnsureRouteIsAdded(cfg) + if err != nil { + t.Errorf("add error: %s", err) + } + + cleanRoute(t, "10.96.0.0/12") +} + +func TestDarwinRouteCleanupIdempontentIntegrationTest(t *testing.T) { + + cfg := &Route{ + Gateway: net.IPv4(192, 168, 1, 1), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + + cleanRoute(t, "10.96.0.0/12") + addRoute(t, "10.96.0.0/12", "192.168.1.1") + err := (&osRouter{}).Cleanup(cfg) + if err != nil { + t.Errorf("cleanup failed with %s", err) + } + err = (&osRouter{}).Cleanup(cfg) + if err != nil { + t.Errorf("cleanup failed with %s", err) + } +} + +func addRoute(t *testing.T, cidr string, gw string) { + command := exec.Command("sudo", "route", "-n", "add", cidr, gw) + _, err := command.CombinedOutput() + if err != nil { + t.Logf("add Route error (should be ok): %s", err) + } +} + +func cleanRoute(t *testing.T, cidr string) { + command := exec.Command("sudo", "route", "-n", "delete", cidr) + _, err := command.CombinedOutput() + if err != nil { + t.Logf("cleanup error (should be ok): %s", err) + } +} + +func TestCIDRPadding(t *testing.T) { + testCases := []struct { + inputCIDR string + paddedCIDR string + }{ + {inputCIDR: "10", paddedCIDR: "10.0.0.0/8"}, + {inputCIDR: "10.96/12", paddedCIDR: "10.96.0.0/12"}, + {inputCIDR: "192.168.43", paddedCIDR: "192.168.43.0/24"}, + {inputCIDR: "192.168.43.1/32", paddedCIDR: "192.168.43.1/32"}, + {inputCIDR: "127.0.0.1", paddedCIDR: "127.0.0.1/32"}, + } + + for _, test := range testCases { + testName := fmt.Sprintf("pad(%s) should be %s", test.inputCIDR, test.paddedCIDR) + t.Run(testName, func(t *testing.T) { + cidr := (&osRouter{}).padCIDR(test.inputCIDR) + if cidr != test.paddedCIDR { + t.Errorf("%s got %s", testName, cidr) + } + }) + } +} + +func TestRoutingTableParser(t *testing.T) { + table := `Routing tables + +Internet: +Destination Gateway Flags Refs Use Netif Expire +127 127.0.0.1 UCS 0 0 lo0 +127.0.0.1 127.0.0.1 UH 13 30917 lo0 +172.16.128/24 link#17 UC 1 0 vmnet1 +192.168.246 link#18 UC 1 0 vmnet8 +224.0.0 link#1 UmCS 0 0 lo0 +` + rt := (&osRouter{}).parseTable([]byte(table)) + + expectedRt := routingTable{ + routingTableLine{ + route: unsafeParseRoute("127.0.0.1", "127.0.0.0/8"), + line: "127 127.0.0.1 UCS 0 0 lo0", + }, + routingTableLine{ + route: unsafeParseRoute("127.0.0.1", "127.0.0.1/32"), + line: "127.0.0.1 127.0.0.1 UH 13 30917 lo0", + }, + } + if !reflect.DeepEqual(rt, expectedRt) { + t.Errorf("expected:\n %s\ngot\n %s", expectedRt.String(), rt.String()) + } +} diff --git a/pkg/minikube/tunnel/route_linux.go b/pkg/minikube/tunnel/route_linux.go new file mode 100644 index 000000000000..df5cbcc615e8 --- /dev/null +++ b/pkg/minikube/tunnel/route_linux.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "net" + "os/exec" + "strings" + + "github.com/golang/glog" +) + +func (router *osRouter) EnsureRouteIsAdded(route *Route) error { + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if exists { + return nil + } + + serviceCIDR := route.DestCIDR.String() + gatewayIP := route.Gateway.String() + + glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP) + command := exec.Command("sudo", "ip", "route", "add", serviceCIDR, "via", gatewayIP) + glog.Infof("About to run command: %s", command.Args) + stdInAndOut, err := command.CombinedOutput() + message := string(stdInAndOut) + if len(message) > 0 { + return fmt.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n"))) + } + glog.Info(stdInAndOut) + if err != nil { + glog.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n"))) + return err + } + return nil +} + +func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) { + cmd := exec.Command("netstat", "-nr", "-f", "inet") + cmd.Env = append(cmd.Env, "LC_ALL=C") + stdInAndOut, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("error running '%v': %s", cmd, err) + return + } + rt := router.parseTable(stdInAndOut) + + exists, conflict, overlaps = rt.Check(route) + + return +} + +func (router *osRouter) parseTable(table []byte) routingTable { + t := routingTable{} + skip := true + for _, line := range strings.Split(string(table), "\n") { + //after first line of header we can start consuming + if strings.HasPrefix(line, "Destination") { + skip = false + continue + } + + fields := strings.Fields(line) + //don't care about the 0.0.0.0 routes + if skip || len(fields) == 0 || len(fields) > 0 && (fields[0] == "default" || fields[0] == "0.0.0.0") { + continue + } + if len(fields) > 2 { + dstCIDRIP := net.ParseIP(fields[0]) + dstCIDRMask := fields[2] + dstMaskIP := net.ParseIP(dstCIDRMask) + gatewayIP := net.ParseIP(fields[1]) + + if dstCIDRIP == nil || gatewayIP == nil || dstMaskIP == nil { + glog.V(8).Infof("skipping line: can't parse: %s", line) + } else { + + dstCIDR := &net.IPNet{ + IP: dstCIDRIP, + Mask: net.IPv4Mask(dstMaskIP[12], dstMaskIP[13], dstMaskIP[14], dstMaskIP[15]), + } + + tableLine := routingTableLine{ + route: &Route{ + DestCIDR: dstCIDR, + Gateway: gatewayIP, + }, + line: line, + } + t = append(t, tableLine) + } + } + } + + return t +} + +func (router *osRouter) Cleanup(route *Route) error { + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if !exists { + return nil + } + serviceCIDR := route.DestCIDR.String() + gatewayIP := route.Gateway.String() + + glog.Infof("Cleaning up Route for CIDR %s to gateway %s\n", serviceCIDR, gatewayIP) + command := exec.Command("sudo", "ip", "route", "delete", serviceCIDR) + stdInAndOut, err := command.CombinedOutput() + message := fmt.Sprintf("%s", stdInAndOut) + glog.Infof("%s", message) + if err != nil { + return fmt.Errorf("error deleting Route: %s, %s", message, err) + } + return nil +} diff --git a/pkg/minikube/tunnel/route_linux_test.go b/pkg/minikube/tunnel/route_linux_test.go new file mode 100644 index 000000000000..c0136df709e2 --- /dev/null +++ b/pkg/minikube/tunnel/route_linux_test.go @@ -0,0 +1,135 @@ +// +build linux,integration + +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "net" + "os/exec" + "testing" +) + +func TestLinuxRouteFailsOnConflictIntegrationTest(t *testing.T) { + r := &osRouter{} + + cleanRoute(t, "10.96.0.0/12") + addRoute(t, "10.96.0.0/12", "127.0.0.1") + err := r.EnsureRouteIsAdded(&Route{ + Gateway: net.IPv4(127, 0, 0, 2), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }}) + if err == nil { + t.Errorf("add should have error, but it is nil") + } + cleanRoute(t, "10.96.0.0/12") +} + +func TestLinuxRouteIdempotentIntegrationTest(t *testing.T) { + r := &osRouter{} + + cleanRoute(t, "10.96.0.0/12") + route := &Route{ + Gateway: net.IPv4(127, 0, 0, 1), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + err := r.EnsureRouteIsAdded(route) + if err != nil { + t.Errorf("add error: %s", err) + } + + err = r.EnsureRouteIsAdded(route) + if err != nil { + t.Errorf("add error: %s", err) + } + + cleanRoute(t, "10.96.0.0/12") +} + +func TestLinuxRouteCleanupIdempontentIntegrationTest(t *testing.T) { + + r := &osRouter{} + route := &Route{ + Gateway: net.IPv4(127, 0, 0, 1), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + + cleanRoute(t, "10.96.0.0/12") + addRoute(t, "10.96.0.0/12", "127.0.0.1") + err := r.Cleanup(route) + if err != nil { + t.Errorf("cleanup failed: %s", err) + } + err = r.Cleanup(route) + if err != nil { + t.Errorf("cleanup failed: %s", err) + } +} + +func TestParseTable(t *testing.T) { + + const table = `Kernel IP routing table +Destination Gateway Genmask Flags MSS Window irtt Iface +0.0.0.0 172.31.126.254 0.0.0.0 UG 0 0 0 eno1 +10.96.0.0 127.0.0.1 255.240.0.0 UG 0 0 0 eno1 +172.31.126.0 0.0.0.0 255.255.255.0 U 0 0 0 eno1 +` + + rt := (&osRouter{}).parseTable([]byte(table)) + + expectedRt := routingTable{ + routingTableLine{ + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + line: "10.96.0.0 127.0.0.1 255.240.0.0 UG 0 0 0 eno1", + }, + routingTableLine{ + route: unsafeParseRoute("0.0.0.0", "172.31.126.0/24"), + line: "172.31.126.0 0.0.0.0 255.255.255.0 U 0 0 0 eno1", + }, + } + if !expectedRt.Equal(&rt) { + t.Errorf("expected:\n %s\ngot\n %s", expectedRt.String(), rt.String()) + } +} + +func addRoute(t *testing.T, cidr string, gw string) { + command := exec.Command("sudo", "ip", "route", "add", cidr, "via", gw) + sout, err := command.CombinedOutput() + if err != nil { + t.Logf("assertion add Route error (should be ok): %s, error: %s", sout, err) + } else { + t.Logf("assertion - successfully added %s -> %s", cidr, gw) + } +} + +func cleanRoute(t *testing.T, cidr string) { + command := exec.Command("sudo", "ip", "route", "delete", cidr) + sout, err := command.CombinedOutput() + if err != nil { + t.Logf("integration test cleanup error (should be ok): %s, error: %s", sout, err) + } else { + t.Logf("integration test successfully cleaned %s", cidr) + } +} diff --git a/pkg/minikube/tunnel/route_test.go b/pkg/minikube/tunnel/route_test.go new file mode 100644 index 000000000000..2d8ee660f46b --- /dev/null +++ b/pkg/minikube/tunnel/route_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "net" + "reflect" + "testing" +) + +func TestRoutingTable(t *testing.T) { + tcs := []struct { + name string + table routingTable + route *Route + + exists bool + conflict string + overlaps []string + }{ + { + name: "doesn't exist, no complication", + table: routingTable{ + { + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + line: "line1", + }, + }, + route: unsafeParseRoute("127.0.0.1", "10.112.0.0/12"), + + exists: false, + conflict: "", + overlaps: []string{}, + }, + + { + name: "doesn't exist, and has overlap and a conflict", + table: routingTable{ + { + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + line: "conflicting line", + }, + { + route: unsafeParseRoute("127.0.0.1", "10.98.0.0/8"), + line: "overlap line1", + }, + { + route: unsafeParseRoute("127.0.0.1", "10.100.0.0/24"), + line: "overlap line2", + }, + { + route: unsafeParseRoute("127.0.0.1", "192.96.0.0/12"), + line: "no overlap", + }, + }, + route: unsafeParseRoute("192.168.1.1", "10.96.0.0/12"), + + exists: false, + conflict: "conflicting line", + overlaps: []string{ + "overlap line1", + "overlap line2", + }, + }, + + { + name: "exists, and has overlap and no conflict", + table: routingTable{ + { + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + line: "same", + }, + { + route: unsafeParseRoute("127.0.0.1", "10.98.0.0/8"), + line: "overlap line1", + }, + { + route: unsafeParseRoute("127.0.0.1", "10.100.0.0/24"), + line: "overlap line2", + }, + { + route: unsafeParseRoute("127.0.0.1", "192.96.0.0/12"), + line: "no overlap", + }, + }, + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + + exists: true, + conflict: "", + overlaps: []string{ + "overlap line1", + "overlap line2", + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + exists, conflict, overlaps := tc.table.Check(tc.route) + if tc.exists != exists || tc.conflict != conflict || !reflect.DeepEqual(tc.overlaps, overlaps) { + t.Errorf(`expected + exists: %v + conflict: %s + overlaps: %s +got + exists: %v + conflict: %s + overlaps: %s +`, tc.exists, tc.conflict, tc.overlaps, + exists, conflict, overlaps) + } + }) + } +} + +func unsafeParseRoute(gatewayIP string, destCIDR string) *Route { + ip := net.ParseIP(gatewayIP) + _, ipNet, _ := net.ParseCIDR(destCIDR) + + expectedRoute := &Route{ + Gateway: ip, + DestCIDR: ipNet, + } + return expectedRoute +} diff --git a/pkg/minikube/tunnel/route_windows.go b/pkg/minikube/tunnel/route_windows.go new file mode 100644 index 000000000000..bdd97a526bef --- /dev/null +++ b/pkg/minikube/tunnel/route_windows.go @@ -0,0 +1,142 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "net" + "os/exec" + "strings" + + "github.com/golang/glog" +) + +func (router *osRouter) EnsureRouteIsAdded(route *Route) error { + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if exists { + return nil + } + + serviceCIDR := route.DestCIDR.String() + destinationIP := route.DestCIDR.IP.String() + destinationMask := fmt.Sprintf("%d.%d.%d.%d", + route.DestCIDR.Mask[0], + route.DestCIDR.Mask[1], + route.DestCIDR.Mask[2], + route.DestCIDR.Mask[3]) + + gatewayIP := route.Gateway.String() + + glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP) + command := exec.Command("route", "ADD", destinationIP, "MASK", destinationMask, gatewayIP) + glog.Infof("About to run command: %s", command.Args) + stdInAndOut, err := command.CombinedOutput() + message := string(stdInAndOut) + if message != " OK!\r\n" { + return fmt.Errorf("error adding route: %s, %d", message, len(strings.Split(message, "\n"))) + } + glog.Infof("%s", stdInAndOut) + if err != nil { + glog.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n"))) + return err + } + return nil +} + +func (router *osRouter) parseTable(table []byte) routingTable { + t := routingTable{} + skip := true + for _, line := range strings.Split(string(table), "\n") { + //after first line of header we can start consuming + if strings.HasPrefix(line, "Network Destination") { + skip = false + continue + } + + fields := strings.Fields(line) + //don't care about the 0.0.0.0 routes + if skip || len(fields) == 0 || len(fields) > 0 && (fields[0] == "default" || fields[0] == "0.0.0.0") { + continue + } + if len(fields) > 2 { + dstCIDRIP := net.ParseIP(fields[0]) + dstCIDRMask := fields[1] + dstMaskIP := net.ParseIP(dstCIDRMask) + gatewayIP := net.ParseIP(fields[2]) + if dstCIDRIP == nil || dstMaskIP == nil || gatewayIP == nil { + glog.V(4).Infof("skipping line: can't parse all IPs from routing table: %s", line) + } else { + tableLine := routingTableLine{ + route: &Route{ + DestCIDR: &net.IPNet{ + IP: dstCIDRIP, + Mask: net.IPMask(dstMaskIP.To4()), + }, + Gateway: gatewayIP, + }, + line: line, + } + glog.V(4).Infof("adding line %s", tableLine) + t = append(t, tableLine) + } + } + } + + return t +} + +func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) { + command := exec.Command("route", "print", "-4") + stdInAndOut, err := command.CombinedOutput() + if err != nil { + err = fmt.Errorf("error running '%s': %s", command.Args, err) + return + } + rt := router.parseTable(stdInAndOut) + + exists, conflict, overlaps = rt.Check(route) + + return +} + +func (router *osRouter) Cleanup(route *Route) error { + exists, err := isValidToAddOrDelete(router, route) + if err != nil { + return err + } + if !exists { + return nil + } + serviceCIDR := route.DestCIDR.String() + gatewayIP := route.Gateway.String() + + glog.Infof("Cleaning up Route for CIDR %s to gateway %s\n", serviceCIDR, gatewayIP) + command := exec.Command("route", "delete", serviceCIDR) + stdInAndOut, err := command.CombinedOutput() + if err != nil { + return err + } + message := string(stdInAndOut) + glog.Infof("'%s'", message) + if message != " OK!\r\n" { + return fmt.Errorf("error deleting route: %s, %d", message, len(strings.Split(message, "\n"))) + } + return nil +} diff --git a/pkg/minikube/tunnel/route_windows_test.go b/pkg/minikube/tunnel/route_windows_test.go new file mode 100644 index 000000000000..41d16d1365c0 --- /dev/null +++ b/pkg/minikube/tunnel/route_windows_test.go @@ -0,0 +1,173 @@ +// +build windows,integration + +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "net" + "os/exec" + "testing" + + "reflect" + "strings" +) + +func TestWindowsRouteFailsOnConflictIntegrationTest(t *testing.T) { + route := &Route{ + Gateway: net.IPv4(1, 2, 3, 4), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + r := &osRouter{} + + cleanRoute(t, "10.96.0.0") + addRoute(t, "10.96.0.0", "255.240.0.0", "1.2.3.5") + err := r.EnsureRouteIsAdded(route) + if err == nil { + t.Errorf("add should have error, but it is nil") + } else if !strings.Contains(err.Error(), "conflict") { + t.Errorf("expected to fail with error containg `conflict`, but failed with wrong error %s", err) + } + cleanRoute(t, "10.96.0.0") +} + +func TestWindowsRouteIdempotentIntegrationTest(t *testing.T) { + route := &Route{ + Gateway: net.IPv4(1, 2, 3, 4), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + r := &osRouter{} + + cleanRoute(t, "10.96.0.0") + err := r.EnsureRouteIsAdded(route) + if err != nil { + t.Errorf("add error: %s", err) + } + + err = r.EnsureRouteIsAdded(route) + if err != nil { + t.Errorf("add error: %s", err) + } + + cleanRoute(t, "10.96.0.0") +} + +func TestWindowsRouteCleanupIdempontentIntegrationTest(t *testing.T) { + route := &Route{ + Gateway: net.IPv4(1, 2, 3, 4), + DestCIDR: &net.IPNet{ + IP: net.IPv4(10, 96, 0, 0), + Mask: net.IPv4Mask(255, 240, 0, 0), + }, + } + r := &osRouter{} + + cleanRoute(t, "10.96.0.0") + addRoute(t, "10.96.0.0", "255.240.0.0", "1.2.3.4") + err := r.Cleanup(route) + if err != nil { + t.Errorf("cleanup failed: %s", err) + } + err = r.Cleanup(route) + if err != nil { + t.Errorf("cleanup failed: %s", err) + } + cleanRoute(t, "10.96.0.0") +} + +func TestRouteTable(t *testing.T) { + const table = `=========================================================================== +Interface List + 14...00 1c 42 8f 70 58 ......Intel(R) 82574L Gigabit Network Connection + 6...0a 00 27 00 00 06 ......VirtualBox Host-Only Ethernet Adapter + 8...0a 00 27 00 00 08 ......VirtualBox Host-Only Ethernet Adapter #2 + 1...........................Software Loopback Interface 1 +=========================================================================== + +IPv4 Route Table +=========================================================================== +Active Routes: +Network Destination Netmask Gateway Interface Metric + 0.0.0.0 0.0.0.0 10.211.55.1 10.211.55.3 25 + 10.96.0.0 255.240.0.0 127.0.0.1 127.0.0.1 281 + 10.211.55.3 255.255.255.255 On-link 10.211.55.3 281 + 10.211.55.255 255.255.255.255 On-link 10.211.55.3 281 + 127.0.0.0 255.0.0.0 On-link 127.0.0.1 331 + 127.0.0.1 255.255.255.255 On-link 127.0.0.1 331 + 127.255.255.255 255.255.255.255 On-link 127.0.0.1 331 + 192.168.56.0 255.255.255.0 On-link 192.168.56.1 281 + 192.168.56.1 255.255.255.255 On-link 192.168.56.1 281 + 192.168.56.255 255.255.255.255 On-link 192.168.56.1 281 + 192.168.99.0 255.255.255.0 On-link 192.168.99.1 281 + 192.168.99.1 255.255.255.255 On-link 192.168.99.1 281 + 10.211.55.0 255.255.255.0 192.168.1.2 10.211.55.3 281 + 192.168.99.255 255.255.255.255 On-link 192.168.99.1 281 + 224.0.0.0 240.0.0.0 On-link 127.0.0.1 331 + 224.0.0.0 240.0.0.0 On-link 10.211.55.3 281 + 224.0.0.0 240.0.0.0 On-link 192.168.56.1 281 + 224.0.0.0 240.0.0.0 On-link 192.168.99.1 281 + 255.255.255.255 255.255.255.255 On-link 127.0.0.1 331 + 255.255.255.255 255.255.255.255 On-link 10.211.55.3 281 + 255.255.255.255 255.255.255.255 On-link 192.168.56.1 281 + 255.255.255.255 255.255.255.255 On-link 192.168.99.1 281 +=========================================================================== +Persistent Routes: + None` + + rt := (&osRouter{}).parseTable([]byte(table)) + + expectedRt := routingTable{ + routingTableLine{ + route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"), + line: " 10.96.0.0 255.240.0.0 127.0.0.1 127.0.0.1 281", + }, + routingTableLine{ + route: unsafeParseRoute("192.168.1.2", "10.211.55.0/24"), + line: " 10.211.55.0 255.255.255.0 192.168.1.2 10.211.55.3 281", + }, + } + if !reflect.DeepEqual(rt.String(), expectedRt.String()) { + t.Errorf("expected:\n %s\ngot\n %s", expectedRt.String(), rt.String()) + } + +} + +func addRoute(t *testing.T, dstIP string, dstMask string, gw string) { + command := exec.Command("route", "ADD", dstIP, "mask", dstMask, gw) + sout, err := command.CombinedOutput() + if err != nil { + t.Logf("assertion add Route error (should be ok): %s, error: %s", sout, err) + } else { + t.Logf("assertion - successfully added %s (%s) -> %s", dstIP, dstMask, gw) + } +} + +func cleanRoute(t *testing.T, dstIP string) { + command := exec.Command("route", "DELETE", dstIP) + sout, err := command.CombinedOutput() + if err != nil { + t.Logf("assertion cleanup error (should be ok): %s, error: %s", sout, err) + } else { + t.Logf("assertion - successfully cleaned %s", dstIP) + } +} diff --git a/pkg/minikube/tunnel/test_doubles.go b/pkg/minikube/tunnel/test_doubles.go new file mode 100644 index 000000000000..dd80aebf87ed --- /dev/null +++ b/pkg/minikube/tunnel/test_doubles.go @@ -0,0 +1,91 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/config" +) + +type recordingReporter struct { + statesRecorded []*Status +} + +func (r *recordingReporter) Report(tunnelState *Status) { + glog.V(4).Infof("recordingReporter.Report: %v", tunnelState) + r.statesRecorded = append(r.statesRecorded, tunnelState) +} + +//simulating idempotent router behavior +//without checking for conflicting routes +type fakeRouter struct { + rt routingTable + errorResponse error +} + +func (r *fakeRouter) EnsureRouteIsAdded(route *Route) error { + glog.V(4).Infof("fakerouter.EnsureRouteIsAdded %s", route) + if r.errorResponse == nil { + exists, err := isValidToAddOrDelete(r, route) + if err != nil { + return err + } + if !exists { + r.rt = append(r.rt, routingTableLine{ + route: route, + line: fmt.Sprintf("fake router line: %s", route), + }) + } + + } + return r.errorResponse +} +func (r *fakeRouter) Cleanup(route *Route) error { + glog.V(4).Infof("fake router cleanup: %v\n", route) + if r.errorResponse == nil { + exists, err := isValidToAddOrDelete(r, route) + if err != nil { + return err + } + if exists { + for i := range r.rt { + if r.rt[i].route.Equal(route) { + r.rt = append(r.rt[:i], r.rt[i+1:]...) + break + } + } + } + } + return r.errorResponse +} + +func (r *fakeRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) { + err = r.errorResponse + exists, conflict, overlaps = r.rt.Check(route) + return +} + +type stubConfigLoader struct { + c config.Config + e error +} + +func (l *stubConfigLoader) LoadConfigFromFile(profile string) (config.Config, error) { + return l.c, l.e +} diff --git a/pkg/minikube/tunnel/tunnel.go b/pkg/minikube/tunnel/tunnel.go new file mode 100644 index 000000000000..561d52d146d8 --- /dev/null +++ b/pkg/minikube/tunnel/tunnel.go @@ -0,0 +1,177 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "os" + + "github.com/docker/machine/libmachine" + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/minikube/pkg/minikube/config" +) + +//tunnel represents the basic API for a tunnel: periodically the state of the tunnel +//can be updated and when the tunnel is not needed, it can be cleaned up +//It was mostly introduced for testability. +type controller interface { + cleanup() *Status + update() *Status +} + +func errorTunnelAlreadyExists(id *ID) error { + return fmt.Errorf("there is already a running tunnel for this machine: %s", id) +} + +func newTunnel(machineName string, + machineAPI libmachine.API, + configLoader config.Loader, + v1Core v1.CoreV1Interface, registry *persistentRegistry, router router) (*tunnel, error) { + ci := &clusterInspector{ + machineName: machineName, + machineAPI: machineAPI, + configLoader: configLoader, + } + state, route, err := ci.getStateAndRoute() + + if err != nil { + return nil, fmt.Errorf("unable to determine cluster info: %s", err) + } + id := ID{ + Route: route, + MachineName: machineName, + Pid: getPid(), + } + runningTunnel, err := registry.IsAlreadyDefinedAndRunning(&id) + if err != nil { + return nil, fmt.Errorf("unable to check tunnel registry for conflict: %s", err) + } + if runningTunnel != nil { + return nil, fmt.Errorf("another tunnel is already running, shut it down first: %s", runningTunnel) + } + + return &tunnel{ + clusterInspector: ci, + router: router, + registry: registry, + loadBalancerEmulator: newLoadBalancerEmulator(v1Core), + status: &Status{ + TunnelID: id, + MinikubeState: state, + }, + reporter: &simpleReporter{ + out: os.Stdout, + }, + }, nil + +} + +type tunnel struct { + //collaborators + clusterInspector *clusterInspector + router router + loadBalancerEmulator loadBalancerEmulator + reporter reporter + registry *persistentRegistry + + status *Status +} + +func (t *tunnel) cleanup() *Status { + glog.V(3).Infof("cleaning up %s", t.status.TunnelID.Route) + err := t.router.Cleanup(t.status.TunnelID.Route) + if err != nil { + t.status.RouteError = errors.Errorf("error cleaning up route: %v", err) + glog.V(3).Infof(t.status.RouteError.Error()) + } else { + err = t.registry.Remove(t.status.TunnelID.Route) + if err != nil { + glog.V(3).Infof("error removing route from registry: %v", err) + } + } + if t.status.MinikubeState == Running { + t.status.PatchedServices, t.status.LoadBalancerEmulatorError = t.loadBalancerEmulator.Cleanup() + } + return t.status +} + +func (t *tunnel) update() *Status { + glog.V(3).Info("updating tunnel status...") + t.status.MinikubeState, _, t.status.MinikubeError = t.clusterInspector.getStateAndHost() + defer t.clusterInspector.machineAPI.Close() + if t.status.MinikubeState == Running { + glog.V(3).Infof("minikube is running, trying to add Route %s", t.status.TunnelID.Route) + setupRoute(t) + if t.status.RouteError == nil { + t.status.PatchedServices, t.status.LoadBalancerEmulatorError = t.loadBalancerEmulator.PatchServices() + } + } + glog.V(3).Infof("sending report %s", t.status) + t.reporter.Report(t.status.Clone()) + return t.status +} + +func setupRoute(t *tunnel) { + exists, conflict, _, err := t.router.Inspect(t.status.TunnelID.Route) + if err != nil { + t.status.RouteError = fmt.Errorf("error checking for route state: %s", err) + return + } + + if !exists && len(conflict) == 0 { + t.status.RouteError = t.router.EnsureRouteIsAdded(t.status.TunnelID.Route) + if t.status.RouteError == nil { + //the route was added successfully, we need to make sure the registry has it too + //this might fail in race conditions, when another process created this tunnel + if err := t.registry.Register(&t.status.TunnelID); err != nil { + glog.Errorf("failed to register tunnel: %s", err) + t.status.RouteError = err + } + } + return + } + + if len(conflict) > 0 { + t.status.RouteError = fmt.Errorf("conflicting route: %s", conflict) + return + } + + //the route exists, make sure that this process owns it in the registry + existingTunnel, err := t.registry.IsAlreadyDefinedAndRunning(&t.status.TunnelID) + if err != nil { + glog.Errorf("failed to check for other tunnels: %s", err) + t.status.RouteError = err + return + } + + if existingTunnel == nil { + //the route exists, but "orphaned", this process will "own it" in the registry + if err := t.registry.Register(&t.status.TunnelID); err != nil { + glog.Errorf("failed to register tunnel: %s", err) + t.status.RouteError = err + } + return + } + + if existingTunnel.Pid != getPid() { + //another running process owns the tunnel + t.status.RouteError = errorTunnelAlreadyExists(existingTunnel) + } + +} diff --git a/pkg/minikube/tunnel/tunnel_manager.go b/pkg/minikube/tunnel/tunnel_manager.go new file mode 100644 index 000000000000..24206331a0e5 --- /dev/null +++ b/pkg/minikube/tunnel/tunnel_manager.go @@ -0,0 +1,144 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "time" + + "context" + "fmt" + + "github.com/docker/machine/libmachine" + "github.com/golang/glog" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" +) + +// Manager can create, start and cleanup a tunnel +// It keeps track of created tunnels for multiple vms so that it can cleanup +// after unclean shutdowns. +type Manager struct { + delay time.Duration + registry *persistentRegistry + router router +} + +//stateCheckInterval defines how frequently the cluster and route states are checked +const stateCheckInterval = 5 * time.Second + +func NewManager() *Manager { + return &Manager{ + delay: stateCheckInterval, + registry: &persistentRegistry{ + path: constants.TunnelRegistryPath(), + }, + router: &osRouter{}, + } +} +func (mgr *Manager) StartTunnel(ctx context.Context, machineName string, machineAPI libmachine.API, configLoader config.Loader, v1Core v1.CoreV1Interface) (done chan bool, err error) { + tunnel, err := newTunnel(machineName, machineAPI, configLoader, v1Core, mgr.registry, mgr.router) + if err != nil { + return nil, fmt.Errorf("error creating tunnel: %s", err) + } + return mgr.startTunnel(ctx, tunnel) + +} +func (mgr *Manager) startTunnel(ctx context.Context, tunnel controller) (done chan bool, err error) { + glog.Info("Setting up tunnel...") + + ready := make(chan bool, 1) + check := make(chan bool, 1) + done = make(chan bool, 1) + + //simulating Ctrl+C so that we can cancel the tunnel programmatically too + go mgr.timerLoop(ready, check) + go mgr.run(ctx, tunnel, ready, check, done) + + glog.Info("Started minikube tunnel.") + return +} + +func (mgr *Manager) timerLoop(ready, check chan bool) { + for { + glog.V(4).Infof("waiting for tunnel to be ready for next check") + <-ready + glog.V(4).Infof("sleep for %s", mgr.delay) + time.Sleep(mgr.delay) + check <- true + } +} + +func (mgr *Manager) run(ctx context.Context, t controller, ready, check, done chan bool) { + defer func() { + done <- true + }() + ready <- true + for { + select { + case <-ctx.Done(): + mgr.cleanup(t) + return + case <-check: + glog.V(4).Info("check receieved") + select { + case <-ctx.Done(): + mgr.cleanup(t) + return + default: + } + status := t.update() + glog.V(4).Infof("minikube status: %s", status) + if status.MinikubeState != Running { + glog.Infof("minikube status: %s, cleaning up and quitting...", status.MinikubeState) + mgr.cleanup(t) + return + } + ready <- true + } + } +} + +func (mgr *Manager) cleanup(t controller) *Status { + return t.cleanup() +} + +func (mgr *Manager) CleanupNotRunningTunnels() error { + tunnels, err := mgr.registry.List() + if err != nil { + return fmt.Errorf("error listing tunnels from registry: %s", err) + } + + for _, tunnel := range tunnels { + isRunning, err := checkIfRunning(tunnel.Pid) + glog.Infof("%v is running: %t", tunnel, isRunning) + if err != nil { + return fmt.Errorf("error checking if tunnel is running: %s", err) + } + if !isRunning { + err = mgr.router.Cleanup(tunnel.Route) + if err != nil { + return err + } + err = mgr.registry.Remove(tunnel.Route) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/minikube/tunnel/tunnel_manager_test.go b/pkg/minikube/tunnel/tunnel_manager_test.go new file mode 100644 index 000000000000..8655e9c1d39a --- /dev/null +++ b/pkg/minikube/tunnel/tunnel_manager_test.go @@ -0,0 +1,282 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "testing" + + "context" + "os" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" +) + +func TestTunnelManagerEventHandling(t *testing.T) { + tcs := []struct { + //tunnel inputs + name string + repeat int + test func(tunnel *tunnelStub, cancel context.CancelFunc, ready, check, done chan bool) error + }{ + { + name: "tunnel quits on stopped minikube", + repeat: 1, + test: func(tunnel *tunnelStub, cancel context.CancelFunc, ready, check, done chan bool) error { + tunnel.mockClusterInfo = &Status{ + MinikubeState: Stopped, + } + glog.Info("waiting for tunnel to be ready.") + <-ready + glog.Info("check!") + check <- true + glog.Info("check done.") + select { + case <-done: + glog.Info("it's done, yay!") + case <-time.After(1 * time.Second): + t.Error("tunnel did not stop on stopped minikube") + } + if tunnel.tunnelExists { + t.Error("tunnel should not have been created") + } + return nil + }, + }, + + { + name: "tunnel quits on ctrlc before doing a check", + repeat: 1, + test: func(tunnel *tunnelStub, cancel context.CancelFunc, ready, check, done chan bool) error { + tunnel.mockClusterInfo = &Status{ + MinikubeState: Stopped, + } + <-ready + cancel() + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Error("tunnel did not stop on ctrl c") + } + + if tunnel.tunnelExists { + t.Error("tunnel should not have been created") + } + return nil + }, + }, + { + name: "tunnel always quits when ctrl c is pressed", + repeat: 100000, + test: func(tunnel *tunnelStub, cancel context.CancelFunc, ready, check, done chan bool) error { + tunnel.mockClusterInfo = &Status{ + MinikubeState: Running, + } + go func() { + <-ready + check <- true + <-ready + check <- true + <-ready + cancel() + check <- true + + }() + + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Error("tunnel did not stop on ctrl c") + return errors.New("error") + } + + if tunnel.tunnelExists { + t.Error("tunnel should not have been created") + return errors.New("error") + } + + if tunnel.timesChecked != 2 { + t.Errorf("expected to have 2 tunnel checks, got %d", tunnel.timesChecked) + return errors.New("error") + } + return nil + }, + }, + } + + //t.Parallel() + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + var err error + for i := 1; i <= tc.repeat && err == nil; i++ { + tunnelManager := &Manager{} + tunnel := &tunnelStub{} + + ready := make(chan bool, 1) + check := make(chan bool, 1) + done := make(chan bool, 1) + + ctx, cancel := context.WithCancel(context.Background()) + go tunnelManager.run(ctx, tunnel, ready, check, done) + err = tc.test(tunnel, cancel, ready, check, done) + if err != nil { + glog.Errorf("error at %d", i) + } + } + }) + + } +} + +func TestTunnelManagerDelayAndContext(t *testing.T) { + tunnelManager := &Manager{ + delay: 1000 * time.Millisecond, + } + tunnel := &tunnelStub{ + mockClusterInfo: &Status{ + MinikubeState: Running, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + done, err := tunnelManager.startTunnel(ctx, tunnel) + if err != nil { + t.Errorf("creating tunnel failed: %s", err) + } + time.Sleep(1100 * time.Millisecond) + cancel() + <-done + + if tunnel.timesChecked < 1 { + t.Errorf("tunnel check did not run at all") + } + + if tunnel.timesChecked > 2 { + t.Errorf("tunnel check ran too many times %d", tunnel.timesChecked) + } +} + +func TestTunnelManagerCleanup(t *testing.T) { + reg, cleanup := createTestRegistry(t) + defer cleanup() + + runningTunnel1 := &ID{ + Route: unsafeParseRoute("1.2.3.4", "5.6.7.8/9"), + Pid: os.Getpid(), + MachineName: "minikube", + } + + runningTunnel2 := &ID{ + Route: unsafeParseRoute("100.2.3.4", "200.6.7.8/9"), + Pid: os.Getpid(), + MachineName: "minikube", + } + + notRunningTunnel1 := &ID{ + Route: unsafeParseRoute("200.2.3.4", "10.6.7.8/9"), + Pid: 12341234, + MachineName: "minikube", + } + + notRunningTunnel2 := &ID{ + Route: unsafeParseRoute("250.2.3.4", "20.6.7.8/9"), + Pid: 12341234, + MachineName: "minikube", + } + err := reg.Register(runningTunnel1) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = reg.Register(runningTunnel2) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = reg.Register(notRunningTunnel1) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = reg.Register(notRunningTunnel2) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + + router := &fakeRouter{} + + err = router.EnsureRouteIsAdded(runningTunnel1.Route) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = router.EnsureRouteIsAdded(runningTunnel2.Route) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = router.EnsureRouteIsAdded(notRunningTunnel1.Route) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + err = router.EnsureRouteIsAdded(notRunningTunnel2.Route) + if err != nil { + t.Errorf("expected no error got: %v", err) + } + + manager := NewManager() + manager.router = router + manager.registry = reg + + err = manager.CleanupNotRunningTunnels() + + if err != nil { + t.Errorf("expected no error got: %v", err) + } + + if len(router.rt) != 2 || + !router.rt[0].route.Equal(runningTunnel1.Route) || + !router.rt[1].route.Equal(runningTunnel2.Route) { + t.Errorf("routes are not cleaned up, expected only running tunnels to stay, got: %s", router.rt.String()) + } + + tunnels, err := reg.List() + + if err != nil { + t.Errorf("expected no error got: %v", err) + } + + if len(tunnels) != 2 || + !tunnels[0].Equal(runningTunnel1) || + !tunnels[1].Equal(runningTunnel2) { + t.Errorf("tunnels are not cleaned up properly, expected only running tunnels to stay, got: %v", tunnels) + } + +} + +type tunnelStub struct { + mockClusterInfo *Status + tunnelExists bool + timesChecked int +} + +func (t *tunnelStub) update() *Status { + t.tunnelExists = t.mockClusterInfo.MinikubeState == Running + t.timesChecked++ + return t.mockClusterInfo +} + +func (t *tunnelStub) cleanup() *Status { + t.tunnelExists = false + return t.mockClusterInfo +} diff --git a/pkg/minikube/tunnel/tunnel_test.go b/pkg/minikube/tunnel/tunnel_test.go new file mode 100644 index 000000000000..9d0e8eb35b65 --- /dev/null +++ b/pkg/minikube/tunnel/tunnel_test.go @@ -0,0 +1,470 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "errors" + + "github.com/docker/machine/libmachine/host" + "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/tests" + + "fmt" + "io/ioutil" + "os" + "reflect" + "strings" + "testing" +) + +func TestTunnel(t *testing.T) { + const RunningPid1 = 1234 + const RunningPid2 = 1235 + const NotRunningPid = 1236 + const NotRunningPid2 = 1237 + + mockPidChecker := func(pid int) (bool, error) { + if pid == NotRunningPid || pid == NotRunningPid2 { + return false, nil + } else if pid == RunningPid1 || pid == RunningPid2 { + return true, nil + } + return false, fmt.Errorf("fake pid checker does not recognize %d", pid) + } + + testCases := []struct { + name string + machineState state.State + serviceCIDR string + machineIP string + configLoaderError error + mockPidHandling bool + call func(tunnel *tunnel) *Status + assertion func(*testing.T, *Status, []*Status, []*Route, []*ID) + }{ + { + name: "simple stopped", + machineState: state.Stopped, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + call: func(tunnel *tunnel) *Status { + return tunnel.update() + }, + assertion: func(t *testing.T, returnedState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedState := &Status{ + MinikubeState: Stopped, + MinikubeError: nil, + TunnelID: ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: os.Getpid(), + }, + } + + if !reflect.DeepEqual(expectedState, returnedState) { + t.Errorf("wrong tunnel status.\nexpected %s\ngot: %s", expectedState, returnedState) + } + + if len(routes) > 0 { + t.Errorf("expected empty routes\n got: %s", routes) + } + expectedReports := []*Status{returnedState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports. expected %s\n got: %s", expectedReports, reportedStates) + } + if len(registeredTunnels) != 0 { + t.Errorf("registry mismatch.\nexpected []\ngot %+v", registeredTunnels) + } + }, + }, + { + name: "tunnel cleanup (ctrl+c before check)", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + call: func(tunnel *tunnel) *Status { + return tunnel.cleanup() + }, + assertion: func(t *testing.T, returnedState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: os.Getpid(), + }, + } + + if !reflect.DeepEqual(expectedState, returnedState) { + t.Errorf("wrong tunnel status.\nexpected %s\ngot: %s", expectedState, returnedState) + } + + if len(routes) > 0 { + t.Errorf("expected empty routes\n got: %s", routes) + } + + if len(reportedStates) > 0 { + t.Errorf("wrong reports. expected no reports, got: %s", reportedStates) + } + + if len(registeredTunnels) > 0 { + t.Errorf("registry mismatch.\nexpected []\ngot %+v", registeredTunnels) + } + }, + }, + + { + name: "tunnel create Route", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + call: func(tunnel *tunnel) *Status { + return tunnel.update() + }, + assertion: func(t *testing.T, returnedState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedRoute := unsafeParseRoute("1.2.3.4", "1.2.3.4/5") + expectedState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: expectedRoute, + MachineName: "testmachine", + Pid: os.Getpid(), + }, + } + + if !reflect.DeepEqual(expectedState, returnedState) { + t.Errorf("wrong tunnel status. expected %s\n got: %s", expectedState, returnedState) + } + + expectedRoutes := []*Route{expectedRoute} + if !reflect.DeepEqual(routes, expectedRoutes) { + t.Errorf("expected %s routes\n got: %s", expectedRoutes, routes) + } + expectedReports := []*Status{returnedState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports. expected %s\n got: %s", expectedReports, reportedStates) + } + + if len(registeredTunnels) != 1 || !registeredTunnels[0].Equal(&expectedState.TunnelID) { + t.Errorf("registry mismatch.\nexpected [%+v]\ngot %+v", &expectedState.TunnelID, registeredTunnels) + } + }, + }, + { + name: "tunnel cleanup error after 1 successful addRoute", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + call: func(tunnel *tunnel) *Status { + tunnel.update() + tunnel.router.(*fakeRouter).errorResponse = errors.New("testerror") + return tunnel.cleanup() + }, + assertion: func(t *testing.T, actualSecondState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedRoute := unsafeParseRoute("1.2.3.4", "1.2.3.4/5") + expectedFirstState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: expectedRoute, + MachineName: "testmachine", + Pid: os.Getpid(), + }, + } + + if actualSecondState.MinikubeState != Running { + t.Errorf("wrong minikube status.\nexpected Running\ngot: %s", actualSecondState.MinikubeState) + } + + substring := "testerror" + if actualSecondState.RouteError == nil || !strings.Contains(actualSecondState.RouteError.Error(), substring) { + t.Errorf("wrong tunnel status. expected Route error to contain '%s' \ngot: %s", substring, actualSecondState.RouteError) + } + + expectedRoutes := []*Route{expectedRoute} + if !reflect.DeepEqual(routes, expectedRoutes) { + t.Errorf("expected %s routes\n got: %s", expectedRoutes, routes) + } + + expectedReports := []*Status{expectedFirstState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports.\nexpected %v\n\ngot: %v", expectedReports, reportedStates) + } + if len(registeredTunnels) != 1 || !registeredTunnels[0].Equal(&expectedFirstState.TunnelID) { + t.Errorf("registry mismatch.\nexpected [%+v]\ngot %+v", &expectedFirstState.TunnelID, registeredTunnels) + } + }, + }, + { + name: "tunnel cleanup", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + call: func(tunnel *tunnel) *Status { + tunnel.update() + return tunnel.cleanup() + }, + assertion: func(t *testing.T, actualSecondState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedRoute := unsafeParseRoute("1.2.3.4", "1.2.3.4/5") + expectedFirstState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: expectedRoute, + MachineName: "testmachine", + Pid: os.Getpid(), + }, + } + + if !reflect.DeepEqual(expectedFirstState, actualSecondState) { + t.Errorf("wrong tunnel status.\nexpected %s\ngot: %s", expectedFirstState, actualSecondState) + } + + if len(routes) > 0 { + t.Errorf("expected empty routes\n got: %s", routes) + } + + expectedReports := []*Status{expectedFirstState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports.\nexpected %v\n\ngot: %v", expectedReports, reportedStates) + } + if len(registeredTunnels) > 0 { + t.Errorf("registry mismatch.\nexpected []\ngot %+v", registeredTunnels) + } + }, + }, + { + name: "race condition: other tunnel registers while in between routing and registration", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + mockPidHandling: true, + call: func(tunnel *tunnel) *Status { + + err := tunnel.registry.Register(&ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: RunningPid2, + }) + if err != nil { + t.Errorf("error registering tunnel: %s", err) + } + tunnel.update() + return tunnel.cleanup() + }, + assertion: func(t *testing.T, actualSecondState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedRoute := unsafeParseRoute("1.2.3.4", "1.2.3.4/5") + expectedFirstState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: expectedRoute, + MachineName: "testmachine", + Pid: RunningPid1, + }, + RouteError: errorTunnelAlreadyExists(&ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: RunningPid2, + }), + } + + if !reflect.DeepEqual(expectedFirstState, actualSecondState) { + t.Errorf("wrong tunnel status.\nexpected %s\ngot: %s", expectedFirstState, actualSecondState) + } + + if len(routes) > 0 { + t.Errorf("expected empty routes\n got: %s", routes) + } + + expectedReports := []*Status{expectedFirstState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports.\nexpected %v\n\ngot: %v", expectedReports, reportedStates) + } + if len(registeredTunnels) > 0 { + t.Errorf("registry mismatch.\nexpected []\ngot %+v", registeredTunnels) + } + }, + }, + { + name: "race condition: other tunnel registers and creates the same route first", + machineState: state.Running, + serviceCIDR: "1.2.3.4/5", + machineIP: "1.2.3.4", + mockPidHandling: true, + call: func(tunnel *tunnel) *Status { + + err := tunnel.registry.Register(&ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: RunningPid2, + }) + if err != nil { + t.Errorf("error registering tunnel: %s", err) + } + tunnel.router.(*fakeRouter).rt = append(tunnel.router.(*fakeRouter).rt, routingTableLine{ + route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + line: "", + }) + tunnel.update() + return tunnel.cleanup() + }, + assertion: func(t *testing.T, actualSecondState *Status, reportedStates []*Status, routes []*Route, registeredTunnels []*ID) { + expectedRoute := unsafeParseRoute("1.2.3.4", "1.2.3.4/5") + expectedFirstState := &Status{ + MinikubeState: Running, + MinikubeError: nil, + TunnelID: ID{ + Route: expectedRoute, + MachineName: "testmachine", + Pid: RunningPid1, + }, + RouteError: errorTunnelAlreadyExists(&ID{ + Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5"), + MachineName: "testmachine", + Pid: RunningPid2, + }), + } + + if !reflect.DeepEqual(expectedFirstState, actualSecondState) { + t.Errorf("wrong tunnel status.\nexpected %s\ngot: %s", expectedFirstState, actualSecondState) + } + + if len(routes) > 0 { + t.Errorf("expected empty routes\n got: %s", routes) + } + + expectedReports := []*Status{expectedFirstState} + + if !reflect.DeepEqual(reportedStates, expectedReports) { + t.Errorf("wrong reports.\nexpected %v\n\ngot: %v", expectedReports, reportedStates) + } + if len(registeredTunnels) > 0 { + t.Errorf("registry mismatch.\nexpected []\ngot %+v", registeredTunnels) + } + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.mockPidHandling { + origPidChecker := checkIfRunning + checkIfRunning = mockPidChecker + defer func() { checkIfRunning = origPidChecker }() + + origPidGetter := getPid + getPid = func() int { + return RunningPid1 + } + defer func() { getPid = origPidGetter }() + } + machineName := "testmachine" + machineAPI := &tests.MockAPI{ + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + machineName: { + Driver: &tests.MockDriver{ + CurrentState: tc.machineState, + IP: tc.machineIP, + }, + }, + }, + }, + } + configLoader := &stubConfigLoader{ + c: config.Config{ + KubernetesConfig: config.KubernetesConfig{ + ServiceCIDR: tc.serviceCIDR, + }}, + e: tc.configLoaderError, + } + + registry, cleanup := createTestRegistry(t) + defer cleanup() + + tunnel, err := newTunnel(machineName, machineAPI, configLoader, newStubCoreClient(nil), registry, &fakeRouter{}) + if err != nil { + t.Errorf("error creating tunnel: %s", err) + return + } + reporter := &recordingReporter{} + tunnel.reporter = reporter + + returnedState := tc.call(tunnel) + tunnels, err := registry.List() + if err != nil { + t.Errorf("error querying registry %s", err) + return + } + var routes []*Route + for _, r := range tunnel.router.(*fakeRouter).rt { + routes = append(routes, r.route) + } + tc.assertion(t, returnedState, reporter.statesRecorded, routes, tunnels) + + }) + } + +} + +func TestErrorCreatingTunnel(t *testing.T) { + machineName := "testmachine" + store := &tests.MockAPI{ + FakeStore: tests.FakeStore{ + Hosts: map[string]*host.Host{ + machineName: { + Driver: &tests.MockDriver{ + CurrentState: state.Stopped, + IP: "1.2.3.5", + }, + }, + }, + }, + } + + configLoader := &stubConfigLoader{ + c: config.Config{ + KubernetesConfig: config.KubernetesConfig{ + ServiceCIDR: "10.96.0.0/12", + }}, + e: errors.New("error loading machine"), + } + + f, err := ioutil.TempFile(os.TempDir(), "reg_") + f.Close() + if err != nil { + t.Errorf("failed to create temp file %s", err) + } + defer os.Remove(f.Name()) + registry := &persistentRegistry{ + path: f.Name(), + } + + _, err = newTunnel(machineName, store, configLoader, newStubCoreClient(nil), registry, &fakeRouter{}) + if err == nil || !strings.Contains(err.Error(), "error loading machine") { + t.Errorf("expected error containing 'error loading machine', got %s", err) + } +} diff --git a/pkg/minikube/tunnel/types.go b/pkg/minikube/tunnel/types.go new file mode 100644 index 000000000000..a8540782e8f3 --- /dev/null +++ b/pkg/minikube/tunnel/types.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tunnel + +import ( + "fmt" + "net" + + "k8s.io/apimachinery/pkg/types" +) + +type Status struct { + TunnelID ID + + MinikubeState HostState + MinikubeError error + + RouteError error + + PatchedServices []string + LoadBalancerEmulatorError error +} + +func (t *Status) Clone() *Status { + return &Status{ + TunnelID: t.TunnelID, + MinikubeState: t.MinikubeState, + MinikubeError: t.MinikubeError, + RouteError: t.RouteError, + PatchedServices: t.PatchedServices, + LoadBalancerEmulatorError: t.LoadBalancerEmulatorError, + } +} + +func (t *Status) String() string { + return fmt.Sprintf("id(%v), minikube(%s, e:%s), route(%s, e:%s), services(%s, e:%s)", + t.TunnelID, + t.MinikubeState, + t.MinikubeError, + t.TunnelID.Route, + t.RouteError, + t.PatchedServices, + t.LoadBalancerEmulatorError) +} + +type Route struct { + Gateway net.IP + DestCIDR *net.IPNet +} + +func (r *Route) String() string { + return fmt.Sprintf("%s -> %s", r.DestCIDR.String(), r.Gateway.String()) +} + +func (r *Route) Equal(other *Route) bool { + return other != nil && r.DestCIDR.IP.Equal(other.DestCIDR.IP) && + r.DestCIDR.Mask.String() == other.DestCIDR.Mask.String() && + r.Gateway.Equal(other.Gateway) +} + +type Patch struct { + Type types.PatchType + NameSpace string + NameSpaceSet bool + Resource string + Subresource string + ResourceName string + BodyContent string +} + +// State represents the status of a host +type HostState int + +const ( + Unknown HostState = iota + Running + Stopped +) + +var states = []string{ + "Unknown", + "Running", + "Stopped", +} + +func (h HostState) String() string { + return states[h] +} diff --git a/test.sh b/test.sh index 9c01eeafdd4d..cd69ebef5cfc 100755 --- a/test.sh +++ b/test.sh @@ -43,7 +43,7 @@ done rm out/$COV_TMP_FILE # Ignore these paths in the following tests. -ignore="vendor\|\_gopath\|assets.go\|out" +ignore="vendor\|\_gopath\|assets.go\|out\/" # Check gofmt echo "Checking gofmt..." diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 0fe7c29fe2a0..2fa3a439c1cf 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -36,6 +36,7 @@ func TestFunctional(t *testing.T) { t.Run("Dashboard", testDashboard) t.Run("ServicesList", testServicesList) t.Run("Provisioning", testProvisioning) + t.Run("Tunnel", testTunnel) if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") { t.Run("EnvVars", testClusterEnv) diff --git a/test/integration/testdata/testsvc.yaml b/test/integration/testdata/testsvc.yaml new file mode 100644 index 000000000000..6bfb64d611f8 --- /dev/null +++ b/test/integration/testdata/testsvc.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + run: nginx-svc + name: nginx-svc + namespace: default +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: nginx-svc + name: nginx-svc + namespace: default +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: nginx-svc + sessionAffinity: None + type: LoadBalancer diff --git a/test/integration/tunnel_test.go b/test/integration/tunnel_test.go new file mode 100644 index 000000000000..864207728390 --- /dev/null +++ b/test/integration/tunnel_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "io/ioutil" + "net/http" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/minikube/pkg/minikube/tunnel" + commonutil "k8s.io/minikube/pkg/util" + "k8s.io/minikube/test/integration/util" +) + +func testTunnel(t *testing.T) { + t.Log("starting tunnel test...") + runner := NewMinikubeRunner(t) + go func() { + output := runner.RunCommand("tunnel --alsologtostderr -v 8", true) + fmt.Println(output) + }() + + err := tunnel.NewManager().CleanupNotRunningTunnels() + + if err != nil { + t.Fatal(errors.Wrap(err, "cleaning up tunnels")) + } + + kubectlRunner := util.NewKubectlRunner(t) + + t.Log("deploying nginx...") + podPath, _ := filepath.Abs("testdata/testsvc.yaml") + if _, err := kubectlRunner.RunCommand([]string{"apply", "-f", podPath}); err != nil { + t.Fatalf("creating nginx ingress resource: %s", err) + } + + client, err := commonutil.GetClient() + + if err != nil { + t.Fatal(errors.Wrap(err, "getting kubernetes client")) + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + t.Fatal(errors.Wrap(err, "waiting for nginx pods")) + } + + if err := commonutil.WaitForService(client, "default", "nginx-svc", true, time.Millisecond*500, time.Minute*10); err != nil { + t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up")) + } + + t.Log("getting nginx ingress...") + + nginxIP := "" + + for i := 1; i < 3 && len(nginxIP) == 0; i++ { + stdout, err := kubectlRunner.RunCommand([]string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"}) + + if err != nil { + t.Fatalf("error listing nginx service: %s", err) + } + nginxIP = string(stdout) + time.Sleep(1 * time.Second) + } + + if len(nginxIP) == 0 { + t.Fatal("svc should have ingress after tunnel is created, but it was empty!") + } + + httpClient := http.DefaultClient + httpClient.Timeout = 1 * time.Second + resp, err := httpClient.Get(fmt.Sprintf("http://%s", nginxIP)) + + if err != nil { + t.Fatalf("error reading from nginx at address(%s): %s", nginxIP, err) + } + + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil || len(body) == 0 { + t.Fatalf("error reading body from nginx at address(%s): error: %s, len bytes read: %d", nginxIP, err, len(body)) + } + + responseBody := string(body) + if !strings.Contains(responseBody, "Welcome to nginx!") { + t.Fatalf("response body doesn't seem like an nginx response:\n%s", responseBody) + } +}