From 5811610697bb274ed68abc4cdde4919eef3835b5 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Sat, 17 Aug 2019 19:05:54 -0700
Subject: [PATCH 1/7] refactor util/kuberentes and ExtraOptions

---
 cmd/minikube/cmd/start.go                     |   7 +-
 pkg/{util/kubernetes.go => kube/kube.go}      | 125 +-----------------
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |  21 +--
 .../bootstrapper/kubeadm/kubeadm_test.go      |  23 ++--
 pkg/minikube/bootstrapper/kubeadm/versions.go |  79 +++--------
 .../bootstrapper/kubeadm/versions_test.go     |   2 +-
 .../config}/extra_options.go                  |  28 +++-
 .../config}/extra_options_test.go             |   2 +-
 pkg/minikube/config/types.go                  |  28 +++-
 pkg/util/utils.go                             |  11 --
 test/integration/containerd_test.go           |  10 +-
 test/integration/fn_addons.go                 |  27 ++--
 test/integration/fn_cluster_dns.go            |   4 +-
 test/integration/fn_mount.go                  |   6 +-
 test/integration/fn_pv.go                     |   6 +-
 test/integration/fn_tunnel.go                 |  12 +-
 test/integration/util/common.go               |   6 +-
 17 files changed, 142 insertions(+), 255 deletions(-)
 rename pkg/{util/kubernetes.go => kube/kube.go} (63%)
 rename pkg/{util => minikube/config}/extra_options.go (80%)
 rename pkg/{util => minikube/config}/extra_options_test.go (99%)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index ffddbccdeb2a..742f329619ce 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -46,6 +46,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
 	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -111,7 +112,7 @@ var (
 	insecureRegistry []string
 	apiServerNames   []string
 	apiServerIPs     []net.IP
-	extraOptions     pkgutil.ExtraOptionSlice
+	extraOptions     config.ExtraOptionSlice
 )
 
 func init() {
@@ -538,8 +539,8 @@ func validateConfig() {
 
 	// check that kubeadm extra args contain only whitelisted parameters
 	for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
-		if !pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
-			!pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
+		if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
+			!cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
 		}
 	}
diff --git a/pkg/util/kubernetes.go b/pkg/kube/kube.go
similarity index 63%
rename from pkg/util/kubernetes.go
rename to pkg/kube/kube.go
index 259149b5ba02..5b859bb6f916 100644
--- a/pkg/util/kubernetes.go
+++ b/pkg/kube/kube.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package kube
 
 import (
 	"context"
@@ -29,12 +29,10 @@ import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
 	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -45,33 +43,11 @@ var (
 	// ReasonableMutateTime is how long to wait for basic object mutations, such as deletions, to show up
 	ReasonableMutateTime = time.Minute * 2
 	// ReasonableStartTime is how long to wait for pods to start
-	ReasonableStartTime = time.Minute * 5
+	ReasonableStartTime = time.Minute * 6
 )
 
-// PodStore stores pods
-type PodStore struct {
-	cache.Store
-	stopCh    chan struct{}
-	Reflector *cache.Reflector
-}
-
-// List lists the pods
-func (s *PodStore) List() []*core.Pod {
-	objects := s.Store.List()
-	pods := make([]*core.Pod, 0)
-	for _, o := range objects {
-		pods = append(pods, o.(*core.Pod))
-	}
-	return pods
-}
-
-// Stop stops the pods
-func (s *PodStore) Stop() {
-	close(s.stopCh)
-}
-
-// GetClient gets the client from config
-func GetClient(kubectlContext ...string) (kubernetes.Interface, error) {
+// Client gets the kuberentes client from default kubeconfig
+func Client(kubectlContext ...string) (kubernetes.Interface, error) {
 	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
 	configOverrides := &clientcmd.ConfigOverrides{}
 	if kubectlContext != nil {
@@ -92,41 +68,6 @@ func GetClient(kubectlContext ...string) (kubernetes.Interface, error) {
 	return client, nil
 }
 
-// NewPodStore creates a new PodStore
-func NewPodStore(c kubernetes.Interface, namespace string, label fmt.Stringer, field fmt.Stringer) *PodStore {
-	lw := &cache.ListWatch{
-		ListFunc: func(options meta.ListOptions) (runtime.Object, error) {
-			options.LabelSelector = label.String()
-			options.FieldSelector = field.String()
-			obj, err := c.CoreV1().Pods(namespace).List(options)
-			return runtime.Object(obj), err
-		},
-		WatchFunc: func(options meta.ListOptions) (watch.Interface, error) {
-			options.LabelSelector = label.String()
-			options.FieldSelector = field.String()
-			return c.CoreV1().Pods(namespace).Watch(options)
-		},
-	}
-	store := cache.NewStore(cache.MetaNamespaceKeyFunc)
-	stopCh := make(chan struct{})
-	reflector := cache.NewReflector(lw, &core.Pod{}, store, 0)
-	go reflector.Run(stopCh)
-	return &PodStore{Store: store, stopCh: stopCh, Reflector: reflector}
-}
-
-// StartPods starts all pods
-func StartPods(c kubernetes.Interface, namespace string, pod core.Pod, waitForRunning bool) error {
-	pod.ObjectMeta.Labels["name"] = pod.Name
-	if waitForRunning {
-		label := labels.SelectorFromSet(labels.Set(map[string]string{"name": pod.Name}))
-		err := WaitForPodsWithLabelRunning(c, namespace, label)
-		if err != nil {
-			return fmt.Errorf("error waiting for pod %s to be running: %v", pod.Name, err)
-		}
-	}
-	return nil
-}
-
 // WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
 func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
 	glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
@@ -135,7 +76,7 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 		listOpts := meta.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
-			glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err)
+			glog.Errorf("error getting Pods with label selector %q [%v]\n", label.String(), err)
 			return false, nil
 		}
 
@@ -158,36 +99,6 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 	})
 }
 
-// WaitForPodDelete waits for a pod to be deleted
-func WaitForPodDelete(c kubernetes.Interface, ns string, label fmt.Stringer) error {
-	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
-		listOpts := meta.ListOptions{LabelSelector: label.String()}
-		pods, err := c.CoreV1().Pods(ns).List(listOpts)
-		if err != nil {
-			glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err)
-			return false, nil
-		}
-		return len(pods.Items) == 0, nil
-	})
-}
-
-// WaitForEvent waits for the given event to appear
-func WaitForEvent(c kubernetes.Interface, ns string, reason string) error {
-	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
-		events, err := c.EventsV1beta1().Events("default").List(meta.ListOptions{})
-		if err != nil {
-			glog.Infof("error getting events: %v", err)
-			return false, nil
-		}
-		for _, e := range events.Items {
-			if e.Reason == reason {
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-}
-
 // WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status.
 func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
 	options := meta.ListOptions{FieldSelector: fields.Set{
@@ -281,32 +192,6 @@ func WaitForService(c kubernetes.Interface, namespace, name string, exist bool,
 	return nil
 }
 
-// WaitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum.
-func WaitForServiceEndpointsNum(c kubernetes.Interface, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error {
-	return wait.Poll(interval, timeout, func() (bool, error) {
-		glog.Infof("Waiting for amount of service:%s endpoints to be %d", serviceName, expectNum)
-		list, err := c.CoreV1().Endpoints(namespace).List(meta.ListOptions{})
-		if err != nil {
-			return false, err
-		}
-
-		for _, e := range list.Items {
-			if e.Name == serviceName && countEndpointsNum(&e) == expectNum {
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-}
-
-func countEndpointsNum(e *core.Endpoints) int {
-	num := 0
-	for _, sub := range e.Subsets {
-		num += len(sub.Addresses)
-	}
-	return num
-}
-
 // IsRetryableAPIError returns if this error is retryable or not
 func IsRetryableAPIError(err error) bool {
 	return apierr.IsTimeout(err) || apierr.IsServerTimeout(err) || apierr.IsTooManyRequests(err) || apierr.IsInternalError(err)
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index a22057324202..7cf462470a5d 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -34,6 +34,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -178,13 +179,13 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string
 }
 
 // createFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the commad linne
-func createFlagsFromExtraArgs(extraOptions util.ExtraOptionSlice) string {
+func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string {
 	kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm)
 
 	// kubeadm allows only a small set of parameters to be supplied from the command line when the --config param
 	// is specified, here we remove those that are not allowed
 	for opt := range kubeadmExtraOpts {
-		if !util.ContainsString(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
+		if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
 			// kubeadmExtraOpts is a copy so safe to delete
 			delete(kubeadmExtraOpts, opt)
 		}
@@ -194,7 +195,7 @@ func createFlagsFromExtraArgs(extraOptions util.ExtraOptionSlice) string {
 
 // StartCluster starts the cluster
 func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
-	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -309,7 +310,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
 	// up. Otherwise, minikube won't start, as "k8s-app" pods are not ready.
 	componentsOnly := k8s.NetworkPlugin == "cni"
 	out.T(out.WaitingPods, "Waiting for:")
-	client, err := util.GetClient()
+	client, err := kube.Client()
 	if err != nil {
 		return errors.Wrap(err, "k8s client")
 	}
@@ -328,7 +329,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
 
 		out.String(" %s", p.name)
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{p.key: p.value}))
-		if err := util.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("waiting for %s=%s", p.key, p.value))
 		}
 	}
@@ -338,7 +339,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
 
 // RestartCluster restarts the Kubernetes cluster configured by kubeadm
 func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
-	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -409,7 +410,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
 
 // PullImages downloads images that will be used by RestartCluster
 func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
-	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -432,7 +433,7 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error {
 // NewKubeletConfig generates a new systemd unit containing a configured kubelet
 // based on the options present in the KubernetesConfig.
 func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) {
-	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return "", errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -532,7 +533,7 @@ sudo systemctl start kubelet
 }
 
 // createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm
-func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) {
+func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) {
 	extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs)
 	if err != nil {
 		return nil, err
@@ -551,7 +552,7 @@ func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semv
 }
 
 func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) {
-	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return "", errors.Wrap(err, "parsing kubernetes version")
 	}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go
index 5e6389f993ea..04f8f823fe2a 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go
@@ -26,7 +26,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
-	"k8s.io/minikube/pkg/util"
 )
 
 func TestGenerateKubeletConfig(t *testing.T) {
@@ -149,29 +148,29 @@ ExecStart=/usr/bin/kubelet --authorization-mode=Webhook --bootstrap-kubeconfig=/
 	}
 }
 
-func getExtraOpts() []util.ExtraOption {
-	return util.ExtraOptionSlice{
-		util.ExtraOption{
+func getExtraOpts() []config.ExtraOption {
+	return config.ExtraOptionSlice{
+		config.ExtraOption{
 			Component: Apiserver,
 			Key:       "fail-no-swap",
 			Value:     "true",
 		},
-		util.ExtraOption{
+		config.ExtraOption{
 			Component: ControllerManager,
 			Key:       "kube-api-burst",
 			Value:     "32",
 		},
-		util.ExtraOption{
+		config.ExtraOption{
 			Component: Scheduler,
 			Key:       "scheduler-name",
 			Value:     "mini-scheduler",
 		},
-		util.ExtraOption{
+		config.ExtraOption{
 			Component: Kubeadm,
 			Key:       "ignore-preflight-errors",
 			Value:     "true",
 		},
-		util.ExtraOption{
+		config.ExtraOption{
 			Component: Kubeadm,
 			Key:       "dry-run",
 			Value:     "true",
@@ -179,9 +178,9 @@ func getExtraOpts() []util.ExtraOption {
 	}
 }
 
-func getExtraOptsPodCidr() []util.ExtraOption {
-	return util.ExtraOptionSlice{
-		util.ExtraOption{
+func getExtraOptsPodCidr() []config.ExtraOption {
+	return config.ExtraOptionSlice{
+		config.ExtraOption{
 			Component: Kubeadm,
 			Key:       "pod-network-cidr",
 			Value:     "192.168.32.0/20",
@@ -233,7 +232,7 @@ func TestGenerateConfig(t *testing.T) {
 		{"crio", "crio", false, config.KubernetesConfig{}},
 		{"options", "docker", false, config.KubernetesConfig{ExtraOptions: extraOpts}},
 		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
-		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: util.ExtraOptionSlice{util.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
+		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: config.ExtraOptionSlice{config.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
 		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
 		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
 		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go
index 2c7c9a97f991..a2e4c1c9e63e 100644
--- a/pkg/minikube/bootstrapper/kubeadm/versions.go
+++ b/pkg/minikube/bootstrapper/kubeadm/versions.go
@@ -27,6 +27,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/util"
 )
@@ -43,7 +44,7 @@ const (
 
 // ExtraConfigForComponent generates a map of flagname-value pairs for a k8s
 // component.
-func ExtraConfigForComponent(component string, opts util.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
+func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
 	versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version)
 	if err != nil {
 		return nil, errors.Wrapf(err, "setting version specific options for %s", component)
@@ -78,7 +79,7 @@ var componentToKubeadmConfigKey = map[string]string{
 }
 
 // NewComponentExtraArgs creates a new ComponentExtraArgs
-func NewComponentExtraArgs(opts util.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) {
+func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) {
 	var kubeadmExtraArgs []ComponentExtraArgs
 	for _, extraOpt := range opts {
 		if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok {
@@ -158,8 +159,8 @@ func Supports(featureName string) bool {
 	return false
 }
 
-// ParseKubernetesVersion parses the kubernetes version
-func ParseKubernetesVersion(version string) (semver.Version, error) {
+// arseKubernetesVersion parses the kubernetes version
+func parseKubernetesVersion(version string) (semver.Version, error) {
 	// Strip leading 'v' prefix from version for semver parsing
 	v, err := semver.Make(version[1:])
 	if err != nil {
@@ -182,45 +183,9 @@ func convertToFlags(opts map[string]string) string {
 	return strings.Join(flags, " ")
 }
 
-// VersionedExtraOption holds information on flags to apply to a specific range
-// of versions
-type VersionedExtraOption struct {
-	// Special Cases:
-	//
-	// If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied
-	// to all versions
-	//
-	// If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that
-	// specific version
-
-	// The flag and component that will be set
-	Option util.ExtraOption
-
-	// This flag will only be applied to versions before or equal to this version
-	// If it is the default value, it will have no upper bound on versions the
-	// flag is applied to
-	LessThanOrEqual semver.Version
-
-	// The flag will only be applied to versions after or equal to this version
-	// If it is the default value, it will have no lower bound on versions the
-	// flag is applied to
-	GreaterThanOrEqual semver.Version
-}
-
-// NewUnversionedOption returns a VersionedExtraOption that applies to all versions.
-func NewUnversionedOption(component, k, v string) VersionedExtraOption {
-	return VersionedExtraOption{
-		Option: util.ExtraOption{
-			Component: component,
-			Key:       k,
-			Value:     v,
-		},
-	}
-}
-
-var versionSpecificOpts = []VersionedExtraOption{
+var versionSpecificOpts = []config.VersionedExtraOption{
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Kubelet,
 			Key:       "fail-swap-on",
 			Value:     "false",
@@ -228,22 +193,22 @@ var versionSpecificOpts = []VersionedExtraOption{
 		GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"),
 	},
 	// Kubeconfig args
-	NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"),
-	NewUnversionedOption(Kubelet, "bootstrap-kubeconfig", "/etc/kubernetes/bootstrap-kubelet.conf"),
+	config.NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"),
+	config.NewUnversionedOption(Kubelet, "bootstrap-kubeconfig", "/etc/kubernetes/bootstrap-kubelet.conf"),
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Kubelet,
 			Key:       "require-kubeconfig",
 			Value:     "true",
 		},
 		LessThanOrEqual: semver.MustParse("1.9.10"),
 	},
-	NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName),
+	config.NewUnversionedOption(Kubelet, "hostname-override", constants.DefaultNodeName),
 
 	// System pods args
-	NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"),
+	config.NewUnversionedOption(Kubelet, "pod-manifest-path", "/etc/kubernetes/manifests"),
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Kubelet,
 			Key:       "allow-privileged",
 			Value:     "true",
@@ -252,17 +217,17 @@ var versionSpecificOpts = []VersionedExtraOption{
 	},
 
 	// Network args
-	NewUnversionedOption(Kubelet, "cluster-dns", "10.96.0.10"),
-	NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"),
+	config.NewUnversionedOption(Kubelet, "cluster-dns", "10.96.0.10"),
+	config.NewUnversionedOption(Kubelet, "cluster-domain", "cluster.local"),
 
 	// Auth args
-	NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"),
-	NewUnversionedOption(Kubelet, "client-ca-file", path.Join(util.DefaultCertPath, "ca.crt")),
+	config.NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"),
+	config.NewUnversionedOption(Kubelet, "client-ca-file", path.Join(util.DefaultCertPath, "ca.crt")),
 
 	// Cgroup args
-	NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),
+	config.NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Apiserver,
 			Key:       "admission-control",
 			Value:     strings.Join(util.DefaultLegacyAdmissionControllers, ","),
@@ -271,7 +236,7 @@ var versionSpecificOpts = []VersionedExtraOption{
 		GreaterThanOrEqual: semver.MustParse("1.9.0-alpha.0"),
 	},
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Apiserver,
 			Key:       "enable-admission-plugins",
 			Value:     strings.Join(util.DefaultLegacyAdmissionControllers, ","),
@@ -280,7 +245,7 @@ var versionSpecificOpts = []VersionedExtraOption{
 		LessThanOrEqual:    semver.MustParse("1.13.1000"),
 	},
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Apiserver,
 			Key:       "enable-admission-plugins",
 			Value:     strings.Join(util.DefaultV114AdmissionControllers, ","),
@@ -289,7 +254,7 @@ var versionSpecificOpts = []VersionedExtraOption{
 	},
 
 	{
-		Option: util.ExtraOption{
+		Option: config.ExtraOption{
 			Component: Kubelet,
 			Key:       "cadvisor-port",
 			Value:     "0",
diff --git a/pkg/minikube/bootstrapper/kubeadm/versions_test.go b/pkg/minikube/bootstrapper/kubeadm/versions_test.go
index cc4ec232b602..40d46c077e63 100644
--- a/pkg/minikube/bootstrapper/kubeadm/versions_test.go
+++ b/pkg/minikube/bootstrapper/kubeadm/versions_test.go
@@ -94,7 +94,7 @@ func TestVersionIsBetween(t *testing.T) {
 }
 
 func TestParseKubernetesVersion(t *testing.T) {
-	version, err := ParseKubernetesVersion("v1.8.0-alpha.5")
+	version, err := parseKubernetesVersion("v1.8.0-alpha.5")
 	if err != nil {
 		t.Fatalf("Error parsing version: %v", err)
 	}
diff --git a/pkg/util/extra_options.go b/pkg/minikube/config/extra_options.go
similarity index 80%
rename from pkg/util/extra_options.go
rename to pkg/minikube/config/extra_options.go
index 2a6015526103..0448750f390e 100644
--- a/pkg/util/extra_options.go
+++ b/pkg/minikube/config/extra_options.go
@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors All rights reserved.
+Copyright 2019 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package config
 
 import (
 	"fmt"
@@ -76,7 +76,7 @@ func (es *ExtraOptionSlice) String() string {
 // component is not specified, all of the components are used.
 func (es *ExtraOptionSlice) Get(key string, component ...string) string {
 	for _, opt := range *es {
-		if component == nil || ContainsString(component, opt.Component) {
+		if component == nil || ContainsParam(component, opt.Component) {
 			if opt.Key == key {
 				return opt.Value
 			}
@@ -107,3 +107,25 @@ func (es *ExtraOptionSlice) Type() string {
 func (cm ComponentExtraOptionMap) Get(component string) map[string]string {
 	return cm[component]
 }
+
+// ContainsParam checks if a given slice of strings contains the provided string.
+// If a modifier func is provided, it is called with the slice item before the comparation.
+func ContainsParam(slice []string, s string) bool {
+	for _, item := range slice {
+		if item == s {
+			return true
+		}
+	}
+	return false
+}
+
+// NewUnversionedOption returns a VersionedExtraOption that applies to all versions.
+func NewUnversionedOption(component, k, v string) VersionedExtraOption {
+	return VersionedExtraOption{
+		Option: ExtraOption{
+			Component: component,
+			Key:       k,
+			Value:     v,
+		},
+	}
+}
diff --git a/pkg/util/extra_options_test.go b/pkg/minikube/config/extra_options_test.go
similarity index 99%
rename from pkg/util/extra_options_test.go
rename to pkg/minikube/config/extra_options_test.go
index 4e54d99d13d0..cce3463ab81b 100644
--- a/pkg/util/extra_options_test.go
+++ b/pkg/minikube/config/extra_options_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package config
 
 import (
 	"flag"
diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go
index 0f7f7708e161..d2ed5e014cfa 100644
--- a/pkg/minikube/config/types.go
+++ b/pkg/minikube/config/types.go
@@ -19,6 +19,7 @@ package config
 import (
 	"net"
 
+	"github.com/blang/semver"
 	"k8s.io/minikube/pkg/util"
 )
 
@@ -82,8 +83,33 @@ type KubernetesConfig struct {
 	FeatureGates      string
 	ServiceCIDR       string
 	ImageRepository   string
-	ExtraOptions      util.ExtraOptionSlice
+	ExtraOptions      ExtraOptionSlice
 
 	ShouldLoadCachedImages bool
 	EnableDefaultCNI       bool
 }
+
+// VersionedExtraOption holds information on flags to apply to a specific range
+// of versions
+type VersionedExtraOption struct {
+	// Special Cases:
+	//
+	// If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied
+	// to all versions
+	//
+	// If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that
+	// specific version
+
+	// The flag and component that will be set
+	Option ExtraOption
+
+	// This flag will only be applied to versions before or equal to this version
+	// If it is the default value, it will have no upper bound on versions the
+	// flag is applied to
+	LessThanOrEqual semver.Version
+
+	// The flag will only be applied to versions after or equal to this version
+	// If it is the default value, it will have no lower bound on versions the
+	// flag is applied to
+	GreaterThanOrEqual semver.Version
+}
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index 1cd596a7369a..c549ca4ebe44 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -200,14 +200,3 @@ func ConcatStrings(src []string, prefix string, postfix string) []string {
 	}
 	return ret
 }
-
-// ContainsString checks if a given slice of strings contains the provided string.
-// If a modifier func is provided, it is called with the slice item before the comparation.
-func ContainsString(slice []string, s string) bool {
-	for _, item := range slice {
-		if item == s {
-			return true
-		}
-	}
-	return false
-}
diff --git a/test/integration/containerd_test.go b/test/integration/containerd_test.go
index 822631730eb7..61a5af4d5e4a 100644
--- a/test/integration/containerd_test.go
+++ b/test/integration/containerd_test.go
@@ -25,7 +25,7 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
-	commonutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/test/integration/util"
 )
 
@@ -107,13 +107,13 @@ func deleteUntrustedWorkload(t *testing.T, profile string) {
 
 // waitForGvisorControllerRunning waits for the gvisor controller pod to be running
 func waitForGvisorControllerRunning(p string) error {
-	client, err := commonutil.GetClient(p)
+	client, err := kube.Client(p)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
-	if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
 	}
 	return nil
@@ -121,13 +121,13 @@ func waitForGvisorControllerRunning(p string) error {
 
 // waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running
 func waitForUntrustedNginxRunning(miniProfile string) error {
-	client, err := commonutil.GetClient(miniProfile)
+	client, err := kube.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
-	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}
 	return nil
diff --git a/test/integration/fn_addons.go b/test/integration/fn_addons.go
index 60af99b14f22..95cfbd7828c3 100644
--- a/test/integration/fn_addons.go
+++ b/test/integration/fn_addons.go
@@ -34,8 +34,7 @@ import (
 
 	retryablehttp "github.com/hashicorp/go-retryablehttp"
 	"k8s.io/apimachinery/pkg/labels"
-	commonutil "k8s.io/minikube/pkg/util"
-	pkgutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -43,12 +42,12 @@ import (
 func testAddons(t *testing.T) {
 	t.Parallel()
 	p := profileName(t)
-	client, err := pkgutil.GetClient(p)
+	client, err := kube.Client(p)
 	if err != nil {
 		t.Fatalf("Could not get kubernetes client: %v", err)
 	}
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-addon-manager"}))
-	if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		t.Errorf("Error waiting for addon manager to be up")
 	}
 }
@@ -193,22 +192,22 @@ func testRegistry(t *testing.T) {
 	p := profileName(t)
 	mk := NewMinikubeRunner(t, p)
 	mk.RunCommand("addons enable registry", true)
-	client, err := pkgutil.GetClient(p)
+	client, err := kube.Client(p)
 	if err != nil {
 		t.Fatalf("getting kubernetes client: %v", err)
 	}
-	if err := pkgutil.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil {
+	if err := kube.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil {
 		t.Fatalf("waiting for registry replicacontroller to stabilize: %v", err)
 	}
 	rs := labels.SelectorFromSet(labels.Set(map[string]string{"actual-registry": "true"}))
-	if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil {
 		t.Fatalf("waiting for registry pods: %v", err)
 	}
 	ps, err := labels.Parse("kubernetes.io/minikube-addons=registry,actual-registry!=true")
 	if err != nil {
 		t.Fatalf("Unable to parse selector: %v", err)
 	}
-	if err := pkgutil.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil {
 		t.Fatalf("waiting for registry-proxy pods: %v", err)
 	}
 	ip, stderr := mk.RunCommand("ip", true)
@@ -265,18 +264,18 @@ func testRegistry(t *testing.T) {
 
 // waitForNginxRunning waits for nginx service to be up
 func waitForNginxRunning(t *testing.T, miniProfile string) error {
-	client, err := commonutil.GetClient(miniProfile)
+	client, err := kube.Client(miniProfile)
 
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
-	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}
 
-	if err := commonutil.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
+	if err := kube.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
 		t.Errorf("Error waiting for nginx service to be up")
 	}
 	return nil
@@ -284,17 +283,17 @@ func waitForNginxRunning(t *testing.T, miniProfile string) error {
 
 // waitForIngressControllerRunning waits until ingress controller pod to be running
 func waitForIngressControllerRunning(miniProfile string) error {
-	client, err := commonutil.GetClient(miniProfile)
+	client, err := kube.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
-	if err := commonutil.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil {
+	if err := kube.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil {
 		return errors.Wrap(err, "waiting for ingress-controller deployment to stabilize")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "nginx-ingress-controller"}))
-	if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		return errors.Wrap(err, "waiting for ingress-controller pods")
 	}
 
diff --git a/test/integration/fn_cluster_dns.go b/test/integration/fn_cluster_dns.go
index d17f76e10324..bfceae335f66 100644
--- a/test/integration/fn_cluster_dns.go
+++ b/test/integration/fn_cluster_dns.go
@@ -26,7 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	pkgutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -34,7 +34,7 @@ import (
 func testClusterDNS(t *testing.T) {
 	t.Parallel()
 	p := profileName(t)
-	client, err := pkgutil.GetClient(p)
+	client, err := kube.Client(p)
 	if err != nil {
 		t.Fatalf("Error getting kubernetes client %v", err)
 	}
diff --git a/test/integration/fn_mount.go b/test/integration/fn_mount.go
index 5d68c18554b0..04db1f007583 100644
--- a/test/integration/fn_mount.go
+++ b/test/integration/fn_mount.go
@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/labels"
-	pkgutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/util/lock"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
@@ -132,12 +132,12 @@ func writeFilesFromHost(mountedDir string, files []string, content string) error
 }
 
 func waitForPods(s map[string]string, profile string) error {
-	client, err := pkgutil.GetClient(profile)
+	client, err := kube.Client(profile)
 	if err != nil {
 		return fmt.Errorf("getting kubernetes client: %v", err)
 	}
 	selector := labels.SelectorFromSet(labels.Set(s))
-	if err := pkgutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return err
 	}
 	return nil
diff --git a/test/integration/fn_pv.go b/test/integration/fn_pv.go
index b1b34d8a3437..ed80ff5ef5ac 100644
--- a/test/integration/fn_pv.go
+++ b/test/integration/fn_pv.go
@@ -29,7 +29,7 @@ import (
 	core "k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	commonutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -73,13 +73,13 @@ func testProvisioning(t *testing.T) {
 	// Check that the storage provisioner pod is running
 
 	checkPodRunning := func() error {
-		client, err := commonutil.GetClient(p)
+		client, err := kube.Client(p)
 		if err != nil {
 			return errors.Wrap(err, "getting kubernetes client")
 		}
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"}))
 
-		if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 			return err
 		}
 		return nil
diff --git a/test/integration/fn_tunnel.go b/test/integration/fn_tunnel.go
index a5eed147730a..a68c53dbc45a 100644
--- a/test/integration/fn_tunnel.go
+++ b/test/integration/fn_tunnel.go
@@ -31,8 +31,8 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/minikube/tunnel"
-	commonutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -70,18 +70,18 @@ func testTunnel(t *testing.T) {
 		t.Fatalf("creating nginx ingress resource: %s", err)
 	}
 
-	client, err := commonutil.GetClient(p)
+	client, err := kube.Client(p)
 
 	if err != nil {
 		t.Fatal(errors.Wrap(err, "getting kubernetes client"))
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"}))
-	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		t.Fatal(errors.Wrap(err, "waiting for nginx pods"))
 	}
 
-	if err := commonutil.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
+	if err := kube.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
 		t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
 	}
 
@@ -114,14 +114,14 @@ func testTunnel(t *testing.T) {
 func getIngress(kr *util.KubectlRunner) (string, error) {
 	nginxIP := ""
 	var ret error
-	err := wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
+	err := wait.PollImmediate(1*time.Second, 2*time.Minute, func() (bool, error) {
 		cmd := []string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"}
 		stdout, err := kr.RunCommand(cmd)
 		switch {
 		case err == nil:
 			nginxIP = string(stdout)
 			return len(stdout) != 0, nil
-		case !commonutil.IsRetryableAPIError(err):
+		case !kube.IsRetryableAPIError(err):
 			ret = fmt.Errorf("`%s` failed with non retriable error: %v", cmd, err)
 			return false, err
 		default:
diff --git a/test/integration/util/common.go b/test/integration/util/common.go
index b3b1143cdba5..9037ae660b0f 100644
--- a/test/integration/util/common.go
+++ b/test/integration/util/common.go
@@ -23,17 +23,17 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
-	commonutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/kube"
 )
 
 // WaitForBusyboxRunning waits until busybox pod to be running
 func WaitForBusyboxRunning(t *testing.T, namespace string, miniProfile string) error {
-	client, err := commonutil.GetClient(miniProfile)
+	client, err := kube.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "busybox"}))
-	return commonutil.WaitForPodsWithLabelRunning(client, namespace, selector)
+	return kube.WaitForPodsWithLabelRunning(client, namespace, selector)
 }
 
 // Logf writes logs to stdout if -v is set.

From a85db2ddb44f5c29ec8e431d009eb3334f67f1d1 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Sat, 17 Aug 2019 23:18:13 -0700
Subject: [PATCH 2/7] use kubeadm consts and add duration metrics logs

---
 pkg/kube/kube.go                             | 13 +++++++++----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 14 ++++++++++----
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/pkg/kube/kube.go b/pkg/kube/kube.go
index 5b859bb6f916..81179bd6e7b2 100644
--- a/pkg/kube/kube.go
+++ b/pkg/kube/kube.go
@@ -35,7 +35,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	watchtools "k8s.io/client-go/tools/watch"
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/minikube/proxy"
 )
 
@@ -43,7 +43,7 @@ var (
 	// ReasonableMutateTime is how long to wait for basic object mutations, such as deletions, to show up
 	ReasonableMutateTime = time.Minute * 2
 	// ReasonableStartTime is how long to wait for pods to start
-	ReasonableStartTime = time.Minute * 6
+	ReasonableStartTime = time.Minute * 5
 )
 
 // Client gets the kuberentes client from default kubeconfig
@@ -70,9 +70,10 @@ func Client(kubectlContext ...string) (kubernetes.Interface, error) {
 
 // WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
 func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
+	start := time.Now()
 	glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
 	lastKnownPodNumber := -1
-	return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) {
+	f := func() (bool, error) {
 		listOpts := meta.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
@@ -96,7 +97,11 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 		}
 
 		return true, nil
-	})
+	}
+	err := wait.PollImmediate(kconst.APICallRetryInterval, ReasonableStartTime, f)
+	elapsed := time.Since(start)
+	glog.Infof("duration metric: took %s to wait for %s ...", elapsed, label)
+	return err
 }
 
 // WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status.
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 7cf462470a5d..975d5a63ed0f 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -34,6 +34,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/kube"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -323,10 +324,9 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
 	}
 
 	for _, p := range PodsByLayer {
-		if componentsOnly && p.key != "component" {
+		if componentsOnly && p.key != "component" { // skip component check if network plugin is cni
 			continue
 		}
-
 		out.String(" %s", p.name)
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{p.key: p.value}))
 		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
@@ -383,8 +383,10 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error {
 
 // waitForAPIServer waits for the apiserver to start up
 func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error {
+	start := time.Now()
 	glog.Infof("Waiting for apiserver ...")
-	return wait.PollImmediate(time.Millisecond*300, time.Minute*3, func() (bool, error) {
+
+	f := func() (bool, error) {
 		status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort)
 		glog.Infof("apiserver status: %s, err: %v", status, err)
 		if err != nil {
@@ -394,7 +396,11 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error {
 			return false, nil
 		}
 		return true, nil
-	})
+	}
+	err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f)
+	elapsed := time.Since(start)
+	glog.Infof("duration metric: took %s to wait for apiserver status ...", elapsed)
+	return err
 }
 
 // DeleteCluster removes the components that were started earlier

From 5464a7c9e2d1ef1ed1f25bc3cea45938bcfd6a1f Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Sun, 18 Aug 2019 01:14:02 -0700
Subject: [PATCH 3/7] adding a new flag wait-timeout and reduce default wait
 from 5 to 3

---
 cmd/minikube/cmd/start.go                    |  4 +++-
 pkg/kube/kube.go                             | 15 ++++++++++-----
 pkg/minikube/bootstrapper/bootstrapper.go    |  3 ++-
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go |  4 ++--
 test/integration/util/minikube_runner.go     |  2 +-
 5 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index 742f329619ce..da2c023c6a1a 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -103,6 +103,7 @@ const (
 	dnsProxy              = "dns-proxy"
 	hostDNSResolver       = "host-dns-resolver"
 	waitUntilHealthy      = "wait"
+	waitTimeout           = "wait-timeout"
 )
 
 var (
@@ -149,6 +150,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
 	startCmd.Flags().Bool(waitUntilHealthy, true, "Wait until Kubernetes core services are healthy before exiting.")
+	startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait for Kubernetes core services to be healthy.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
@@ -322,7 +324,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	// special ops for none driver, like change minikube directory.
 	prepareNone(viper.GetString(vmDriver))
 	if viper.GetBool(waitUntilHealthy) {
-		if err := bs.WaitCluster(config.KubernetesConfig); err != nil {
+		if err := bs.WaitCluster(config.KubernetesConfig, viper.GetDuration(waitTimeout)); err != nil {
 			exit.WithError("Wait failed", err)
 		}
 	}
diff --git a/pkg/kube/kube.go b/pkg/kube/kube.go
index 81179bd6e7b2..db17136b03bc 100644
--- a/pkg/kube/kube.go
+++ b/pkg/kube/kube.go
@@ -69,7 +69,7 @@ func Client(kubectlContext ...string) (kubernetes.Interface, error) {
 }
 
 // WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
-func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
+func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector, timeOut ...time.Duration) error {
 	start := time.Now()
 	glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
 	lastKnownPodNumber := -1
@@ -77,7 +77,7 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 		listOpts := meta.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
-			glog.Errorf("error getting Pods with label selector %q [%v]\n", label.String(), err)
+			glog.Infof("temproary error: getting Pods with label selector %q : [%v]\n", label.String(), err)
 			return false, nil
 		}
 
@@ -92,19 +92,24 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 
 		for _, pod := range pods.Items {
 			if pod.Status.Phase != core.PodRunning {
+				glog.Infof("temporary error: for Pod with label %q expected status to be running but got %s : [%v]\n", label.String(), pod.Status.Phase, err)
 				return false, nil
 			}
 		}
 
 		return true, nil
 	}
-	err := wait.PollImmediate(kconst.APICallRetryInterval, ReasonableStartTime, f)
+	t := ReasonableStartTime
+	if timeOut != nil {
+		t = timeOut[0]
+	}
+	err := wait.PollImmediate(kconst.APICallRetryInterval, t, f)
 	elapsed := time.Since(start)
 	glog.Infof("duration metric: took %s to wait for %s ...", elapsed, label)
 	return err
 }
 
-// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status.
+// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. used by integration tests
 func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
 	options := meta.ListOptions{FieldSelector: fields.Set{
 		"metadata.name":      name,
@@ -138,7 +143,7 @@ func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.
 	return err
 }
 
-// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status.
+// WaitForDeploymentToStabilize waits till the Deployment has a matching generation/replica count between spec and status. used by integrationt tests
 func WaitForDeploymentToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
 	options := meta.ListOptions{FieldSelector: fields.Set{
 		"metadata.name":      name,
diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go
index b3ff31095d76..2da2e5265493 100644
--- a/pkg/minikube/bootstrapper/bootstrapper.go
+++ b/pkg/minikube/bootstrapper/bootstrapper.go
@@ -18,6 +18,7 @@ package bootstrapper
 
 import (
 	"net"
+	"time"
 
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -39,7 +40,7 @@ type Bootstrapper interface {
 	UpdateCluster(config.KubernetesConfig) error
 	RestartCluster(config.KubernetesConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	WaitCluster(config.KubernetesConfig) error
+	WaitCluster(config.KubernetesConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
 	SetupCerts(cfg config.KubernetesConfig) error
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index 975d5a63ed0f..e29b37393af8 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -305,7 +305,7 @@ func addAddons(files *[]assets.CopyableFile, data interface{}) error {
 }
 
 // WaitCluster blocks until Kubernetes appears to be healthy.
-func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
+func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
 	// Do not wait for "k8s-app" pods in the case of CNI, as they are managed
 	// by a CNI plugin which is usually started after minikube has been brought
 	// up. Otherwise, minikube won't start, as "k8s-app" pods are not ready.
@@ -329,7 +329,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig) error {
 		}
 		out.String(" %s", p.name)
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{p.key: p.value}))
-		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector, timeout); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("waiting for %s=%s", p.key, p.value))
 		}
 	}
diff --git a/test/integration/util/minikube_runner.go b/test/integration/util/minikube_runner.go
index 19c766e2480e..25c71e88302e 100644
--- a/test/integration/util/minikube_runner.go
+++ b/test/integration/util/minikube_runner.go
@@ -244,7 +244,7 @@ func (m *MinikubeRunner) TearDown(t *testing.T) {
 	profileArg := fmt.Sprintf("-p=%s", m.Profile)
 	path, _ := filepath.Abs(m.BinaryPath)
 	cmd := exec.Command(path, profileArg, "delete")
-	err := cmd.Start()
+	err := cmd.Start() // don't wait for it to finish
 	if err != nil {
 		t.Errorf("error tearing down minikube %s : %v", profileArg, err)
 	}

From af16d336daab4fc2c77d3b4ee7f34b4cc5de54ca Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Sun, 18 Aug 2019 01:39:42 -0700
Subject: [PATCH 4/7] Fixed givsor test setup and added time out to integration
 test

---
 hack/jenkins/common.sh    | 5 +++++
 test/integration/flags.go | 7 ++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 8e0c7c0b2aa4..8e4efb7e32c0 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -241,6 +241,11 @@ export MINIKUBE_HOME="${TEST_HOME}/.minikube"
 export MINIKUBE_WANTREPORTERRORPROMPT=False
 export KUBECONFIG="${TEST_HOME}/kubeconfig"
 
+# Build the gvisor image. This will be copied into minikube and loaded by ctr.
+# Used by TestContainerd for Gvisor Test.
+docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile out
+
+
 # Display the default image URL
 echo ""
 echo ">> ISO URL"
diff --git a/test/integration/flags.go b/test/integration/flags.go
index f8ad39211e0f..f58019534a95 100644
--- a/test/integration/flags.go
+++ b/test/integration/flags.go
@@ -18,6 +18,7 @@ package integration
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -32,7 +33,7 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-var startTimeout = flag.Int("timeout", 25, "number of minutes to wait for minikube start")
+var startTimeout = flag.Duration("timeout", 25*time.Minute, "max duration to wait for a full minikube start")
 var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
 var globalArgs = flag.String("minikube-args", "", "Arguments to pass to minikube")
 var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")
@@ -45,10 +46,10 @@ func NewMinikubeRunner(t *testing.T, profile string, extraStartArgs ...string) u
 	return util.MinikubeRunner{
 		Profile:      profile,
 		BinaryPath:   *binaryPath,
-		StartArgs:    *startArgs + " " + strings.Join(extraStartArgs, " "),
+		StartArgs:    *startArgs + fmt.Sprintf(" --wait-timeout=%s ", *startTimeout/2) + strings.Join(extraStartArgs, " "), // adding timeout per component
 		GlobalArgs:   *globalArgs,
 		MountArgs:    *mountArgs,
-		TimeOutStart: time.Duration(*startTimeout) * time.Minute,
+		TimeOutStart: *startTimeout, // timeout for all start
 		T:            t,
 	}
 }

From 6554a5146f742ffc410e5976609d907f20cedcc2 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Sun, 18 Aug 2019 02:36:18 -0700
Subject: [PATCH 5/7] make TestVersionUpgrade campatible with the new flag

---
 test/integration/flags.go                |  3 +--
 test/integration/version_upgrade_test.go | 14 +++++++++-----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/test/integration/flags.go b/test/integration/flags.go
index f58019534a95..5b0c69931052 100644
--- a/test/integration/flags.go
+++ b/test/integration/flags.go
@@ -18,7 +18,6 @@ package integration
 
 import (
 	"flag"
-	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -46,7 +45,7 @@ func NewMinikubeRunner(t *testing.T, profile string, extraStartArgs ...string) u
 	return util.MinikubeRunner{
 		Profile:      profile,
 		BinaryPath:   *binaryPath,
-		StartArgs:    *startArgs + fmt.Sprintf(" --wait-timeout=%s ", *startTimeout/2) + strings.Join(extraStartArgs, " "), // adding timeout per component
+		StartArgs:    *startArgs + " --wait-timeout=13m " + strings.Join(extraStartArgs, " "), // adding timeout per component
 		GlobalArgs:   *globalArgs,
 		MountArgs:    *mountArgs,
 		TimeOutStart: *startTimeout, // timeout for all start
diff --git a/test/integration/version_upgrade_test.go b/test/integration/version_upgrade_test.go
index 700161ea31de..7e45ec28fa65 100644
--- a/test/integration/version_upgrade_test.go
+++ b/test/integration/version_upgrade_test.go
@@ -73,10 +73,14 @@ func TestVersionUpgrade(t *testing.T) {
 	}
 	defer os.Remove(fname)
 
-	mkCurrent := NewMinikubeRunner(t, p)
-	defer mkCurrent.TearDown(t)
+	mkHead := NewMinikubeRunner(t, p) // minikube from HEAD.
+	defer mkHead.TearDown(t)
 
-	mkRelease := NewMinikubeRunner(t, p)
+	mkRelease := NewMinikubeRunner(t, p) // lastest publicly released version minikbue.
+
+	// because the --wait-timeout is a new flag and the current latest release (1.3.1) doesn't have it
+	// this won't be necessary after we release the change with --wait-timeout flag
+	mkRelease.StartArgs = strings.Replace("mkRelease.StartArgs", "--wait-timeout=13m", "", 1)
 	mkRelease.BinaryPath = fname
 	// For full coverage: also test upgrading from oldest to newest supported k8s release
 	stdout, stderr, err := mkRelease.Start(fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion))
@@ -89,9 +93,9 @@ func TestVersionUpgrade(t *testing.T) {
 	mkRelease.CheckStatus(state.Stopped.String())
 
 	// Trim the leading "v" prefix to assert that we handle it properly.
-	stdout, stderr, err = mkCurrent.Start(fmt.Sprintf("--kubernetes-version=%s", strings.TrimPrefix(constants.NewestKubernetesVersion, "v")))
+	stdout, stderr, err = mkHead.Start(fmt.Sprintf("--kubernetes-version=%s", strings.TrimPrefix(constants.NewestKubernetesVersion, "v")))
 	if err != nil {
 		t.Fatalf("TestVersionUpgrade mkCurrent.Start start failed : %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
 	}
-	mkCurrent.CheckStatus(state.Running.String())
+	mkHead.CheckStatus(state.Running.String())
 }

From f75b558ac21e92c1600d5256c9e159793ddd0f93 Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 19 Aug 2019 16:11:38 -0700
Subject: [PATCH 6/7] rename kube pkg to kapi

---
 cmd/minikube/cmd/start.go                     |  2 +-
 pkg/{kube/kube.go => kapi/kapi.go}            |  9 +++----
 pkg/minikube/bootstrapper/kubeadm/kubeadm.go  |  9 +++----
 pkg/minikube/bootstrapper/kubeadm/versions.go |  2 +-
 test/integration/containerd_test.go           | 10 +++----
 test/integration/fn_addons.go                 | 26 +++++++++----------
 test/integration/fn_cluster_dns.go            |  4 +--
 test/integration/fn_mount.go                  |  6 ++---
 test/integration/fn_pv.go                     |  6 ++---
 test/integration/fn_tunnel.go                 | 10 +++----
 test/integration/util/common.go               |  6 ++---
 11 files changed, 44 insertions(+), 46 deletions(-)
 rename pkg/{kube/kube.go => kapi/kapi.go} (95%)

diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go
index da2c023c6a1a..1c0a9fa168dd 100644
--- a/cmd/minikube/cmd/start.go
+++ b/cmd/minikube/cmd/start.go
@@ -150,7 +150,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
 	startCmd.Flags().Bool(waitUntilHealthy, true, "Wait until Kubernetes core services are healthy before exiting.")
-	startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait for Kubernetes core services to be healthy.")
+	startCmd.Flags().Duration(waitTimeout, 3*time.Minute, "max time to wait per Kubernetes core services to be healthy.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
diff --git a/pkg/kube/kube.go b/pkg/kapi/kapi.go
similarity index 95%
rename from pkg/kube/kube.go
rename to pkg/kapi/kapi.go
index db17136b03bc..b480f3200a8f 100644
--- a/pkg/kube/kube.go
+++ b/pkg/kapi/kapi.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kube
+package kapi
 
 import (
 	"context"
@@ -77,7 +77,7 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 		listOpts := meta.ListOptions{LabelSelector: label.String()}
 		pods, err := c.CoreV1().Pods(ns).List(listOpts)
 		if err != nil {
-			glog.Infof("temproary error: getting Pods with label selector %q : [%v]\n", label.String(), err)
+			glog.Infof("temporary error: getting Pods with label selector %q : [%v]\n", label.String(), err)
 			return false, nil
 		}
 
@@ -92,7 +92,7 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 
 		for _, pod := range pods.Items {
 			if pod.Status.Phase != core.PodRunning {
-				glog.Infof("temporary error: for Pod with label %q expected status to be running but got %s : [%v]\n", label.String(), pod.Status.Phase, err)
+				glog.Infof("waiting for pod %q, current state: %s: [%v]\n", label.String(), pod.Status.Phase, err)
 				return false, nil
 			}
 		}
@@ -104,8 +104,7 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 		t = timeOut[0]
 	}
 	err := wait.PollImmediate(kconst.APICallRetryInterval, t, f)
-	elapsed := time.Since(start)
-	glog.Infof("duration metric: took %s to wait for %s ...", elapsed, label)
+	glog.Infof("duration metric: took %s to wait for %s ...", time.Since(start), label)
 	return err
 }
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
index e29b37393af8..736cdd4f2515 100644
--- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
+++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -35,7 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -311,7 +311,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur
 	// up. Otherwise, minikube won't start, as "k8s-app" pods are not ready.
 	componentsOnly := k8s.NetworkPlugin == "cni"
 	out.T(out.WaitingPods, "Waiting for:")
-	client, err := kube.Client()
+	client, err := kapi.Client()
 	if err != nil {
 		return errors.Wrap(err, "k8s client")
 	}
@@ -329,7 +329,7 @@ func (k *Bootstrapper) WaitCluster(k8s config.KubernetesConfig, timeout time.Dur
 		}
 		out.String(" %s", p.name)
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{p.key: p.value}))
-		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector, timeout); err != nil {
+		if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector, timeout); err != nil {
 			return errors.Wrap(err, fmt.Sprintf("waiting for %s=%s", p.key, p.value))
 		}
 	}
@@ -398,8 +398,7 @@ func (k *Bootstrapper) waitForAPIServer(k8s config.KubernetesConfig) error {
 		return true, nil
 	}
 	err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, f)
-	elapsed := time.Since(start)
-	glog.Infof("duration metric: took %s to wait for apiserver status ...", elapsed)
+	glog.Infof("duration metric: took %s to wait for apiserver status ...", time.Since(start))
 	return err
 }
 
diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go
index a2e4c1c9e63e..2abe97e853cd 100644
--- a/pkg/minikube/bootstrapper/kubeadm/versions.go
+++ b/pkg/minikube/bootstrapper/kubeadm/versions.go
@@ -159,7 +159,7 @@ func Supports(featureName string) bool {
 	return false
 }
 
-// arseKubernetesVersion parses the kubernetes version
+// parseKubernetesVersion parses the kubernetes version
 func parseKubernetesVersion(version string) (semver.Version, error) {
 	// Strip leading 'v' prefix from version for semver parsing
 	v, err := semver.Make(version[1:])
diff --git a/test/integration/containerd_test.go b/test/integration/containerd_test.go
index 61a5af4d5e4a..7c72a1349166 100644
--- a/test/integration/containerd_test.go
+++ b/test/integration/containerd_test.go
@@ -25,7 +25,7 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/test/integration/util"
 )
 
@@ -107,13 +107,13 @@ func deleteUntrustedWorkload(t *testing.T, profile string) {
 
 // waitForGvisorControllerRunning waits for the gvisor controller pod to be running
 func waitForGvisorControllerRunning(p string) error {
-	client, err := kube.Client(p)
+	client, err := kapi.Client(p)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
 	}
 	return nil
@@ -121,13 +121,13 @@ func waitForGvisorControllerRunning(p string) error {
 
 // waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running
 func waitForUntrustedNginxRunning(miniProfile string) error {
-	client, err := kube.Client(miniProfile)
+	client, err := kapi.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}
 	return nil
diff --git a/test/integration/fn_addons.go b/test/integration/fn_addons.go
index 95cfbd7828c3..0ecc1ecba488 100644
--- a/test/integration/fn_addons.go
+++ b/test/integration/fn_addons.go
@@ -34,7 +34,7 @@ import (
 
 	retryablehttp "github.com/hashicorp/go-retryablehttp"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -42,12 +42,12 @@ import (
 func testAddons(t *testing.T) {
 	t.Parallel()
 	p := profileName(t)
-	client, err := kube.Client(p)
+	client, err := kapi.Client(p)
 	if err != nil {
 		t.Fatalf("Could not get kubernetes client: %v", err)
 	}
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"component": "kube-addon-manager"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		t.Errorf("Error waiting for addon manager to be up")
 	}
 }
@@ -192,22 +192,22 @@ func testRegistry(t *testing.T) {
 	p := profileName(t)
 	mk := NewMinikubeRunner(t, p)
 	mk.RunCommand("addons enable registry", true)
-	client, err := kube.Client(p)
+	client, err := kapi.Client(p)
 	if err != nil {
 		t.Fatalf("getting kubernetes client: %v", err)
 	}
-	if err := kube.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil {
+	if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*5); err != nil {
 		t.Fatalf("waiting for registry replicacontroller to stabilize: %v", err)
 	}
 	rs := labels.SelectorFromSet(labels.Set(map[string]string{"actual-registry": "true"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", rs); err != nil {
 		t.Fatalf("waiting for registry pods: %v", err)
 	}
 	ps, err := labels.Parse("kubernetes.io/minikube-addons=registry,actual-registry!=true")
 	if err != nil {
 		t.Fatalf("Unable to parse selector: %v", err)
 	}
-	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", ps); err != nil {
 		t.Fatalf("waiting for registry-proxy pods: %v", err)
 	}
 	ip, stderr := mk.RunCommand("ip", true)
@@ -264,18 +264,18 @@ func testRegistry(t *testing.T) {
 
 // waitForNginxRunning waits for nginx service to be up
 func waitForNginxRunning(t *testing.T, miniProfile string) error {
-	client, err := kube.Client(miniProfile)
+	client, err := kapi.Client(miniProfile)
 
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}
 
-	if err := kube.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
+	if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
 		t.Errorf("Error waiting for nginx service to be up")
 	}
 	return nil
@@ -283,17 +283,17 @@ func waitForNginxRunning(t *testing.T, miniProfile string) error {
 
 // waitForIngressControllerRunning waits until ingress controller pod to be running
 func waitForIngressControllerRunning(miniProfile string) error {
-	client, err := kube.Client(miniProfile)
+	client, err := kapi.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 
-	if err := kube.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil {
+	if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*10); err != nil {
 		return errors.Wrap(err, "waiting for ingress-controller deployment to stabilize")
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"app.kubernetes.io/name": "nginx-ingress-controller"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 		return errors.Wrap(err, "waiting for ingress-controller pods")
 	}
 
diff --git a/test/integration/fn_cluster_dns.go b/test/integration/fn_cluster_dns.go
index bfceae335f66..e27301e134ee 100644
--- a/test/integration/fn_cluster_dns.go
+++ b/test/integration/fn_cluster_dns.go
@@ -26,7 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -34,7 +34,7 @@ import (
 func testClusterDNS(t *testing.T) {
 	t.Parallel()
 	p := profileName(t)
-	client, err := kube.Client(p)
+	client, err := kapi.Client(p)
 	if err != nil {
 		t.Fatalf("Error getting kubernetes client %v", err)
 	}
diff --git a/test/integration/fn_mount.go b/test/integration/fn_mount.go
index 04db1f007583..ce5d45f6c09c 100644
--- a/test/integration/fn_mount.go
+++ b/test/integration/fn_mount.go
@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/util/lock"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
@@ -132,12 +132,12 @@ func writeFilesFromHost(mountedDir string, files []string, content string) error
 }
 
 func waitForPods(s map[string]string, profile string) error {
-	client, err := kube.Client(profile)
+	client, err := kapi.Client(profile)
 	if err != nil {
 		return fmt.Errorf("getting kubernetes client: %v", err)
 	}
 	selector := labels.SelectorFromSet(labels.Set(s))
-	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return err
 	}
 	return nil
diff --git a/test/integration/fn_pv.go b/test/integration/fn_pv.go
index ed80ff5ef5ac..c938400610a1 100644
--- a/test/integration/fn_pv.go
+++ b/test/integration/fn_pv.go
@@ -29,7 +29,7 @@ import (
 	core "k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
 )
@@ -73,13 +73,13 @@ func testProvisioning(t *testing.T) {
 	// Check that the storage provisioner pod is running
 
 	checkPodRunning := func() error {
-		client, err := kube.Client(p)
+		client, err := kapi.Client(p)
 		if err != nil {
 			return errors.Wrap(err, "getting kubernetes client")
 		}
 		selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"}))
 
-		if err := kube.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+		if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
 			return err
 		}
 		return nil
diff --git a/test/integration/fn_tunnel.go b/test/integration/fn_tunnel.go
index a68c53dbc45a..d3721c71bffa 100644
--- a/test/integration/fn_tunnel.go
+++ b/test/integration/fn_tunnel.go
@@ -31,7 +31,7 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/tunnel"
 	"k8s.io/minikube/pkg/util/retry"
 	"k8s.io/minikube/test/integration/util"
@@ -70,18 +70,18 @@ func testTunnel(t *testing.T) {
 		t.Fatalf("creating nginx ingress resource: %s", err)
 	}
 
-	client, err := kube.Client(p)
+	client, err := kapi.Client(p)
 
 	if err != nil {
 		t.Fatal(errors.Wrap(err, "getting kubernetes client"))
 	}
 
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"}))
-	if err := kube.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		t.Fatal(errors.Wrap(err, "waiting for nginx pods"))
 	}
 
-	if err := kube.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
+	if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
 		t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
 	}
 
@@ -121,7 +121,7 @@ func getIngress(kr *util.KubectlRunner) (string, error) {
 		case err == nil:
 			nginxIP = string(stdout)
 			return len(stdout) != 0, nil
-		case !kube.IsRetryableAPIError(err):
+		case !kapi.IsRetryableAPIError(err):
 			ret = fmt.Errorf("`%s` failed with non retriable error: %v", cmd, err)
 			return false, err
 		default:
diff --git a/test/integration/util/common.go b/test/integration/util/common.go
index 9037ae660b0f..c313837155fd 100644
--- a/test/integration/util/common.go
+++ b/test/integration/util/common.go
@@ -23,17 +23,17 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/minikube/pkg/kube"
+	"k8s.io/minikube/pkg/kapi"
 )
 
 // WaitForBusyboxRunning waits until busybox pod to be running
 func WaitForBusyboxRunning(t *testing.T, namespace string, miniProfile string) error {
-	client, err := kube.Client(miniProfile)
+	client, err := kapi.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "busybox"}))
-	return kube.WaitForPodsWithLabelRunning(client, namespace, selector)
+	return kapi.WaitForPodsWithLabelRunning(client, namespace, selector)
 }
 
 // Logf writes logs to stdout if -v is set.

From dd94e15f7bab5cf888b7762c9e0a527bef7ee30b Mon Sep 17 00:00:00 2001
From: Medya Gh <medya@google.com>
Date: Mon, 19 Aug 2019 16:23:40 -0700
Subject: [PATCH 7/7] move the gvisor test setup before the minikube start

---
 hack/jenkins/common.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh
index 8e4efb7e32c0..db0df5816604 100755
--- a/hack/jenkins/common.sh
+++ b/hack/jenkins/common.sh
@@ -279,9 +279,6 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true
 rmdir "${TEST_HOME}"
 echo ">> ${TEST_HOME} completed at $(date)"
 
-# Build the gvisor image. This will be copied into minikube and loaded by ctr.
-docker build -t gcr.io/k8s-minikube/gvisor-addon:latest -f testdata/gvisor-addon-Dockerfile out
-
 if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then
   readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
   curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \