diff --git a/Makefile b/Makefile index a3f7e48b4ced..3c166fd91c0b 100755 --- a/Makefile +++ b/Makefile @@ -150,6 +150,10 @@ endif test-iso: go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660" +.PHONY: test-pkg +test-pkg/%: + go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)" + .PHONY: integration integration: out/minikube go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 925ef77a51f3..cd237637603c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -20,7 +20,6 @@ import ( "bytes" "crypto" "fmt" - "html/template" "os" "path/filepath" "strings" @@ -44,48 +43,6 @@ type KubeadmBootstrapper struct { c bootstrapper.CommandRunner } -// TODO(r2d4): template this with bootstrapper.KubernetesConfig -const kubeletSystemdConf = ` -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" -Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" -Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" -Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" -Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs" -ExecStart= -ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS -` - -const kubeletService = ` -[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ - -[Service] -ExecStart=/usr/bin/kubelet -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target -` - -const kubeadmConfigTmpl = ` -apiVersion: kubeadm.k8s.io/v1alpha1 -kind: MasterConfiguration -api: - advertiseAddress: {{.AdvertiseAddress}} - bindPort: {{.APIServerPort}} -kubernetesVersion: {{.KubernetesVersion}} -certificatesDir: {{.CertDir}} -networking: - serviceSubnet: {{.ServiceCIDR}} -etcd: - dataDir: {{.EtcdDataDir}} -nodeName: {{.NodeName}} -` - func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) { h, err := api.Load(config.GetMachineName()) if err != nil { @@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) { func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error { // We use --skip-preflight-checks since we have our own custom addons // that we also stick in /etc/kubernetes/manifests - kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks" - t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl)) b := bytes.Buffer{} - if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil { + if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil { return err } @@ -197,14 +152,6 @@ func addAddons(files *[]assets.CopyableFile) error { } func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error { - restoreTmpl := ` - sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} && - sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} && - sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} && - sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}} - ` - t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl)) - opts := struct { KubeadmConfigFile string }{ @@ -212,7 +159,7 @@ func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) } b := bytes.Buffer{} - if err := t.Execute(&b, opts); err != nil { + if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil { return err } @@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro return bootstrapper.SetupCerts(k.c, k8s) } +func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) { + version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return "", errors.Wrap(err, "parsing kubernetes version") + } + + extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version) + if err != nil { + return "", errors.Wrap(err, "generating extra configuration for kubelet") + } + + extraFlags := convertToFlags(extraOpts) + b := bytes.Buffer{} + if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil { + return "", err + } + + return b.String(), nil +} + func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error { if cfg.ShouldLoadCachedImages { // Make best effort to load any cached images go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir) } - kubeadmCfg, err := k.generateConfig(cfg) + kubeadmCfg, err := generateConfig(cfg) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") } + kubeletCfg, err := NewKubeletConfig(cfg) + if err != nil { + return errors.Wrap(err, "generating kubelet config") + } + files := []assets.CopyableFile{ assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"), - assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"), + assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"), assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"), } @@ -290,9 +262,17 @@ sudo systemctl start kubelet return nil } -func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { - t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl)) +func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) { + version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return "", errors.Wrap(err, "parsing kubernetes version") + } + // generates a map of component to extra args for apiserver, controller-manager, and scheduler + extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version) + if err != nil { + return "", errors.Wrap(err, "generating extra component config for kubeadm") + } opts := struct { CertDir string ServiceCIDR string @@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) KubernetesVersion string EtcdDataDir string NodeName string + ExtraArgs []ComponentExtraArgs }{ CertDir: util.DefaultCertPath, ServiceCIDR: util.DefaultInsecureRegistry, @@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) KubernetesVersion: k8s.KubernetesVersion, EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted NodeName: k8s.NodeName, + ExtraArgs: extraComponentConfig, } b := bytes.Buffer{} - if err := t.Execute(&b, opts); err != nil { + if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil { return "", err } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go new file mode 100644 index 000000000000..e15e56fb26b0 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go @@ -0,0 +1,118 @@ +package kubeadm + +import ( + "testing" + + "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/util" +) + +func TestGenerateConfig(t *testing.T) { + tests := []struct { + description string + cfg bootstrapper.KubernetesConfig + expectedCfg string + shouldErr bool + }{ + { + description: "no extra args", + cfg: bootstrapper.KubernetesConfig{ + NodeIP: "192.168.1.100", + KubernetesVersion: "v1.8.0", + NodeName: "minikube", + }, + expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +api: + advertiseAddress: 192.168.1.100 + bindPort: 8443 +kubernetesVersion: v1.8.0 +certificatesDir: /var/lib/localkube/certs/ +networking: + serviceSubnet: 10.0.0.0/24 +etcd: + dataDir: /data +nodeName: minikube +`, + }, + { + description: "extra args all components", + cfg: bootstrapper.KubernetesConfig{ + NodeIP: "192.168.1.101", + KubernetesVersion: "v1.8.0-alpha.0", + NodeName: "extra-args-minikube", + ExtraOptions: util.ExtraOptionSlice{ + util.ExtraOption{ + Component: Apiserver, + Key: "fail-no-swap", + Value: "true", + }, + util.ExtraOption{ + Component: ControllerManager, + Key: "kube-api-burst", + Value: "32", + }, + util.ExtraOption{ + Component: Scheduler, + Key: "scheduler-name", + Value: "mini-scheduler", + }, + }, + }, + expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +api: + advertiseAddress: 192.168.1.101 + bindPort: 8443 +kubernetesVersion: v1.8.0-alpha.0 +certificatesDir: /var/lib/localkube/certs/ +networking: + serviceSubnet: 10.0.0.0/24 +etcd: + dataDir: /data +nodeName: extra-args-minikube +apiServerExtraArgs: + fail-no-swap: true +controllerManagerExtraArgs: + kube-api-burst: 32 +schedulerExtraArgs: + scheduler-name: mini-scheduler +`, + }, + { + // Unknown components should fail silently + description: "unknown component", + cfg: bootstrapper.KubernetesConfig{ + NodeIP: "192.168.1.101", + KubernetesVersion: "v1.8.0-alpha.0", + NodeName: "extra-args-minikube", + ExtraOptions: util.ExtraOptionSlice{ + util.ExtraOption{ + Component: "not-a-real-component", + Key: "killswitch", + Value: "true", + }, + }, + }, + shouldErr: true, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + actualCfg, err := generateConfig(test.cfg) + if err != nil && !test.shouldErr { + t.Errorf("got unexpected error generating config: %s", err) + return + } + if err == nil && test.shouldErr { + t.Errorf("expected error but got none, config: %s", actualCfg) + return + } + if actualCfg != test.expectedCfg { + t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg) + return + } + }) + } +} diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go new file mode 100644 index 000000000000..a9f0bd358aa0 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -0,0 +1,54 @@ +package kubeadm + +import "html/template" + +var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +api: + advertiseAddress: {{.AdvertiseAddress}} + bindPort: {{.APIServerPort}} +kubernetesVersion: {{.KubernetesVersion}} +certificatesDir: {{.CertDir}} +networking: + serviceSubnet: {{.ServiceCIDR}} +etcd: + dataDir: {{.EtcdDataDir}} +nodeName: {{.NodeName}} +{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}} + {{$key}}: {{$value}} +{{end}}{{end}}`)) + +var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true" +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true" +Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" +Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" +Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs" +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}} +`)) + +const kubeletService = ` +[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +ExecStart=/usr/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target +` + +var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(` +sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} && +sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} && +sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} && +sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}} +`)) + +var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks")) diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go new file mode 100644 index 000000000000..e266f547f3fa --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -0,0 +1,153 @@ +package kubeadm + +import ( + "fmt" + "strings" + + "github.com/blang/semver" + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/util" +) + +// These are the components that can be configured +// through the "extra-config" +const ( + Kubelet = "kubelet" + Apiserver = "apiserver" + Scheduler = "scheduler" + ControllerManager = "controller-manager" +) + +// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s +// component. +func ExtraConfigForComponent(component string, opts util.ExtraOptionSlice, version semver.Version) (map[string]string, error) { + versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version) + if err != nil { + return nil, errors.Wrapf(err, "setting version specific options for %s", component) + } + + for _, opt := range opts { + if opt.Component == component { + if val, ok := versionedOpts[opt.Key]; ok { + glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component) + } + versionedOpts[opt.Key] = opt.Value + } + } + + return versionedOpts, nil +} + +type ComponentExtraArgs struct { + Component string + Options map[string]string +} + +var componentToKubeadmConfigKey = map[string]string{ + Apiserver: "apiServerExtraArgs", + Scheduler: "schedulerExtraArgs", + ControllerManager: "controllerManagerExtraArgs", +} + +func NewComponentExtraArgs(opts util.ExtraOptionSlice, version semver.Version) ([]ComponentExtraArgs, error) { + var kubeadmExtraArgs []ComponentExtraArgs + for _, extraOpt := range opts { + kubeadmKey, ok := componentToKubeadmConfigKey[extraOpt.Component] + if !ok { + return nil, fmt.Errorf("Unknown component %s. Valid components and kubeadm config are %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey) + } + extraConfig, err := ExtraConfigForComponent(extraOpt.Component, opts, version) + if err != nil { + return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", extraOpt.Component) + } + if len(extraConfig) > 0 { + kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{ + Component: kubeadmKey, + Options: extraConfig, + }) + } + } + + return kubeadmExtraArgs, nil +} + +func ParseKubernetesVersion(version string) (semver.Version, error) { + // Strip leading 'v' prefix from version for semver parsing + v, err := semver.Make(version[1:]) + if err != nil { + return semver.Version{}, errors.Wrap(err, "parsing kubernetes version") + } + + return v, nil +} + +func convertToFlags(opts map[string]string) string { + var flags []string + for k, v := range opts { + flags = append(flags, fmt.Sprintf("--%s=%s", k, v)) + } + return strings.Join(flags, " ") +} + +// VersionedExtraOption holds information on flags to apply to a specific range +// of versions +type VersionedExtraOption struct { + // Special Cases: + // + // If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied + // to all versions + // + // If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that + // specific version + + // The flag and component that will be set + Option util.ExtraOption + + // This flag will only be applied to versions before or equal to this version + // If it is the default value, it will have no upper bound on versions the + // flag is applied to + LessThanOrEqual semver.Version + + // The flag will only be applied to versions after or equal to this version + // If it is the default value, it will have no lower bound on versions the + // flag is applied to + GreaterThanOrEqual semver.Version +} + +var versionSpecificOpts = []VersionedExtraOption{ + VersionedExtraOption{ + Option: util.ExtraOption{ + Component: Kubelet, + Key: "fail-swap-on", + Value: "false", + }, + GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"), + }, +} + +func VersionIsBetween(version, gte, lte semver.Version) bool { + if gte.NE(semver.Version{}) && !version.GTE(gte) { + return false + } + if lte.NE(semver.Version{}) && !version.LTE(lte) { + return false + } + + return true +} + +func DefaultOptionsForComponentAndVersion(component string, version semver.Version) (map[string]string, error) { + versionedOpts := map[string]string{} + for _, opts := range versionSpecificOpts { + if opts.Option.Component == component { + if VersionIsBetween(version, opts.GreaterThanOrEqual, opts.LessThanOrEqual) { + if val, ok := versionedOpts[opts.Option.Key]; ok { + return nil, fmt.Errorf("Flag %s=%s already set %s=%s", opts.Option.Key, opts.Option.Value, opts.Option.Key, val) + } + versionedOpts[opts.Option.Key] = opts.Option.Value + } + } + } + return versionedOpts, nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/versions_test.go b/pkg/minikube/bootstrapper/kubeadm/versions_test.go new file mode 100644 index 000000000000..fdcdc6457200 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/versions_test.go @@ -0,0 +1,87 @@ +package kubeadm + +import ( + "testing" + + "github.com/blang/semver" +) + +func TestVersionIsBetween(t *testing.T) { + tests := []struct { + description string + ver semver.Version + gte semver.Version + lte semver.Version + expected bool + }{ + { + description: "between", + ver: semver.MustParse("1.8.0"), + gte: semver.MustParse("1.7.0"), + lte: semver.MustParse("1.9.0"), + expected: true, + }, + { + description: "less than minimum version", + ver: semver.MustParse("1.6.0"), + gte: semver.MustParse("1.7.0"), + lte: semver.MustParse("1.9.0"), + expected: false, + }, + { + description: "greather than max version", + ver: semver.MustParse("2.8.0"), + gte: semver.MustParse("1.7.0"), + lte: semver.MustParse("1.9.0"), + expected: true, + }, + { + description: "equal to max version", + ver: semver.MustParse("1.9.0"), + gte: semver.MustParse("1.7.0"), + lte: semver.MustParse("1.9.0"), + expected: true, + }, + { + description: "equal to min version", + ver: semver.MustParse("1.7.0"), + gte: semver.MustParse("1.7.0"), + lte: semver.MustParse("1.9.0"), + expected: true, + }, + { + description: "alpha between", + ver: semver.MustParse("1.8.0-alpha.0"), + gte: semver.MustParse("1.8.0"), + lte: semver.MustParse("1.9.0"), + expected: true, + }, + { + description: "beta greater than alpha", + ver: semver.MustParse("1.8.0-beta.1"), + gte: semver.MustParse("1.8.0"), + lte: semver.MustParse("1.8.0-alpha.0"), + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + t.Parallel() + between := VersionIsBetween(test.ver, test.gte, test.lte) + if between != test.expected { + t.Errorf("Expected: %t, Actual: %t", test.expected, between) + } + }) + } +} + +func TestParseKubernetesVersion(t *testing.T) { + version, err := ParseKubernetesVersion("v1.8.0-alpha.5") + if err != nil { + t.Fatalf("Error parsing version: %s", err) + } + if version.NE(semver.MustParse("1.8.0-alpha.5")) { + t.Errorf("Expected: %s, Actual:%s", "1.8.0-alpha.5", version) + } +}