Skip to content

Commit

Permalink
Implement extra-config for kubeadm kubelet
Browse files Browse the repository at this point in the history
  • Loading branch information
r2d4 committed Oct 4, 2017
1 parent 93323c6 commit 312bec6
Show file tree
Hide file tree
Showing 6 changed files with 458 additions and 60 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ endif
test-iso:
go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"

.PHONY: test-pkg
test-pkg/%:
go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)"

.PHONY: integration
integration: out/minikube
go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS)
Expand Down
102 changes: 42 additions & 60 deletions pkg/minikube/bootstrapper/kubeadm/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"bytes"
"crypto"
"fmt"
"html/template"
"os"
"path/filepath"
"strings"
Expand All @@ -44,48 +43,6 @@ type KubeadmBootstrapper struct {
c bootstrapper.CommandRunner
}

// TODO(r2d4): template this with bootstrapper.KubernetesConfig
const kubeletSystemdConf = `
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
`

const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`

const kubeadmConfigTmpl = `
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
`

func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
h, err := api.Load(config.GetMachineName())
if err != nil {
Expand Down Expand Up @@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) {
func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error {
// We use --skip-preflight-checks since we have our own custom addons
// that we also stick in /etc/kubernetes/manifests
kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"
t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl))
b := bytes.Buffer{}
if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
return err
}

Expand Down Expand Up @@ -197,22 +152,14 @@ func addAddons(files *[]assets.CopyableFile) error {
}

func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error {
restoreTmpl := `
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`
t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl))

opts := struct {
KubeadmConfigFile string
}{
KubeadmConfigFile: constants.KubeadmConfigFile,
}

b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil {
return err
}

Expand All @@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
return bootstrapper.SetupCerts(k.c, k8s)
}

func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}

extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra configuration for kubelet")
}

extraFlags := convertToFlags(extraOpts)
b := bytes.Buffer{}
if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
return "", err
}

return b.String(), nil
}

func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error {
if cfg.ShouldLoadCachedImages {
// Make best effort to load any cached images
go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir)
}
kubeadmCfg, err := k.generateConfig(cfg)
kubeadmCfg, err := generateConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubeadm cfg")
}

kubeletCfg, err := NewKubeletConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubelet config")
}

files := []assets.CopyableFile{
assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"),
}

Expand Down Expand Up @@ -290,9 +262,17 @@ sudo systemctl start kubelet
return nil
}

func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl))
func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}

// generates a map of component to extra args for apiserver, controller-manager, and scheduler
extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra component config for kubeadm")
}
opts := struct {
CertDir string
ServiceCIDR string
Expand All @@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion string
EtcdDataDir string
NodeName string
ExtraArgs []ComponentExtraArgs
}{
CertDir: util.DefaultCertPath,
ServiceCIDR: util.DefaultInsecureRegistry,
Expand All @@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion: k8s.KubernetesVersion,
EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted
NodeName: k8s.NodeName,
ExtraArgs: extraComponentConfig,
}

b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
return "", err
}

Expand Down
118 changes: 118 additions & 0 deletions pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package kubeadm

import (
"testing"

"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/util"
)

func TestGenerateConfig(t *testing.T) {
tests := []struct {
description string
cfg bootstrapper.KubernetesConfig
expectedCfg string
shouldErr bool
}{
{
description: "no extra args",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.100",
KubernetesVersion: "v1.8.0",
NodeName: "minikube",
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.100
bindPort: 8443
kubernetesVersion: v1.8.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: minikube
`,
},
{
description: "extra args all components",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: Apiserver,
Key: "fail-no-swap",
Value: "true",
},
util.ExtraOption{
Component: ControllerManager,
Key: "kube-api-burst",
Value: "32",
},
util.ExtraOption{
Component: Scheduler,
Key: "scheduler-name",
Value: "mini-scheduler",
},
},
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.101
bindPort: 8443
kubernetesVersion: v1.8.0-alpha.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: extra-args-minikube
apiServerExtraArgs:
fail-no-swap: true
controllerManagerExtraArgs:
kube-api-burst: 32
schedulerExtraArgs:
scheduler-name: mini-scheduler
`,
},
{
// Unknown components should fail silently
description: "unknown component",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: "not-a-real-component",
Key: "killswitch",
Value: "true",
},
},
},
shouldErr: true,
},
}

for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
actualCfg, err := generateConfig(test.cfg)
if err != nil && !test.shouldErr {
t.Errorf("got unexpected error generating config: %s", err)
return
}
if err == nil && test.shouldErr {
t.Errorf("expected error but got none, config: %s", actualCfg)
return
}
if actualCfg != test.expectedCfg {
t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg)
return
}
})
}
}
54 changes: 54 additions & 0 deletions pkg/minikube/bootstrapper/kubeadm/templates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package kubeadm

import "html/template"

var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
{{$key}}: {{$value}}
{{end}}{{end}}`))

var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
`))

const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`

var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(`
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`))

var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))
Loading

0 comments on commit 312bec6

Please sign in to comment.