Skip to content

Commit

Permalink
Implement extra-config for kubeadm kubelet
Browse files Browse the repository at this point in the history
  • Loading branch information
r2d4 committed Oct 4, 2017
1 parent 93323c6 commit a139ccd
Show file tree
Hide file tree
Showing 6 changed files with 522 additions and 60 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ endif
test-iso:
go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"

.PHONY: test-pkg
test-pkg/%:
go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)"

.PHONY: integration
integration: out/minikube
go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS)
Expand Down
102 changes: 42 additions & 60 deletions pkg/minikube/bootstrapper/kubeadm/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"bytes"
"crypto"
"fmt"
"html/template"
"os"
"path/filepath"
"strings"
Expand All @@ -44,48 +43,6 @@ type KubeadmBootstrapper struct {
c bootstrapper.CommandRunner
}

// TODO(r2d4): template this with bootstrapper.KubernetesConfig
const kubeletSystemdConf = `
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
`

const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`

const kubeadmConfigTmpl = `
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
`

func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
h, err := api.Load(config.GetMachineName())
if err != nil {
Expand Down Expand Up @@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) {
func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error {
// We use --skip-preflight-checks since we have our own custom addons
// that we also stick in /etc/kubernetes/manifests
kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"
t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl))
b := bytes.Buffer{}
if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
return err
}

Expand Down Expand Up @@ -197,22 +152,14 @@ func addAddons(files *[]assets.CopyableFile) error {
}

func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error {
restoreTmpl := `
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`
t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl))

opts := struct {
KubeadmConfigFile string
}{
KubeadmConfigFile: constants.KubeadmConfigFile,
}

b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil {
return err
}

Expand All @@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
return bootstrapper.SetupCerts(k.c, k8s)
}

func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}

extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra configuration for kubelet")
}

extraFlags := convertToFlags(extraOpts)
b := bytes.Buffer{}
if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
return "", err
}

return b.String(), nil
}

func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error {
if cfg.ShouldLoadCachedImages {
// Make best effort to load any cached images
go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir)
}
kubeadmCfg, err := k.generateConfig(cfg)
kubeadmCfg, err := generateConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubeadm cfg")
}

kubeletCfg, err := NewKubeletConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubelet config")
}

files := []assets.CopyableFile{
assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"),
}

Expand Down Expand Up @@ -290,9 +262,17 @@ sudo systemctl start kubelet
return nil
}

func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl))
func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}

// generates a map of component to extra args for apiserver, controller-manager, and scheduler
extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra component config for kubeadm")
}
opts := struct {
CertDir string
ServiceCIDR string
Expand All @@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion string
EtcdDataDir string
NodeName string
ExtraArgs []ComponentExtraArgs
}{
CertDir: util.DefaultCertPath,
ServiceCIDR: util.DefaultInsecureRegistry,
Expand All @@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion: k8s.KubernetesVersion,
EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted
NodeName: k8s.NodeName,
ExtraArgs: extraComponentConfig,
}

b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
return "", err
}

Expand Down
134 changes: 134 additions & 0 deletions pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubeadm

import (
"testing"

"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/util"
)

func TestGenerateConfig(t *testing.T) {
tests := []struct {
description string
cfg bootstrapper.KubernetesConfig
expectedCfg string
shouldErr bool
}{
{
description: "no extra args",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.100",
KubernetesVersion: "v1.8.0",
NodeName: "minikube",
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.100
bindPort: 8443
kubernetesVersion: v1.8.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: minikube
`,
},
{
description: "extra args all components",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: Apiserver,
Key: "fail-no-swap",
Value: "true",
},
util.ExtraOption{
Component: ControllerManager,
Key: "kube-api-burst",
Value: "32",
},
util.ExtraOption{
Component: Scheduler,
Key: "scheduler-name",
Value: "mini-scheduler",
},
},
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.101
bindPort: 8443
kubernetesVersion: v1.8.0-alpha.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: extra-args-minikube
apiServerExtraArgs:
fail-no-swap: true
controllerManagerExtraArgs:
kube-api-burst: 32
schedulerExtraArgs:
scheduler-name: mini-scheduler
`,
},
{
// Unknown components should fail silently
description: "unknown component",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: "not-a-real-component",
Key: "killswitch",
Value: "true",
},
},
},
shouldErr: true,
},
}

for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
actualCfg, err := generateConfig(test.cfg)
if err != nil && !test.shouldErr {
t.Errorf("got unexpected error generating config: %s", err)
return
}
if err == nil && test.shouldErr {
t.Errorf("expected error but got none, config: %s", actualCfg)
return
}
if actualCfg != test.expectedCfg {
t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg)
return
}
})
}
}
70 changes: 70 additions & 0 deletions pkg/minikube/bootstrapper/kubeadm/templates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubeadm

import "html/template"

var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
{{$key}}: {{$value}}
{{end}}{{end}}`))

var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
`))

const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`

var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(`
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`))

var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))
Loading

0 comments on commit a139ccd

Please sign in to comment.