Remove creation of static node bootstrapToken (gardener#4824)

* Remove creation of static node bootstrapToken This commit is part of issue [gardener#3898](gardener#3898) which replaces the long-valid bootstrap-token shared between nodes with a short-lived token unique for each node. The new flow of using smaller-scoped, short-lived tokens was already active once you updated to compatible versions of the infrastructure-extension, operatingsystem-extension and of gardener/gardener as specified in: gardener#3898 With this commit we are now removing the old secret from the Shoot which means you need to run supported versions of the os-extensions and the infrastructure-provider-extensions when upgrading gardener to this version. * Apply suggestions from rfranzke Co-authored-by: Rafael Franzke <rafael.franzke@sap.com> * Change shipped feature to v1.35 Co-authored-by: Rafael Franzke <rafael.franzke@sap.com> Co-authored-by: Rafael Franzke <rafael.franzke@sap.com>
krgostev · Apr 21, 2022 · 47896ef · 47896ef
1 parent 1acfc2b
commit 47896ef
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 159 deletions.
diff --git a/docs/extensions/operatingsystemconfig.md b/docs/extensions/operatingsystemconfig.md
@@ -55,10 +55,25 @@ One implementation of that is depicted in the picture where the machine-controll
 As part of the user-data the bootstrap-token is placed on the newly created VM under a defined path.
 The cloud-config-script will then refer to the file path of the added bootstrap token in the kubelet-bootstrap script.
 
-If either the `Worker` controller does not replace the `<<BOOTSTRAP_TOKEN>>` or the OS extension does not support the `transmitUnencoded` flag then the bootstrap flow will fall back to the old flow of using a shared bootstrap token between the worker nodes of a shoot cluster.
-
 ![Bootstrap flow with shortlived bootstrapTokens](./images/bootstrap_token.png)
 
+### Compatibility matrix for node bootstrap-token
+
+With Gardener v1.23, we replaced the long-valid bootstrap-token shared between nodes with a short-lived token unique for each node, ref: [#3898](https://github.com/gardener/gardener/issues/3898).
+
+❗ When updating to Gardener version >=1.35 the old bootstrap-token will be removed. You are required to update your extensions to the following versions when updating Gardener:
+
+| Extension   | Version   | Release Date   | Pull Request |
+|---|---|---|---|
+| os-gardenlinux   |  v0.9.0  |   2 Jul  |  https://github.com/gardener/gardener-extension-os-gardenlinux/pull/29  |
+| os-suse-chost   | v1.11.0 |  2 Jul  |  https://github.com/gardener/gardener-extension-os-suse-chost/pull/41 |
+| os-ubuntu   | v1.11.0   |  2 Jul   |  https://github.com/gardener/gardener-extension-os-ubuntu/pull/42 |
+| os-flatcar   |  v1.7.0  |  2 Jul   |   https://github.com/gardener/gardener-extension-os-coreos/pull/24 |
+| infrastructure-provider using Machine Controller Manager |  varies | ~ end of 2019   | https://github.com/gardener/machine-controller-manager/pull/351  |
+
+⚠️ If you run a provider extension that does not use Machine Controller Manager (MCM) you need to implement the functionality of creating a temporary bootstrap-token before updating your Gardener version to v1.35 or higher.
+All provider extensions maintained in https://github.com/gardener/ use MCM.
+
 ## How does Gardener update the user-data on already existing machines?
 
 With ongoing development and new releases of Gardener some new components could be required to get installed onto every shoot worker VM, or existing components need to be changed.

diff --git a/pkg/operation/botanist/component/extensions/operatingsystemconfig/executor/executor.go b/pkg/operation/botanist/component/extensions/operatingsystemconfig/executor/executor.go
@@ -96,7 +96,6 @@ const (
 
 // Script returns the executor script that applies the downloaded cloud-config user-data.
 func Script(
-	bootstrapToken string,
 	cloudConfigUserData []byte,
 	hyperkubeImage *imagevector.Image,
 	kubernetesVersion string,
@@ -122,7 +121,6 @@ func Script(
 		"annotationKeyChecksum":                    AnnotationKeyChecksum,
 		"pathKubeletDirectory":                     kubelet.PathKubeletDirectory,
 		"pathDownloadsDirectory":                   downloader.PathDownloadsDirectory,
-		"bootstrapToken":                           bootstrapToken,
 		"pathBinaries":                             v1beta1constants.OperatingSystemConfigFilePathBinaries,
 		"pathBootstrapToken":                       downloader.PathBootstrapToken,
 		"pathCCDScript":                            downloader.PathCCDScript,
@@ -142,7 +140,6 @@ func Script(
 		"pathLastDownloadedHyperkubeImage":         PathLastDownloadedHyperkubeImage,
 		"pathScriptCopyKubernetesBinary":           kubelet.PathScriptCopyKubernetesBinary,
 		"bootstrapTokenPlaceholder":                downloader.BootstrapTokenPlaceholder,
-		"bootstrapTokenPlaceholderB64":             utils.EncodeBase64([]byte(downloader.BootstrapTokenPlaceholder)),
 		"cloudConfigUserData":                      utils.EncodeBase64(cloudConfigUserData),
 		"cloudConfigDownloaderName":                downloader.Name,
 		"executionMinDelaySeconds":                 downloader.UnitRestartSeconds,

diff --git a/pkg/operation/botanist/component/extensions/operatingsystemconfig/executor/executor_test.go b/pkg/operation/botanist/component/extensions/operatingsystemconfig/executor/executor_test.go
@@ -34,7 +34,6 @@ import (
 var _ = Describe("Executor", func() {
 	Describe("#Script", func() {
 		var (
-			bootstrapToken      string
 			cloudConfigUserData []byte
 			hyperkubeImage      *imagevector.Image
 			reloadConfigCommand string
@@ -45,7 +44,6 @@ var _ = Describe("Executor", func() {
 		)
 
 		BeforeEach(func() {
-			bootstrapToken = "token"
 			cloudConfigUserData = []byte("user-data")
 			hyperkubeImage = &imagevector.Image{Repository: "bar", Tag: pointer.String("v1.0")}
 			reloadConfigCommand = "/var/bin/reload"
@@ -62,9 +60,9 @@ var _ = Describe("Executor", func() {
 
 		DescribeTable("should correctly render the executor script",
 			func(kubernetesVersion string, copyKubernetesBinariesFn func(*imagevector.Image) string, kubeletDataVol *gardencorev1beta1.DataVolume, kubeletDataVolSize *string) {
-				script, err := executor.Script(bootstrapToken, cloudConfigUserData, hyperkubeImage, kubernetesVersion, kubeletDataVol, reloadConfigCommand, units)
+				script, err := executor.Script(cloudConfigUserData, hyperkubeImage, kubernetesVersion, kubeletDataVol, reloadConfigCommand, units)
 				Expect(err).ToNot(HaveOccurred())
-				testScript := scriptFor(bootstrapToken, cloudConfigUserData, hyperkubeImage, copyKubernetesBinariesFn, kubeletDataVolSize, reloadConfigCommand, units)
+				testScript := scriptFor(cloudConfigUserData, hyperkubeImage, copyKubernetesBinariesFn, kubeletDataVolSize, reloadConfigCommand, units)
 				Expect(string(script)).To(Equal(testScript))
 			},
 
@@ -93,7 +91,7 @@ var _ = Describe("Executor", func() {
 		It("should return an error because the data volume size cannot be parsed", func() {
 			kubeletDataVolume := &gardencorev1beta1.DataVolume{VolumeSize: "not-parsable"}
 
-			script, err := executor.Script(bootstrapToken, cloudConfigUserData, hyperkubeImage, "1.2.3", kubeletDataVolume, reloadConfigCommand, units)
+			script, err := executor.Script(cloudConfigUserData, hyperkubeImage, "1.2.3", kubeletDataVolume, reloadConfigCommand, units)
 			Expect(script).To(BeNil())
 			Expect(err).To(MatchError(ContainSubstring("quantities must match the regular expression")))
 		})
@@ -129,7 +127,6 @@ var _ = Describe("Executor", func() {
 })
 
 func scriptFor(
-	bootstrapToken string,
 	cloudConfigUserData []byte,
 	hyperkubeImage *imagevector.Image,
 	copyKubernetesBinariesFn func(*imagevector.Image) string,
@@ -259,15 +256,6 @@ fi
 md5sum ${PATH_CCD_SCRIPT} > ${PATH_CCD_SCRIPT_CHECKSUM}
 
 if [[ ! -f "/var/lib/kubelet/kubeconfig-real" ]] || [[ ! -f "/var/lib/kubelet/pki/kubelet-client-current.pem" ]]; then
-  BOOTSTRAP_TOKEN="` + bootstrapToken + `"
-  # If a bootstrap token file exists and the placeholder got replaced by the Worker extension then use it
-  if [[ -f "/var/lib/cloud-config-downloader/credentials/bootstrap-token" ]]; then
-    FILE_CONTENT="$(cat "/var/lib/cloud-config-downloader/credentials/bootstrap-token")"
-    if [[ $FILE_CONTENT != "<<BOOTSTRAP_TOKEN>>" ]] && [[ $FILE_CONTENT != "PDxCT09UU1RSQVBfVE9LRU4+Pg==" ]]; then
-      BOOTSTRAP_TOKEN="$FILE_CONTENT"
-    fi
-  fi
-
   cat <<EOF > "/var/lib/kubelet/kubeconfig-bootstrap"
 ---
 apiVersion: v1
@@ -287,7 +275,7 @@ users:
 - name: kubelet-bootstrap
   user:
     as-user-extra: {}
-    token: "$BOOTSTRAP_TOKEN"
+    token: "$(cat "/var/lib/cloud-config-downloader/credentials/bootstrap-token")"
 EOF
 
 else

diff --git a/...t/extensions/operatingsystemconfig/executor/templates/scripts/execute-cloud-config.tpl.sh b/...t/extensions/operatingsystemconfig/executor/templates/scripts/execute-cloud-config.tpl.sh
@@ -125,15 +125,6 @@ fi
 md5sum ${PATH_CCD_SCRIPT} > ${PATH_CCD_SCRIPT_CHECKSUM}
 
 if [[ ! -f "{{ .pathKubeletKubeconfigReal }}" ]] || [[ ! -f "{{ .pathKubeletDirectory }}/pki/kubelet-client-current.pem" ]]; then
-  BOOTSTRAP_TOKEN="{{ .bootstrapToken }}"
-  # If a bootstrap token file exists and the placeholder got replaced by the Worker extension then use it
-  if [[ -f "{{ .pathBootstrapToken }}" ]]; then
-    FILE_CONTENT="$(cat "{{ .pathBootstrapToken }}")"
-    if [[ $FILE_CONTENT != "{{ .bootstrapTokenPlaceholder }}" ]] && [[ $FILE_CONTENT != "{{ .bootstrapTokenPlaceholderB64 }}" ]]; then
-      BOOTSTRAP_TOKEN="$FILE_CONTENT"
-    fi
-  fi
-
   cat <<EOF > "{{ .pathKubeletKubeconfigBootstrap }}"
 ---
 apiVersion: v1
@@ -153,7 +144,7 @@ users:
 - name: kubelet-bootstrap
   user:
     as-user-extra: {}
-    token: "$BOOTSTRAP_TOKEN"
+    token: "$(cat "{{ .pathBootstrapToken }}")"
 EOF
 
 else

diff --git a/pkg/operation/botanist/operatingsystemconfig.go b/pkg/operation/botanist/operatingsystemconfig.go
@@ -17,7 +17,6 @@ package botanist
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"github.com/gardener/gardener/charts"
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
@@ -28,11 +27,9 @@ import (
 	"github.com/gardener/gardener/pkg/operation/botanist/component/extensions/operatingsystemconfig/executor"
 	"github.com/gardener/gardener/pkg/operation/botanist/component/extensions/operatingsystemconfig/original/components"
 	"github.com/gardener/gardener/pkg/operation/common"
-	"github.com/gardener/gardener/pkg/utils"
 	"github.com/gardener/gardener/pkg/utils/flow"
 	"github.com/gardener/gardener/pkg/utils/imagevector"
 	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
-	"github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken"
 	"github.com/gardener/gardener/pkg/utils/managedresources"
 	"github.com/gardener/gardener/pkg/utils/secrets"
 
@@ -146,13 +143,6 @@ var (
 // 1. A secret containing the dedicated cloud config execution script for each worker group
 // 2. A secret containing some shared RBAC policies for downloading the cloud config execution script
 func (b *Botanist) DeployManagedResourceForCloudConfigExecutor(ctx context.Context) error {
-	// creates a new token every day
-	bootstrapTokenSecret, err := bootstraptoken.ComputeBootstrapToken(ctx, b.K8sShootClient.Client(), utils.ComputeSHA256Hex([]byte(time.Now().Format("2006-01-02")))[:6], "A bootstrap token generated by Gardener.", 48*time.Hour)
-	if err != nil {
-		return fmt.Errorf("error computing bootstrap token for shoot cloud config: %+v", err)
-	}
-	bootstrapToken := bootstraptoken.FromSecretData(bootstrapTokenSecret.Data)
-
 	hyperkubeImage, err := b.ImageVector.FindImage(charts.ImageNameHyperkube, imagevector.RuntimeVersion(b.ShootVersion()), imagevector.TargetVersion(b.ShootVersion()))
 	if err != nil {
 		return err
@@ -178,7 +168,7 @@ func (b *Botanist) DeployManagedResourceForCloudConfigExecutor(ctx context.Conte
 			return fmt.Errorf("did not find osc data for worker pool %q", worker.Name)
 		}
 
-		secretName, data, err := b.generateCloudConfigExecutorResourcesForWorker(worker, oscData.Original, bootstrapToken, hyperkubeImage)
+		secretName, data, err := b.generateCloudConfigExecutorResourcesForWorker(worker, oscData.Original, hyperkubeImage)
 		if err != nil {
 			return err
 		}
@@ -238,7 +228,6 @@ func (b *Botanist) DeployManagedResourceForCloudConfigExecutor(ctx context.Conte
 func (b *Botanist) generateCloudConfigExecutorResourcesForWorker(
 	worker gardencorev1beta1.Worker,
 	oscDataOriginal operatingsystemconfig.Data,
-	bootstrapToken string,
 	hyperkubeImage *imagevector.Image,
 ) (
 	string,
@@ -261,7 +250,7 @@ func (b *Botanist) generateCloudConfigExecutorResourcesForWorker(
 		}
 	}
 
-	executorScript, err := ExecutorScriptFn(bootstrapToken, []byte(oscDataOriginal.Content), hyperkubeImage, b.Shoot.KubernetesVersion.String(), kubeletDataVolume, *oscDataOriginal.Command, oscDataOriginal.Units)
+	executorScript, err := ExecutorScriptFn([]byte(oscDataOriginal.Content), hyperkubeImage, b.Shoot.KubernetesVersion.String(), kubeletDataVolume, *oscDataOriginal.Command, oscDataOriginal.Units)
 	if err != nil {
 		return "", nil, err
 	}