feat: support kubelet credentialprovider config

Support configuring kubelet credential provider config.

Partially fixes: #7911

Signed-off-by: Noel Georgi <git@frezbo.dev>

Dockerfile

@@ -568,7 +568,7 @@ COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
 RUN <<END
     cleanup.sh /rootfs
     mkdir -pv /rootfs/{boot/EFI,etc/cri/conf.d/hosts,lib/firmware,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
-    mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes}
+    mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders}
     mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
 END
 COPY --chmod=0644 hack/zoneinfo/Etc/UTC /rootfs/usr/share/zoneinfo/Etc/UTC
@@ -632,7 +632,7 @@ COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
 RUN <<END
     cleanup.sh /rootfs
     mkdir -pv /rootfs/{boot/EFI,etc/cri/conf.d/hosts,lib/firmware,usr/local/share,usr/share/zoneinfo/Etc,mnt,system,opt,.extra}
-    mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes}
+    mkdir -pv /rootfs/{etc/kubernetes/manifests,etc/cni/net.d,usr/libexec/kubernetes,/usr/local/lib/kubelet/credentialproviders}
     mkdir -pv /rootfs/opt/{containerd/bin,containerd/lib}
 END
 COPY --chmod=0644 hack/zoneinfo/Etc/UTC /rootfs/usr/share/zoneinfo/Etc/UTC

Makefile

@@ -47,6 +47,8 @@ TEXTLINT_VERSION ?= 13.3.3
 TEXTLINT_FILTER_RULE_COMMENTS_VERSION ?= 1.2.2
 # renovate: datasource=npm depName=textlint-rule-one-sentence-per-line
 TEXTLINT_RULE_ONE_SENTENCE_PER_LINE_VERSION ?= 2.0.0
+# renovate: datasource=docker depName=klakegg/hugo
+HUGO_VERSION ?= 0.99.1-ext-alpine
 OPERATING_SYSTEM := $(shell uname -s | tr "[:upper:]" "[:lower:]")
 ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
 TALOSCTL_DEFAULT_TARGET := talosctl-$(OPERATING_SYSTEM)
@@ -235,9 +237,10 @@ docs: ## Generates the documentation for machine config, and talosctl.
 .PHONY: docs-preview
 docs-preview: ## Starts a local preview of the documentation using Hugo in docker
 	@docker run --rm --interactive --tty \
+	--user $(shell id -u):$(shell id -g) \
 	--volume $(PWD):/src --workdir /src/website \
 	--publish 1313:1313 \
-	klakegg/hugo:0.95.0-ext-alpine \
+	klakegg/hugo:$(HUGO_VERSION) \
 	server
 
 # Local Artifacts

api/resource/definitions/k8s/k8s.proto

@@ -143,6 +143,7 @@ message KubeletConfigSpec {
   string static_pod_list_url = 10;
   bool disable_manifests_directory = 11;
   bool enable_fs_quota_monitoring = 12;
+  google.protobuf.Struct credential_provider_config = 13;
 }
 
 // KubeletSpecSpec holds the source of kubelet configuration.
@@ -152,6 +153,7 @@ message KubeletSpecSpec {
   repeated talos.resource.definitions.proto.Mount extra_mounts = 3;
   string expected_nodename = 4;
   google.protobuf.Struct config = 5;
+  google.protobuf.Struct credential_provider_config = 6;
 }
 
 // ManifestSpec holds the Kubernetes resources spec.

hack/release.toml

@@ -21,6 +21,15 @@ Starting with Talos 1.6, there is no Linux firmware included in the initramfs.
 Customers who need Linux firmware can pull them as extension during install time using the image factory service.
 If the initial boot requires firmware, a custom iso can be built with the firmware included using the image factory service.
 This also ensures that the linux-firmware is not tied to a specific Talos version.
+"""
+
+    [notes.Kubelet]
+        title = "Kubelet Credential Provider Configuration"
+        description = """\
+Talos now supports specifying the kubelet credential provider configuration in the Talos configuration file.
+It can be set under `machine.kubelet.credentialProviderConfig` and kubelet will be automatically configured to with the correct flags.
+The credential binaries are expected to be present under `/usr/local/lib/kubelet/credentialproviders`.
+Talos System Extensions can be used to install the credential binaries.
 """
 
     [notes.updates]

internal/app/machined/pkg/controllers/k8s/kubelet_config.go

@@ -77,6 +77,7 @@ func NewKubeletConfigController() *KubeletConfigController {
 				kubeletConfig.StaticPodListURL = staticPodURL.TypedSpec().URL
 				kubeletConfig.DisableManifestsDirectory = cfgProvider.Machine().Kubelet().DisableManifestsDirectory()
 				kubeletConfig.EnableFSQuotaMonitoring = cfgProvider.Machine().Features().DiskQuotaSupportEnabled()
+				kubeletConfig.CredentialProviderConfig = cfgProvider.Machine().Kubelet().CredentialProviderConfig()
 
 				return nil
 			},

internal/app/machined/pkg/controllers/k8s/kubelet_service.go

@@ -179,6 +179,10 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt
 			return fmt.Errorf("error writing kubelet configuration: %w", err)
 		}
 
+		if err = ctrl.writeKubeletCredentialProviderConfig(cfgSpec); err != nil {
+			return fmt.Errorf("error writing kubelet credential provider configuration: %w", err)
+		}
+
 		_, running, err := ctrl.V1Alpha1Services.IsRunning("kubelet")
 		if err != nil {
 			ctrl.V1Alpha1Services.Load(&services.Kubelet{})
@@ -369,6 +373,35 @@ func (ctrl *KubeletServiceController) writeConfig(cfgSpec *k8s.KubeletSpecSpec)
 	return os.WriteFile("/etc/kubernetes/kubelet.yaml", buf.Bytes(), 0o600)
 }
 
+func (ctrl *KubeletServiceController) writeKubeletCredentialProviderConfig(cfgSpec *k8s.KubeletSpecSpec) error {
+	if cfgSpec.CredentialProviderConfig == nil {
+		return os.RemoveAll(constants.KubeletCredentialProviderConfig)
+	}
+
+	var kubeletCredentialProviderConfig kubeletconfig.CredentialProviderConfig
+
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(cfgSpec.CredentialProviderConfig, &kubeletCredentialProviderConfig); err != nil {
+		return fmt.Errorf("error converting kubelet credentialprovider configuration from unstructured: %w", err)
+	}
+
+	serializer := json.NewSerializerWithOptions(
+		json.DefaultMetaFactory,
+		nil,
+		nil,
+		json.SerializerOptions{
+			Yaml: true,
+		},
+	)
+
+	var buf bytes.Buffer
+
+	if err := serializer.Encode(&kubeletCredentialProviderConfig, &buf); err != nil {
+		return err
+	}
+
+	return os.WriteFile(constants.KubeletCredentialProviderConfig, buf.Bytes(), 0o600)
+}
+
 // updateKubeconfig updates the kubeconfig of kubelet with the given endpoint if it exists.
 func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, logger *zap.Logger) error {
 	config, err := clientcmd.LoadFromFile(constants.KubeletKubeconfig)

internal/app/machined/pkg/controllers/k8s/kubelet_spec.go

@@ -77,7 +77,7 @@ func (ctrl *KubeletSpecController) Outputs() []controller.Output {
 
 // Run implements controller.Controller interface.
 //
-//nolint:gocyclo
+//nolint:gocyclo,cyclop
 func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
 	for {
 		select {
@@ -166,6 +166,12 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime
 			return fmt.Errorf("error merging arguments: %w", err)
 		}
 
+		// these flags are present from v1.24
+		if cfgSpec.CredentialProviderConfig != nil {
+			args["image-credential-provider-bin-dir"] = constants.KubeletCredentialProviderBinDir
+			args["image-credential-provider-config"] = constants.KubeletCredentialProviderConfig
+		}
+
 		kubeletConfig, err := NewKubeletConfiguration(cfgSpec, kubeletVersion)
 		if err != nil {
 			return fmt.Errorf("error creating kubelet configuration: %w", err)
@@ -195,6 +201,7 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime
 				kubeletSpec.Args = args.Args()
 				kubeletSpec.Config = unstructuredConfig
 				kubeletSpec.ExpectedNodename = expectedNodename
+				kubeletSpec.CredentialProviderConfig = cfgSpec.CredentialProviderConfig
 
 				return nil
 			},

internal/app/machined/pkg/system/services/kubelet.go

@@ -123,6 +123,7 @@ func (k *Kubelet) Runner(r runtime.Runtime) (runner.Runner, error) {
 		{Type: "bind", Destination: constants.CgroupMountPath, Source: constants.CgroupMountPath, Options: []string{"rbind", "rshared", "rw"}},
 		{Type: "bind", Destination: "/lib/modules", Source: "/lib/modules", Options: []string{"bind", "ro"}},
 		{Type: "bind", Destination: "/etc/kubernetes", Source: "/etc/kubernetes", Options: []string{"bind", "rshared", "rw"}},
+		{Type: "bind", Destination: constants.KubeletCredentialProviderBinDir, Source: constants.KubeletCredentialProviderBinDir, Options: []string{"bind", "ro"}},
 		{Type: "bind", Destination: "/etc/nfsmount.conf", Source: "/etc/nfsmount.conf", Options: []string{"bind", "ro"}},
 		{Type: "bind", Destination: "/etc/machine-id", Source: "/etc/machine-id", Options: []string{"bind", "ro"}},
 		{Type: "bind", Destination: "/etc/os-release", Source: "/etc/os-release", Options: []string{"bind", "ro"}},