Skip to content

Commit

Permalink
fix: don't reload control plane pods on cert SANs changes
Browse files Browse the repository at this point in the history 
Fixes #7159

The change looks big, but it's actually pretty simple inside: the static
pods had an annotation which tracks a version of the secrets which
forced control plane pods to reload on a change. At the same time
`kube-apiserver` can reload certificate inputs automatically from files
without restart.

So the inputs were split: the dynamic (for kube-apiserver) inputs don't
need to be reloaded, so its version is not tracked in static pod
annotation, so they don't cause a reload. The previous non-dynamic
resource still causes a reload, but it doesn't get updated when e.g.
node addresses change.

There might be many more refactoring done, the resource chain is a bit
of a mess there, but I wanted to keep number of changes minimal to keep
this backportable.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
  • Loading branch information
@smira
smira committed May 5, 2023
1 parent d43c61e commit 860002c
Show file tree
Hide file tree
Showing 19 changed files with 1,063 additions and 576 deletions.
Binary file modified api/api.descriptors
View file
Binary file not shown.
10 changes: 7 additions & 3 deletions api/resource/definitions/secrets/secrets.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,19 @@ message KubeletSpec {

// KubernetesCertsSpec describes generated Kubernetes certificates.
message KubernetesCertsSpec {
common.PEMEncodedCertificateAndKey api_server = 1;
common.PEMEncodedCertificateAndKey api_server_kubelet_client = 2;
common.PEMEncodedCertificateAndKey front_proxy = 3;
string scheduler_kubeconfig = 4;
string controller_manager_kubeconfig = 5;
string localhost_admin_kubeconfig = 6;
string admin_kubeconfig = 7;
}

// KubernetesDynamicCertsSpec describes generated KubernetesCerts certificates.
message KubernetesDynamicCertsSpec {
common.PEMEncodedCertificateAndKey api_server = 1;
common.PEMEncodedCertificateAndKey api_server_kubelet_client = 2;
common.PEMEncodedCertificateAndKey front_proxy = 3;
}

// KubernetesRootSpec describes root Kubernetes secrets.
message KubernetesRootSpec {
string name = 1;
Expand Down
31 changes: 18 additions & 13 deletions internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/safe"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/go-pointer"
"go.uber.org/zap"
Expand Down Expand Up @@ -47,8 +48,8 @@ func (ctrl *KubeletStaticPodController) Inputs() []controller.Input {
},
{
Namespace: secrets.NamespaceName,
Type: secrets.KubernetesType,
ID: pointer.To(secrets.KubernetesID),
Type: secrets.KubernetesDynamicCertsType,
ID: pointer.To(secrets.KubernetesDynamicCertsID),
Kind: controller.InputWeak,
},
{
Expand Down Expand Up @@ -94,7 +95,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru
case <-r.EventCh():
}

kubeletResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "kubelet", resource.VersionUndefined))
kubeletService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "kubelet", resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
kubeletClient = nil
Expand All @@ -109,7 +110,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru
return err
}

if !kubeletResource.(*v1alpha1.Service).TypedSpec().Running {
if !kubeletService.TypedSpec().Running {
kubeletClient = nil

if err = ctrl.teardownStatuses(ctx, r); err != nil {
Expand All @@ -121,7 +122,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru

// on worker nodes, there's no way to connect to the kubelet to fetch the pod status (only API server can do that)
// on control plane nodes, use API servers' client kubelet certificate to fetch statuses
rootSecretResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined))
rootSecrets, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
kubeletClient = nil
Expand All @@ -132,9 +133,10 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru
return err
}

rootSecrets := rootSecretResource.(*secrets.KubernetesRoot).TypedSpec()

secretsResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined))
certsResource, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts](
ctx, r,
resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined),
)
if err != nil {
if state.IsNotFoundError(err) {
kubeletClient = nil
Expand All @@ -145,17 +147,20 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru
return err
}

secrets := secretsResource.(*secrets.Kubernetes).TypedSpec()
certs := certsResource.TypedSpec()

nodenameResource, err := r.Get(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined))
nodename, err := safe.ReaderGet[*k8s.Nodename](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined))
if err != nil {
// nodename should exist if the kubelet is running
return err
}

nodename := nodenameResource.(*k8s.Nodename).TypedSpec().Nodename

kubeletClient, err = kubelet.NewClient(nodename, secrets.APIServerKubeletClient.Crt, secrets.APIServerKubeletClient.Key, rootSecrets.CA.Crt)
kubeletClient, err = kubelet.NewClient(
nodename.TypedSpec().Nodename,
certs.APIServerKubeletClient.Crt,
certs.APIServerKubeletClient.Key,
rootSecrets.TypedSpec().CA.Crt,
)
if err != nil {
return fmt.Errorf("error building kubelet client: %w", err)
}
Expand Down
42 changes: 31 additions & 11 deletions internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/safe"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/crypto/x509"
"github.com/siderolabs/go-pointer"
Expand Down Expand Up @@ -53,6 +54,12 @@ func (ctrl *RenderSecretsStaticPodController) Inputs() []controller.Input { //no
ID: pointer.To(secrets.KubernetesID),
Kind: controller.InputWeak,
},
{
Namespace: secrets.NamespaceName,
Type: secrets.KubernetesDynamicCertsType,
ID: pointer.To(secrets.KubernetesDynamicCertsID),
Kind: controller.InputWeak,
},
{
Namespace: secrets.NamespaceName,
Type: secrets.EtcdType,
Expand Down Expand Up @@ -83,7 +90,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
case <-r.EventCh():
}

secretsRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined))
secretsRes, err := safe.ReaderGet[*secrets.Kubernetes](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
Expand All @@ -92,7 +99,19 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
return fmt.Errorf("error getting secrets resource: %w", err)
}

etcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined))
certsRes, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts](
ctx, r,
resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined),
)
if err != nil {
if state.IsNotFoundError(err) {
continue
}

return fmt.Errorf("error getting certificates resource: %w", err)
}

etcdRes, err := safe.ReaderGet[*secrets.Etcd](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
Expand All @@ -101,7 +120,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
return fmt.Errorf("error getting secrets resource: %w", err)
}

rootEtcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined))
rootEtcdRes, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
Expand All @@ -110,7 +129,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
return fmt.Errorf("error getting secrets resource: %w", err)
}

rootK8sRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined))
rootK8sRes, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
Expand All @@ -119,10 +138,11 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
return fmt.Errorf("error getting secrets resource: %w", err)
}

rootEtcdSecrets := rootEtcdRes.(*secrets.EtcdRoot).TypedSpec()
rootK8sSecrets := rootK8sRes.(*secrets.KubernetesRoot).TypedSpec()
etcdSecrets := etcdRes.(*secrets.Etcd).TypedSpec()
k8sSecrets := secretsRes.(*secrets.Kubernetes).TypedSpec()
rootEtcdSecrets := rootEtcdRes.TypedSpec()
rootK8sSecrets := rootK8sRes.TypedSpec()
etcdSecrets := etcdRes.TypedSpec()
k8sSecrets := secretsRes.TypedSpec()
k8sCerts := certsRes.TypedSpec()

serviceAccountKey, err := rootK8sSecrets.ServiceAccount.GetKey()
if err != nil {
Expand Down Expand Up @@ -168,12 +188,12 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
certFilename: "ca.crt",
},
{
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServer },
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServer },
certFilename: "apiserver.crt",
keyFilename: "apiserver.key",
},
{
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServerKubeletClient },
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServerKubeletClient },
certFilename: "apiserver-kubelet-client.crt",
keyFilename: "apiserver-kubelet-client.key",
},
Expand All @@ -192,7 +212,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
certFilename: "aggregator-ca.crt",
},
{
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.FrontProxy },
getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.FrontProxy },
certFilename: "front-proxy-client.crt",
keyFilename: "front-proxy-client.key",
},
Expand Down
Loading

0 comments on commit 860002c

Please sign in to comment.