From 860002c7352bedd10845e11da37c80685ff0e720 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 5 May 2023 13:56:11 +0400 Subject: [PATCH] fix: don't reload control plane pods on cert SANs changes Fixes #7159 The change looks big, but it's actually pretty simple inside: the static pods had an annotation which tracks a version of the secrets which forced control plane pods to reload on a change. At the same time `kube-apiserver` can reload certificate inputs automatically from files without restart. So the inputs were split: the dynamic (for kube-apiserver) inputs don't need to be reloaded, so its version is not tracked in static pod annotation, so they don't cause a reload. The previous non-dynamic resource still causes a reload, but it doesn't get updated when e.g. node addresses change. There might be many more refactoring done, the resource chain is a bit of a mess there, but I wanted to keep number of changes minimal to keep this backportable. Signed-off-by: Andrey Smirnov --- api/api.descriptors | Bin 73042 -> 76976 bytes .../definitions/secrets/secrets.proto | 10 +- .../pkg/controllers/k8s/kubelet_static_pod.go | 31 +- .../k8s/render_secrets_static_pod.go | 42 +- .../pkg/controllers/secrets/kubernetes.go | 156 +------ .../secrets/kubernetes_dynamic_certs.go | 248 +++++++++++ .../secrets/kubernetes_dynamic_certs_test.go | 201 +++++++++ .../controllers/secrets/kubernetes_test.go | 151 ++----- .../runtime/v1alpha2/v1alpha2_controller.go | 1 + .../pkg/runtime/v1alpha2/v1alpha2_state.go | 1 + internal/integration/api/apply-config.go | 8 +- .../definitions/secrets/secrets.pb.go | 386 ++++++++++-------- .../definitions/secrets/secrets_vtproto.pb.go | 300 +++++++++----- .../resources/secrets/deep_copy.generated.go | 8 +- pkg/machinery/resources/secrets/kubernetes.go | 7 +- .../resources/secrets/kubernetes_certs.go | 65 +++ pkg/machinery/resources/secrets/secrets.go | 2 +- .../resources/secrets/secrets_test.go | 1 + website/content/v1.5/reference/api.md | 21 +- 19 files changed, 1063 insertions(+), 576 deletions(-) create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go create mode 100644 internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go create mode 100644 pkg/machinery/resources/secrets/kubernetes_certs.go diff --git a/api/api.descriptors b/api/api.descriptors index 1628d6c0a55b15f8f7679668a4eb0fefb5bce64b..fbad017e460c02d004f08615b4bac1c4431b593b 100644 GIT binary patch delta 3752 zcmbUkTWlOxm6?6b-t}WLP2)Tk&w1}8P2s3R2ouwaN)oJEb8A*Z9i9}X zlQaZ7*3jFd;s+5gHU4mGlT1}v(`FUZWmTijcFoFx@omRy7&|pquNoDru~V;ATt`iT z5@|tH1)Awe6u7_&@aK?yG355lGcS3M&ZWHdvlHT>)VX=~Md9nyWT?uvTf2tMs%FJi zWz={%wqw>EMpK+bIYn6|Nt?N>!M#5LY$G!lykE+vykEo<7qZ1B;%Uf0d%Q}H#9h+Km=#>f-pd3seV8|7>38mevwxIrhDAFF*6oGPJHbY8y-R*oxN(@LdzAVkpDe#u;3D=f*;5T z99Poh)N!lU;CZ^KVqiAqs9mT|~X zvEjWw(G-6mdY?^f{ts0d|9`5o^I-C3O#D#vrl%_6Uj^^Z)c#3@63JovCe&O-jq&1{ z^S&{4;Yzj?U}=}0nI=+*RU{JCi@F9?ksy(hR)%CHNo>B5pD)Q`=buyW5%G_|>fF2Z zASK=xJ@d-jZRN{EX*tX`sx{|^W4M-atJY*{bTR^6I*ODfB$TZ!Kr1sOx^Gok;L}st zeY3JxYp^V34s$8H*RxMt{3`yj`i8eT^$RyIN zRe7)y#bf%uKoWr+q`H-6NQI-KkSKiSc#sn1XXI*Co7{KM0Y~J5&m8v(xbTeA9lrIN z>lcPZ+aV0SDg~rDmA41&!l>LfZ>b5050t|ZmBVL_rxujsIenePc!qF>+S+Dz-iCK_n7tUNHaX;O?(i|qq#iFiji)C#I>Jq?09x7GO z>sMibejF#muvr?c0)?%ed<`@eLc3ZveCzF$`idnBU`K=Y39Z?#BNU;M7HXKj_5DbZ~ttJ6EGLsoJ(he^tTdwUhd%ohV)8 zxAs^=C0t<-;m7UT4QJowD?W;G`!Fv>DCZ#_aHLh^7mvEEmdhU%=(GMhy0JB%FJFSq+6T_SLIn_?2b>I>6n4X6p;*#jaSpek z?D33}urE_ED(1oJ;;Nif2#NbWAQL1YA9h13lOw6V3}X)!b2)H1PQ;?_H_xi3t-}J^ z#CHL>isreA2wGpK#RW}{W?Cc>hc1U=^d8MVv-~_6uJ}C$k3r}$=$;6q;M8E1g+)dO znKLhtQD2wqy|QxtrQl2@SR9VI%bd(*BCj#m+_GU}1xpKaa)TZ4sRB8HBj)jCvko1B zQ-6s1K;}7;?82hcMd)tEe`cUWg8$XJy6^N8?>{z8Ht0-p) zsnlBzbgU69wpuqXS6PW50M?SR)_U!xO+Bp_N+PlaeA0T;7mn1%xc*Z~R^lOUQEvr(LEKmwgem+=e% z7nh9!0SB|)+*%j2mi0^rvqSx?4*}V?@C*US0=L}{0TTiN(YNst0mlOl-vbI0OLbyp va&BdGWpkIs76J2@&lLgTx49Pqu@3>imun{hR072Ux4|a?2m!YUDFLVhl6GtX diff --git a/api/resource/definitions/secrets/secrets.proto b/api/resource/definitions/secrets/secrets.proto index 61eaf12301..48e7941db8 100755 --- a/api/resource/definitions/secrets/secrets.proto +++ b/api/resource/definitions/secrets/secrets.proto @@ -43,15 +43,19 @@ message KubeletSpec { // KubernetesCertsSpec describes generated Kubernetes certificates. message KubernetesCertsSpec { - common.PEMEncodedCertificateAndKey api_server = 1; - common.PEMEncodedCertificateAndKey api_server_kubelet_client = 2; - common.PEMEncodedCertificateAndKey front_proxy = 3; string scheduler_kubeconfig = 4; string controller_manager_kubeconfig = 5; string localhost_admin_kubeconfig = 6; string admin_kubeconfig = 7; } +// KubernetesDynamicCertsSpec describes generated KubernetesCerts certificates. +message KubernetesDynamicCertsSpec { + common.PEMEncodedCertificateAndKey api_server = 1; + common.PEMEncodedCertificateAndKey api_server_kubelet_client = 2; + common.PEMEncodedCertificateAndKey front_proxy = 3; +} + // KubernetesRootSpec describes root Kubernetes secrets. message KubernetesRootSpec { string name = 1; diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go index 213ba772fc..e57de80f56 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go @@ -11,6 +11,7 @@ import ( "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" "github.com/cosi-project/runtime/pkg/state" "github.com/siderolabs/go-pointer" "go.uber.org/zap" @@ -47,8 +48,8 @@ func (ctrl *KubeletStaticPodController) Inputs() []controller.Input { }, { Namespace: secrets.NamespaceName, - Type: secrets.KubernetesType, - ID: pointer.To(secrets.KubernetesID), + Type: secrets.KubernetesDynamicCertsType, + ID: pointer.To(secrets.KubernetesDynamicCertsID), Kind: controller.InputWeak, }, { @@ -94,7 +95,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru case <-r.EventCh(): } - kubeletResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "kubelet", resource.VersionUndefined)) + kubeletService, err := safe.ReaderGet[*v1alpha1.Service](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "kubelet", resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { kubeletClient = nil @@ -109,7 +110,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru return err } - if !kubeletResource.(*v1alpha1.Service).TypedSpec().Running { + if !kubeletService.TypedSpec().Running { kubeletClient = nil if err = ctrl.teardownStatuses(ctx, r); err != nil { @@ -121,7 +122,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru // on worker nodes, there's no way to connect to the kubelet to fetch the pod status (only API server can do that) // on control plane nodes, use API servers' client kubelet certificate to fetch statuses - rootSecretResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + rootSecrets, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { kubeletClient = nil @@ -132,9 +133,10 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru return err } - rootSecrets := rootSecretResource.(*secrets.KubernetesRoot).TypedSpec() - - secretsResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + certsResource, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts]( + ctx, r, + resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined), + ) if err != nil { if state.IsNotFoundError(err) { kubeletClient = nil @@ -145,17 +147,20 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru return err } - secrets := secretsResource.(*secrets.Kubernetes).TypedSpec() + certs := certsResource.TypedSpec() - nodenameResource, err := r.Get(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined)) + nodename, err := safe.ReaderGet[*k8s.Nodename](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.NodenameType, k8s.NodenameID, resource.VersionUndefined)) if err != nil { // nodename should exist if the kubelet is running return err } - nodename := nodenameResource.(*k8s.Nodename).TypedSpec().Nodename - - kubeletClient, err = kubelet.NewClient(nodename, secrets.APIServerKubeletClient.Crt, secrets.APIServerKubeletClient.Key, rootSecrets.CA.Crt) + kubeletClient, err = kubelet.NewClient( + nodename.TypedSpec().Nodename, + certs.APIServerKubeletClient.Crt, + certs.APIServerKubeletClient.Key, + rootSecrets.TypedSpec().CA.Crt, + ) if err != nil { return fmt.Errorf("error building kubelet client: %w", err) } diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go index 05e18f4d5c..b90661587b 100644 --- a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go @@ -14,6 +14,7 @@ import ( "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" "github.com/cosi-project/runtime/pkg/state" "github.com/siderolabs/crypto/x509" "github.com/siderolabs/go-pointer" @@ -53,6 +54,12 @@ func (ctrl *RenderSecretsStaticPodController) Inputs() []controller.Input { //no ID: pointer.To(secrets.KubernetesID), Kind: controller.InputWeak, }, + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesDynamicCertsType, + ID: pointer.To(secrets.KubernetesDynamicCertsID), + Kind: controller.InputWeak, + }, { Namespace: secrets.NamespaceName, Type: secrets.EtcdType, @@ -83,7 +90,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control case <-r.EventCh(): } - secretsRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + secretsRes, err := safe.ReaderGet[*secrets.Kubernetes](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { continue @@ -92,7 +99,19 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - etcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) + certsRes, err := safe.ReaderGet[*secrets.KubernetesDynamicCerts]( + ctx, r, + resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, secrets.KubernetesDynamicCertsID, resource.VersionUndefined), + ) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting certificates resource: %w", err) + } + + etcdRes, err := safe.ReaderGet[*secrets.Etcd](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { continue @@ -101,7 +120,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - rootEtcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined)) + rootEtcdRes, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { continue @@ -110,7 +129,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - rootK8sRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + rootK8sRes, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { continue @@ -119,10 +138,11 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - rootEtcdSecrets := rootEtcdRes.(*secrets.EtcdRoot).TypedSpec() - rootK8sSecrets := rootK8sRes.(*secrets.KubernetesRoot).TypedSpec() - etcdSecrets := etcdRes.(*secrets.Etcd).TypedSpec() - k8sSecrets := secretsRes.(*secrets.Kubernetes).TypedSpec() + rootEtcdSecrets := rootEtcdRes.TypedSpec() + rootK8sSecrets := rootK8sRes.TypedSpec() + etcdSecrets := etcdRes.TypedSpec() + k8sSecrets := secretsRes.TypedSpec() + k8sCerts := certsRes.TypedSpec() serviceAccountKey, err := rootK8sSecrets.ServiceAccount.GetKey() if err != nil { @@ -168,12 +188,12 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control certFilename: "ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServer }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServer }, certFilename: "apiserver.crt", keyFilename: "apiserver.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServerKubeletClient }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.APIServerKubeletClient }, certFilename: "apiserver-kubelet-client.crt", keyFilename: "apiserver-kubelet-client.key", }, @@ -192,7 +212,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control certFilename: "aggregator-ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.FrontProxy }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sCerts.FrontProxy }, certFilename: "front-proxy-client.crt", keyFilename: "front-proxy-client.key", }, diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes.go b/internal/app/machined/pkg/controllers/secrets/kubernetes.go index b8ca9aa89c..f178f22939 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes.go @@ -7,13 +7,13 @@ package secrets import ( "bytes" "context" - stdlibx509 "crypto/x509" "fmt" "net/url" "time" "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" "github.com/cosi-project/runtime/pkg/state" "github.com/siderolabs/crypto/x509" "github.com/siderolabs/go-pointer" @@ -22,7 +22,6 @@ import ( "github.com/siderolabs/talos/pkg/kubeconfig" "github.com/siderolabs/talos/pkg/machinery/config" "github.com/siderolabs/talos/pkg/machinery/constants" - "github.com/siderolabs/talos/pkg/machinery/resources/network" "github.com/siderolabs/talos/pkg/machinery/resources/secrets" timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" @@ -45,9 +44,15 @@ func (ctrl *KubernetesController) Name() string { func (ctrl *KubernetesController) Inputs() []controller.Input { return []controller.Input{ { - Namespace: network.NamespaceName, - Type: network.StatusType, - ID: pointer.To(network.StatusID), + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: pointer.To(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: pointer.To(timeresource.StatusID), Kind: controller.InputWeak, }, } @@ -67,56 +72,6 @@ func (ctrl *KubernetesController) Outputs() []controller.Output { // //nolint:gocyclo,cyclop func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { - // wait for the network to be ready first, then switch to regular inputs - for { - select { - case <-ctx.Done(): - return nil - case <-r.EventCh(): - } - // wait for network to be ready as it might change IPs/hostname - networkResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) - if err != nil { - if state.IsNotFoundError(err) { - continue - } - - return err - } - - networkStatus := networkResource.(*network.Status).TypedSpec() - - if networkStatus.AddressReady && networkStatus.HostnameReady { - break - } - } - - // switch to regular inputs once the network is ready - if err := r.UpdateInputs([]controller.Input{ - { - Namespace: secrets.NamespaceName, - Type: secrets.KubernetesRootType, - ID: pointer.To(secrets.KubernetesRootID), - Kind: controller.InputWeak, - }, - { - Namespace: v1alpha1.NamespaceName, - Type: timeresource.StatusType, - ID: pointer.To(timeresource.StatusID), - Kind: controller.InputWeak, - }, - { - Namespace: secrets.NamespaceName, - Type: secrets.CertSANType, - ID: pointer.To(secrets.CertSANKubernetesID), - Kind: controller.InputWeak, - }, - }); err != nil { - return fmt.Errorf("error updating inputs: %w", err) - } - - r.QueueReconcile() - refreshTicker := time.NewTicker(KubernetesCertificateValidityDuration / 2) defer refreshTicker.Stop() @@ -128,7 +83,7 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, case <-refreshTicker.C: } - k8sRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + k8sRoot, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { if err = ctrl.teardownAll(ctx, r); err != nil { @@ -141,10 +96,8 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, return fmt.Errorf("error getting root k8s secrets: %w", err) } - k8sRoot := k8sRootRes.(*secrets.KubernetesRoot).TypedSpec() - // wait for time sync as certs depend on current time - timeSyncResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, timeresource.StatusType, timeresource.StatusID, resource.VersionUndefined)) + timeSync, err := safe.ReaderGet[*timeresource.Status](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, timeresource.StatusType, timeresource.StatusID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { continue @@ -153,23 +106,12 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, return err } - if !timeSyncResource.(*timeresource.Status).TypedSpec().Synced { + if !timeSync.TypedSpec().Synced { continue } - certSANResource, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, secrets.CertSANKubernetesID, resource.VersionUndefined)) - if err != nil { - if state.IsNotFoundError(err) { - continue - } - - return err - } - - certSANs := certSANResource.(*secrets.CertSAN).TypedSpec() - - if err = r.Modify(ctx, secrets.NewKubernetes(), func(r resource.Resource) error { - return ctrl.updateSecrets(k8sRoot, r.(*secrets.Kubernetes).TypedSpec(), certSANs) + if err = safe.WriterModify(ctx, r, secrets.NewKubernetes(), func(r *secrets.Kubernetes) error { + return ctrl.updateSecrets(k8sRoot.TypedSpec(), r.TypedSpec()) }); err != nil { return err } @@ -178,68 +120,10 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, } } -func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sSecrets *secrets.KubernetesCertsSpec, - certSANs *secrets.CertSANSpec, -) error { - ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.CA) - if err != nil { - return fmt.Errorf("failed to parse CA certificate: %w", err) - } - - apiServer, err := x509.NewKeyPair(ca, - x509.IPAddresses(certSANs.StdIPs()), - x509.DNSNames(certSANs.DNSNames), - x509.CommonName("kube-apiserver"), - x509.Organization("kube-master"), - x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), - x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), - x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ - stdlibx509.ExtKeyUsageServerAuth, - }), - ) - if err != nil { - return fmt.Errorf("failed to generate api-server cert: %w", err) - } - - k8sSecrets.APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) - - apiServerKubeletClient, err := x509.NewKeyPair(ca, - x509.CommonName(constants.KubernetesAPIServerKubeletClientCommonName), - x509.Organization(constants.KubernetesAdminCertOrganization), - x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), - x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), - x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ - stdlibx509.ExtKeyUsageClientAuth, - }), - ) - if err != nil { - return fmt.Errorf("failed to generate api-server cert: %w", err) - } - - k8sSecrets.APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) - - aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.AggregatorCA) - if err != nil { - return fmt.Errorf("failed to parse aggregator CA: %w", err) - } - - frontProxy, err := x509.NewKeyPair(aggregatorCA, - x509.CommonName("front-proxy-client"), - x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), - x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), - x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ - stdlibx509.ExtKeyUsageClientAuth, - }), - ) - if err != nil { - return fmt.Errorf("failed to generate aggregator cert: %w", err) - } - - k8sSecrets.FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) - +func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sSecrets *secrets.KubernetesCertsSpec) error { var buf bytes.Buffer - if err = kubeconfig.Generate(&kubeconfig.GenerateInput{ + if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ ClusterName: k8sRoot.Name, CA: k8sRoot.CA, @@ -259,7 +143,7 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootS buf.Reset() - if err = kubeconfig.Generate(&kubeconfig.GenerateInput{ + if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ ClusterName: k8sRoot.Name, CA: k8sRoot.CA, @@ -279,7 +163,7 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootS buf.Reset() - if err = kubeconfig.GenerateAdmin(&generateAdminAdapter{ + if err := kubeconfig.GenerateAdmin(&generateAdminAdapter{ k8sRoot: k8sRoot, endpoint: k8sRoot.Endpoint, }, &buf); err != nil { @@ -290,7 +174,7 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootS buf.Reset() - if err = kubeconfig.GenerateAdmin(&generateAdminAdapter{ + if err := kubeconfig.GenerateAdmin(&generateAdminAdapter{ k8sRoot: k8sRoot, endpoint: k8sRoot.LocalEndpoint, }, &buf); err != nil { diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go new file mode 100644 index 0000000000..eeb7756cf9 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + stdlibx509 "crypto/x509" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/controller" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-pointer" + "go.uber.org/zap" + + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// KubernetesDynamicCertsController manages secrets.KubernetesDynamicCerts based on configuration. +type KubernetesDynamicCertsController struct{} + +// Name implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Name() string { + return "secrets.KubernetesDynamicCertsController" +} + +// Inputs implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Inputs() []controller.Input { + return nil +} + +// Outputs implements controller.Controller interface. +func (ctrl *KubernetesDynamicCertsController) Outputs() []controller.Output { + return []controller.Output{ + { + Type: secrets.KubernetesDynamicCertsType, + Kind: controller.OutputExclusive, + }, + } +} + +// Run implements controller.Controller interface. +// +//nolint:gocyclo,cyclop +func (ctrl *KubernetesDynamicCertsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + // wait for the network to be ready first, then switch to regular inputs + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: network.NamespaceName, + Type: network.StatusType, + ID: pointer.To(network.StatusID), + Kind: controller.InputWeak, + }, + }); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + // wait for network to be ready as it might change IPs/hostname + networkStatus, err := safe.ReaderGet[*network.Status](ctx, r, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if networkStatus.TypedSpec().AddressReady && networkStatus.TypedSpec().HostnameReady { + break + } + } + + // switch to regular inputs once the network is ready + if err := r.UpdateInputs([]controller.Input{ + { + Namespace: secrets.NamespaceName, + Type: secrets.KubernetesRootType, + ID: pointer.To(secrets.KubernetesRootID), + Kind: controller.InputWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: timeresource.StatusType, + ID: pointer.To(timeresource.StatusID), + Kind: controller.InputWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.CertSANType, + ID: pointer.To(secrets.CertSANKubernetesID), + Kind: controller.InputWeak, + }, + }); err != nil { + return fmt.Errorf("error updating inputs: %w", err) + } + + r.QueueReconcile() + + refreshTicker := time.NewTicker(KubernetesCertificateValidityDuration / 2) + defer refreshTicker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + case <-refreshTicker.C: + } + + k8sRoot, err := safe.ReaderGet[*secrets.KubernetesRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesRootType, secrets.KubernetesRootID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting root k8s secrets: %w", err) + } + + // wait for time sync as certs depend on current time + timeSync, err := safe.ReaderGet[*timeresource.Status](ctx, r, resource.NewMetadata(v1alpha1.NamespaceName, timeresource.StatusType, timeresource.StatusID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSync.TypedSpec().Synced { + continue + } + + certSANs, err := safe.ReaderGet[*secrets.CertSAN](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.CertSANType, secrets.CertSANKubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if err = safe.WriterModify(ctx, r, secrets.NewKubernetesDynamicCerts(), func(r *secrets.KubernetesDynamicCerts) error { + return ctrl.updateSecrets(k8sRoot.TypedSpec(), r.TypedSpec(), certSANs.TypedSpec()) + }); err != nil { + return err + } + + r.ResetRestartBackoff() + } +} + +func (ctrl *KubernetesDynamicCertsController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sCerts *secrets.KubernetesDynamicCertsSpec, + certSANs *secrets.CertSANSpec, +) error { + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.CA) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + + apiServer, err := x509.NewKeyPair(ca, + x509.IPAddresses(certSANs.StdIPs()), + x509.DNSNames(certSANs.DNSNames), + x509.CommonName("kube-apiserver"), + x509.Organization("kube-master"), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageServerAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate api-server cert: %w", err) + } + + k8sCerts.APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) + + apiServerKubeletClient, err := x509.NewKeyPair(ca, + x509.CommonName(constants.KubernetesAPIServerKubeletClientCommonName), + x509.Organization(constants.KubernetesAdminCertOrganization), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate api-server cert: %w", err) + } + + k8sCerts.APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) + + aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.AggregatorCA) + if err != nil { + return fmt.Errorf("failed to parse aggregator CA: %w", err) + } + + frontProxy, err := x509.NewKeyPair(aggregatorCA, + x509.CommonName("front-proxy-client"), + x509.NotAfter(time.Now().Add(KubernetesCertificateValidityDuration)), + x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment), + x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{ + stdlibx509.ExtKeyUsageClientAuth, + }), + ) + if err != nil { + return fmt.Errorf("failed to generate aggregator cert: %w", err) + } + + k8sCerts.FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) + + return nil +} + +func (ctrl *KubernetesDynamicCertsController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesDynamicCertsType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go new file mode 100644 index 0000000000..756cf74779 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go @@ -0,0 +1,201 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//nolint:dupl +package secrets_test + +import ( + stdlibx509 "crypto/x509" + "fmt" + "net/netip" + "net/url" + "testing" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" + "github.com/siderolabs/crypto/x509" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" + secretsctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/secrets" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/config" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" +) + +func TestKubernetesDynamicCertsSuite(t *testing.T) { + suite.Run(t, &KubernetesDynamicCertsSuite{ + DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, + AfterSetup: func(suite *ctest.DefaultSuite) { + suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.KubernetesDynamicCertsController{})) + }, + }, + }) +} + +type KubernetesDynamicCertsSuite struct { + ctest.DefaultSuite +} + +func (suite *KubernetesDynamicCertsSuite) TestReconcile() { + rootSecrets := secrets.NewKubernetesRoot(secrets.KubernetesRootID) + + k8sCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + aggregatorCA, err := x509.NewSelfSignedCertificateAuthority( + x509.Organization("kubernetes"), + x509.ECDSA(true), + ) + suite.Require().NoError(err) + + serviceAccount, err := x509.NewECDSAKey() + suite.Require().NoError(err) + + rootSecrets.TypedSpec().Name = "cluster1" + rootSecrets.TypedSpec().Endpoint, err = url.Parse("https://some.url:6443/") + suite.Require().NoError(err) + rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") + suite.Require().NoError(err) + + rootSecrets.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{ + Crt: k8sCA.CrtPEM, + Key: k8sCA.KeyPEM, + } + rootSecrets.TypedSpec().AggregatorCA = &x509.PEMEncodedCertificateAndKey{ + Crt: aggregatorCA.CrtPEM, + Key: aggregatorCA.KeyPEM, + } + rootSecrets.TypedSpec().ServiceAccount = &x509.PEMEncodedKey{ + Key: serviceAccount.KeyPEM, + } + rootSecrets.TypedSpec().CertSANs = []string{"example.com"} + rootSecrets.TypedSpec().APIServerIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} + rootSecrets.TypedSpec().DNSDomain = "cluster.remote" + suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) + + machineType := config.NewMachineType() + machineType.SetMachineType(machine.TypeControlPlane) + suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) + + networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) + networkStatus.TypedSpec().AddressReady = true + networkStatus.TypedSpec().HostnameReady = true + suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) + + certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANKubernetesID) + certSANs.TypedSpec().Append( + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.url", + "10.2.1.3", + "10.4.3.2", + "172.16.0.1", + ) + suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) + + timeSync := timeresource.NewStatus() + *timeSync.TypedSpec() = timeresource.StatusSpec{ + Synced: true, + } + suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync)) + + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesDynamicCertsID}, + func(certs *secrets.KubernetesDynamicCerts, assertion *assert.Assertions) { + kubernetesCerts := certs.TypedSpec() + + apiCert, err := kubernetesCerts.APIServer.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Equal( + []string{ + "example.com", + "foo", + "foo.example.com", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.remote", + "localhost", + "some.url", + }, apiCert.DNSNames, + ) + assertion.Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", apiCert.IPAddresses)) + + assertion.Equal("kube-apiserver", apiCert.Subject.CommonName) + assertion.Equal([]string{"kube-master"}, apiCert.Subject.Organization) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + apiCert.KeyUsage, + ) + assertion.Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, apiCert.ExtKeyUsage) + + clientCert, err := kubernetesCerts.APIServerKubeletClient.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Empty(clientCert.DNSNames) + assertion.Empty(clientCert.IPAddresses) + + assertion.Equal( + constants.KubernetesAPIServerKubeletClientCommonName, + clientCert.Subject.CommonName, + ) + assertion.Equal( + []string{constants.KubernetesAdminCertOrganization}, + clientCert.Subject.Organization, + ) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + clientCert.KeyUsage, + ) + assertion.Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, clientCert.ExtKeyUsage) + + frontProxyCert, err := kubernetesCerts.FrontProxy.GetCert() + assertion.NoError(err) + + if err != nil { + return + } + + assertion.Empty(frontProxyCert.DNSNames) + assertion.Empty(frontProxyCert.IPAddresses) + + assertion.Equal("front-proxy-client", frontProxyCert.Subject.CommonName) + assertion.Empty(frontProxyCert.Subject.Organization) + + assertion.Equal( + stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, + frontProxyCert.KeyUsage, + ) + assertion.Equal( + []stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, + frontProxyCert.ExtKeyUsage, + ) + }) +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go index c6cc4b7766..da51a0ce4f 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go @@ -6,26 +6,22 @@ package secrets_test import ( - stdlibx509 "crypto/x509" - "fmt" "net/netip" "net/url" "testing" "time" "github.com/cosi-project/runtime/pkg/resource" - "github.com/cosi-project/runtime/pkg/state" + "github.com/cosi-project/runtime/pkg/resource/rtestutils" "github.com/siderolabs/crypto/x509" - "github.com/siderolabs/go-retry/retry" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "k8s.io/client-go/tools/clientcmd" "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest" secretsctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/secrets" "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine" - "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/resources/config" - "github.com/siderolabs/talos/pkg/machinery/resources/network" "github.com/siderolabs/talos/pkg/machinery/resources/secrets" timeresource "github.com/siderolabs/talos/pkg/machinery/resources/time" ) @@ -33,6 +29,7 @@ import ( func TestKubernetesSuite(t *testing.T) { suite.Run(t, &KubernetesSuite{ DefaultSuite: ctest.DefaultSuite{ + Timeout: 5 * time.Second, AfterSetup: func(suite *ctest.DefaultSuite) { suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.KubernetesController{})) }, @@ -81,139 +78,37 @@ func (suite *KubernetesSuite) TestReconcile() { } rootSecrets.TypedSpec().CertSANs = []string{"example.com"} rootSecrets.TypedSpec().APIServerIPs = []netip.Addr{netip.MustParseAddr("10.4.3.2"), netip.MustParseAddr("10.2.1.3")} - rootSecrets.TypedSpec().DNSDomain = "cluster.remote" + rootSecrets.TypedSpec().DNSDomain = "cluster.svc" suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets)) machineType := config.NewMachineType() machineType.SetMachineType(machine.TypeControlPlane) suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType)) - networkStatus := network.NewStatus(network.NamespaceName, network.StatusID) - networkStatus.TypedSpec().AddressReady = true - networkStatus.TypedSpec().HostnameReady = true - suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus)) - - certSANs := secrets.NewCertSAN(secrets.NamespaceName, secrets.CertSANKubernetesID) - certSANs.TypedSpec().Append( - "example.com", - "foo", - "foo.example.com", - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.remote", - "localhost", - "some.url", - "10.2.1.3", - "10.4.3.2", - "172.16.0.1", - ) - suite.Require().NoError(suite.State().Create(suite.Ctx(), certSANs)) - timeSync := timeresource.NewStatus() *timeSync.TypedSpec() = timeresource.StatusSpec{ Synced: true, } suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync)) - suite.AssertWithin(10*time.Second, 100*time.Millisecond, func() error { - certs, err := ctest.Get[*secrets.Kubernetes]( - suite, - resource.NewMetadata( - secrets.NamespaceName, - secrets.KubernetesType, - secrets.KubernetesID, - resource.VersionUndefined, - ), - ) - if err != nil { - if state.IsNotFoundError(err) { - return retry.ExpectedError(err) - } + rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesID}, + func(certs *secrets.Kubernetes, assertion *assert.Assertions) { + kubernetesCerts := certs.TypedSpec() - return err - } - - kubernetesCerts := certs.TypedSpec() - - apiCert, err := kubernetesCerts.APIServer.GetCert() - suite.Require().NoError(err) - - suite.Assert().Equal( - []string{ - "example.com", - "foo", - "foo.example.com", - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.remote", - "localhost", - "some.url", - }, apiCert.DNSNames, - ) - suite.Assert().Equal("[10.2.1.3 10.4.3.2 172.16.0.1]", fmt.Sprintf("%v", apiCert.IPAddresses)) - - suite.Assert().Equal("kube-apiserver", apiCert.Subject.CommonName) - suite.Assert().Equal([]string{"kube-master"}, apiCert.Subject.Organization) - - suite.Assert().Equal( - stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, - apiCert.KeyUsage, - ) - suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageServerAuth}, apiCert.ExtKeyUsage) - - clientCert, err := kubernetesCerts.APIServerKubeletClient.GetCert() - suite.Require().NoError(err) - - suite.Assert().Empty(clientCert.DNSNames) - suite.Assert().Empty(clientCert.IPAddresses) - - suite.Assert().Equal( - constants.KubernetesAPIServerKubeletClientCommonName, - clientCert.Subject.CommonName, - ) - suite.Assert().Equal( - []string{constants.KubernetesAdminCertOrganization}, - clientCert.Subject.Organization, - ) - - suite.Assert().Equal( - stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, - clientCert.KeyUsage, - ) - suite.Assert().Equal([]stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, clientCert.ExtKeyUsage) - - frontProxyCert, err := kubernetesCerts.FrontProxy.GetCert() - suite.Require().NoError(err) - - suite.Assert().Empty(frontProxyCert.DNSNames) - suite.Assert().Empty(frontProxyCert.IPAddresses) - - suite.Assert().Equal("front-proxy-client", frontProxyCert.Subject.CommonName) - suite.Assert().Empty(frontProxyCert.Subject.Organization) - - suite.Assert().Equal( - stdlibx509.KeyUsageDigitalSignature|stdlibx509.KeyUsageKeyEncipherment, - frontProxyCert.KeyUsage, - ) - suite.Assert().Equal( - []stdlibx509.ExtKeyUsage{stdlibx509.ExtKeyUsageClientAuth}, - frontProxyCert.ExtKeyUsage, - ) - - for _, kubeconfig := range []string{ - kubernetesCerts.ControllerManagerKubeconfig, - kubernetesCerts.SchedulerKubeconfig, - kubernetesCerts.LocalhostAdminKubeconfig, - kubernetesCerts.AdminKubeconfig, - } { - config, err := clientcmd.Load([]byte(kubeconfig)) - suite.Require().NoError(err) - - suite.Assert().NoError(clientcmd.ConfirmUsable(*config, config.CurrentContext)) - } - - return nil - }) + for _, kubeconfig := range []string{ + kubernetesCerts.ControllerManagerKubeconfig, + kubernetesCerts.SchedulerKubeconfig, + kubernetesCerts.LocalhostAdminKubeconfig, + kubernetesCerts.AdminKubeconfig, + } { + config, err := clientcmd.Load([]byte(kubeconfig)) + assertion.NoError(err) + + if err != nil { + return + } + + assertion.NoError(clientcmd.ConfirmUsable(*config, config.CurrentContext)) + } + }) } diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index 314b197a6d..250c9952f1 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -247,6 +247,7 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error &secrets.EtcdController{}, &secrets.KubeletController{}, &secrets.KubernetesCertSANsController{}, + &secrets.KubernetesDynamicCertsController{}, &secrets.KubernetesController{}, &secrets.RootController{}, &secrets.TrustdController{}, diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go index 184304577e..b8caf2797a 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -177,6 +177,7 @@ func NewState() (*State, error) { &secrets.EtcdRoot{}, &secrets.Kubelet{}, &secrets.Kubernetes{}, + &secrets.KubernetesDynamicCerts{}, &secrets.KubernetesRoot{}, &secrets.OSRoot{}, &secrets.Trustd{}, diff --git a/internal/integration/api/apply-config.go b/internal/integration/api/apply-config.go index 135dcf68b8..b2e381cc05 100644 --- a/internal/integration/api/apply-config.go +++ b/internal/integration/api/apply-config.go @@ -239,11 +239,11 @@ func (suite *ApplyConfigSuite) TestApplyConfigRotateEncryptionSecrets() { encryption := machineConfig.MachineConfig.MachineSystemDiskEncryption - if encryption == nil { + if encryption == nil { //nolint:staticcheck suite.T().Skip("skipped in not encrypted mode") } - cfg := encryption.EphemeralPartition + cfg := encryption.EphemeralPartition //nolint:staticcheck if cfg == nil { suite.T().Skip("skipped in not encrypted mode") @@ -251,10 +251,6 @@ func (suite *ApplyConfigSuite) TestApplyConfigRotateEncryptionSecrets() { provider.Machine().SystemDiskEncryption().Get(constants.EphemeralPartitionLabel) - if encryption == nil { - suite.T().Skip("skipped in not encrypted mode") - } - suite.WaitForBootDone(suite.ctx) keySets := [][]*v1alpha1.EncryptionKey{ diff --git a/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go b/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go index a1b1eebe64..6ff24b9af9 100644 --- a/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go +++ b/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go @@ -349,13 +349,10 @@ type KubernetesCertsSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ApiServer *common.PEMEncodedCertificateAndKey `protobuf:"bytes,1,opt,name=api_server,json=apiServer,proto3" json:"api_server,omitempty"` - ApiServerKubeletClient *common.PEMEncodedCertificateAndKey `protobuf:"bytes,2,opt,name=api_server_kubelet_client,json=apiServerKubeletClient,proto3" json:"api_server_kubelet_client,omitempty"` - FrontProxy *common.PEMEncodedCertificateAndKey `protobuf:"bytes,3,opt,name=front_proxy,json=frontProxy,proto3" json:"front_proxy,omitempty"` - SchedulerKubeconfig string `protobuf:"bytes,4,opt,name=scheduler_kubeconfig,json=schedulerKubeconfig,proto3" json:"scheduler_kubeconfig,omitempty"` - ControllerManagerKubeconfig string `protobuf:"bytes,5,opt,name=controller_manager_kubeconfig,json=controllerManagerKubeconfig,proto3" json:"controller_manager_kubeconfig,omitempty"` - LocalhostAdminKubeconfig string `protobuf:"bytes,6,opt,name=localhost_admin_kubeconfig,json=localhostAdminKubeconfig,proto3" json:"localhost_admin_kubeconfig,omitempty"` - AdminKubeconfig string `protobuf:"bytes,7,opt,name=admin_kubeconfig,json=adminKubeconfig,proto3" json:"admin_kubeconfig,omitempty"` + SchedulerKubeconfig string `protobuf:"bytes,4,opt,name=scheduler_kubeconfig,json=schedulerKubeconfig,proto3" json:"scheduler_kubeconfig,omitempty"` + ControllerManagerKubeconfig string `protobuf:"bytes,5,opt,name=controller_manager_kubeconfig,json=controllerManagerKubeconfig,proto3" json:"controller_manager_kubeconfig,omitempty"` + LocalhostAdminKubeconfig string `protobuf:"bytes,6,opt,name=localhost_admin_kubeconfig,json=localhostAdminKubeconfig,proto3" json:"localhost_admin_kubeconfig,omitempty"` + AdminKubeconfig string `protobuf:"bytes,7,opt,name=admin_kubeconfig,json=adminKubeconfig,proto3" json:"admin_kubeconfig,omitempty"` } func (x *KubernetesCertsSpec) Reset() { @@ -390,53 +387,96 @@ func (*KubernetesCertsSpec) Descriptor() ([]byte, []int) { return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{5} } -func (x *KubernetesCertsSpec) GetApiServer() *common.PEMEncodedCertificateAndKey { +func (x *KubernetesCertsSpec) GetSchedulerKubeconfig() string { if x != nil { - return x.ApiServer + return x.SchedulerKubeconfig } - return nil + return "" } -func (x *KubernetesCertsSpec) GetApiServerKubeletClient() *common.PEMEncodedCertificateAndKey { +func (x *KubernetesCertsSpec) GetControllerManagerKubeconfig() string { if x != nil { - return x.ApiServerKubeletClient + return x.ControllerManagerKubeconfig } - return nil + return "" } -func (x *KubernetesCertsSpec) GetFrontProxy() *common.PEMEncodedCertificateAndKey { +func (x *KubernetesCertsSpec) GetLocalhostAdminKubeconfig() string { if x != nil { - return x.FrontProxy + return x.LocalhostAdminKubeconfig } - return nil + return "" } -func (x *KubernetesCertsSpec) GetSchedulerKubeconfig() string { +func (x *KubernetesCertsSpec) GetAdminKubeconfig() string { if x != nil { - return x.SchedulerKubeconfig + return x.AdminKubeconfig } return "" } -func (x *KubernetesCertsSpec) GetControllerManagerKubeconfig() string { +// KubernetesDynamicCertsSpec describes generated KubernetesCerts certificates. +type KubernetesDynamicCertsSpec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ApiServer *common.PEMEncodedCertificateAndKey `protobuf:"bytes,1,opt,name=api_server,json=apiServer,proto3" json:"api_server,omitempty"` + ApiServerKubeletClient *common.PEMEncodedCertificateAndKey `protobuf:"bytes,2,opt,name=api_server_kubelet_client,json=apiServerKubeletClient,proto3" json:"api_server_kubelet_client,omitempty"` + FrontProxy *common.PEMEncodedCertificateAndKey `protobuf:"bytes,3,opt,name=front_proxy,json=frontProxy,proto3" json:"front_proxy,omitempty"` +} + +func (x *KubernetesDynamicCertsSpec) Reset() { + *x = KubernetesDynamicCertsSpec{} + if protoimpl.UnsafeEnabled { + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *KubernetesDynamicCertsSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KubernetesDynamicCertsSpec) ProtoMessage() {} + +func (x *KubernetesDynamicCertsSpec) ProtoReflect() protoreflect.Message { + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KubernetesDynamicCertsSpec.ProtoReflect.Descriptor instead. +func (*KubernetesDynamicCertsSpec) Descriptor() ([]byte, []int) { + return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{6} +} + +func (x *KubernetesDynamicCertsSpec) GetApiServer() *common.PEMEncodedCertificateAndKey { if x != nil { - return x.ControllerManagerKubeconfig + return x.ApiServer } - return "" + return nil } -func (x *KubernetesCertsSpec) GetLocalhostAdminKubeconfig() string { +func (x *KubernetesDynamicCertsSpec) GetApiServerKubeletClient() *common.PEMEncodedCertificateAndKey { if x != nil { - return x.LocalhostAdminKubeconfig + return x.ApiServerKubeletClient } - return "" + return nil } -func (x *KubernetesCertsSpec) GetAdminKubeconfig() string { +func (x *KubernetesDynamicCertsSpec) GetFrontProxy() *common.PEMEncodedCertificateAndKey { if x != nil { - return x.AdminKubeconfig + return x.FrontProxy } - return "" + return nil } // KubernetesRootSpec describes root Kubernetes secrets. @@ -463,7 +503,7 @@ type KubernetesRootSpec struct { func (x *KubernetesRootSpec) Reset() { *x = KubernetesRootSpec{} if protoimpl.UnsafeEnabled { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[6] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -476,7 +516,7 @@ func (x *KubernetesRootSpec) String() string { func (*KubernetesRootSpec) ProtoMessage() {} func (x *KubernetesRootSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[6] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -489,7 +529,7 @@ func (x *KubernetesRootSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use KubernetesRootSpec.ProtoReflect.Descriptor instead. func (*KubernetesRootSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{6} + return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{7} } func (x *KubernetesRootSpec) GetName() string { @@ -598,7 +638,7 @@ type OSRootSpec struct { func (x *OSRootSpec) Reset() { *x = OSRootSpec{} if protoimpl.UnsafeEnabled { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[7] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -611,7 +651,7 @@ func (x *OSRootSpec) String() string { func (*OSRootSpec) ProtoMessage() {} func (x *OSRootSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[7] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -624,7 +664,7 @@ func (x *OSRootSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use OSRootSpec.ProtoReflect.Descriptor instead. func (*OSRootSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{7} + return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{8} } func (x *OSRootSpec) GetCa() *common.PEMEncodedCertificateAndKey { @@ -668,7 +708,7 @@ type TrustdCertsSpec struct { func (x *TrustdCertsSpec) Reset() { *x = TrustdCertsSpec{} if protoimpl.UnsafeEnabled { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[8] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -681,7 +721,7 @@ func (x *TrustdCertsSpec) String() string { func (*TrustdCertsSpec) ProtoMessage() {} func (x *TrustdCertsSpec) ProtoReflect() protoreflect.Message { - mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[8] + mi := &file_resource_definitions_secrets_secrets_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -694,7 +734,7 @@ func (x *TrustdCertsSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustdCertsSpec.ProtoReflect.Descriptor instead. func (*TrustdCertsSpec) Descriptor() ([]byte, []int) { - return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{8} + return file_resource_definitions_secrets_secrets_proto_rawDescGZIP(), []int{9} } func (x *TrustdCertsSpec) GetCa() *common.PEMEncodedCertificateAndKey { @@ -774,104 +814,105 @@ var file_resource_definitions_secrets_secrets_proto_rawDesc = []byte{ 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0xdf, 0x03, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, - 0x65, 0x74, 0x65, 0x73, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x42, 0x0a, - 0x0a, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, - 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x12, 0x5e, 0x0a, 0x19, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, - 0x6b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, - 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x16, 0x61, 0x70, 0x69, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x12, 0x44, 0x0a, 0x0b, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, - 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0a, 0x66, 0x72, 0x6f, - 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x31, 0x0a, 0x14, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x1d, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, - 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, - 0x0a, 0x1a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x61, 0x64, 0x6d, 0x69, - 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x18, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x41, 0x64, 0x6d, - 0x69, 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, 0x0a, 0x10, - 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x75, 0x62, - 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x94, 0x05, 0x0a, 0x12, 0x4b, 0x75, 0x62, 0x65, - 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, - 0x4c, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x0e, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0xf5, 0x01, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, + 0x65, 0x74, 0x65, 0x73, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x31, 0x0a, + 0x14, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x73, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x42, 0x0a, 0x1d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x1a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, + 0x74, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, + 0x6f, 0x73, 0x74, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x64, + 0x6d, 0x69, 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x86, 0x02, + 0x0a, 0x1a, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x44, 0x79, 0x6e, 0x61, + 0x6d, 0x69, 0x63, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x42, 0x0a, 0x0a, + 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, + 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, + 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x09, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x12, 0x5e, 0x0a, 0x19, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6b, + 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, + 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x16, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x12, 0x44, 0x0a, 0x0b, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, + 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0a, 0x66, 0x72, 0x6f, 0x6e, + 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x22, 0x94, 0x05, 0x0a, 0x12, 0x4b, 0x75, 0x62, 0x65, 0x72, + 0x6e, 0x65, 0x74, 0x65, 0x73, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x27, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, - 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, - 0x1c, 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x5f, 0x6e, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x4e, 0x73, 0x12, 0x1d, 0x0a, - 0x0a, 0x64, 0x6e, 0x73, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x64, 0x6e, 0x73, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x33, 0x0a, 0x02, - 0x63, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, + 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x0e, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, 0x52, + 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1c, + 0x0a, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x5f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x4e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, + 0x64, 0x6e, 0x73, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x64, 0x6e, 0x73, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x33, 0x0a, 0x02, 0x63, + 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, 0x61, + 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, + 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, + 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x48, 0x0a, 0x0d, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, + 0x61, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0c, 0x61, 0x67, + 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x61, 0x12, 0x38, 0x0a, 0x18, 0x61, 0x65, + 0x73, 0x63, 0x62, 0x63, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x61, 0x65, + 0x73, 0x63, 0x62, 0x63, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, + 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x62, 0x6f, 0x78, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x73, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x62, 0x6f, 0x78, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x33, 0x0a, 0x0e, 0x61, 0x70, 0x69, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x70, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, + 0x0c, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x70, 0x73, 0x22, 0xb4, 0x01, + 0x0a, 0x0a, 0x4f, 0x53, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, 0x02, + 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, - 0x61, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, - 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x4b, 0x65, - 0x79, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x48, 0x0a, 0x0d, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x5f, - 0x63, 0x61, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0c, 0x61, - 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x61, 0x12, 0x38, 0x0a, 0x18, 0x61, - 0x65, 0x73, 0x63, 0x62, 0x63, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x61, - 0x65, 0x73, 0x63, 0x62, 0x63, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, - 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, - 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x62, 0x6f, 0x78, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x62, 0x6f, 0x78, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x33, 0x0a, 0x0e, 0x61, 0x70, 0x69, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x70, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, - 0x52, 0x0c, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x70, 0x73, 0x22, 0xb4, - 0x01, 0x0a, 0x0a, 0x4f, 0x53, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, - 0x02, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, - 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, - 0x63, 0x61, 0x12, 0x2f, 0x0a, 0x0c, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x69, 0x5f, - 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, - 0x69, 0x50, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x64, - 0x6e, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, - 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x64, 0x6e, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, - 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x83, 0x01, 0x0a, 0x0f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x64, - 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, 0x02, 0x63, 0x61, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, - 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, 0x61, 0x12, 0x3b, - 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, - 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, - 0x4b, 0x65, 0x79, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x4c, 0x5a, 0x4a, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x69, 0x64, 0x65, 0x72, 0x6f, - 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x61, 0x12, 0x2f, 0x0a, 0x0c, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x69, 0x5f, 0x70, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x0a, 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x69, + 0x50, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x64, 0x6e, + 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x63, + 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x64, 0x6e, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x83, 0x01, 0x0a, 0x0f, 0x54, 0x72, 0x75, 0x73, 0x74, 0x64, 0x43, + 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, + 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, 0x61, 0x12, 0x3b, 0x0a, + 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, + 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, + 0x65, 0x79, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x4c, 0x5a, 0x4a, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x69, 0x64, 0x65, 0x72, 0x6f, 0x6c, + 0x61, 0x62, 0x73, 0x2f, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x61, + 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x2f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -886,7 +927,7 @@ func file_resource_definitions_secrets_secrets_proto_rawDescGZIP() []byte { return file_resource_definitions_secrets_secrets_proto_rawDescData } -var file_resource_definitions_secrets_secrets_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_resource_definitions_secrets_secrets_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_resource_definitions_secrets_secrets_proto_goTypes = []interface{}{ (*APICertsSpec)(nil), // 0: talos.resource.definitions.secrets.APICertsSpec (*CertSANSpec)(nil), // 1: talos.resource.definitions.secrets.CertSANSpec @@ -894,39 +935,40 @@ var file_resource_definitions_secrets_secrets_proto_goTypes = []interface{}{ (*EtcdRootSpec)(nil), // 3: talos.resource.definitions.secrets.EtcdRootSpec (*KubeletSpec)(nil), // 4: talos.resource.definitions.secrets.KubeletSpec (*KubernetesCertsSpec)(nil), // 5: talos.resource.definitions.secrets.KubernetesCertsSpec - (*KubernetesRootSpec)(nil), // 6: talos.resource.definitions.secrets.KubernetesRootSpec - (*OSRootSpec)(nil), // 7: talos.resource.definitions.secrets.OSRootSpec - (*TrustdCertsSpec)(nil), // 8: talos.resource.definitions.secrets.TrustdCertsSpec - (*common.PEMEncodedCertificateAndKey)(nil), // 9: common.PEMEncodedCertificateAndKey - (*common.NetIP)(nil), // 10: common.NetIP - (*common.URL)(nil), // 11: common.URL - (*common.PEMEncodedKey)(nil), // 12: common.PEMEncodedKey + (*KubernetesDynamicCertsSpec)(nil), // 6: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec + (*KubernetesRootSpec)(nil), // 7: talos.resource.definitions.secrets.KubernetesRootSpec + (*OSRootSpec)(nil), // 8: talos.resource.definitions.secrets.OSRootSpec + (*TrustdCertsSpec)(nil), // 9: talos.resource.definitions.secrets.TrustdCertsSpec + (*common.PEMEncodedCertificateAndKey)(nil), // 10: common.PEMEncodedCertificateAndKey + (*common.NetIP)(nil), // 11: common.NetIP + (*common.URL)(nil), // 12: common.URL + (*common.PEMEncodedKey)(nil), // 13: common.PEMEncodedKey } var file_resource_definitions_secrets_secrets_proto_depIdxs = []int32{ - 9, // 0: talos.resource.definitions.secrets.APICertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 9, // 1: talos.resource.definitions.secrets.APICertsSpec.client:type_name -> common.PEMEncodedCertificateAndKey - 9, // 2: talos.resource.definitions.secrets.APICertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey - 10, // 3: talos.resource.definitions.secrets.CertSANSpec.i_ps:type_name -> common.NetIP - 9, // 4: talos.resource.definitions.secrets.EtcdCertsSpec.etcd:type_name -> common.PEMEncodedCertificateAndKey - 9, // 5: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_peer:type_name -> common.PEMEncodedCertificateAndKey - 9, // 6: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_admin:type_name -> common.PEMEncodedCertificateAndKey - 9, // 7: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_api_server:type_name -> common.PEMEncodedCertificateAndKey - 9, // 8: talos.resource.definitions.secrets.EtcdRootSpec.etcd_ca:type_name -> common.PEMEncodedCertificateAndKey - 11, // 9: talos.resource.definitions.secrets.KubeletSpec.endpoint:type_name -> common.URL - 9, // 10: talos.resource.definitions.secrets.KubeletSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 9, // 11: talos.resource.definitions.secrets.KubernetesCertsSpec.api_server:type_name -> common.PEMEncodedCertificateAndKey - 9, // 12: talos.resource.definitions.secrets.KubernetesCertsSpec.api_server_kubelet_client:type_name -> common.PEMEncodedCertificateAndKey - 9, // 13: talos.resource.definitions.secrets.KubernetesCertsSpec.front_proxy:type_name -> common.PEMEncodedCertificateAndKey - 11, // 14: talos.resource.definitions.secrets.KubernetesRootSpec.endpoint:type_name -> common.URL - 11, // 15: talos.resource.definitions.secrets.KubernetesRootSpec.local_endpoint:type_name -> common.URL - 9, // 16: talos.resource.definitions.secrets.KubernetesRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 12, // 17: talos.resource.definitions.secrets.KubernetesRootSpec.service_account:type_name -> common.PEMEncodedKey - 9, // 18: talos.resource.definitions.secrets.KubernetesRootSpec.aggregator_ca:type_name -> common.PEMEncodedCertificateAndKey - 10, // 19: talos.resource.definitions.secrets.KubernetesRootSpec.api_server_ips:type_name -> common.NetIP - 9, // 20: talos.resource.definitions.secrets.OSRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 10, // 21: talos.resource.definitions.secrets.OSRootSpec.cert_sani_ps:type_name -> common.NetIP - 9, // 22: talos.resource.definitions.secrets.TrustdCertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 9, // 23: talos.resource.definitions.secrets.TrustdCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey + 10, // 0: talos.resource.definitions.secrets.APICertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 10, // 1: talos.resource.definitions.secrets.APICertsSpec.client:type_name -> common.PEMEncodedCertificateAndKey + 10, // 2: talos.resource.definitions.secrets.APICertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey + 11, // 3: talos.resource.definitions.secrets.CertSANSpec.i_ps:type_name -> common.NetIP + 10, // 4: talos.resource.definitions.secrets.EtcdCertsSpec.etcd:type_name -> common.PEMEncodedCertificateAndKey + 10, // 5: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_peer:type_name -> common.PEMEncodedCertificateAndKey + 10, // 6: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_admin:type_name -> common.PEMEncodedCertificateAndKey + 10, // 7: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_api_server:type_name -> common.PEMEncodedCertificateAndKey + 10, // 8: talos.resource.definitions.secrets.EtcdRootSpec.etcd_ca:type_name -> common.PEMEncodedCertificateAndKey + 12, // 9: talos.resource.definitions.secrets.KubeletSpec.endpoint:type_name -> common.URL + 10, // 10: talos.resource.definitions.secrets.KubeletSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 10, // 11: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.api_server:type_name -> common.PEMEncodedCertificateAndKey + 10, // 12: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.api_server_kubelet_client:type_name -> common.PEMEncodedCertificateAndKey + 10, // 13: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.front_proxy:type_name -> common.PEMEncodedCertificateAndKey + 12, // 14: talos.resource.definitions.secrets.KubernetesRootSpec.endpoint:type_name -> common.URL + 12, // 15: talos.resource.definitions.secrets.KubernetesRootSpec.local_endpoint:type_name -> common.URL + 10, // 16: talos.resource.definitions.secrets.KubernetesRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 13, // 17: talos.resource.definitions.secrets.KubernetesRootSpec.service_account:type_name -> common.PEMEncodedKey + 10, // 18: talos.resource.definitions.secrets.KubernetesRootSpec.aggregator_ca:type_name -> common.PEMEncodedCertificateAndKey + 11, // 19: talos.resource.definitions.secrets.KubernetesRootSpec.api_server_ips:type_name -> common.NetIP + 10, // 20: talos.resource.definitions.secrets.OSRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 11, // 21: talos.resource.definitions.secrets.OSRootSpec.cert_sani_ps:type_name -> common.NetIP + 10, // 22: talos.resource.definitions.secrets.TrustdCertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 10, // 23: talos.resource.definitions.secrets.TrustdCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey 24, // [24:24] is the sub-list for method output_type 24, // [24:24] is the sub-list for method input_type 24, // [24:24] is the sub-list for extension type_name @@ -1013,7 +1055,7 @@ func file_resource_definitions_secrets_secrets_proto_init() { } } file_resource_definitions_secrets_secrets_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KubernetesRootSpec); i { + switch v := v.(*KubernetesDynamicCertsSpec); i { case 0: return &v.state case 1: @@ -1025,7 +1067,7 @@ func file_resource_definitions_secrets_secrets_proto_init() { } } file_resource_definitions_secrets_secrets_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OSRootSpec); i { + switch v := v.(*KubernetesRootSpec); i { case 0: return &v.state case 1: @@ -1037,6 +1079,18 @@ func file_resource_definitions_secrets_secrets_proto_init() { } } file_resource_definitions_secrets_secrets_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*OSRootSpec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_resource_definitions_secrets_secrets_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TrustdCertsSpec); i { case 0: return &v.state @@ -1055,7 +1109,7 @@ func file_resource_definitions_secrets_secrets_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_resource_definitions_secrets_secrets_proto_rawDesc, NumEnums: 0, - NumMessages: 9, + NumMessages: 10, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go b/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go index 185c49b5ca..ad7a911685 100644 --- a/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go +++ b/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go @@ -519,6 +519,39 @@ func (m *KubernetesCertsSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x22 } + return len(dAtA) - i, nil +} + +func (m *KubernetesDynamicCertsSpec) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *KubernetesDynamicCertsSpec) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *KubernetesDynamicCertsSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } if m.FrontProxy != nil { if vtmsg, ok := interface{}(m.FrontProxy).(interface { MarshalToSizedBufferVT([]byte) (int, error) @@ -1170,6 +1203,32 @@ func (m *KubeletSpec) SizeVT() (n int) { } func (m *KubernetesCertsSpec) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SchedulerKubeconfig) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } + l = len(m.ControllerManagerKubeconfig) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } + l = len(m.LocalhostAdminKubeconfig) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } + l = len(m.AdminKubeconfig) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *KubernetesDynamicCertsSpec) SizeVT() (n int) { if m == nil { return 0 } @@ -1205,22 +1264,6 @@ func (m *KubernetesCertsSpec) SizeVT() (n int) { } n += 1 + l + sov(uint64(l)) } - l = len(m.SchedulerKubeconfig) - if l > 0 { - n += 1 + l + sov(uint64(l)) - } - l = len(m.ControllerManagerKubeconfig) - if l > 0 { - n += 1 + l + sov(uint64(l)) - } - l = len(m.LocalhostAdminKubeconfig) - if l > 0 { - n += 1 + l + sov(uint64(l)) - } - l = len(m.AdminKubeconfig) - if l > 0 { - n += 1 + l + sov(uint64(l)) - } n += len(m.unknownFields) return n } @@ -2299,11 +2342,11 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { return fmt.Errorf("proto: KubernetesCertsSpec: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: + case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ApiServer", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SchedulerKubeconfig", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2313,41 +2356,29 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - if m.ApiServer == nil { - m.ApiServer = &common.PEMEncodedCertificateAndKey{} - } - if unmarshal, ok := interface{}(m.ApiServer).(interface { - UnmarshalVT([]byte) error - }); ok { - if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.ApiServer); err != nil { - return err - } - } + m.SchedulerKubeconfig = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ApiServerKubeletClient", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ControllerManagerKubeconfig", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2357,41 +2388,29 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - if m.ApiServerKubeletClient == nil { - m.ApiServerKubeletClient = &common.PEMEncodedCertificateAndKey{} - } - if unmarshal, ok := interface{}(m.ApiServerKubeletClient).(interface { - UnmarshalVT([]byte) error - }); ok { - if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.ApiServerKubeletClient); err != nil { - return err - } - } + m.ControllerManagerKubeconfig = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 6: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FrontProxy", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field LocalhostAdminKubeconfig", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2401,39 +2420,27 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - if m.FrontProxy == nil { - m.FrontProxy = &common.PEMEncodedCertificateAndKey{} - } - if unmarshal, ok := interface{}(m.FrontProxy).(interface { - UnmarshalVT([]byte) error - }); ok { - if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.FrontProxy); err != nil { - return err - } - } + m.LocalhostAdminKubeconfig = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: + case 7: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SchedulerKubeconfig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field AdminKubeconfig", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2461,13 +2468,64 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.SchedulerKubeconfig = string(dAtA[iNdEx:postIndex]) + m.AdminKubeconfig = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 5: + default: + iNdEx = preIndex + skippy, err := skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *KubernetesDynamicCertsSpec) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: KubernetesDynamicCertsSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: KubernetesDynamicCertsSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ControllerManagerKubeconfig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ApiServer", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2477,29 +2535,41 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.ControllerManagerKubeconfig = string(dAtA[iNdEx:postIndex]) + if m.ApiServer == nil { + m.ApiServer = &common.PEMEncodedCertificateAndKey{} + } + if unmarshal, ok := interface{}(m.ApiServer).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.ApiServer); err != nil { + return err + } + } iNdEx = postIndex - case 6: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field LocalhostAdminKubeconfig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ApiServerKubeletClient", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2509,29 +2579,41 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.LocalhostAdminKubeconfig = string(dAtA[iNdEx:postIndex]) + if m.ApiServerKubeletClient == nil { + m.ApiServerKubeletClient = &common.PEMEncodedCertificateAndKey{} + } + if unmarshal, ok := interface{}(m.ApiServerKubeletClient).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.ApiServerKubeletClient); err != nil { + return err + } + } iNdEx = postIndex - case 7: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AdminKubeconfig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field FrontProxy", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflow @@ -2541,23 +2623,35 @@ func (m *KubernetesCertsSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.AdminKubeconfig = string(dAtA[iNdEx:postIndex]) + if m.FrontProxy == nil { + m.FrontProxy = &common.PEMEncodedCertificateAndKey{} + } + if unmarshal, ok := interface{}(m.FrontProxy).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.FrontProxy); err != nil { + return err + } + } iNdEx = postIndex default: iNdEx = preIndex diff --git a/pkg/machinery/resources/secrets/deep_copy.generated.go b/pkg/machinery/resources/secrets/deep_copy.generated.go index 91fc265d5c..de9c7feb86 100644 --- a/pkg/machinery/resources/secrets/deep_copy.generated.go +++ b/pkg/machinery/resources/secrets/deep_copy.generated.go @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -// Code generated by "deep-copy -type APICertsSpec -type CertSANSpec -type EtcdCertsSpec -type EtcdRootSpec -type KubeletSpec -type KubernetesCertsSpec -type KubernetesRootSpec -type OSRootSpec -type TrustdCertsSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go ."; DO NOT EDIT. +// Code generated by "deep-copy -type APICertsSpec -type CertSANSpec -type EtcdCertsSpec -type EtcdRootSpec -type KubeletSpec -type KubernetesCertsSpec -type KubernetesDynamicCertsSpec -type KubernetesRootSpec -type OSRootSpec -type TrustdCertsSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go ."; DO NOT EDIT. package secrets @@ -87,6 +87,12 @@ func (o KubeletSpec) DeepCopy() KubeletSpec { // DeepCopy generates a deep copy of KubernetesCertsSpec. func (o KubernetesCertsSpec) DeepCopy() KubernetesCertsSpec { var cp KubernetesCertsSpec = o + return cp +} + +// DeepCopy generates a deep copy of KubernetesDynamicCertsSpec. +func (o KubernetesDynamicCertsSpec) DeepCopy() KubernetesDynamicCertsSpec { + var cp KubernetesDynamicCertsSpec = o if o.APIServer != nil { cp.APIServer = o.APIServer.DeepCopy() } diff --git a/pkg/machinery/resources/secrets/kubernetes.go b/pkg/machinery/resources/secrets/kubernetes.go index 85f5e789e3..703398ca01 100644 --- a/pkg/machinery/resources/secrets/kubernetes.go +++ b/pkg/machinery/resources/secrets/kubernetes.go @@ -9,7 +9,6 @@ import ( "github.com/cosi-project/runtime/pkg/resource/meta" "github.com/cosi-project/runtime/pkg/resource/protobuf" "github.com/cosi-project/runtime/pkg/resource/typed" - "github.com/siderolabs/crypto/x509" "github.com/siderolabs/talos/pkg/machinery/proto" ) @@ -21,16 +20,14 @@ const KubernetesType = resource.Type("KubernetesSecrets.secrets.talos.dev") const KubernetesID = resource.ID("k8s-certs") // Kubernetes contains K8s generated secrets. +// +// Kubernetes resource contains secrets which require reload of the control plane pods if updated. type Kubernetes = typed.Resource[KubernetesCertsSpec, KubernetesExtension] // KubernetesCertsSpec describes generated Kubernetes certificates. // //gotagsrewrite:gen type KubernetesCertsSpec struct { - APIServer *x509.PEMEncodedCertificateAndKey `yaml:"apiServer" protobuf:"1"` - APIServerKubeletClient *x509.PEMEncodedCertificateAndKey `yaml:"apiServerKubeletClient" protobuf:"2"` - FrontProxy *x509.PEMEncodedCertificateAndKey `yaml:"frontProxy" protobuf:"3"` - SchedulerKubeconfig string `yaml:"schedulerKubeconfig" protobuf:"4"` ControllerManagerKubeconfig string `yaml:"controllerManagerKubeconfig" protobuf:"5"` diff --git a/pkg/machinery/resources/secrets/kubernetes_certs.go b/pkg/machinery/resources/secrets/kubernetes_certs.go new file mode 100644 index 0000000000..1817cc3056 --- /dev/null +++ b/pkg/machinery/resources/secrets/kubernetes_certs.go @@ -0,0 +1,65 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/meta" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/cosi-project/runtime/pkg/resource/typed" + "github.com/siderolabs/crypto/x509" + + "github.com/siderolabs/talos/pkg/machinery/proto" +) + +// KubernetesDynamicCertsType is type of KubernetesCerts resource. +const KubernetesDynamicCertsType = resource.Type("KubernetesDynamicCerts.secrets.talos.dev") + +// KubernetesDynamicCertsID is a resource ID of singleton instance. +const KubernetesDynamicCertsID = resource.ID("k8s-dynamic-certs") + +// KubernetesDynamicCerts contains K8s generated secrets. +// +// KubernetesDynamicCerts resource contains secrets which do not require reload when updated. +type KubernetesDynamicCerts = typed.Resource[KubernetesDynamicCertsSpec, KubernetesDynamicCertsExtension] + +// KubernetesDynamicCertsSpec describes generated KubernetesCerts certificates. +// +//gotagsrewrite:gen +type KubernetesDynamicCertsSpec struct { + APIServer *x509.PEMEncodedCertificateAndKey `yaml:"apiServer" protobuf:"1"` + APIServerKubeletClient *x509.PEMEncodedCertificateAndKey `yaml:"apiServerKubeletClient" protobuf:"2"` + FrontProxy *x509.PEMEncodedCertificateAndKey `yaml:"frontProxy" protobuf:"3"` +} + +// NewKubernetesDynamicCerts initializes a KubernetesCerts resource. +func NewKubernetesDynamicCerts() *KubernetesDynamicCerts { + return typed.NewResource[KubernetesDynamicCertsSpec, KubernetesDynamicCertsExtension]( + resource.NewMetadata(NamespaceName, KubernetesDynamicCertsType, KubernetesDynamicCertsID, resource.VersionUndefined), + KubernetesDynamicCertsSpec{}, + ) +} + +// KubernetesDynamicCertsExtension provides auxiliary methods for KubernetesCerts. +type KubernetesDynamicCertsExtension struct{} + +// ResourceDefinition implements meta.ResourceDefinitionProvider interface. +func (KubernetesDynamicCertsExtension) ResourceDefinition() meta.ResourceDefinitionSpec { + return meta.ResourceDefinitionSpec{ + Type: KubernetesDynamicCertsType, + Aliases: []resource.Type{}, + DefaultNamespace: NamespaceName, + Sensitivity: meta.Sensitive, + } +} + +func init() { + proto.RegisterDefaultTypes() + + err := protobuf.RegisterDynamic[KubernetesDynamicCertsSpec](KubernetesDynamicCertsType, &KubernetesDynamicCerts{}) + if err != nil { + panic(err) + } +} diff --git a/pkg/machinery/resources/secrets/secrets.go b/pkg/machinery/resources/secrets/secrets.go index adfb8f85e7..27f85d07b5 100644 --- a/pkg/machinery/resources/secrets/secrets.go +++ b/pkg/machinery/resources/secrets/secrets.go @@ -11,4 +11,4 @@ import "github.com/cosi-project/runtime/pkg/resource" const NamespaceName resource.Namespace = "secrets" //nolint:lll -//go:generate deep-copy -type APICertsSpec -type CertSANSpec -type EtcdCertsSpec -type EtcdRootSpec -type KubeletSpec -type KubernetesCertsSpec -type KubernetesRootSpec -type OSRootSpec -type TrustdCertsSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go . +//go:generate deep-copy -type APICertsSpec -type CertSANSpec -type EtcdCertsSpec -type EtcdRootSpec -type KubeletSpec -type KubernetesCertsSpec -type KubernetesDynamicCertsSpec -type KubernetesRootSpec -type OSRootSpec -type TrustdCertsSpec -header-file ../../../../hack/boilerplate.txt -o deep_copy.generated.go . diff --git a/pkg/machinery/resources/secrets/secrets_test.go b/pkg/machinery/resources/secrets/secrets_test.go index 55153c70f4..abe93d1087 100644 --- a/pkg/machinery/resources/secrets/secrets_test.go +++ b/pkg/machinery/resources/secrets/secrets_test.go @@ -31,6 +31,7 @@ func TestRegisterResource(t *testing.T) { &secrets.EtcdRoot{}, &secrets.Kubelet{}, &secrets.Kubernetes{}, + &secrets.KubernetesDynamicCerts{}, &secrets.KubernetesRoot{}, &secrets.OSRoot{}, &secrets.Trustd{}, diff --git a/website/content/v1.5/reference/api.md b/website/content/v1.5/reference/api.md index b0218b3785..77d245ab95 100644 --- a/website/content/v1.5/reference/api.md +++ b/website/content/v1.5/reference/api.md @@ -192,6 +192,7 @@ description: Talos gRPC API reference. - [EtcdRootSpec](#talos.resource.definitions.secrets.EtcdRootSpec) - [KubeletSpec](#talos.resource.definitions.secrets.KubeletSpec) - [KubernetesCertsSpec](#talos.resource.definitions.secrets.KubernetesCertsSpec) + - [KubernetesDynamicCertsSpec](#talos.resource.definitions.secrets.KubernetesDynamicCertsSpec) - [KubernetesRootSpec](#talos.resource.definitions.secrets.KubernetesRootSpec) - [OSRootSpec](#talos.resource.definitions.secrets.OSRootSpec) - [TrustdCertsSpec](#talos.resource.definitions.secrets.TrustdCertsSpec) @@ -3480,9 +3481,6 @@ KubernetesCertsSpec describes generated Kubernetes certificates. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| api_server | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | -| api_server_kubelet_client | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | -| front_proxy | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | | scheduler_kubeconfig | [string](#string) | | | | controller_manager_kubeconfig | [string](#string) | | | | localhost_admin_kubeconfig | [string](#string) | | | @@ -3493,6 +3491,23 @@ KubernetesCertsSpec describes generated Kubernetes certificates. + + +### KubernetesDynamicCertsSpec +KubernetesDynamicCertsSpec describes generated KubernetesCerts certificates. + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| api_server | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | +| api_server_kubelet_client | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | +| front_proxy | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | + + + + + + ### KubernetesRootSpec