From b9ecd0bd6699be684c0570b1dd28807489e6363b Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Mon, 9 Oct 2023 20:41:32 +0200 Subject: [PATCH] [node-agent] Introduce `cmd/gardener-node-agent/app.go` with `manager.Manager` initialization (#8627) * Add `Server` configuration to `node-agent` config API * Introduce `main()` function for `gardener-node-agent` If it will be started with a kubeconfig pointing to the well-known location of the bootstrap token file, it will use it to fetch the access token and write it to the disk. Afterwards, it will overwrite the kubeconfig to point to the access token file instead of the bootstrap token file. The rest config will be modified ad-hoc, hence, the bootstrap token will only be used for fetching the "real" access token. * Drop validation regarding kubeconfig Otherwise, it's not possible to specify the kubeconfig via the `KUBECONFIG` environment variable * Add GNA to `Dockerfile` and `Makefile` * Address PR review feedback --- Dockerfile | 6 + Makefile | 5 + cmd/gardener-node-agent/app/app.go | 226 ++++++++++++++++++ cmd/gardener-node-agent/app/options.go | 78 ++++++ cmd/gardener-node-agent/main.go | 36 +++ example/node-agent/10-componentconfig.yaml | 5 + pkg/nodeagent/apis/config/types.go | 18 ++ pkg/nodeagent/apis/config/v1alpha1/types.go | 63 +++-- .../v1alpha1/zz_generated.conversion.go | 70 ++++++ .../config/v1alpha1/zz_generated.deepcopy.go | 43 ++++ .../apis/config/validation/validation.go | 4 - .../apis/config/validation/validation_test.go | 12 - .../apis/config/zz_generated.deepcopy.go | 43 ++++ pkg/nodeagent/controller/add.go | 26 ++ pkg/nodeagent/features/features.go | 26 ++ 15 files changed, 622 insertions(+), 39 deletions(-) create mode 100644 cmd/gardener-node-agent/app/app.go create mode 100644 cmd/gardener-node-agent/app/options.go create mode 100644 cmd/gardener-node-agent/main.go create mode 100644 pkg/nodeagent/controller/add.go create mode 100644 pkg/nodeagent/features/features.go diff --git a/Dockerfile b/Dockerfile index 54ee7f776a3..cbfd30dfa66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,12 @@ COPY --from=builder /go/bin/gardener-resource-manager /gardener-resource-manager WORKDIR / ENTRYPOINT ["/gardener-resource-manager"] +# node-agent +FROM distroless-static AS node-agent +COPY --from=builder /go/bin/gardener-node-agent /gardener-node-agent +WORKDIR / +ENTRYPOINT ["/gardener-node-agent"] + # operator FROM distroless-static AS operator COPY --from=builder /go/bin/gardener-operator /gardener-operator diff --git a/Makefile b/Makefile index a3a99b1f317..4e02f47f49a 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ CONTROLLER_MANAGER_IMAGE_REPOSITORY := $(REGISTRY)/controller-manager SCHEDULER_IMAGE_REPOSITORY := $(REGISTRY)/scheduler ADMISSION_IMAGE_REPOSITORY := $(REGISTRY)/admission-controller RESOURCE_MANAGER_IMAGE_REPOSITORY := $(REGISTRY)/resource-manager +NODE_AGENT_IMAGE_REPOSITORY := $(REGISTRY)/node-agent OPERATOR_IMAGE_REPOSITORY := $(REGISTRY)/operator GARDENLET_IMAGE_REPOSITORY := $(REGISTRY)/gardenlet EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY := $(REGISTRY)/extensions/provider-local @@ -96,6 +97,7 @@ docker-images: @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(SCHEDULER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(SCHEDULER_IMAGE_REPOSITORY):latest -f Dockerfile --target scheduler . @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target admission-controller . @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target resource-manager . + @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(NODE_AGENT_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(NODE_AGENT_IMAGE_REPOSITORY):latest -f Dockerfile --target node-agent . @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target operator . @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenlet . @docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-provider-local . @@ -107,6 +109,7 @@ docker-push: @if ! docker images $(SCHEDULER_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(SCHEDULER_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi @if ! docker images $(ADMISSION_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(ADMISSION_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi @if ! docker images $(RESOURCE_MANAGER_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(RESOURCE_MANAGER_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi + @if ! docker images $(NODE_AGENT_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(NODE_AGENT_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi @if ! docker images $(GARDENLET_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(GARDENLET_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi @if ! docker images $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY) | awk '{ print $$2 }' | grep -q -F $(EFFECTIVE_VERSION); then echo "$(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY) version $(EFFECTIVE_VERSION) is not yet built. Please run 'make docker-images'"; false; fi @docker push $(APISERVER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) @@ -119,6 +122,8 @@ docker-push: @if [[ "$(PUSH_LATEST_TAG)" == "true" ]]; then docker push $(ADMISSION_IMAGE_REPOSITORY):latest; fi @docker push $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) @if [[ "$(PUSH_LATEST_TAG)" == "true" ]]; then docker push $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest; fi + @docker push $(NODE_AGENT_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) + @if [[ "$(PUSH_LATEST_TAG)" == "true" ]]; then docker push $(NODE_AGENT_IMAGE_REPOSITORY):latest; fi @docker push $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) @if [[ "$(PUSH_LATEST_TAG)" == "true" ]]; then docker push $(GARDENLET_IMAGE_REPOSITORY):latest; fi @docker push $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) diff --git a/cmd/gardener-node-agent/app/app.go b/cmd/gardener-node-agent/app/app.go new file mode 100644 index 00000000000..b0de270cd8a --- /dev/null +++ b/cmd/gardener-node-agent/app/app.go @@ -0,0 +1,226 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + goruntime "runtime" + "strconv" + "time" + + "github.com/go-logr/logr" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" + clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/component-base/version" + "k8s.io/component-base/version/verflag" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/healthz" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/controllerutils/routes" + "github.com/gardener/gardener/pkg/features" + gardenerhealthz "github.com/gardener/gardener/pkg/healthz" + "github.com/gardener/gardener/pkg/logger" + "github.com/gardener/gardener/pkg/nodeagent/apis/config" + nodeagentv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" + "github.com/gardener/gardener/pkg/nodeagent/controller" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +// Name is a const for the name of this component. +const Name = "gardener-node-agent" + +// NewCommand creates a new cobra.Command for running gardener-node-agent. +func NewCommand() *cobra.Command { + opts := &options{} + + cmd := &cobra.Command{ + Use: Name, + Short: "Launch the " + Name, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + verflag.PrintAndExitIfRequested() + + if err := opts.complete(); err != nil { + return err + } + if err := opts.validate(); err != nil { + return err + } + + log, err := logger.NewZapLogger(opts.config.LogLevel, opts.config.LogFormat) + if err != nil { + return fmt.Errorf("error instantiating zap logger: %w", err) + } + + logf.SetLogger(log) + klog.SetLogger(log) + + log.Info("Starting "+Name, "version", version.Get()) + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + log.Info(fmt.Sprintf("FLAG: --%s=%s", flag.Name, flag.Value)) //nolint:logcheck + }) + + // don't output usage on further errors raised during execution + cmd.SilenceUsage = true + // further errors will be logged properly, don't duplicate + cmd.SilenceErrors = true + + return run(cmd.Context(), log, opts.config) + }, + } + + flags := cmd.Flags() + verflag.AddFlags(flags) + opts.addFlags(flags) + + return cmd +} + +func run(ctx context.Context, log logr.Logger, cfg *config.NodeAgentConfiguration) error { + log.Info("Feature Gates", "featureGates", features.DefaultFeatureGate) + + log.Info("Getting rest config") + if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + cfg.ClientConnection.Kubeconfig = kubeconfig + } + if cfg.ClientConnection.Kubeconfig == "" { + return fmt.Errorf("must specify path to a kubeconfig (either via \"KUBECONFIG\" environment variable of via .clientConnection.kubeconfig in component config)") + } + + restConfig, err := kubernetes.RESTConfigFromClientConnectionConfiguration(&cfg.ClientConnection, nil, kubernetes.AuthTokenFile) + if err != nil { + return err + } + + if restConfig.BearerTokenFile == nodeagentv1alpha1.BootstrapTokenFilePath { + log.Info("Kubeconfig points to the bootstrap token file") + if err := fetchAccessTokenViaBootstrapToken(ctx, log, restConfig, cfg); err != nil { + return fmt.Errorf("failed fetching access token via bootstrap token: %w", err) + } + } + + var extraHandlers map[string]http.Handler + if cfg.Debugging != nil && cfg.Debugging.EnableProfiling { + extraHandlers = routes.ProfilingHandlers + if cfg.Debugging.EnableContentionProfiling { + goruntime.SetBlockProfileRate(1) + } + } + + log.Info("Setting up manager") + mgr, err := manager.New(restConfig, manager.Options{ + Logger: log, + Scheme: kubernetes.SeedScheme, + GracefulShutdownTimeout: pointer.Duration(5 * time.Second), + + HealthProbeBindAddress: net.JoinHostPort(cfg.Server.HealthProbes.BindAddress, strconv.Itoa(cfg.Server.HealthProbes.Port)), + Metrics: metricsserver.Options{ + BindAddress: net.JoinHostPort(cfg.Server.Metrics.BindAddress, strconv.Itoa(cfg.Server.Metrics.Port)), + ExtraHandlers: extraHandlers, + }, + + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Namespaces: map[string]cache.Config{metav1.NamespaceSystem: {}}, + }, + }, + }, + LeaderElection: false, + Controller: controllerconfig.Controller{ + RecoverPanic: pointer.Bool(true), + }, + }) + if err != nil { + return err + } + + log.Info("Setting up health check endpoints") + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + return err + } + if err := mgr.AddReadyzCheck("informer-sync", gardenerhealthz.NewCacheSyncHealthz(mgr.GetCache())); err != nil { + return err + } + + log.Info("Adding controllers to manager") + if err := controller.AddToManager(mgr, cfg); err != nil { + return fmt.Errorf("failed adding controllers to manager: %w", err) + } + + log.Info("Starting manager") + return mgr.Start(ctx) +} + +func fetchAccessTokenViaBootstrapToken(ctx context.Context, log logr.Logger, restConfig *rest.Config, cfg *config.NodeAgentConfiguration) error { + c, err := client.New(restConfig, client.Options{}) + if err != nil { + return fmt.Errorf("unable to create client with bootstrap token: %w", err) + } + + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: cfg.AccessTokenSecretName, Namespace: metav1.NamespaceSystem}} + log.Info("Fetching access token secret", "secret", client.ObjectKeyFromObject(secret)) + if err := c.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + return fmt.Errorf("failed fetching access token from API server: %w", err) + } + + token := secret.Data[resourcesv1alpha1.DataKeyToken] + if len(token) == 0 { + return fmt.Errorf("secret key %q does not exist or empty", resourcesv1alpha1.DataKeyToken) + } + + restConfig.BearerTokenFile = nodeagentv1alpha1.TokenFilePath + kubeconfigRaw, err := runtime.Encode(clientcmdlatest.Codec, kubernetesutils.NewKubeconfig( + Name, + clientcmdv1.Cluster{Server: restConfig.Host, CertificateAuthorityData: restConfig.CAData}, + clientcmdv1.AuthInfo{TokenFile: nodeagentv1alpha1.TokenFilePath}, + )) + if err != nil { + return fmt.Errorf("failed encoding kubeconfig: %w", err) + } + + log.Info("Writing downloaded access token to disk", "path", nodeagentv1alpha1.TokenFilePath) + if err := os.WriteFile(nodeagentv1alpha1.TokenFilePath, token, 0600); err != nil { + return fmt.Errorf("unable to write access token to %s: %w", nodeagentv1alpha1.TokenFilePath, err) + } + log.Info("Token written to disk") + + log.Info("Overwriting kubeconfig on disk to no longer use bootstrap token file", "path", cfg.ClientConnection.Kubeconfig) + if err := os.WriteFile(cfg.ClientConnection.Kubeconfig, kubeconfigRaw, 0600); err != nil { + return fmt.Errorf("unable to write kubeconfig to %s: %w", cfg.ClientConnection.Kubeconfig, err) + } + log.Info("Kubeconfig written to disk") + + return nil +} diff --git a/cmd/gardener-node-agent/app/options.go b/cmd/gardener-node-agent/app/options.go new file mode 100644 index 00000000000..6e91bf70c3f --- /dev/null +++ b/cmd/gardener-node-agent/app/options.go @@ -0,0 +1,78 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "fmt" + "os" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + "github.com/gardener/gardener/pkg/features" + "github.com/gardener/gardener/pkg/nodeagent/apis/config" + nodeagentv1alpha1 "github.com/gardener/gardener/pkg/nodeagent/apis/config/v1alpha1" + nodeagentvalidation "github.com/gardener/gardener/pkg/nodeagent/apis/config/validation" +) + +var configDecoder runtime.Decoder + +func init() { + configScheme := runtime.NewScheme() + schemeBuilder := runtime.NewSchemeBuilder( + config.AddToScheme, + nodeagentv1alpha1.AddToScheme, + ) + utilruntime.Must(schemeBuilder.AddToScheme(configScheme)) + configDecoder = serializer.NewCodecFactory(configScheme).UniversalDecoder() +} + +type options struct { + configFile string + config *config.NodeAgentConfiguration +} + +func (o *options) addFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.configFile, "config", o.configFile, "Path to configuration file.") +} + +func (o *options) complete() error { + if len(o.configFile) == 0 { + return fmt.Errorf("missing config file") + } + + data, err := os.ReadFile(o.configFile) + if err != nil { + return fmt.Errorf("error reading config file: %w", err) + } + + o.config = &config.NodeAgentConfiguration{} + if err = runtime.DecodeInto(configDecoder, data, o.config); err != nil { + return fmt.Errorf("error decoding config: %w", err) + } + + // Set feature gates immediately after decoding the config. + // Feature gates might influence the next steps, e.g., validating the config. + return features.DefaultFeatureGate.SetFromMap(o.config.FeatureGates) +} + +func (o *options) validate() error { + if errs := nodeagentvalidation.ValidateNodeAgentConfiguration(o.config); len(errs) > 0 { + return errs.ToAggregate() + } + return nil +} diff --git a/cmd/gardener-node-agent/main.go b/cmd/gardener-node-agent/main.go new file mode 100644 index 00000000000..6ff96fc999b --- /dev/null +++ b/cmd/gardener-node-agent/main.go @@ -0,0 +1,36 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + "github.com/gardener/gardener/cmd/gardener-node-agent/app" + "github.com/gardener/gardener/cmd/utils" + "github.com/gardener/gardener/pkg/nodeagent/features" +) + +func main() { + utils.DeduplicateWarnings() + features.RegisterFeatureGates() + + if err := app.NewCommand().ExecuteContext(signals.SetupSignalHandler()); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/example/node-agent/10-componentconfig.yaml b/example/node-agent/10-componentconfig.yaml index 6cd37511b88..6c4d1c9da7e 100644 --- a/example/node-agent/10-componentconfig.yaml +++ b/example/node-agent/10-componentconfig.yaml @@ -7,6 +7,11 @@ clientConnection: kubeconfig: path/to/kubeconfig logLevel: info logFormat: text +server: + healthProbes: + port: 2751 + metrics: + port: 2752 debugging: enableProfiling: false enableContentionProfiling: false diff --git a/pkg/nodeagent/apis/config/types.go b/pkg/nodeagent/apis/config/types.go index 7da65f3b327..b9e73f60b44 100644 --- a/pkg/nodeagent/apis/config/types.go +++ b/pkg/nodeagent/apis/config/types.go @@ -32,6 +32,8 @@ type NodeAgentConfiguration struct { LogLevel string // LogFormat is the output format for the logs. Must be one of [text,json]. LogFormat string + // Server defines the configuration of the HTTP server. + Server ServerConfiguration // Debugging holds configuration for Debugging related features. Debugging *componentbaseconfig.DebuggingConfiguration // FeatureGates is a map of feature names to bools that enable or disable alpha/experimental features. This field @@ -55,3 +57,19 @@ type NodeAgentConfiguration struct { // /var/lib on the worker. KubeletDataVolumeSize *int64 } + +// ServerConfiguration contains details for the HTTP(S) servers. +type ServerConfiguration struct { + // HealthProbes is the configuration for serving the healthz and readyz endpoints. + HealthProbes *Server + // Metrics is the configuration for serving the metrics endpoint. + Metrics *Server +} + +// Server contains information for HTTP(S) server configuration. +type Server struct { + // BindAddress is the IP address on which to listen for the specified port. + BindAddress string + // Port is the port on which to serve requests. + Port int +} diff --git a/pkg/nodeagent/apis/config/v1alpha1/types.go b/pkg/nodeagent/apis/config/v1alpha1/types.go index 9b29a490e3f..643ccf652e2 100644 --- a/pkg/nodeagent/apis/config/v1alpha1/types.go +++ b/pkg/nodeagent/apis/config/v1alpha1/types.go @@ -21,37 +21,34 @@ import ( ) const ( - // NodeAgentBaseDir is the directory on the worker node that contains gardener-node-agent - // relevant files. - NodeAgentBaseDir = "/var/lib/gardener-node-agent" - // NodeAgentConfigPath is the file path on the worker node that contains the configuration + // BaseDir is the directory on the worker node that contains gardener-node-agent relevant files. + BaseDir = "/var/lib/gardener-node-agent" + + // CredentialsDir is the directory on the worker node that contains credentials for the gardener-node-agent. + CredentialsDir = BaseDir + "/credentials" + // BootstrapTokenFilePath is the file path on the worker node that contains the bootstrap token for the node. + BootstrapTokenFilePath = CredentialsDir + "/bootstrap-token" + // TokenFilePath is the file path on the worker node that contains the access token of the gardener-node-agent. + TokenFilePath = CredentialsDir + "/token" + + // ConfigPath is the file path on the worker node that contains the configuration // of the gardener-node-agent. - NodeAgentConfigPath = NodeAgentBaseDir + "/configuration.yaml" - // NodeAgentInitScriptPath is the file path on the worker node that contains the init script + ConfigPath = BaseDir + "/config.yaml" + // InitScriptPath is the file path on the worker node that contains the init script // of the gardener-node-agent. - NodeAgentInitScriptPath = NodeAgentBaseDir + "/gardener-node-init.sh" + InitScriptPath = BaseDir + "/gardener-node-init.sh" // NodeInitUnitName is the name of the gardener-node-init systemd service. NodeInitUnitName = "gardener-node-init.service" - // NodeAgentUnitName is the name of the gardener-node-agent systemd service. - NodeAgentUnitName = "gardener-node-agent.service" + // UnitName is the name of the gardener-node-agent systemd service. + UnitName = "gardener-node-agent.service" - // NodeAgentOSCSecretKey is the key inside the gardener-node-agent osc secret to access + // OSCSecretKey is the key inside the gardener-node-agent osc secret to access // the encoded osc. - NodeAgentOSCSecretKey = "gardener-node-agent" - // NodeAgentOSCOldConfigPath is the file path on the worker node that contains the + OSCSecretKey = "gardener-node-agent" + // OSCOldConfigPath is the file path on the worker node that contains the // previous content of the osc - NodeAgentOSCOldConfigPath = NodeAgentBaseDir + "/previous-osc.yaml" - - // NodeAgentTokenFilePath is the file path on the worker node that contains the shoot access - // token of the gardener-node-agent. - NodeAgentTokenFilePath = NodeAgentBaseDir + "/token" - // NodeAgentTokenSecretName is the name of the secret that contains the shoot access - // token of the gardener-node-agent. - NodeAgentTokenSecretName = "gardener-node-agent" - // NodeAgentTokenSecretKey is the key inside the gardener-node-agent token secret to access - // the token. - NodeAgentTokenSecretKey = "token" + OSCOldConfigPath = BaseDir + "/previous-osc.yaml" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -66,6 +63,8 @@ type NodeAgentConfiguration struct { LogLevel string `json:"logLevel"` // LogFormat is the output format for the logs. Must be one of [text,json]. LogFormat string `json:"logFormat"` + // Server defines the configuration of the HTTP server. + Server ServerConfiguration `json:"server"` // Debugging holds configuration for Debugging related features. // +optional Debugging *componentbaseconfigv1alpha1.DebuggingConfiguration `json:"debugging,omitempty"` @@ -92,3 +91,21 @@ type NodeAgentConfiguration struct { // +optional KubeletDataVolumeSize *int64 `json:"kubeletDataVolumeSize,omitempty"` } + +// ServerConfiguration contains details for the HTTP(S) servers. +type ServerConfiguration struct { + // HealthProbes is the configuration for serving the healthz and readyz endpoints. + // +optional + HealthProbes *Server `json:"healthProbes,omitempty"` + // Metrics is the configuration for serving the metrics endpoint. + // +optional + Metrics *Server `json:"metrics,omitempty"` +} + +// Server contains information for HTTP(S) server configuration. +type Server struct { + // BindAddress is the IP address on which to listen for the specified port. + BindAddress string `json:"bindAddress"` + // Port is the port on which to serve requests. + Port int `json:"port"` +} diff --git a/pkg/nodeagent/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/nodeagent/apis/config/v1alpha1/zz_generated.conversion.go index 8311cbdfda5..6f19a6abe4f 100644 --- a/pkg/nodeagent/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/nodeagent/apis/config/v1alpha1/zz_generated.conversion.go @@ -49,6 +49,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*Server)(nil), (*config.Server)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_Server_To_config_Server(a.(*Server), b.(*config.Server), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.Server)(nil), (*Server)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_Server_To_v1alpha1_Server(a.(*config.Server), b.(*Server), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ServerConfiguration)(nil), (*config.ServerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration(a.(*ServerConfiguration), b.(*config.ServerConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.ServerConfiguration)(nil), (*ServerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration(a.(*config.ServerConfiguration), b.(*ServerConfiguration), scope) + }); err != nil { + return err + } return nil } @@ -58,6 +78,9 @@ func autoConvert_v1alpha1_NodeAgentConfiguration_To_config_NodeAgentConfiguratio } out.LogLevel = in.LogLevel out.LogFormat = in.LogFormat + if err := Convert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration(&in.Server, &out.Server, s); err != nil { + return err + } if in.Debugging != nil { in, out := &in.Debugging, &out.Debugging *out = new(componentbaseconfig.DebuggingConfiguration) @@ -88,6 +111,9 @@ func autoConvert_config_NodeAgentConfiguration_To_v1alpha1_NodeAgentConfiguratio } out.LogLevel = in.LogLevel out.LogFormat = in.LogFormat + if err := Convert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration(&in.Server, &out.Server, s); err != nil { + return err + } if in.Debugging != nil { in, out := &in.Debugging, &out.Debugging *out = new(configv1alpha1.DebuggingConfiguration) @@ -111,3 +137,47 @@ func autoConvert_config_NodeAgentConfiguration_To_v1alpha1_NodeAgentConfiguratio func Convert_config_NodeAgentConfiguration_To_v1alpha1_NodeAgentConfiguration(in *config.NodeAgentConfiguration, out *NodeAgentConfiguration, s conversion.Scope) error { return autoConvert_config_NodeAgentConfiguration_To_v1alpha1_NodeAgentConfiguration(in, out, s) } + +func autoConvert_v1alpha1_Server_To_config_Server(in *Server, out *config.Server, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.Port = in.Port + return nil +} + +// Convert_v1alpha1_Server_To_config_Server is an autogenerated conversion function. +func Convert_v1alpha1_Server_To_config_Server(in *Server, out *config.Server, s conversion.Scope) error { + return autoConvert_v1alpha1_Server_To_config_Server(in, out, s) +} + +func autoConvert_config_Server_To_v1alpha1_Server(in *config.Server, out *Server, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.Port = in.Port + return nil +} + +// Convert_config_Server_To_v1alpha1_Server is an autogenerated conversion function. +func Convert_config_Server_To_v1alpha1_Server(in *config.Server, out *Server, s conversion.Scope) error { + return autoConvert_config_Server_To_v1alpha1_Server(in, out, s) +} + +func autoConvert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration(in *ServerConfiguration, out *config.ServerConfiguration, s conversion.Scope) error { + out.HealthProbes = (*config.Server)(unsafe.Pointer(in.HealthProbes)) + out.Metrics = (*config.Server)(unsafe.Pointer(in.Metrics)) + return nil +} + +// Convert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration(in *ServerConfiguration, out *config.ServerConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_ServerConfiguration_To_config_ServerConfiguration(in, out, s) +} + +func autoConvert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration(in *config.ServerConfiguration, out *ServerConfiguration, s conversion.Scope) error { + out.HealthProbes = (*Server)(unsafe.Pointer(in.HealthProbes)) + out.Metrics = (*Server)(unsafe.Pointer(in.Metrics)) + return nil +} + +// Convert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration is an autogenerated conversion function. +func Convert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration(in *config.ServerConfiguration, out *ServerConfiguration, s conversion.Scope) error { + return autoConvert_config_ServerConfiguration_To_v1alpha1_ServerConfiguration(in, out, s) +} diff --git a/pkg/nodeagent/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/nodeagent/apis/config/v1alpha1/zz_generated.deepcopy.go index 4ddc2ea8fac..b81bb60f535 100644 --- a/pkg/nodeagent/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/nodeagent/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -32,6 +32,7 @@ func (in *NodeAgentConfiguration) DeepCopyInto(out *NodeAgentConfiguration) { *out = *in out.TypeMeta = in.TypeMeta out.ClientConnection = in.ClientConnection + in.Server.DeepCopyInto(&out.Server) if in.Debugging != nil { in, out := &in.Debugging, &out.Debugging *out = new(configv1alpha1.DebuggingConfiguration) @@ -74,3 +75,45 @@ func (in *NodeAgentConfiguration) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Server) DeepCopyInto(out *Server) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Server. +func (in *Server) DeepCopy() *Server { + if in == nil { + return nil + } + out := new(Server) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServerConfiguration) DeepCopyInto(out *ServerConfiguration) { + *out = *in + if in.HealthProbes != nil { + in, out := &in.HealthProbes, &out.HealthProbes + *out = new(Server) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Server) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerConfiguration. +func (in *ServerConfiguration) DeepCopy() *ServerConfiguration { + if in == nil { + return nil + } + out := new(ServerConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/nodeagent/apis/config/validation/validation.go b/pkg/nodeagent/apis/config/validation/validation.go index 98e4fc6d6a0..13be17ecb4d 100644 --- a/pkg/nodeagent/apis/config/validation/validation.go +++ b/pkg/nodeagent/apis/config/validation/validation.go @@ -25,10 +25,6 @@ import ( func ValidateNodeAgentConfiguration(conf *config.NodeAgentConfiguration) field.ErrorList { allErrs := field.ErrorList{} - if conf.ClientConnection.Kubeconfig == "" { - allErrs = append(allErrs, field.Required(field.NewPath("clientConnection").Child("kubeconfig"), "must provide a path to a kubeconfig")) - } - if conf.HyperkubeImage == "" { allErrs = append(allErrs, field.Required(field.NewPath("hyperkubeImage"), "must provide a hyperkube image")) } diff --git a/pkg/nodeagent/apis/config/validation/validation_test.go b/pkg/nodeagent/apis/config/validation/validation_test.go index a4fe7a5f5d8..524cb338a92 100644 --- a/pkg/nodeagent/apis/config/validation/validation_test.go +++ b/pkg/nodeagent/apis/config/validation/validation_test.go @@ -20,7 +20,6 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "k8s.io/apimachinery/pkg/util/validation/field" - componentbaseconfig "k8s.io/component-base/config" . "github.com/gardener/gardener/pkg/nodeagent/apis/config" . "github.com/gardener/gardener/pkg/nodeagent/apis/config/validation" @@ -31,7 +30,6 @@ var _ = Describe("#ValidateNodeAgentConfiguration", func() { BeforeEach(func() { config = &NodeAgentConfiguration{ - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: "path/to/kubeconfig"}, KubernetesVersion: semver.MustParse("v1.27.0"), HyperkubeImage: "registry.com/hyperkube:v1.27.0", Image: "registry.com/node-agent:v1.73.0", @@ -40,16 +38,6 @@ var _ = Describe("#ValidateNodeAgentConfiguration", func() { } }) - It("should fail because clientConnection.kubeconfig config is not specified", func() { - config.ClientConnection.Kubeconfig = "" - Expect(ValidateNodeAgentConfiguration(config)).To(ConsistOf( - PointTo(MatchFields(IgnoreExtras, Fields{ - "Type": Equal(field.ErrorTypeRequired), - "Field": Equal("clientConnection.kubeconfig"), - })), - )) - }) - It("should pass because all necessary fields is specified", func() { Expect(ValidateNodeAgentConfiguration(config)).To(BeEmpty()) }) diff --git a/pkg/nodeagent/apis/config/zz_generated.deepcopy.go b/pkg/nodeagent/apis/config/zz_generated.deepcopy.go index 448532748e9..7fa60c0d4fe 100644 --- a/pkg/nodeagent/apis/config/zz_generated.deepcopy.go +++ b/pkg/nodeagent/apis/config/zz_generated.deepcopy.go @@ -32,6 +32,7 @@ func (in *NodeAgentConfiguration) DeepCopyInto(out *NodeAgentConfiguration) { *out = *in out.TypeMeta = in.TypeMeta out.ClientConnection = in.ClientConnection + in.Server.DeepCopyInto(&out.Server) if in.Debugging != nil { in, out := &in.Debugging, &out.Debugging *out = new(componentbaseconfig.DebuggingConfiguration) @@ -74,3 +75,45 @@ func (in *NodeAgentConfiguration) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Server) DeepCopyInto(out *Server) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Server. +func (in *Server) DeepCopy() *Server { + if in == nil { + return nil + } + out := new(Server) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServerConfiguration) DeepCopyInto(out *ServerConfiguration) { + *out = *in + if in.HealthProbes != nil { + in, out := &in.HealthProbes, &out.HealthProbes + *out = new(Server) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Server) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerConfiguration. +func (in *ServerConfiguration) DeepCopy() *ServerConfiguration { + if in == nil { + return nil + } + out := new(ServerConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/nodeagent/controller/add.go b/pkg/nodeagent/controller/add.go new file mode 100644 index 00000000000..cd184ce6191 --- /dev/null +++ b/pkg/nodeagent/controller/add.go @@ -0,0 +1,26 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/gardener/gardener/pkg/nodeagent/apis/config" +) + +// AddToManager adds all controllers to the given manager. +func AddToManager(_ manager.Manager, _ *config.NodeAgentConfiguration) error { + return nil +} diff --git a/pkg/nodeagent/features/features.go b/pkg/nodeagent/features/features.go new file mode 100644 index 00000000000..d670cf8049f --- /dev/null +++ b/pkg/nodeagent/features/features.go @@ -0,0 +1,26 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package features + +import ( + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + "github.com/gardener/gardener/pkg/features" +) + +// RegisterFeatureGates registers the feature gates of gardener-node-agent. +func RegisterFeatureGates() { + utilruntime.Must(features.DefaultFeatureGate.Add(features.GetFeatures())) +}