Add spec.api.onlyBindToAddress configuration

cmd/controller/certificates.go

@@ -20,16 +20,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
-	"os"
-	"path/filepath"
-
 	"github.com/k0sproject/k0s/internal/pkg/file"
 	"github.com/k0sproject/k0s/internal/pkg/users"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/certificate"
 	"github.com/k0sproject/k0s/pkg/config"
 	"github.com/k0sproject/k0s/pkg/constant"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
 
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -65,15 +65,15 @@ func (c *Certificates) Init(ctx context.Context) error {
 	}
 	c.CACert = string(cert)
 	// Changing the URL here also requires changes in the "k0s kubeconfig admin" subcommand.
-	kubeConfigAPIUrl := fmt.Sprintf("https://localhost:%d", c.ClusterSpec.API.Port)
+	apiAddress := net.JoinHostPort(c.ClusterSpec.API.Address, strconv.Itoa(c.ClusterSpec.API.Port))
+	kubeConfigAPIUrl := fmt.Sprintf("https://%s", apiAddress)
 
 	apiServerUID, err := users.LookupUID(constant.ApiserverUser)
 	if err != nil {
 		err = fmt.Errorf("failed to lookup UID for %q: %w", constant.ApiserverUser, err)
 		apiServerUID = users.RootUID
 		logrus.WithError(err).Warn("Files with key material for kube-apiserver user will be owned by root")
 	}
-
 	eg.Go(func() error {
 		// Front proxy CA
 		if err := c.CertManager.EnsureCA("front-proxy-ca", "kubernetes-front-proxy-ca"); err != nil {

docs/configuration.md

@@ -112,14 +112,15 @@ spec:
 
 ### `spec.api`
 
-| Element                  | Description                                                                                                                                                                                                                 |
-| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `externalAddress`        | The loadbalancer address (for k0s controllers running behind a loadbalancer). Configures all cluster components to connect to this address and also configures this address for use  when joining new nodes to the cluster. |
-| `address`                | Local address on which to bind an API. Also serves as one of the addresses pushed on the k0s create service certificate on the API. Defaults to first non-local address found on the node.                                  |
-| `sans`                   | List of additional addresses to push to API servers serving the certificate.                                                                                                                                                |
-| `extraArgs`              | Map of key-values (strings) for any extra arguments to pass down to Kubernetes api-server process.                                                                                                                          |
-| `port`¹                  | Custom port for kube-api server to listen on (default: 6443)                                                                                                                                                                |
-| `k0sApiPort`¹            | Custom port for k0s-api server to listen on (default: 9443)                                                                                                                                                                 |
+| Element             | Description                                                                                                                                                                                                                                                                |
+|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `address`           | IP Address used by cluster components to talk to the API server. Also serves as one of the addresses pushed on the k0s create service certificate on the API. Defaults to first non-local address found on the node.                                                       |
+| `onlyBindToAddress` | The API server binds too all interfaces by default. With this option set to `true`, the API server will only listen on the IP address configured by the `address` option (first non-local address by default). This can be necessary with multi-homed control plane nodes. |
+| `externalAddress`   | The loadbalancer address (for k0s controllers running behind a loadbalancer). Configures all cluster components to connect to this address and also configures this address for use when joining new nodes to the cluster.                                                 |
+| `sans`              | List of additional addresses to push to API servers serving the certificate.                                                                                                                                                                                               |
+| `extraArgs`         | Map of key-values (strings) for any extra arguments to pass down to Kubernetes api-server process.                                                                                                                                                                         |
+| `port`¹             | Custom port for kube-api server to listen on (default: 6443)                                                                                                                                                                                                               |
+| `k0sApiPort`¹       | Custom port for k0s-api server to listen on (default: 9443)                                                                                                                                                                                                                |
 
 ¹ If `port` and `k0sApiPort` are used with the `externalAddress` element, the loadbalancer serving at `externalAddress` must listen on the same ports.
 

inttest/Makefile.variables

@@ -15,6 +15,7 @@ smoketests := \
 	check-ap-updater-periodic \
 	check-backup \
 	check-basic \
+	check-bind-address \
 	check-byocri \
 	check-calico \
 	check-capitalhostnames \

inttest/bind-address/bind_address_test.go

@@ -0,0 +1,180 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bind_address
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/k0sproject/k0s/inttest/common"
+	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	kubeletv1beta1 "k8s.io/kubelet/config/v1beta1"
+
+	testifysuite "github.com/stretchr/testify/suite"
+	"golang.org/x/sync/errgroup"
+	"sigs.k8s.io/yaml"
+)
+
+const kubeSystem = "kube-system"
+
+type suite struct {
+	common.BootlooseSuite
+}
+
+func (s *suite) TestCustomizedBindAddress() {
+	const controllerArgs = "--kube-controller-manager-extra-args='--node-monitor-period=3s --node-monitor-grace-period=9s'"
+
+	ctx := s.Context()
+
+	{
+		for i := 0; i < s.ControllerCount; i++ {
+			config, err := yaml.Marshal(&v1beta1.ClusterConfig{
+				Spec: &v1beta1.ClusterSpec{
+					API: func() *v1beta1.APISpec {
+						apiSpec := v1beta1.DefaultAPISpec()
+						apiSpec.Address = s.GetIPAddress(s.ControllerNode(i))
+						apiSpec.OnlyBindToAddress = true
+						return apiSpec
+					}(),
+					WorkerProfiles: v1beta1.WorkerProfiles{
+						v1beta1.WorkerProfile{
+							Name: "default",
+							Config: func() *runtime.RawExtension {
+								kubeletConfig := kubeletv1beta1.KubeletConfiguration{
+									NodeStatusUpdateFrequency: metav1.Duration{Duration: 3 * time.Second},
+								}
+								bytes, err := json.Marshal(kubeletConfig)
+								s.Require().NoError(err)
+								return &runtime.RawExtension{Raw: bytes}
+							}(),
+						},
+					},
+				},
+			})
+			s.Require().NoError(err)
+			s.WriteFileContent(s.ControllerNode(i), "/tmp/k0s.yaml", config)
+		}
+	}
+
+	s.Run("controller_and_workers_get_up", func() {
+		s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", controllerArgs))
+
+		s.T().Logf("Starting workers and waiting for cluster to become ready")
+
+		token, err := s.GetJoinToken("worker")
+		s.Require().NoError(err)
+		s.Require().NoError(s.RunWorkersWithToken(token))
+
+		clients, err := s.KubeClient(s.ControllerNode(0))
+		s.Require().NoError(err)
+
+		eg, _ := errgroup.WithContext(ctx)
+		for i := 0; i < s.WorkerCount; i++ {
+			nodeName := s.WorkerNode(i)
+			eg.Go(func() error {
+				if err := s.WaitForNodeReady(nodeName, clients); err != nil {
+					return fmt.Errorf("Node %s is not ready: %w", nodeName, err)
+				}
+				return nil
+			})
+		}
+		s.Require().NoError(eg.Wait())
+
+		s.Require().NoError(s.checkClusterReadiness(ctx, clients, 1))
+	})
+
+	s.Run("join_new_controllers", func() {
+		token, err := s.GetJoinToken("controller")
+		s.Require().NoError(err)
+
+		eg, _ := errgroup.WithContext(ctx)
+		eg.Go(func() error { return s.InitController(1, "--config=/tmp/k0s.yaml", controllerArgs, token) })
+		eg.Go(func() error { return s.InitController(2, "--config=/tmp/k0s.yaml", controllerArgs, token) })
+
+		s.Require().NoError(eg.Wait())
+
+		clients, err := s.KubeClient(s.ControllerNode(1))
+		s.Require().NoError(err)
+
+		s.T().Logf("Checking if HA cluster is ready")
+		s.Require().NoError(s.checkClusterReadiness(ctx, clients, s.ControllerCount))
+	})
+}
+
+func (s *suite) checkClusterReadiness(ctx context.Context, clients *kubernetes.Clientset, numControllers int, degradedControllers ...string) error {
+	eg, ctx := errgroup.WithContext(ctx)
+
+	eg.Go(func() error {
+		if err := common.WaitForKubeRouterReady(ctx, clients); err != nil {
+			return fmt.Errorf("kube-router did not start: %w", err)
+		}
+		s.T().Logf("kube-router is ready")
+		return nil
+	})
+
+	for _, lease := range []string{"kube-scheduler", "kube-controller-manager"} {
+		lease := lease
+		eg.Go(func() error {
+			id, err := common.WaitForLease(ctx, clients, lease, kubeSystem)
+			if err != nil {
+				return fmt.Errorf("%s has no leader: %w", lease, err)
+			}
+			s.T().Logf("%s has a leader: %q", lease, id)
+			return nil
+		})
+	}
+
+	for _, daemonSet := range []string{"kube-proxy", "konnectivity-agent"} {
+		daemonSet := daemonSet
+		eg.Go(func() error {
+			if err := common.WaitForDaemonSet(ctx, clients, daemonSet, "kube-system"); err != nil {
+				return fmt.Errorf("%s is not ready: %w", daemonSet, err)
+			}
+			s.T().Log(daemonSet, "is ready")
+			return nil
+		})
+	}
+
+	for _, deployment := range []string{"coredns", "metrics-server"} {
+		deployment := deployment
+		eg.Go(func() error {
+			if err := common.WaitForDeployment(ctx, clients, deployment, "kube-system"); err != nil {
+				return fmt.Errorf("%s did not become ready: %w", deployment, err)
+			}
+			s.T().Log(deployment, "is ready")
+			return nil
+		})
+	}
+
+	return eg.Wait()
+}
+
+func TestCustomizedBindAddressSuite(t *testing.T) {
+	s := suite{
+		common.BootlooseSuite{
+			ControllerCount: 3,
+			WorkerCount:     1,
+		},
+	}
+	testifysuite.Run(t, &s)
+}

pkg/apis/k0s/v1beta1/api.go

@@ -37,6 +37,10 @@ type APISpec struct {
 	// Address on which to connect to the API server.
 	Address string `json:"address,omitempty"`
 
+	// Whether to only bind to the IP given by the address option.
+	// +optional
+	OnlyBindToAddress bool `json:"onlyBindToAddress,omitempty"`
+
 	// The loadbalancer address (for k0s controllers running behind a loadbalancer)
 	ExternalAddress string `json:"externalAddress,omitempty"`
 
@@ -102,7 +106,7 @@ func (a *APISpec) getExternalURIForPort(port int) string {
 	return fmt.Sprintf("https://%s:%d", addr, port)
 }
 
-// Sans return the given SANS plus all local adresses and externalAddress if given
+// Sans return the given SANS plus all local addresses and externalAddress if given
 func (a *APISpec) Sans() []string {
 	sans, _ := iface.AllAddresses()
 	sans = append(sans, a.Address)
@@ -114,6 +118,10 @@ func (a *APISpec) Sans() []string {
 	return stringslice.Unique(sans)
 }
 
+func isAnyAddress(address string) bool {
+	return address == "0.0.0.0" || address == "::"
+}
+
 // Validate validates APISpec struct
 func (a *APISpec) Validate() []error {
 	if a == nil {
@@ -125,6 +133,9 @@ func (a *APISpec) Validate() []error {
 	if !govalidator.IsIP(a.Address) {
 		errors = append(errors, field.Invalid(field.NewPath("address"), a.Address, "invalid IP address"))
 	}
+	if isAnyAddress(a.Address) {
+		errors = append(errors, field.Invalid(field.NewPath("address"), a.Address, "invalid INADDR_ANY"))
+	}
 
 	validateIPAddressOrDNSName := func(path *field.Path, san string) {
 		if govalidator.IsIP(san) || govalidator.IsDNSName(san) {
@@ -135,6 +146,9 @@ func (a *APISpec) Validate() []error {
 
 	if a.ExternalAddress != "" {
 		validateIPAddressOrDNSName(field.NewPath("externalAddress"), a.ExternalAddress)
+		if isAnyAddress(a.ExternalAddress) {
+			errors = append(errors, field.Invalid(field.NewPath("externalAddress"), a.Address, "invalid INADDR_ANY"))
+		}
 	}
 
 	for _, msg := range validation.IsValidPortNum(a.K0sAPIPort) {

pkg/component/controller/apiserver.go

@@ -22,11 +22,13 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/sirupsen/logrus"
@@ -127,6 +129,10 @@ func (a *APIServer) Start(_ context.Context) error {
 		"enable-admission-plugins":         "NodeRestriction",
 	}
 
+	if a.ClusterConfig.Spec.API.OnlyBindToAddress {
+		args["bind-address"] = a.ClusterConfig.Spec.API.Address
+	}
+
 	apiAudiences := []string{"https://kubernetes.default.svc"}
 
 	if a.EnableKonnectivity {
@@ -232,7 +238,8 @@ func (a *APIServer) Ready() error {
 		TLSClientConfig: tlsConfig,
 	}
 	client := &http.Client{Transport: tr}
-	resp, err := client.Get(fmt.Sprintf("https://localhost:%d/readyz?verbose", a.ClusterConfig.Spec.API.Port))
+	apiAddress := net.JoinHostPort(a.ClusterConfig.Spec.API.Address, strconv.Itoa(a.ClusterConfig.Spec.API.Port))
+	resp, err := client.Get(fmt.Sprintf("https://%s/readyz?verbose", apiAddress))
 	if err != nil {
 		return err
 	}

static/_crds/k0s/k0s.k0sproject.io_clusterconfigs.yaml

@@ -62,6 +62,10 @@ spec:
                     maximum: 65535
                     minimum: 1
                     type: integer
+                  onlyBindToAddress:
+                    description: Whether to only bind to the IP given by the address
+                      option.
+                    type: boolean
                   port:
                     default: 6443
                     description: 'Custom port for kube-api server to listen on (default: