refactor: improve the kubernetes upgrade flow

Use new version of go-kubernetes, and move the `kube-proxy` DaemonSet
update to follow common logic of bootstrap manifests update.

This fixes a confusing behavior when after `k8s-upgrade` the version of
`kube-proxy` is not updated in the machine config.

See siderolabs/go-kubernetes#3

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

cmd/talosctl/cmd/talos/upgrade-k8s.go

@@ -84,5 +84,5 @@ func upgradeKubernetes(ctx context.Context, c *client.Client) error {
 		return fmt.Errorf("error creating upgrade path %w", err)
 	}
 
-	return k8s.UpgradeTalosManaged(ctx, &state, upgradeOptions)
+	return k8s.Upgrade(ctx, &state, upgradeOptions)
 }

go.mod

@@ -98,7 +98,7 @@ require (
 	github.com/siderolabs/go-debug v0.2.2
 	github.com/siderolabs/go-kmsg v0.1.2
 	github.com/siderolabs/go-kubeconfig v0.1.0
-	github.com/siderolabs/go-kubernetes v0.1.0
+	github.com/siderolabs/go-kubernetes v0.1.1
 	github.com/siderolabs/go-loadbalancer v0.2.1
 	github.com/siderolabs/go-pcidb v0.1.0
 	github.com/siderolabs/go-pointer v1.0.0

go.sum

@@ -1176,8 +1176,8 @@ github.com/siderolabs/go-kmsg v0.1.2 h1:ft5dJ/Qn3m5KbqwOImR4mCEW9R9SUYqWknhGfv/Q
 github.com/siderolabs/go-kmsg v0.1.2/go.mod h1:fVM55Sm3Y92NCGNNK7o7k9ej2ec2wmA6LhXnQwXiIl8=
 github.com/siderolabs/go-kubeconfig v0.1.0 h1:t/2oMWkLSdWHXglKPMz8ySXnx6ZjHckeGY79NaDcBTo=
 github.com/siderolabs/go-kubeconfig v0.1.0/go.mod h1:eM3mO02Td6wYDvdi9zTbMrj1Q4WqEFN8XQ6pNjCUWkI=
-github.com/siderolabs/go-kubernetes v0.1.0 h1:/nFc8hr+5DeqmoCo31gCxN89m0rar36R4Iro3wjfmT4=
-github.com/siderolabs/go-kubernetes v0.1.0/go.mod h1:oNMxB93Yu606+h+k1zfVGt0jUfO+WK3kDmR/pajgBdY=
+github.com/siderolabs/go-kubernetes v0.1.1 h1:n2YedX6r2FyVwlhYUGVB1fmNHyZZaahRltVf3q767gw=
+github.com/siderolabs/go-kubernetes v0.1.1/go.mod h1:oNMxB93Yu606+h+k1zfVGt0jUfO+WK3kDmR/pajgBdY=
 github.com/siderolabs/go-loadbalancer v0.2.1 h1:zjkBvKM6j/KIPQeitDWtP02G7Ew5ZnnrDdp+k6zcEBc=
 github.com/siderolabs/go-loadbalancer v0.2.1/go.mod h1:yVZxpjjbDgvbp71z9inTTjT0rZtY11wPDbB2kbH/m8Q=
 github.com/siderolabs/go-pcidb v0.1.0 h1:6cJPBBmHlIF4GouYR/1g3JXS/niAON+6lIOfKl/t794=

internal/integration/provision/upgrade.go

@@ -662,7 +662,7 @@ func (suite *UpgradeSuite) upgradeKubernetes(fromVersion, toVersion string, skip
 		UpgradeKubelet: !skipKubeletUpgrade,
 	}
 
-	suite.Require().NoError(kubernetes.UpgradeTalosManaged(suite.ctx, suite.clusterAccess, options))
+	suite.Require().NoError(kubernetes.Upgrade(suite.ctx, suite.clusterAccess, options))
 }
 
 func (suite *UpgradeSuite) untaint(name string) {

pkg/cluster/kubernetes/talos_managed.go

@@ -18,7 +18,6 @@ import (
 	"github.com/siderolabs/go-retry/retry"
 	"google.golang.org/grpc/metadata"
 	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
@@ -37,10 +36,10 @@ type UpgradeProvider interface {
 	cluster.K8sProvider
 }
 
-// UpgradeTalosManaged the Kubernetes control plane.
+// Upgrade the Kubernetes control plane components, manifests, kubelets.
 //
 //nolint:gocyclo,cyclop
-func UpgradeTalosManaged(ctx context.Context, cluster UpgradeProvider, options UpgradeOptions) error {
+func Upgrade(ctx context.Context, cluster UpgradeProvider, options UpgradeOptions) error {
 	if !options.Path.IsSupported() {
 		return fmt.Errorf("unsupported upgrade path %s (from %q to %q)", options.Path, options.Path.FromVersion(), options.Path.ToVersion())
 	}
@@ -95,12 +94,8 @@ func UpgradeTalosManaged(ctx context.Context, cluster UpgradeProvider, options U
 		}
 	}
 
-	if err = upgradeDaemonset(ctx, k8sClient.Clientset, kubeProxy, options); err != nil {
-		if apierrors.IsNotFound(err) {
-			options.Log("kube-proxy skipped as DaemonSet was not found")
-		} else {
-			return fmt.Errorf("error updating kube-proxy: %w", err)
-		}
+	if err = upgradeKubeProxy(ctx, cluster, options); err != nil {
+		return fmt.Errorf("failed updating kube-proxy: %w", err)
 	}
 
 	if err = upgradeKubelet(ctx, cluster, options); err != nil {
@@ -127,6 +122,34 @@ func upgradeStaticPod(ctx context.Context, cluster UpgradeProvider, options Upgr
 	return nil
 }
 
+func upgradeKubeProxy(ctx context.Context, cluster UpgradeProvider, options UpgradeOptions) error {
+	options.Log("updating kube-proxy to version %q", options.Path.ToVersion())
+
+	for _, node := range options.controlPlaneNodes {
+		if err := patchNodeConfig(ctx, cluster, node, patchKubeProxy(options)); err != nil {
+			return fmt.Errorf("error updating node %q: %w", node, err)
+		}
+	}
+
+	return nil
+}
+
+func patchKubeProxy(options UpgradeOptions) func(config *v1alpha1config.Config) error {
+	return func(config *v1alpha1config.Config) error {
+		if config.ClusterConfig == nil {
+			config.ClusterConfig = &v1alpha1config.ClusterConfig{}
+		}
+
+		if config.ClusterConfig.ProxyConfig == nil {
+			config.ClusterConfig.ProxyConfig = &v1alpha1config.ProxyConfig{}
+		}
+
+		config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", constants.KubeProxyImage, options.Path.ToVersion())
+
+		return nil
+	}
+}
+
 func controlplaneConfigResourceType(service string) resource.Type {
 	switch service {
 	case kubeAPIServer:
@@ -337,14 +360,7 @@ func getManifests(ctx context.Context, cluster UpgradeProvider) ([]*unstructured
 		ctx = client.WithNode(ctx, nodes[0])
 	}
 
-	return manifests.GetBootstrapManifests(ctx, talosclient.COSI, func(obj manifests.Manifest) bool {
-		// kubeproxy daemon set is updated as part of a different flow
-		if obj.GetName() == kubeProxy && obj.GetKind() == "DaemonSet" {
-			return false
-		}
-
-		return true
-	})
+	return manifests.GetBootstrapManifests(ctx, talosclient.COSI, nil)
 }
 
 //nolint:gocyclo
@@ -354,7 +370,7 @@ func syncManifests(ctx context.Context, objects []*unstructured.Unstructured, cl
 		return err
 	}
 
-	return manifests.Sync(ctx, objects, config, options.DryRun, options.Log)
+	return manifests.SyncWithLog(ctx, objects, config, options.DryRun, options.Log)
 }
 
 //nolint:gocyclo

pkg/cluster/kubernetes/upgrade.go

@@ -9,7 +9,6 @@ import (
 	"io"
 
 	"github.com/siderolabs/go-kubernetes/kubernetes/upgrade"
-	appsv1 "k8s.io/api/apps/v1"
 )
 
 const (
@@ -18,7 +17,6 @@ const (
 	kubeAPIServer         = "kube-apiserver"
 	kubeControllerManager = "kube-controller-manager"
 	kubeScheduler         = "kube-scheduler"
-	kubeProxy             = "kube-proxy"
 )
 
 // UpgradeOptions represents Kubernetes control plane upgrade settings.
@@ -30,7 +28,6 @@ type UpgradeOptions struct {
 	UpgradeKubelet       bool
 	DryRun               bool
 
-	extraUpdaters     []daemonsetUpdater
 	controlPlaneNodes []string
 	workerNodes       []string
 }
@@ -45,5 +42,3 @@ func (options *UpgradeOptions) Log(line string, args ...interface{}) {
 
 	fmt.Printf(line+"\n", args...)
 }
-
-type daemonsetUpdater func(ds string, daemonset *appsv1.DaemonSet) error