admission: validate minimumKubeletVersion

Signed-off-by: Peter Hunt <pehunt@redhat.com>

openshift-kube-apiserver/admission/customresourcevalidation/node/validate_node_config.go

@@ -5,14 +5,21 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/blang/semver/v4"
+	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/library-go/pkg/apiserver/admission/admissionrestconfig"
+
 	"k8s.io/apimachinery/pkg/api/validation"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/admission"
-
-	configv1 "github.com/openshift/api/config/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
 	"k8s.io/kubernetes/openshift-kube-apiserver/admission/customresourcevalidation"
+	"k8s.io/kubernetes/openshift-kube-apiserver/authorization/minimumkubeletversion"
 )
 
 var rejectionScenarios = []struct {
@@ -25,18 +32,28 @@ var rejectionScenarios = []struct {
 	{fromProfile: configv1.LowUpdateSlowReaction, toProfile: configv1.DefaultUpdateDefaultReaction},
 }
 
-const PluginName = "config.openshift.io/RestrictExtremeWorkerLatencyProfile"
+const PluginName = "config.openshift.io/ValidateConfigNodeV1"
 
 // Register registers a plugin
 func Register(plugins *admission.Plugins) {
 	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
-		return customresourcevalidation.NewValidator(
+		ret := &validateCustomResourceWithClient{}
+
+		delegate, err := customresourcevalidation.NewValidator(
 			map[schema.GroupResource]bool{
 				configv1.Resource("nodes"): true,
 			},
 			map[schema.GroupVersionKind]customresourcevalidation.ObjectValidator{
-				configv1.GroupVersion.WithKind("Node"): configNodeV1{},
+				configv1.GroupVersion.WithKind("Node"): configNodeV1{
+					nodesGetter: ret.getNodesGetter,
+				},
 			})
+		if err != nil {
+			return nil, err
+		}
+		ret.ValidationInterface = delegate
+
+		return ret, nil
 	})
 }
 
@@ -57,7 +74,9 @@ func toConfigNodeV1(uncastObj runtime.Object) (*configv1.Node, field.ErrorList)
 	return obj, nil
 }
 
-type configNodeV1 struct{}
+type configNodeV1 struct {
+	nodesGetter func() corev1client.NodesGetter
+}
 
 func validateConfigNodeForExtremeLatencyProfile(obj, oldObj *configv1.Node) *field.Error {
 	fromProfile := oldObj.Spec.WorkerLatencyProfile
@@ -78,18 +97,47 @@ func validateConfigNodeForExtremeLatencyProfile(obj, oldObj *configv1.Node) *fie
 	return nil
 }
 
-func (configNodeV1) ValidateCreate(_ context.Context, uncastObj runtime.Object) field.ErrorList {
+func validateMinimumKubeletVersion(nodesGetter corev1client.NodesGetter, obj *configv1.Node) *field.Error {
+	// unset, no error
+	if obj.Spec.MinimumKubeletVersion == "" {
+		return nil
+	}
+
+	fieldPath := field.NewPath("spec", "minimumKubeletVersion")
+	nodes, err := nodesGetter.Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return field.Forbidden(fieldPath, fmt.Sprintf("Getting nodes to compare minimum version %v", err.Error()))
+	}
+
+	version, err := semver.Parse(obj.Spec.MinimumKubeletVersion)
+	if err != nil {
+		return field.Invalid(fieldPath, obj.Spec.MinimumKubeletVersion, fmt.Sprintf("Failed to parse submitted version %s %v", obj.Spec.MinimumKubeletVersion, err.Error()))
+	}
+
+	for _, node := range nodes.Items {
+		_, errStr := minimumkubeletversion.IsKubeletVersionTooOld(&node, &version)
+		if errStr != "" {
+			return field.Invalid(fieldPath, obj.Spec.MinimumKubeletVersion, errStr)
+		}
+	}
+	return nil
+}
+
+func (c configNodeV1) ValidateCreate(_ context.Context, uncastObj runtime.Object) field.ErrorList {
 	obj, allErrs := toConfigNodeV1(uncastObj)
 	if len(allErrs) > 0 {
 		return allErrs
 	}
 
 	allErrs = append(allErrs, validation.ValidateObjectMeta(&obj.ObjectMeta, false, customresourcevalidation.RequireNameCluster, field.NewPath("metadata"))...)
+	if err := validateMinimumKubeletVersion(c.nodesGetter(), obj); err != nil {
+		allErrs = append(allErrs, err)
+	}
 
 	return allErrs
 }
 
-func (configNodeV1) ValidateUpdate(_ context.Context, uncastObj runtime.Object, uncastOldObj runtime.Object) field.ErrorList {
+func (c configNodeV1) ValidateUpdate(_ context.Context, uncastObj runtime.Object, uncastOldObj runtime.Object) field.ErrorList {
 	obj, allErrs := toConfigNodeV1(uncastObj)
 	if len(allErrs) > 0 {
 		return allErrs
@@ -103,6 +151,9 @@ func (configNodeV1) ValidateUpdate(_ context.Context, uncastObj runtime.Object,
 	if err := validateConfigNodeForExtremeLatencyProfile(obj, oldObj); err != nil {
 		allErrs = append(allErrs, err)
 	}
+	if err := validateMinimumKubeletVersion(c.nodesGetter(), obj); err != nil {
+		allErrs = append(allErrs, err)
+	}
 
 	return allErrs
 }
@@ -122,3 +173,32 @@ func (configNodeV1) ValidateStatusUpdate(_ context.Context, uncastObj runtime.Ob
 
 	return errs
 }
+
+type validateCustomResourceWithClient struct {
+	admission.ValidationInterface
+
+	nodesGetter corev1client.NodesGetter
+}
+
+var _ admissionrestconfig.WantsRESTClientConfig = validateCustomResourceWithClient{}
+
+func (a validateCustomResourceWithClient) SetRESTClientConfig(restClientConfig rest.Config) {
+	var err error
+
+	a.nodesGetter, err = corev1client.NewForConfig(&restClientConfig)
+	if err != nil {
+		utilruntime.HandleError(err)
+	}
+}
+
+func (a validateCustomResourceWithClient) ValidateInitialization() error {
+	if a.nodesGetter == nil {
+		return fmt.Errorf("%s needs a nodes", PluginName)
+	}
+
+	return nil
+}
+
+func (a validateCustomResourceWithClient) getNodesGetter() corev1client.NodesGetter {
+	return a.nodesGetter
+}

openshift-kube-apiserver/admission/customresourcevalidation/node/validate_node_config_test.go

@@ -7,6 +7,13 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	configv1 "github.com/openshift/api/config/v1"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
 )
 
 func TestValidateConfigNodeForExtremeLatencyProfile(t *testing.T) {
@@ -66,3 +73,214 @@ func TestValidateConfigNodeForExtremeLatencyProfile(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateConfigNodeForMinimumKubeletVersion(t *testing.T) {
+	testCases := []struct {
+		name         string
+		version      string
+		shouldReject bool
+		nodes        []v1.Node
+		nodeListErr  error
+		errType      field.ErrorType
+		errMsg       string
+	}{
+		// no rejections
+		{
+			name:         "should not reject when minimum kubelet version is empty",
+			version:      "",
+			shouldReject: false,
+		},
+		{
+			name:         "should reject when list nodes fails",
+			version:      "1.30.0",
+			shouldReject: true,
+			nodeListErr:  fmt.Errorf("Failed to list nodes"),
+			errType:      field.ErrorTypeForbidden,
+			errMsg:       "Getting nodes to compare minimum version",
+		},
+		{
+			name:         "should reject when min kubelet version bogus",
+			version:      "bogus",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.30.0",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "Failed to parse submitted version bogus No Major.Minor.Patch elements found",
+		},
+		{
+			name:         "should reject when kubelet version is bogus",
+			version:      "1.30.0",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "bogus",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "failed to parse node version bogus: No Major.Minor.Patch elements found",
+		},
+		{
+			name:         "should reject when kubelet version is too old",
+			version:      "1.30.0",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.29.0",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "kubelet version of node node is 1.29.0, which is lower than minimumKubeletVersion of 1.30.0",
+		},
+		{
+			name:         "should reject when one kubelet version is too old",
+			version:      "1.30.0",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.30.0",
+						},
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node2",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.29.0",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "kubelet version of node node2 is 1.29.0, which is lower than minimumKubeletVersion of 1.30.0",
+		},
+		{
+			name:         "should not reject when kubelet version is equal",
+			version:      "1.30.0",
+			shouldReject: false,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.30.0",
+						},
+					},
+				},
+			},
+		},
+		{
+			name:         "should reject when min version incomplete",
+			version:      "1.30",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.30.0",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "Failed to parse submitted version 1.30 No Major.Minor.Patch elements found",
+		},
+		{
+			name:         "should reject when kubelet version incomplete",
+			version:      "1.30.0",
+			shouldReject: true,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.30",
+						},
+					},
+				},
+			},
+			errType: field.ErrorTypeInvalid,
+			errMsg:  "failed to parse node version 1.30: No Major.Minor.Patch elements found",
+		},
+		{
+			name:         "should not reject when kubelet version is new enough",
+			version:      "1.30.0",
+			shouldReject: false,
+			nodes: []v1.Node{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "node",
+					},
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeletVersion: "1.31.0",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		c := fake.NewSimpleClientset()
+		c.PrependReactor("list", "nodes", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+			nodeList := &v1.NodeList{Items: testCase.nodes}
+			return true, nodeList, testCase.nodeListErr
+		})
+
+		shouldStr := "should not be"
+		if testCase.shouldReject {
+			shouldStr = "should be"
+		}
+		t.Run(testCase.name, func(t *testing.T) {
+			obj := configv1.Node{
+				Spec: configv1.NodeSpec{
+					MinimumKubeletVersion: testCase.version,
+				},
+			}
+
+			fieldErr := validateMinimumKubeletVersion(c.CoreV1(), &obj)
+			assert.Equal(t, testCase.shouldReject, fieldErr != nil, "minimum kubelet version %q %s rejected", testCase.version, shouldStr)
+
+			if testCase.shouldReject {
+				assert.Equal(t, "spec.minimumKubeletVersion", fieldErr.Field, "field name during for mininumKubeletVersion should be spec.mininumKubeletVersion")
+				assert.Equal(t, fieldErr.Type, testCase.errType, "error type should be %q", testCase.errType)
+				assert.Contains(t, fieldErr.Detail, testCase.errMsg, "error message should contain %q", testCase.errMsg)
+			}
+		})
+	}
+}