Add support for allow-dynamic-schema-drop annotation

pkg/resources/provisioning.cattle.io/v1/cluster/mutator.go

@@ -42,11 +42,12 @@ const (
 	// MountPath is where the admission control configuration file will be mounted in the control plane nodes
 	mountPath = "/etc/rancher/%s/config/rancher-psact.yaml"
 
-	controlPlaneRoleLabel = "rke.cattle.io/control-plane-role"
-	secretAnnotation      = "rke.cattle.io/object-authorized-for-clusters"
-	runtimeK3S            = "k3s"
-	runtimeRKE2           = "rke2"
-	runtimeRKE            = "rke"
+	controlPlaneRoleLabel            = "rke.cattle.io/control-plane-role"
+	secretAnnotation                 = "rke.cattle.io/object-authorized-for-clusters"
+	allowDynamicSchemaDropAnnotation = "provisioning.cattle.io/allow-dynamic-schema-drop"
+	runtimeK3S                       = "k3s"
+	runtimeRKE2                      = "rke2"
+	runtimeRKE                       = "rke"
 )
 
 var (
@@ -133,13 +134,66 @@ func (m *ProvisioningClusterMutator) Admit(request *admission.Request) (*admissi
 		return response, nil
 	}
 
+	response, err = m.handleDynamicSchemaDrop(request, cluster)
+	if err != nil {
+		return nil, err
+	}
+	if response.Result != nil {
+		return response, nil
+	}
+
 	response.Allowed = true
 	if err = patch.CreatePatch(clusterJSON, cluster, response); err != nil {
 		return nil, fmt.Errorf("failed to create patch: %w", err)
 	}
 	return response, nil
 }
 
+// handleDynamicSchemaDrop watches for provisioning cluster updates, and reinserts the previous value of the
+// dynamicSchemaSpec field for a machine pool if the "provisioning.cattle.io/allow-dynamic-schema-drop" annotation is
+// not present and true on the cluster. If the value of the annotation is true, no mutation is performed.
+func (m *ProvisioningClusterMutator) handleDynamicSchemaDrop(request *admission.Request, cluster *v1.Cluster) (*admissionv1.AdmissionResponse, error) {
+	if cluster.Name == "local" || cluster.Spec.RKEConfig == nil {
+		return admission.ResponseAllowed(), nil
+	}
+
+	if request.Operation != admissionv1.Update {
+		return admission.ResponseAllowed(), nil
+	}
+
+	oldCluster, newCluster, err := objectsv1.ClusterOldAndNewFromRequest(&request.AdmissionRequest)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract old and new cluster objects: %w", err)
+	}
+
+	if newCluster.Annotations[allowDynamicSchemaDropAnnotation] == "true" {
+		return admission.ResponseAllowed(), nil
+	}
+
+	oldClusterPools := map[string]*v1.RKEMachinePool{}
+	for _, mp := range oldCluster.Spec.RKEConfig.MachinePools {
+		oldClusterPools[mp.Name] = &mp
+	}
+
+	newClusterPools := map[string]*v1.RKEMachinePool{}
+	for _, mp := range newCluster.Spec.RKEConfig.MachinePools {
+		newClusterPools[mp.Name] = &mp
+	}
+
+	for name, newPool := range newClusterPools {
+		oldPool, ok := oldClusterPools[name]
+		if !ok {
+			logrus.Debugf("new machine pool, skipping validation of dynamic schema spec")
+			continue
+		}
+		if oldPool.DynamicSchemaSpec != "" && newPool.DynamicSchemaSpec == "" {
+			logrus.Infof("cluster %s machine pool %s dynamic schema spec mutated without supplying annotation %s, reverting", cluster.Name, name, "provisioning.cattle.io/allow-dynamic-schema-drop")
+			newPool.DynamicSchemaSpec = oldPool.DynamicSchemaSpec
+		}
+	}
+	return admission.ResponseAllowed(), nil
+}
+
 // handlePSACT updates the cluster and an underlying secret to support PSACT.
 // If a PSACT is set in the cluster, handlePSACT generates an admission configuration file, mounts the file into a secret,
 // updates the cluster's spec to mount the secret to the control plane nodes, and configures kube-apisever to use the admission configuration file;

pkg/resources/provisioning.cattle.io/v1/cluster/mutator_test.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"encoding/json"
+	"reflect"
 	"testing"
 
 	v1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
@@ -617,3 +618,109 @@ func TestAdmitPreserveUnknownFields(t *testing.T) {
 	assert.Nil(t, response.Patch)
 
 }
+
+func objToRaw[T any](obj *T) ([]byte, error) {
+	data, err := data2.Convert(obj)
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	return raw, nil
+}
+
+func TestDynamicSchemaDrop(t *testing.T) {
+	cluster := &v1.Cluster{
+		Spec: v1.ClusterSpec{
+			RKEConfig: &v1.RKEConfig{
+				MachinePools: []v1.RKEMachinePool{
+					{
+						Name: "A",
+					},
+				},
+			},
+		},
+	}
+	raw, err := json.Marshal(cluster)
+	assert.Nil(t, err)
+
+	m := ProvisioningClusterMutator{}
+
+	request := &admission.Request{
+		AdmissionRequest: admissionv1.AdmissionRequest{
+			Object: runtime.RawExtension{
+				Raw: raw,
+			},
+			OldObject: runtime.RawExtension{
+				Raw: raw,
+			},
+		},
+	}
+
+	// Always allowed on Create
+	request.Operation = admissionv1.Create
+	output := cluster.DeepCopy()
+	response, err := m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	assert.True(t, reflect.DeepEqual(cluster, output))
+
+	// Allowed on update if no schema present
+	request.Operation = admissionv1.Update
+	output = cluster.DeepCopy()
+	response, err = m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	assert.True(t, reflect.DeepEqual(cluster, output))
+
+	// Allowed on update if schemas match
+	cluster.Spec.RKEConfig.MachinePools[0].DynamicSchemaSpec = "a"
+	raw, err = json.Marshal(cluster)
+	assert.Nil(t, err)
+	request.AdmissionRequest.OldObject.Raw = raw
+	request.AdmissionRequest.Object.Raw = raw
+	output = cluster.DeepCopy()
+	response, err = m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	assert.True(t, reflect.DeepEqual(cluster, output))
+
+	// Allowed on update if new pool is added
+	cluster.Spec.RKEConfig.MachinePools = append(cluster.Spec.RKEConfig.MachinePools, v1.RKEMachinePool{Name: "B"})
+	raw, err = json.Marshal(cluster)
+	assert.Nil(t, err)
+	request.AdmissionRequest.Object.Raw = raw
+	output = cluster.DeepCopy()
+	response, err = m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	assert.True(t, reflect.DeepEqual(cluster, output))
+
+	// Rejected on update if schema is removed
+	cluster.Spec.RKEConfig.MachinePools[0].DynamicSchemaSpec = "b"
+	raw, err = json.Marshal(cluster)
+	assert.Nil(t, err)
+	request.AdmissionRequest.Object.Raw = raw
+	output = cluster.DeepCopy()
+	response, err = m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	expected := cluster.DeepCopy()
+	expected.Spec.RKEConfig.MachinePools[0].DynamicSchemaSpec = "a"
+	assert.True(t, reflect.DeepEqual(cluster, output))
+
+	// Allowed on update if annotation is present and true
+	cluster.Annotations = map[string]string{allowDynamicSchemaDropAnnotation: "true"}
+	cluster.Spec.RKEConfig.MachinePools[0].DynamicSchemaSpec = "b"
+	raw, err = json.Marshal(cluster)
+	assert.Nil(t, err)
+	request.AdmissionRequest.Object.Raw = raw
+	output = cluster.DeepCopy()
+	response, err = m.handleDynamicSchemaDrop(request, output)
+	assert.Nil(t, err)
+	assert.True(t, response.Allowed)
+	assert.True(t, reflect.DeepEqual(cluster, output))
+}