Implementation of Configurable Resource Transformers (#3026)

* Implementation of KEP-2937: Configurable Resource Transformers * Add field Workload.Status.ResourceRequests * linter fix * adjust new Status API based on review comments in the KEP * update KEP metadata; fix spelling of excludeResourcePrefixes * clear Status.DesiredResources when workload is admitted * Apply suggestions from code review Co-authored-by: Michał Woźniak <mimowo@users.noreply.github.com> * Address review comments that impact API * Rename functions per review suggestions * review comment: eliminate redundant condition in if * rewiew comment: enhance resource transformation integration test * address review comments * review comment: lowercasing field name in comments * fix bad rebase * review comment: add KEP PR discussion about config validation to KEP Graduation Criteria * regenerate apiref * add feature gate WorkloadResourceRequestsSummary * address review comments related to api conventions * feature table: add WorkloadResourceRequestsSummary and ConfigurableResourceTransformations * Apply suggestions from code review use propagated gomega instance Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * cleanup: use symbolic name for default strategy * integration tests to verify features can be disabled * Update test/integration/scheduler/workload_controller_test.go Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Update test/integration/scheduler/workload_controller_test.go Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Update apis/config/v1beta1/configuration_types.go Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * fix typo * use util.ExpectWorkloadsToHaveQuotaReservation * focused enablement of feature gate --------- Co-authored-by: Michał Woźniak <mimowo@users.noreply.github.com> Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com>
kubernetes-sigs · Oct 17, 2024 · 7dfe921 · 7dfe921
1 parent 399abac
commit 7dfe921
Show file tree

Hide file tree

Showing 28 changed files with 1,035 additions and 63 deletions.
diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go
@@ -19,6 +19,7 @@ package v1beta1
 import (
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
 )
@@ -372,6 +373,28 @@ type ClusterQueueVisibility struct {
 type Resources struct {
 	// ExcludedResourcePrefixes defines which resources should be ignored by Kueue
 	ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"`
+
+	// Transformations defines how to transform PodSpec resources into Workload resource requests.
+	// This is intended to be a map with Input as the key (enforced by validation code)
+	Transformations []ResourceTransformation `json:"transformations,omitempty"`
+}
+
+type ResourceTransformationStrategy string
+
+const Retain ResourceTransformationStrategy = "Retain"
+const Replace ResourceTransformationStrategy = "Replace"
+
+type ResourceTransformation struct {
+	// Input is the name of the input resource.
+	Input corev1.ResourceName `json:"input"`
+
+	// Strategy specifies if the input resource should be replaced or retained.
+	// Defaults to Retain
+	Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"`
+
+	// Outputs specifies the output resources and quantities per unit of input resource.
+	// An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.
+	Outputs corev1.ResourceList `json:"outputs,omitempty"`
 }
 
 type PreemptionStrategy string

diff --git a/apis/config/v1beta1/defaults.go b/apis/config/v1beta1/defaults.go
@@ -49,6 +49,7 @@ const (
 	DefaultMultiKueueWorkerLostTimeout                  = 15 * time.Minute
 	DefaultRequeuingBackoffBaseSeconds                  = 60
 	DefaultRequeuingBackoffMaxSeconds                   = 3600
+	DefaultResourceTransformationStrategy               = Retain
 )
 
 func getOperatorNamespace() string {
@@ -193,4 +194,12 @@ func SetDefaults_Configuration(cfg *Configuration) {
 	if fs := cfg.FairSharing; fs != nil && fs.Enable && len(fs.PreemptionStrategies) == 0 {
 		fs.PreemptionStrategies = []PreemptionStrategy{LessThanOrEqualToFinalShare, LessThanInitialShare}
 	}
+
+	if cfg.Resources != nil {
+		for idx := range cfg.Resources.Transformations {
+			if ptr.Deref(cfg.Resources.Transformations[idx].Strategy, "") == "" {
+				cfg.Resources.Transformations[idx].Strategy = ptr.To(DefaultResourceTransformationStrategy)
+			}
+		}
+	}
 }
diff --git a/apis/config/v1beta1/defaults_test.go b/apis/config/v1beta1/defaults_test.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	componentconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
 	"k8s.io/utils/ptr"
@@ -579,6 +580,36 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				},
 			},
 		},
+		"resources.transformations strategy": {
+			original: &Configuration{
+				InternalCertManagement: &InternalCertManagement{
+					Enable: ptr.To(false),
+				},
+				Resources: &Resources{
+					Transformations: []ResourceTransformation{
+						{Input: corev1.ResourceCPU},
+						{Input: corev1.ResourceMemory, Strategy: ptr.To(Replace)},
+					},
+				},
+			},
+			want: &Configuration{
+				Namespace:         ptr.To(DefaultNamespace),
+				ControllerManager: defaultCtrlManagerConfigurationSpec,
+				InternalCertManagement: &InternalCertManagement{
+					Enable: ptr.To(false),
+				},
+				ClientConnection: defaultClientConnection,
+				Integrations:     defaultIntegrations,
+				QueueVisibility:  defaultQueueVisibility,
+				MultiKueue:       defaultMultiKueue,
+				Resources: &Resources{
+					Transformations: []ResourceTransformation{
+						{Input: corev1.ResourceCPU, Strategy: ptr.To(DefaultResourceTransformationStrategy)},
+						{Input: corev1.ResourceMemory, Strategy: ptr.To(Replace)},
+					},
+				},
+			},
+		},
 	}
 
 	for name, tc := range testCases {

diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go
diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go
@@ -294,6 +294,17 @@ type WorkloadStatus struct {
 	// +patchMergeKey=name
 	// +kubebuilder:validation:MaxItems=8
 	AdmissionChecks []AdmissionCheckState `json:"admissionChecks,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+
+	// resourceRequests provides a detailed view of the resources that were
+	// requested by a non-admitted workload when it was considered for admission.
+	// If admission is non-null, resourceRequests will be empty because
+	// admission.resourceUsage contains the detailed information.
+	//
+	// +optional
+	// +listType=map
+	// +listMapKey=name
+	// +kubebuilder:validation:MaxItems=8
+	ResourceRequests []PodSetRequest `json:"resourceRequests,omitempty"`
 }
 
 type RequeueState struct {
@@ -382,6 +393,24 @@ type ReclaimablePod struct {
 	Count int32 `json:"count"`
 }
 
+type PodSetRequest struct {
+	// name is the name of the podSet. It should match one of the names in .spec.podSets.
+	// +kubebuilder:default=main
+	// +required
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern="^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
+	Name string `json:"name"`
+
+	// resources is the total resources all the pods in the podset need to run.
+	//
+	// Beside what is provided in podSet's specs, this value also takes into account
+	// the LimitRange defaults and RuntimeClass overheads at the moment of consideration
+	// and the application of resource.excludeResourcePrefixes and resource.transformations.
+	// +optional
+	Resources corev1.ResourceList `json:"resources,omitempty"`
+}
+
 const (
 	// WorkloadAdmitted means that the Workload has reserved quota and all the admissionChecks
 	// defined in the ClusterQueue are satisfied.

diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml
@@ -8647,6 +8647,43 @@ spec:
                     format: date-time
                     type: string
                 type: object
+              resourceRequests:
+                description: |-
+                  resourceRequests provides a detailed view of the resources that were
+                  requested by a non-admitted workload when it was considered for admission.
+                  If admission is non-null, resourceRequests will be empty because
+                  admission.resourceUsage contains the detailed information.
+                items:
+                  properties:
+                    name:
+                      default: main
+                      description: name is the name of the podSet. It should match
+                        one of the names in .spec.podSets.
+                      maxLength: 63
+                      pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                      type: string
+                    resources:
+                      additionalProperties:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      description: |-
+                        resources is the total resources all the pods in the podset need to run.
+
+                        Beside what is provided in podSet's specs, this value also takes into account
+                        the LimitRange defaults and RuntimeClass overheads at the moment of consideration
+                        and the application of resource.excludeResourcePrefixes and resource.transformations.
+                      type: object
+                  required:
+                  - name
+                  type: object
+                maxItems: 8
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
             type: object
         type: object
         x-kubernetes-validations:

diff --git a/client-go/applyconfiguration/kueue/v1beta1/podsetrequest.go b/client-go/applyconfiguration/kueue/v1beta1/podsetrequest.go
diff --git a/client-go/applyconfiguration/kueue/v1beta1/workloadstatus.go b/client-go/applyconfiguration/kueue/v1beta1/workloadstatus.go
diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go
diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go
@@ -159,6 +159,10 @@ func main() {
 		cacheOptions = append(cacheOptions, cache.WithExcludedResourcePrefixes(cfg.Resources.ExcludeResourcePrefixes))
 		queueOptions = append(queueOptions, queue.WithExcludedResourcePrefixes(cfg.Resources.ExcludeResourcePrefixes))
 	}
+	if features.Enabled(features.ConfigurableResourceTransformations) && cfg.Resources != nil && len(cfg.Resources.Transformations) > 0 {
+		cacheOptions = append(cacheOptions, cache.WithResourceTransformations(cfg.Resources.Transformations))
+		queueOptions = append(queueOptions, queue.WithResourceTransformations(cfg.Resources.Transformations))
+	}
 	if cfg.FairSharing != nil {
 		cacheOptions = append(cacheOptions, cache.WithFairSharing(cfg.FairSharing.Enable))
 	}