From fa96518c293473480b0d1f45f642af23e433302e Mon Sep 17 00:00:00 2001 From: Aldo Culquicondor <1299064+alculquicondor@users.noreply.github.com> Date: Tue, 20 Feb 2024 11:03:26 -0500 Subject: [PATCH] Exclude labels, image, preemptionPolicy from pod role hash calculation (#1758) Change-Id: I19ab4c259818b0ba0d25dd3346025c343a757ccc --- pkg/controller/jobs/pod/pod_controller.go | 4 - .../jobs/pod/pod_controller_test.go | 148 +++++++++--------- pkg/controller/jobs/pod/pod_webhook.go | 13 -- pkg/controller/jobs/pod/pod_webhook_test.go | 4 +- .../jobs/pod/pod_controller_test.go | 90 +++++++---- 5 files changed, 139 insertions(+), 120 deletions(-) diff --git a/pkg/controller/jobs/pod/pod_controller.go b/pkg/controller/jobs/pod/pod_controller.go index 9cc4a0be44..fafaaee379 100644 --- a/pkg/controller/jobs/pod/pod_controller.go +++ b/pkg/controller/jobs/pod/pod_controller.go @@ -554,9 +554,6 @@ func getRoleHash(p corev1.Pod) (string, error) { } shape := map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": omitKueueLabels(p.ObjectMeta.Labels), - }, "spec": map[string]interface{}{ "initContainers": containersShape(p.Spec.InitContainers), "containers": containersShape(p.Spec.Containers), @@ -565,7 +562,6 @@ func getRoleHash(p corev1.Pod) (string, error) { "tolerations": p.Spec.Tolerations, "runtimeClassName": p.Spec.RuntimeClassName, "priority": p.Spec.Priority, - "preemptionPolicy": p.Spec.PreemptionPolicy, "topologySpreadConstraints": p.Spec.TopologySpreadConstraints, "overhead": p.Spec.Overhead, "resourceClaims": p.Spec.ResourceClaims, diff --git a/pkg/controller/jobs/pod/pod_controller_test.go b/pkg/controller/jobs/pod/pod_controller_test.go index 3e6a67706b..ec43209ccf 100644 --- a/pkg/controller/jobs/pod/pod_controller_test.go +++ b/pkg/controller/jobs/pod/pod_controller_test.go @@ -532,7 +532,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Obj(), @@ -596,7 +596,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Obj(), @@ -610,7 +610,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Obj(), @@ -635,7 +635,6 @@ func TestReconciler(t *testing.T) { KueueSchedulingGate(). Group("test-group"). GroupTotalCount("2"). - Annotation("kueue.x-k8s.io/role-hash", "60bc72d3"). Obj(), *basePodWrapper. Clone(). @@ -645,7 +644,6 @@ func TestReconciler(t *testing.T) { KueueSchedulingGate(). Group("test-group"). GroupTotalCount("2"). - Annotation("kueue.x-k8s.io/role-hash", "60bc72d3"). Obj(), }, wantPods: []corev1.Pod{ @@ -655,7 +653,6 @@ func TestReconciler(t *testing.T) { KueueFinalizer(). Group("test-group"). GroupTotalCount("2"). - Annotation("kueue.x-k8s.io/role-hash", "60bc72d3"). NodeSelector("kubernetes.io/arch", "arm64"). Obj(), *basePodWrapper. @@ -665,17 +662,16 @@ func TestReconciler(t *testing.T) { KueueFinalizer(). Group("test-group"). GroupTotalCount("2"). - Annotation("kueue.x-k8s.io/role-hash", "60bc72d3"). NodeSelector("kubernetes.io/arch", "arm64"). Obj(), }, workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). ReserveQuota( - utiltesting.MakeAdmission("cq", "60bc72d3"). + utiltesting.MakeAdmission("cq", "dc85db45"). Assignment(corev1.ResourceCPU, "unit-test-flavor", "2"). AssignmentPodCount(2). Obj(), @@ -685,11 +681,11 @@ func TestReconciler(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). ReserveQuota( - utiltesting.MakeAdmission("cq", "60bc72d3"). + utiltesting.MakeAdmission("cq", "dc85db45"). Assignment(corev1.ResourceCPU, "unit-test-flavor", "2"). AssignmentPodCount(2). Obj(), @@ -754,7 +750,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -768,7 +764,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -777,7 +773,7 @@ func TestReconciler(t *testing.T) { OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). Admitted(true). - ReclaimablePods(kueue.ReclaimablePod{Name: "60bc72d3", Count: 1}). + ReclaimablePods(kueue.ReclaimablePod{Name: "dc85db45", Count: 1}). Obj(), }, workloadCmpOpts: defaultWorkloadCmpOpts, @@ -822,7 +818,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -836,7 +832,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -881,7 +877,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -894,7 +890,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -951,7 +947,7 @@ func TestReconciler(t *testing.T) { }, workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). Queue("test-queue"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). @@ -966,7 +962,7 @@ func TestReconciler(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). Queue("test-queue"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). @@ -1004,7 +1000,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1023,7 +1019,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1121,7 +1117,7 @@ func TestReconciler(t *testing.T) { wantPods: []corev1.Pod{}, workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). ControllerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). ControllerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). Queue("test-queue"). @@ -1192,28 +1188,28 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). Queue("user-queue"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). - ReserveQuota(utiltesting.MakeAdmission("cq", "60bc72d3").AssignmentPodCount(1).Obj()). + ReserveQuota(utiltesting.MakeAdmission("cq", "dc85db45").AssignmentPodCount(1).Obj()). Admitted(true). Obj(), }, wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). Queue("user-queue"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). - ReserveQuota(utiltesting.MakeAdmission("cq", "60bc72d3").AssignmentPodCount(1).Obj()). + ReserveQuota(utiltesting.MakeAdmission("cq", "dc85db45").AssignmentPodCount(1).Obj()). Admitted(true). Obj(), }, @@ -1268,17 +1264,17 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). Queue("user-queue"). - ReserveQuota(utiltesting.MakeAdmission("cq", "60bc72d3").AssignmentPodCount(3).Obj()). + ReserveQuota(utiltesting.MakeAdmission("cq", "dc85db45").AssignmentPodCount(3).Obj()). Admitted(true). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod3", "test-uid"). - ReclaimablePods(kueue.ReclaimablePod{Name: "60bc72d3", Count: 1}). + ReclaimablePods(kueue.ReclaimablePod{Name: "dc85db45", Count: 1}). Obj(), }, wantPods: []corev1.Pod{ @@ -1320,14 +1316,14 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). Queue("user-queue"). - ReserveQuota(utiltesting.MakeAdmission("cq", "60bc72d3").AssignmentPodCount(3).Obj()). + ReserveQuota(utiltesting.MakeAdmission("cq", "dc85db45").AssignmentPodCount(3).Obj()). ReclaimablePods(kueue.ReclaimablePod{ - Name: "60bc72d3", + Name: "dc85db45", Count: 1, }). Admitted(true). @@ -1335,7 +1331,7 @@ func TestReconciler(t *testing.T) { OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod3", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "replacement", "test-uid"). - ReclaimablePods(kueue.ReclaimablePod{Name: "60bc72d3", Count: 1}). + ReclaimablePods(kueue.ReclaimablePod{Name: "dc85db45", Count: 1}). Obj(), }, workloadCmpOpts: defaultWorkloadCmpOpts, @@ -1411,7 +1407,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1426,7 +1422,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1480,7 +1476,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1500,7 +1496,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1545,7 +1541,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1565,7 +1561,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1655,7 +1651,7 @@ func TestReconciler(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - PodSets(*utiltesting.MakePodSet("60bc72d3", 2).Request(corev1.ResourceCPU, "1").Obj()). + PodSets(*utiltesting.MakePodSet("dc85db45", 2).Request(corev1.ResourceCPU, "1").Obj()). Queue("test-queue"). Priority(0). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). @@ -1716,7 +1712,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1730,7 +1726,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1770,7 +1766,7 @@ func TestReconciler(t *testing.T) { *utiltesting.MakePodSet("absent-pod-role", 1). Request(corev1.ResourceCPU, "1"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1786,7 +1782,7 @@ func TestReconciler(t *testing.T) { *utiltesting.MakePodSet("absent-pod-role", 1). Request(corev1.ResourceCPU, "1"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -1882,7 +1878,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Request(corev1.ResourceCPU, "1"). Obj(), @@ -1980,7 +1976,7 @@ func TestReconciler(t *testing.T) { Label("kueue.x-k8s.io/managed", "true"). KueueFinalizer(). Group("test-group"). - Image("test-image-role2", nil). + Request(corev1.ResourceMemory, "1Gi"). GroupTotalCount("3"). StatusPhase(corev1.PodFailed). Obj(), @@ -2007,7 +2003,7 @@ func TestReconciler(t *testing.T) { Name("pod3"). Label("kueue.x-k8s.io/managed", "true"). Group("test-group"). - Image("test-image-role2", nil). + Request(corev1.ResourceMemory, "1Gi"). GroupTotalCount("3"). StatusPhase(corev1.PodFailed). Obj(), @@ -2015,10 +2011,11 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("4389b941", 1). + *utiltesting.MakePodSet("a119f908", 1). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1Gi"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2033,10 +2030,11 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("4389b941", 1). + *utiltesting.MakePodSet("a119f908", 1). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1Gi"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2091,7 +2089,7 @@ func TestReconciler(t *testing.T) { Label("kueue.x-k8s.io/managed", "true"). KueueFinalizer(). Group("test-group"). - Image("test-image-role2", nil). + Request(corev1.ResourceMemory, "1Gi"). GroupTotalCount("3"). StatusPhase(corev1.PodSucceeded). Obj(), @@ -2120,7 +2118,7 @@ func TestReconciler(t *testing.T) { Label("kueue.x-k8s.io/managed", "true"). KueueFinalizer(). Group("test-group"). - Image("test-image-role2", nil). + Request(corev1.ResourceMemory, "1Gi"). GroupTotalCount("3"). StatusPhase(corev1.PodSucceeded). Obj(), @@ -2128,10 +2126,11 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("007b32e9", 1). + *utiltesting.MakePodSet("4ebdd4a6", 1). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1Gi"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2146,10 +2145,11 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("007b32e9", 1). + *utiltesting.MakePodSet("4ebdd4a6", 1). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1Gi"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2159,7 +2159,7 @@ func TestReconciler(t *testing.T) { OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod2", "test-uid"). OwnerReference(corev1.SchemeGroupVersion.WithKind("Pod"), "pod3", "test-uid"). - ReclaimablePods(kueue.ReclaimablePod{Name: "007b32e9", Count: 1}). + ReclaimablePods(kueue.ReclaimablePod{Name: "4ebdd4a6", Count: 1}). Obj(), }, workloadCmpOpts: defaultWorkloadCmpOpts, @@ -2201,7 +2201,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Obj(), @@ -2284,7 +2284,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2297,7 +2297,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2356,7 +2356,7 @@ func TestReconciler(t *testing.T) { *utiltesting.MakePodSet("aaf269e6", 1). Request(corev1.ResourceCPU, "1"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2372,7 +2372,7 @@ func TestReconciler(t *testing.T) { *utiltesting.MakePodSet("aaf269e6", 1). Request(corev1.ResourceCPU, "1"). Obj(), - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2462,7 +2462,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2474,7 +2474,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2543,7 +2543,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2558,7 +2558,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2631,7 +2631,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns").Finalizers(kueue.ResourceInUseFinalizerName). PodSets( - *utiltesting.MakePodSet("60bc72d3", 1). + *utiltesting.MakePodSet("dc85db45", 1). Request(corev1.ResourceCPU, "1"). SchedulingGates(corev1.PodSchedulingGate{Name: "kueue.x-k8s.io/admission"}). Obj(), @@ -2760,7 +2760,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2774,7 +2774,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2866,7 +2866,7 @@ func TestReconciler(t *testing.T) { workloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -2886,7 +2886,7 @@ func TestReconciler(t *testing.T) { wantWorkloads: []kueue.Workload{ *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 3). + *utiltesting.MakePodSet("dc85db45", 3). Request(corev1.ResourceCPU, "1"). Obj(), ). @@ -3222,7 +3222,7 @@ func TestReconciler_DeletePodAfterTransientErrorsOnUpdateOrDeleteOps(t *testing. wl := *utiltesting.MakeWorkload("test-group", "ns"). PodSets( - *utiltesting.MakePodSet("60bc72d3", 2). + *utiltesting.MakePodSet("dc85db45", 2). Request(corev1.ResourceCPU, "1"). Obj(), ). diff --git a/pkg/controller/jobs/pod/pod_webhook.go b/pkg/controller/jobs/pod/pod_webhook.go index 3a090680f3..b81ae06563 100644 --- a/pkg/controller/jobs/pod/pod_webhook.go +++ b/pkg/controller/jobs/pod/pod_webhook.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/validation" @@ -105,21 +104,9 @@ func getPodOptions(integrationOpts map[string]any) (configapi.PodIntegrationOpti var _ webhook.CustomDefaulter = &PodWebhook{} -func omitKueueLabels(l map[string]string) map[string]string { - result := map[string]string{} - - for key, value := range l { - if !strings.HasPrefix(key, "kueue.x-k8s.io/") { - result[key] = value - } - } - return result -} - func containersShape(containers []corev1.Container) (result []map[string]interface{}) { for _, c := range containers { result = append(result, map[string]interface{}{ - "image": c.Image, "resources": map[string]interface{}{ "requests": c.Resources.Requests, }, diff --git a/pkg/controller/jobs/pod/pod_webhook_test.go b/pkg/controller/jobs/pod/pod_webhook_test.go index 2bd4f6759d..f2f8d696ba 100644 --- a/pkg/controller/jobs/pod/pod_webhook_test.go +++ b/pkg/controller/jobs/pod/pod_webhook_test.go @@ -233,7 +233,7 @@ func TestDefault(t *testing.T) { want: testingpod.MakePod("test-pod", defaultNamespace.Name). Queue("test-queue"). Group("test-group"). - RoleHash("3b4907a8"). + RoleHash("a9f06f3a"). Label("kueue.x-k8s.io/managed", "true"). KueueSchedulingGate(). KueueFinalizer(). @@ -250,7 +250,7 @@ func TestDefault(t *testing.T) { want: testingpod.MakePod("test-pod", defaultNamespace.Name). Queue("test-queue"). Group("test-group"). - RoleHash("3b4907a8"). + RoleHash("a9f06f3a"). Label("kueue.x-k8s.io/managed", "true"). KueueSchedulingGate(). KueueFinalizer(). diff --git a/test/integration/controller/jobs/pod/pod_controller_test.go b/test/integration/controller/jobs/pod/pod_controller_test.go index 8a0f42a469..5e5333113f 100644 --- a/test/integration/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/controller/jobs/pod/pod_controller_test.go @@ -29,6 +29,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -503,7 +504,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu pod2 := testingpod.MakePod("test-pod2", ns.Name). Group("test-group"). GroupTotalCount("2"). - Image("test-image", nil). + Request(corev1.ResourceCPU, "1"). Queue("test-queue"). Obj() pod1LookupKey := client.ObjectKeyFromObject(pod1) @@ -529,15 +530,27 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdWorkload.Spec.QueueName).To(gomega.Equal("test-queue"), "The Workload should have .spec.queueName set") ginkgo.By("checking that all pods in group are unsuspended when workload is admitted", func() { - admission := testing.MakeAdmission(clusterQueue.Name, "7c214403", "41b86f81"). - Assignment(corev1.ResourceCPU, "default", "1"). - AssignmentPodCount(2). - Obj() + admission := testing.MakeAdmission(clusterQueue.Name).PodSets( + kueue.PodSetAssignment{ + Name: "4b0469f7", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + kueue.PodSetAssignment{ + Name: "bf90803c", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + ).Obj() gomega.Expect(util.SetQuotaReservation(ctx, k8sClient, createdWorkload, admission)).Should(gomega.Succeed()) util.SyncAdmittedConditionForWorkloads(ctx, k8sClient, createdWorkload) util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod1LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) - util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, nil) + util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) gomega.Expect(createdWorkload.Status.Conditions).Should(gomega.BeComparableTo( @@ -602,7 +615,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdWorkload.Spec.QueueName).To(gomega.Equal("test-queue"), "The Workload should have .spec.queueName set") originalWorkloadUID := createdWorkload.UID - admission := testing.MakeAdmission(clusterQueue.Name, "7c214403"). + admission := testing.MakeAdmission(clusterQueue.Name, "bf90803c"). Assignment(corev1.ResourceCPU, "default", "1"). AssignmentPodCount(createdWorkload.Spec.PodSets[0].Count). Obj() @@ -724,7 +737,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdWorkload.Spec.QueueName).To(gomega.Equal("test-queue"), "The Workload should have .spec.queueName set") ginkgo.By("checking that pod is unsuspended when workload is admitted") - admission := testing.MakeAdmission(clusterQueue.Name, "7c214403"). + admission := testing.MakeAdmission(clusterQueue.Name, "bf90803c"). Assignment(corev1.ResourceCPU, "default", "1"). AssignmentPodCount(createdWorkload.Spec.PodSets[0].Count). Obj() @@ -787,7 +800,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu pod2 := testingpod.MakePod("test-pod2", ns.Name). Group("test-group"). GroupTotalCount("2"). - Image("test-image", nil). + Request(corev1.ResourceCPU, "1"). Queue("test-queue"). Obj() pod1LookupKey := client.ObjectKeyFromObject(pod1) @@ -813,15 +826,27 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu createdPod := &corev1.Pod{} ginkgo.By("checking that all pods in group are unsuspended when workload is admitted", func() { - admission := testing.MakeAdmission(clusterQueue.Name, "7c214403", "41b86f81"). - Assignment(corev1.ResourceCPU, "default", "1"). - AssignmentPodCount(2). - Obj() + admission := testing.MakeAdmission(clusterQueue.Name).PodSets( + kueue.PodSetAssignment{ + Name: "4b0469f7", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + kueue.PodSetAssignment{ + Name: "bf90803c", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + ).Obj() gomega.Expect(util.SetQuotaReservation(ctx, k8sClient, createdWorkload, admission)).Should(gomega.Succeed()) util.SyncAdmittedConditionForWorkloads(ctx, k8sClient, createdWorkload) util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod1LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) - util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, nil) + util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) gomega.Expect(createdWorkload.Status.Conditions).Should(gomega.BeComparableTo( @@ -861,7 +886,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu replacementPod2LookupKey := client.ObjectKeyFromObject(replacementPod2) ginkgo.By("checking that unretriable replacement pod is allowed to run", func() { - util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, replacementPod2LookupKey, nil) + util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, replacementPod2LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) }) ginkgo.By("checking that pod group is finalized when unretriable pod has failed", func() { @@ -894,13 +919,13 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu pod2 := testingpod.MakePod("test-pod2", ns.Name). Group("test-group"). GroupTotalCount("2"). - Image("test-image", nil). + Request(corev1.ResourceCPU, "1"). Queue("test-queue"). Obj() excessBasePod := testingpod.MakePod("excess-pod", ns.Name). Group("test-group"). GroupTotalCount("2"). - Image("test-image", nil). + Request(corev1.ResourceCPU, "1"). Queue("test-queue") pod1LookupKey := client.ObjectKeyFromObject(pod1) @@ -940,15 +965,27 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }) ginkgo.By("checking that all pods in group are unsuspended when workload is admitted", func() { - admission := testing.MakeAdmission(clusterQueue.Name, "7c214403", "41b86f81"). - Assignment(corev1.ResourceCPU, "default", "1"). - AssignmentPodCount(2). - Obj() + admission := testing.MakeAdmission(clusterQueue.Name).PodSets( + kueue.PodSetAssignment{ + Name: "4b0469f7", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + kueue.PodSetAssignment{ + Name: "bf90803c", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "default", + }, + Count: ptr.To[int32](1), + }, + ).Obj() gomega.Expect(util.SetQuotaReservation(ctx, k8sClient, createdWorkload, admission)).Should(gomega.Succeed()) util.SyncAdmittedConditionForWorkloads(ctx, k8sClient, createdWorkload) util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod1LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) - util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, nil) + util.ExpectPodUnsuspendedWithNodeSelectors(ctx, k8sClient, pod2LookupKey, map[string]string{"kubernetes.io/arch": "arm64"}) gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) gomega.Expect(createdWorkload.Status.Conditions).Should(gomega.BeComparableTo( @@ -975,12 +1012,13 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu pod1 := testingpod.MakePod("test-pod1", ns.Name). Group("test-group"). GroupTotalCount("2"). + Request(corev1.ResourceCPU, "1"). Queue("test-queue"). Obj() pod2 := testingpod.MakePod("test-pod2", ns.Name). Group("test-group"). GroupTotalCount("2"). - Image("test-image", nil). + Request(corev1.ResourceCPU, "2"). Queue("test-queue"). Obj() @@ -1128,22 +1166,20 @@ var _ = ginkgo.Describe("Pod controller interacting with scheduler", ginkgo.Orde role1Pod1 := basePod. Clone(). Name("role1-pod1"). - Image("role1-image", nil). Obj() role1Pod2 := basePod. Clone(). Name("role1-pod2"). - Image("role1-image", nil). Obj() role2Pod1 := basePod. Clone(). Name("role2-pod1"). - Image("role2-image", nil). + Request(corev1.ResourceCPU, "1.5"). Obj() role2Pod2 := basePod. Clone(). Name("role2-pod2"). - Image("role2-image", nil). + Request(corev1.ResourceCPU, "1.5"). Obj() ginkgo.By("creating the pods", func() {