Clean up WorkersToDelete field during the CI test (ray-project#1763)

ray-operator/controllers/ray/raycluster_controller_test.go

@@ -244,16 +244,8 @@ var _ = Context("Inside the default namespace", func() {
 			Eventually(
 				listResourceFunc(ctx, &workerPods, workerFilterLabels, &client.ListOptions{Namespace: "default"}),
 				time.Second*15, time.Millisecond*500).Should(Equal(2), fmt.Sprintf("workerGroup %v", workerPods.Items))
-			// Updating WorkersToDelete is the responsibility of the Ray Autoscaler. Here, we simulate the Ray
-			// Autoscaler's behavior after the scale-down process is completed.
-			err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-				Eventually(
-					getResourceFunc(ctx, client.ObjectKey{Name: myRayCluster.Name, Namespace: "default"}, myRayCluster),
-					time.Second*9, time.Millisecond*500).Should(BeNil(), "My raycluster = %v", myRayCluster)
-				myRayCluster.Spec.WorkerGroupSpecs[0].ScaleStrategy.WorkersToDelete = []string{}
-				return k8sClient.Update(ctx, myRayCluster)
-			})
-			Expect(err).NotTo(HaveOccurred(), "failed to update test RayCluster resource")
+
+			cleanUpWorkersToDelete(ctx, myRayCluster, 0)
 		})
 
 		It("should increase replicas past maxReplicas", func() {
@@ -450,3 +442,16 @@ func isAllPodsRunning(ctx context.Context, podlist corev1.PodList, filterLabels
 	}
 	return true
 }
+
+func cleanUpWorkersToDelete(ctx context.Context, rayCluster *rayv1.RayCluster, workerGroupIndex int) {
+	// Updating WorkersToDelete is the responsibility of the Ray Autoscaler. In this function,
+	// we simulate the behavior of the Ray Autoscaler after the scaling process has finished.
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		Eventually(
+			getResourceFunc(ctx, client.ObjectKey{Name: rayCluster.Name, Namespace: "default"}, rayCluster),
+			time.Second*9, time.Millisecond*500).Should(BeNil(), "raycluster = %v", rayCluster)
+		rayCluster.Spec.WorkerGroupSpecs[workerGroupIndex].ScaleStrategy.WorkersToDelete = []string{}
+		return k8sClient.Update(ctx, rayCluster)
+	})
+	Expect(err).NotTo(HaveOccurred(), "failed to clean up WorkersToDelete")
+}

ray-operator/controllers/ray/rayservice_controller_test.go

@@ -431,6 +431,8 @@ applications:
 			Eventually(
 				getResourceFunc(ctx, client.ObjectKey{Name: myRayService.Status.ActiveServiceStatus.RayClusterName, Namespace: "default"}, myRayCluster),
 				time.Second*3, time.Millisecond*500).Should(BeNil(), "My myRayCluster  = %v", myRayCluster.Name)
+
+			cleanUpWorkersToDelete(ctx, myRayCluster, 0)
 		})
 
 		It("Autoscaler updates the pending RayCluster and should not switch to a new RayCluster", func() {
@@ -482,6 +484,8 @@ applications:
 			Eventually(
 				getRayClusterNameFunc(ctx, myRayService),
 				time.Second*15, time.Millisecond*500).Should(Equal(initialPendingClusterName), "New active RayCluster name = %v", myRayService.Status.ActiveServiceStatus.RayClusterName)
+
+			cleanUpWorkersToDelete(ctx, myRayCluster, 0)
 		})
 
 		It("Status should be updated if the differences are not only LastUpdateTime and HealthLastUpdateTime fields.", func() {