chore(charts): remove useless taskSplitSize argument in charts (#1874)

remove useless taskSplitSize argument in charts
star-whale · Feb 22, 2023 · b2e0af5 · b2e0af5
1 parent 6db8dcb
commit b2e0af5
Show file tree

Hide file tree

Showing 11 changed files with 10 additions and 27 deletions.
diff --git a/docker/charts/README.md b/docker/charts/README.md
@@ -81,7 +81,6 @@ The `update` command will update the information of available charts locally fro
 
 | Name | Description | Default Value |
 |------|-------------|---------------|
-| `controller.taskSplitSize`| task split size| `2`|
 | `controller.auth.username`| login username for console web| `starwhale`|
 | `controller.auth.password`| login password for console web| `abcd1234`|
 | `controller.ingress.enabled`| enable ingress for Starwhale controller | `true` |

diff --git a/docker/charts/templates/controller-deployment.yaml b/docker/charts/templates/controller-deployment.yaml
@@ -81,8 +81,6 @@ spec:
               value: "{{ .Values.image.registry }}/{{ .Values.image.org }}/{{ .Values.image.base.repo }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
             - name: SW_DOCKER_REGISTRY_URL
               value: "{{ .Values.image.registry }}"
-            - name: SW_TASK_SPLIT_SIZE
-              value: "{{ .Values.controller.taskSplitSize }}"
             - name: SW_HOST_IP
               valueFrom:
                 fieldRef:

diff --git a/docker/charts/values.minikube.cn.yaml b/docker/charts/values.minikube.cn.yaml
@@ -16,7 +16,6 @@ minio:
     enabled: false
 
 controller:
-  taskSplitSize: 1
   containerPort: 8082
   ingress:
     enabled: false

diff --git a/docker/charts/values.minikube.global.yaml b/docker/charts/values.minikube.global.yaml
@@ -15,7 +15,6 @@ minio:
     enabled: false
 
 controller:
-  taskSplitSize: 1
   containerPort: 8082
   ingress:
     enabled: false

diff --git a/docker/charts/values.yaml b/docker/charts/values.yaml
@@ -69,7 +69,6 @@ externalOSS:
   type: "s3" # supported type: s3, minio and aliyun
 
 controller:
-  taskSplitSize: 2
   jwt:
     tokenExpireMinutes: 43200
   upload:

diff --git a/docker/compose/compose.yaml b/docker/compose/compose.yaml
@@ -31,7 +31,6 @@ services:
     ports:
       - 8082:8082
     environment:
-      - SW_TASK_SPLIT_SIZE=2
       - JAR=controller
       - SW_HOST_IP=127.0.0.1
       - SW_JWT_TOKEN_EXPIRE_MINUTES=144000

diff --git a/docs/docs/guides/install/helm-charts.md b/docs/docs/guides/install/helm-charts.md
@@ -85,7 +85,6 @@ The `update` command will update the information of available charts locally fro
 
 | Name | Description | Default Value |
 |------|-------------|---------------|
-| `controller.taskSplitSize`| task split size| `2`|
 | `controller.auth.username`| login username for console web| `starwhale`|
 | `controller.auth.password`| login password for console web| `abcd1234`|
 | `controller.ingress.enabled`| enable ingress for Starwhale controller | `true` |

diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md
@@ -76,7 +76,6 @@ helm repo update starwhale
 
 |字段|描述|默认值|
 |---|---|-----|
-| `controller.taskSplitSize`| task split size | `2`|
 | `controller.auth.username`| console web 用户名| `starwhale`|
 | `controller.auth.password`| console web 密码| `abcd1234`|
 | `controller.ingress.enabled`| 使用Ingress | `true` |

diff --git a/scripts/e2e_test/start_test.sh b/scripts/e2e_test/start_test.sh
@@ -167,7 +167,6 @@ start_starwhale() {
   --set resources.controller.requests.cpu=700m \
   --set mysql.resources.requests.cpu=300m \
   --set minio.resources.requests.cpu=200m \
-  --set controller.taskSplitSize=1 \
   --set minikube.enabled=true \
   --set image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER \
   --set image.tag=$PYPI_RELEASE_VERSION \

diff --git a/...ontroller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java b/...ontroller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java
@@ -44,7 +44,6 @@
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.StringUtils;
@@ -65,11 +64,6 @@ public class JobSpliteratorEvaluation implements JobSpliterator {
     private final JobDao jobDao;
     private final StepMapper stepMapper;
     private final JobSpecParser jobSpecParser;
-    /**
-     * when task amount exceeds 1000, batch insertion will emit an error
-     */
-    @Value("${sw.task.size}")
-    Integer amountOfTasks = 256;
 
     public JobSpliteratorEvaluation(StoragePathCoordinator storagePathCoordinator,
             TaskMapper taskMapper,
@@ -83,7 +77,8 @@ public JobSpliteratorEvaluation(StoragePathCoordinator storagePathCoordinator,
     }
 
     /**
-     * split job into two steps transactional jobStatus->READY firstStepTaskStatus->READY
+     * split job into two steps transactional jobStatus->READY
+     * firstStepTaskStatus->READY
      * followerStepTaskStatus->CREATED
      */
     @Override
@@ -131,7 +126,8 @@ public List<StepEntity> split(Job job) {
                 // the current implementation is serial, so dependency only one
                 stepEntity.setLastStepId(nameMapping.get(dependency)._1().getId());
             }
-            // TODO: replace this implement with only send ds uri and task index to container
+            // TODO: replace this implement with only send ds uri and task index to
+            // container
             List<TaskEntity> taskEntities = new LinkedList<>();
             for (int i = 0; i < stepEntity.getTaskNum(); i++) {
                 final String taskUuid = UUID.randomUUID().toString();
@@ -140,14 +136,12 @@ public List<StepEntity> split(Job job) {
                         .outputPath(
                                 storagePathCoordinator.allocateTaskResultPath(job.getUuid(), taskUuid))
                         .taskRequest(JSONUtil.toJsonStr(
-                                        TaskRequest.builder()
-                                                .total(stepEntity.getTaskNum())
-                                                .index(i)
-                                                .runtimeResources(
-                                                        nameMapping.get(stepEntity.getName())._2.getResources())
-                                                .build()
-                                )
-                        )
+                                TaskRequest.builder()
+                                        .total(stepEntity.getTaskNum())
+                                        .index(i)
+                                        .runtimeResources(
+                                                nameMapping.get(stepEntity.getName())._2.getResources())
+                                        .build()))
                         .taskStatus(TaskStatus.valueOf(stepEntity.getStatus().name()))
                         .taskUuid(taskUuid)
                         .build());

diff --git a/server/controller/src/main/resources/application.yaml b/server/controller/src/main/resources/application.yaml
@@ -27,7 +27,6 @@ sw:
     load:
       batch-size: ${DATASET_CONSUMPTION_BATCH_SIZE:50}
   task:
-    size: ${SW_TASK_SPLIT_SIZE:1}
     deletion-delay-minutes: ${SW_TASK_DELETION_DELAY_MINUTES:30}
   online-eval:
     max-time-to-live-in-seconds: ${SW_ONLINE_EVAL_MAX_TTL_SECS:43200} # 12h