diff --git a/docker/charts/README.md b/docker/charts/README.md index 02778237f4..5e62131fe8 100644 --- a/docker/charts/README.md +++ b/docker/charts/README.md @@ -81,7 +81,6 @@ The `update` command will update the information of available charts locally fro | Name | Description | Default Value | |------|-------------|---------------| -| `controller.taskSplitSize`| task split size| `2`| | `controller.auth.username`| login username for console web| `starwhale`| | `controller.auth.password`| login password for console web| `abcd1234`| | `controller.ingress.enabled`| enable ingress for Starwhale controller | `true` | diff --git a/docker/charts/templates/controller-deployment.yaml b/docker/charts/templates/controller-deployment.yaml index 2d21b10c4a..8d75968f4b 100644 --- a/docker/charts/templates/controller-deployment.yaml +++ b/docker/charts/templates/controller-deployment.yaml @@ -81,8 +81,6 @@ spec: value: "{{ .Values.image.registry }}/{{ .Values.image.org }}/{{ .Values.image.base.repo }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - name: SW_DOCKER_REGISTRY_URL value: "{{ .Values.image.registry }}" - - name: SW_TASK_SPLIT_SIZE - value: "{{ .Values.controller.taskSplitSize }}" - name: SW_HOST_IP valueFrom: fieldRef: diff --git a/docker/charts/values.minikube.cn.yaml b/docker/charts/values.minikube.cn.yaml index 70a6b0d4fd..508da85067 100644 --- a/docker/charts/values.minikube.cn.yaml +++ b/docker/charts/values.minikube.cn.yaml @@ -16,7 +16,6 @@ minio: enabled: false controller: - taskSplitSize: 1 containerPort: 8082 ingress: enabled: false diff --git a/docker/charts/values.minikube.global.yaml b/docker/charts/values.minikube.global.yaml index 32b077b7c9..bfd1321168 100644 --- a/docker/charts/values.minikube.global.yaml +++ b/docker/charts/values.minikube.global.yaml @@ -15,7 +15,6 @@ minio: enabled: false controller: - taskSplitSize: 1 containerPort: 8082 ingress: enabled: false diff --git a/docker/charts/values.yaml b/docker/charts/values.yaml index ace2e0a0f2..0eba563593 100644 --- a/docker/charts/values.yaml +++ b/docker/charts/values.yaml @@ -69,7 +69,6 @@ externalOSS: type: "s3" # supported type: s3, minio and aliyun controller: - taskSplitSize: 2 jwt: tokenExpireMinutes: 43200 upload: diff --git a/docker/compose/compose.yaml b/docker/compose/compose.yaml index d7c81cf609..89ce9b5942 100644 --- a/docker/compose/compose.yaml +++ b/docker/compose/compose.yaml @@ -31,7 +31,6 @@ services: ports: - 8082:8082 environment: - - SW_TASK_SPLIT_SIZE=2 - JAR=controller - SW_HOST_IP=127.0.0.1 - SW_JWT_TOKEN_EXPIRE_MINUTES=144000 diff --git a/docs/docs/guides/install/helm-charts.md b/docs/docs/guides/install/helm-charts.md index cac33cfbcd..ef088f3f06 100644 --- a/docs/docs/guides/install/helm-charts.md +++ b/docs/docs/guides/install/helm-charts.md @@ -85,7 +85,6 @@ The `update` command will update the information of available charts locally fro | Name | Description | Default Value | |------|-------------|---------------| -| `controller.taskSplitSize`| task split size| `2`| | `controller.auth.username`| login username for console web| `starwhale`| | `controller.auth.password`| login password for console web| `abcd1234`| | `controller.ingress.enabled`| enable ingress for Starwhale controller | `true` | diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md index e3e78df7c9..2155ce97fc 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/guides/install/helm-charts.md @@ -76,7 +76,6 @@ helm repo update starwhale |字段|描述|默认值| |---|---|-----| -| `controller.taskSplitSize`| task split size | `2`| | `controller.auth.username`| console web 用户名| `starwhale`| | `controller.auth.password`| console web 密码| `abcd1234`| | `controller.ingress.enabled`| 使用Ingress | `true` | diff --git a/scripts/e2e_test/start_test.sh b/scripts/e2e_test/start_test.sh index 02ee0ae2ba..e2903acb79 100644 --- a/scripts/e2e_test/start_test.sh +++ b/scripts/e2e_test/start_test.sh @@ -167,7 +167,6 @@ start_starwhale() { --set resources.controller.requests.cpu=700m \ --set mysql.resources.requests.cpu=300m \ --set minio.resources.requests.cpu=200m \ - --set controller.taskSplitSize=1 \ --set minikube.enabled=true \ --set image.registry=$NEXUS_HOSTNAME:$PORT_NEXUS_DOCKER \ --set image.tag=$PYPI_RELEASE_VERSION \ diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java index 6be4cbced5..ce64e26ee0 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/split/JobSpliteratorEvaluation.java @@ -44,7 +44,6 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; @@ -65,11 +64,6 @@ public class JobSpliteratorEvaluation implements JobSpliterator { private final JobDao jobDao; private final StepMapper stepMapper; private final JobSpecParser jobSpecParser; - /** - * when task amount exceeds 1000, batch insertion will emit an error - */ - @Value("${sw.task.size}") - Integer amountOfTasks = 256; public JobSpliteratorEvaluation(StoragePathCoordinator storagePathCoordinator, TaskMapper taskMapper, @@ -83,7 +77,8 @@ public JobSpliteratorEvaluation(StoragePathCoordinator storagePathCoordinator, } /** - * split job into two steps transactional jobStatus->READY firstStepTaskStatus->READY + * split job into two steps transactional jobStatus->READY + * firstStepTaskStatus->READY * followerStepTaskStatus->CREATED */ @Override @@ -131,7 +126,8 @@ public List split(Job job) { // the current implementation is serial, so dependency only one stepEntity.setLastStepId(nameMapping.get(dependency)._1().getId()); } - // TODO: replace this implement with only send ds uri and task index to container + // TODO: replace this implement with only send ds uri and task index to + // container List taskEntities = new LinkedList<>(); for (int i = 0; i < stepEntity.getTaskNum(); i++) { final String taskUuid = UUID.randomUUID().toString(); @@ -140,14 +136,12 @@ public List split(Job job) { .outputPath( storagePathCoordinator.allocateTaskResultPath(job.getUuid(), taskUuid)) .taskRequest(JSONUtil.toJsonStr( - TaskRequest.builder() - .total(stepEntity.getTaskNum()) - .index(i) - .runtimeResources( - nameMapping.get(stepEntity.getName())._2.getResources()) - .build() - ) - ) + TaskRequest.builder() + .total(stepEntity.getTaskNum()) + .index(i) + .runtimeResources( + nameMapping.get(stepEntity.getName())._2.getResources()) + .build())) .taskStatus(TaskStatus.valueOf(stepEntity.getStatus().name())) .taskUuid(taskUuid) .build()); diff --git a/server/controller/src/main/resources/application.yaml b/server/controller/src/main/resources/application.yaml index 02418dfb3a..68a4f00f0f 100644 --- a/server/controller/src/main/resources/application.yaml +++ b/server/controller/src/main/resources/application.yaml @@ -27,7 +27,6 @@ sw: load: batch-size: ${DATASET_CONSUMPTION_BATCH_SIZE:50} task: - size: ${SW_TASK_SPLIT_SIZE:1} deletion-delay-minutes: ${SW_TASK_DELETION_DELAY_MINUTES:30} online-eval: max-time-to-live-in-seconds: ${SW_ONLINE_EVAL_MAX_TTL_SECS:43200} # 12h