refactor(controller): deprecate resource version url in job request (#…

…2939)
star-whale · Nov 7, 2023 · 00d7ced · 00d7ced
1 parent b6f5301
commit 00d7ced
Show file tree

Hide file tree

Showing 25 changed files with 796 additions and 467 deletions.
diff --git a/client/starwhale/base/client/models/models.py b/client/starwhale/base/client/models/models.py
@@ -206,8 +206,11 @@ class DevWay(Enum):
 
 
 class JobRequest(SwBaseModel):
+    model_version_id: Optional[int] = Field(None, alias='modelVersionId')
+    dataset_version_ids: Optional[List[int]] = Field(None, alias='datasetVersionIds')
+    runtime_version_id: Optional[int] = Field(None, alias='runtimeVersionId')
     time_to_live_in_sec: Optional[int] = Field(None, alias='timeToLiveInSec')
-    model_version_url: str = Field(..., alias='modelVersionUrl')
+    model_version_url: Optional[str] = Field(None, alias='modelVersionUrl')
     dataset_version_urls: Optional[str] = Field(None, alias='datasetVersionUrls')
     runtime_version_url: Optional[str] = Field(None, alias='runtimeVersionUrl')
     comment: Optional[str] = None

diff --git a/client/starwhale/base/uri/resource.py b/client/starwhale/base/uri/resource.py
@@ -231,7 +231,7 @@ def _parse_by_version(self, ver: str, detect_typ: bool = False) -> None:
                     raise NoMatchException(ver, list(m))
         else:
             # TODO use api to check if it is a name or version
-            # assume is is name for now
+            # assume it is name for now
             self.name = ver
 
     def _refine_remote_rc_info(self) -> None:

diff --git a/scripts/client_test/cli_test.py b/scripts/client_test/cli_test.py
@@ -24,7 +24,6 @@
 from cmds.artifacts_cmd import Model, Dataset, Runtime
 
 from starwhale.utils import config
-from starwhale.base.type import DatasetChangeMode
 from starwhale.utils.debug import init_logger
 from starwhale.base.uri.resource import Resource, ResourceType
 from starwhale.api._impl.data_store import TableDesc, LocalDataStore, RemoteDataStore
@@ -358,14 +357,6 @@ def test_simple(self) -> None:
         model_uri = self.build_model(workdir, "simple", runtime=str(venv_runtime_uri))
         dataset_uri = self.build_dataset("simple", workdir, DatasetExpl("", ""))
 
-        if self.server_url:
-            self.dataset_api.copy(
-                src_uri=dataset_uri.full_uri,
-                target_project=self.cloud_target_project_uri,
-                force=True,
-                mode=DatasetChangeMode.OVERWRITE,
-            )
-
         remote_job_ids = []
         if self.server_url:
             remote_job_ids = self.run_model_in_server(

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/JobController.java
@@ -43,6 +43,7 @@
 import ai.starwhale.mlops.domain.job.JobServiceForWeb;
 import ai.starwhale.mlops.domain.job.ModelServingService;
 import ai.starwhale.mlops.domain.job.RuntimeSuggestionService;
+import ai.starwhale.mlops.domain.job.converter.UserJobConverter;
 import ai.starwhale.mlops.domain.run.RunService;
 import ai.starwhale.mlops.domain.task.TaskService;
 import ai.starwhale.mlops.exception.SwProcessException;
@@ -89,6 +90,7 @@ public class JobController {
     private final EventService eventService;
 
     private final RunService runService;
+    private final UserJobConverter userJobConverter;
 
     public JobController(
             JobServiceForWeb jobServiceForWeb,
@@ -99,7 +101,8 @@ public JobController(
             DagQuerier dagQuerier,
             FeaturesProperties featuresProperties,
             EventService eventService,
-            RunService runService
+            RunService runService,
+            UserJobConverter userJobConverter
     ) {
         this.jobServiceForWeb = jobServiceForWeb;
         this.taskService = taskService;
@@ -110,6 +113,8 @@ public JobController(
         this.featuresProperties = featuresProperties;
         this.eventService = eventService;
         this.runService = runService;
+        this.userJobConverter = userJobConverter;
+
         var actions = InvokerManager.<String, String>create()
                 .addInvoker("cancel", jobServiceForWeb::cancelJob);
         if (featuresProperties.isJobPauseEnabled()) {
@@ -202,19 +207,9 @@ public ResponseEntity<ResponseMessage<String>> createJob(
             throw new StarwhaleApiException(new SwValidationException(ValidSubject.JOB, "dev mode is not enabled"),
                     HttpStatus.BAD_REQUEST);
         }
-        Long jobId = jobServiceForWeb.createJob(projectUrl,
-                jobRequest.getModelVersionUrl(),
-                jobRequest.getDatasetVersionUrls(),
-                jobRequest.getRuntimeVersionUrl(),
-                jobRequest.getComment(),
-                jobRequest.getResourcePool(),
-                jobRequest.getHandler(),
-                jobRequest.getStepSpecOverWrites(),
-                jobRequest.getType(),
-                jobRequest.getDevWay(),
-                jobRequest.isDevMode(),
-                jobRequest.getDevPassword(),
-                jobRequest.getTimeToLiveInSec());
+
+        var req = userJobConverter.convert(projectUrl, jobRequest);
+        Long jobId = jobServiceForWeb.createJob(req);
 
         return ResponseEntity.ok(Code.success.asResponse(idConvertor.convert(jobId)));
     }

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/job/JobRequest.java b/server/controller/src/main/java/ai/starwhale/mlops/api/protocol/job/JobRequest.java
@@ -20,6 +20,7 @@
 import ai.starwhale.mlops.domain.job.JobType;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.io.Serializable;
+import java.util.List;
 import javax.validation.constraints.NotNull;
 import lombok.Data;
 import org.springframework.validation.annotation.Validated;
@@ -28,16 +29,22 @@
 @Validated
 public class JobRequest implements Serializable {
 
-    @NotNull
+    @Deprecated
     @JsonProperty("modelVersionUrl")
     private String modelVersionUrl;
 
+    @Deprecated
     @JsonProperty("datasetVersionUrls")
     private String datasetVersionUrls;
 
+    @Deprecated
     @JsonProperty("runtimeVersionUrl")
     private String runtimeVersionUrl;
 
+    private Long modelVersionId;
+    private List<Long> datasetVersionIds;
+    private Long runtimeVersionId;
+
     @JsonProperty("comment")
     private String comment;
 

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/bundle/BundleVersionAccessor.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/bundle/BundleVersionAccessor.java
@@ -18,13 +18,13 @@
 
 import ai.starwhale.mlops.domain.bundle.base.BundleVersionEntity;
 
-public interface BundleVersionAccessor {
+public interface BundleVersionAccessor<T extends BundleVersionEntity> {
 
-    BundleVersionEntity findVersionById(Long bundleVersionId);
+    T findVersionById(Long bundleVersionId);
 
-    BundleVersionEntity findVersionByAliasAndBundleId(String alias, Long bundleId);
+    T findVersionByAliasAndBundleId(String alias, Long bundleId);
 
-    BundleVersionEntity findVersionByNameAndBundleId(String name, Long bundleId);
+    T findVersionByNameAndBundleId(String name, Long bundleId);
 
-    BundleVersionEntity findLatestVersionByBundleId(Long bundleId);
+    T findLatestVersionByBundleId(Long bundleId);
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java
@@ -57,6 +57,7 @@
 import ai.starwhale.mlops.domain.job.JobCreator;
 import ai.starwhale.mlops.domain.job.JobType;
 import ai.starwhale.mlops.domain.job.bo.Job;
+import ai.starwhale.mlops.domain.job.bo.JobCreateRequest;
 import ai.starwhale.mlops.domain.job.spec.Env;
 import ai.starwhale.mlops.domain.job.spec.JobSpecParser;
 import ai.starwhale.mlops.domain.job.spec.StepSpec;
@@ -506,10 +507,15 @@ public void build(CreateBuildRecordRequest request) {
             throw new SwProcessException(ErrorType.SYSTEM, "error occurs while writing ds build step specs to string",
                     e);
         }
-        Job job = jobCreator.createJob(project, null, null, null, null,
-                systemSettingService.getRunTimeProperties().getDatasetBuild().getResourcePool(), null,
-                stepSpecOverWrites,
-                JobType.BUILT_IN, null, false, null, null, userService.currentUserDetail());
+        var jobReq = JobCreateRequest.builder()
+                .project(project)
+                .resourcePool(systemSettingService.getRunTimeProperties().getDatasetBuild().getResourcePool())
+                .stepSpecOverWrites(stepSpecOverWrites)
+                .jobType(JobType.BUILT_IN)
+                .user(userService.currentUserDetail())
+                .build();
+        Job job = jobCreator.createJob(jobReq);
+
         var entity = BuildRecordEntity.builder()
                 .projectId(project.getId())
                 .taskId(job.getSteps().get(0).getTasks().get(0).getId())