From ee535540a3ac8fed9bd06bfe9728cfc0b58e2fce Mon Sep 17 00:00:00 2001 From: renyanda <781905270@qq.com> Date: Wed, 15 Nov 2023 19:01:31 +0800 Subject: [PATCH] feat(controller): fill SFT list fields (#2980) --- .../mlops/domain/dataset/DatasetService.java | 2 +- .../mlops/domain/ft/FineTuneAppService.java | 40 ++++++++++++++----- .../mlops/domain/ft/vo/FineTuneVo.java | 30 ++++---------- .../domain/ft/FineTuneAppServiceTest.java | 13 ++++-- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java index 2103cdcc83..2aa134c7b3 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/DatasetService.java @@ -370,7 +370,7 @@ public PageInfo listDatasetVersionHistory(DatasetVersionQuery } public List findDatasetsByVersionIds(List versionIds) { - if (versionIds.isEmpty()) { + if (CollectionUtils.isEmpty(versionIds)) { return List.of(); } List versions = datasetVersionMapper.findByIds(Joiner.on(",").join(versionIds)); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java index 7f01ac000b..b9ec2b8d49 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/FineTuneAppService.java @@ -17,11 +17,13 @@ package ai.starwhale.mlops.domain.ft; import ai.starwhale.mlops.api.protocol.ft.FineTuneCreateRequest; +import ai.starwhale.mlops.api.protocol.model.ModelVo; import ai.starwhale.mlops.common.Constants; import ai.starwhale.mlops.common.IdConverter; import ai.starwhale.mlops.configuration.FeaturesProperties; import ai.starwhale.mlops.domain.bundle.base.BundleEntity; import ai.starwhale.mlops.domain.dataset.DatasetDao; +import ai.starwhale.mlops.domain.dataset.DatasetService; import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion; import ai.starwhale.mlops.domain.ft.mapper.FineTuneMapper; import ai.starwhale.mlops.domain.ft.mapper.FineTuneSpaceMapper; @@ -31,6 +33,7 @@ import ai.starwhale.mlops.domain.job.JobCreator; import ai.starwhale.mlops.domain.job.JobType; import ai.starwhale.mlops.domain.job.bo.Job; +import ai.starwhale.mlops.domain.job.converter.JobConverter; import ai.starwhale.mlops.domain.job.converter.UserJobConverter; import ai.starwhale.mlops.domain.job.mapper.JobMapper; import ai.starwhale.mlops.domain.job.po.JobEntity; @@ -38,6 +41,7 @@ import ai.starwhale.mlops.domain.job.spec.JobSpecParser; import ai.starwhale.mlops.domain.job.spec.StepSpec; import ai.starwhale.mlops.domain.model.ModelDao; +import ai.starwhale.mlops.domain.model.ModelService; import ai.starwhale.mlops.domain.model.bo.ModelVersion; import ai.starwhale.mlops.domain.model.po.ModelEntity; import ai.starwhale.mlops.domain.model.po.ModelVersionEntity; @@ -88,6 +92,12 @@ public class FineTuneAppService { final UserJobConverter userJobConverter; + final JobConverter jobConverter; + + final ModelService modelService; + + final DatasetService datasetService; + public FineTuneAppService( FeaturesProperties featuresProperties, JobCreator jobCreator, @@ -99,7 +109,10 @@ public FineTuneAppService( @Value("${sw.instance-uri}") String instanceUri, DatasetDao datasetDao, FineTuneSpaceMapper fineTuneSpaceMapper, - UserJobConverter userJobConverter + UserJobConverter userJobConverter, + JobConverter jobConverter, + ModelService modelService, + DatasetService datasetService ) { this.featuresProperties = featuresProperties; this.jobCreator = jobCreator; @@ -112,6 +125,9 @@ public FineTuneAppService( this.instanceUri = instanceUri; this.fineTuneSpaceMapper = fineTuneSpaceMapper; this.userJobConverter = userJobConverter; + this.jobConverter = jobConverter; + this.modelService = modelService; + this.datasetService = datasetService; } @@ -200,17 +216,21 @@ public PageInfo list(Long spaceId, Integer pageNum, Integer pageSize fineTuneEntity.getEvalDatasets(); fineTuneEntity.getTrainDatasets(); fineTuneEntity.getBaseModelVersionId(); - fineTuneEntity.getTargetModelVersionId(); + ModelVo mv = null; + Long targetModelVersionId = fineTuneEntity.getTargetModelVersionId(); + if (null != targetModelVersionId) { + List modelVos = modelService.findModelByVersionId(List.of(targetModelVersionId)); + if (!CollectionUtils.isEmpty(modelVos)) { + mv = modelVos.get(0); + } + } + return FineTuneVo.builder() .id(fineTuneEntity.getId()) - .jobId(jobId) - .status(job.getJobStatus()) - .startTime(job.getCreatedTime().getTime()) - .endTime(null != job.getFinishedTime() ? job.getFinishedTime().getTime() : null) - .evalDatasets(List.of())//TODO - .trainDatasets(List.of())//TODO - .baseModel(null)//TODO - .targetModel(null)//TODO + .job(jobConverter.convert(job)) + .evalDatasets(datasetService.findDatasetsByVersionIds(fineTuneEntity.getTrainDatasets())) + .trainDatasets(datasetService.findDatasetsByVersionIds(fineTuneEntity.getEvalDatasets())) + .targetModel(mv) .build(); }).collect(Collectors.toList())); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/vo/FineTuneVo.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/vo/FineTuneVo.java index dc1c8c2cc6..5f3936cd19 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/vo/FineTuneVo.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/ft/vo/FineTuneVo.java @@ -16,7 +16,9 @@ package ai.starwhale.mlops.domain.ft.vo; -import ai.starwhale.mlops.domain.job.status.JobStatus; +import ai.starwhale.mlops.api.protocol.dataset.DatasetVo; +import ai.starwhale.mlops.api.protocol.job.JobVo; +import ai.starwhale.mlops.api.protocol.model.ModelVo; import java.util.List; import javax.validation.constraints.NotNull; import lombok.AllArgsConstructor; @@ -33,29 +35,11 @@ public class FineTuneVo { @NotNull Long id; @NotNull - Long jobId; + JobVo job; @NotNull - JobStatus status; + List trainDatasets; + List evalDatasets; @NotNull - Long startTime; - Long endTime; - List trainDatasets; - List evalDatasets; - @NotNull - ModelInfo baseModel; - ModelInfo targetModel; - - public static class DsInfo { - String name; - String version; - Long id; - } - - public static class ModelInfo { - String name; - String version; - Long id; - } - + ModelVo targetModel; } diff --git a/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java b/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java index 20e7ad6005..3d6d4ec033 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/domain/ft/FineTuneAppServiceTest.java @@ -27,6 +27,7 @@ import ai.starwhale.mlops.common.IdConverter; import ai.starwhale.mlops.configuration.FeaturesProperties; import ai.starwhale.mlops.domain.dataset.DatasetDao; +import ai.starwhale.mlops.domain.dataset.DatasetService; import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion; import ai.starwhale.mlops.domain.ft.mapper.FineTuneMapper; import ai.starwhale.mlops.domain.ft.mapper.FineTuneSpaceMapper; @@ -35,12 +36,14 @@ import ai.starwhale.mlops.domain.job.JobCreator; import ai.starwhale.mlops.domain.job.bo.Job; import ai.starwhale.mlops.domain.job.bo.UserJobCreateRequest; +import ai.starwhale.mlops.domain.job.converter.JobConverter; import ai.starwhale.mlops.domain.job.converter.UserJobConverter; import ai.starwhale.mlops.domain.job.mapper.JobMapper; import ai.starwhale.mlops.domain.job.po.JobEntity; import ai.starwhale.mlops.domain.job.spec.JobSpecParser; import ai.starwhale.mlops.domain.job.spec.StepSpec; import ai.starwhale.mlops.domain.model.ModelDao; +import ai.starwhale.mlops.domain.model.ModelService; import ai.starwhale.mlops.domain.model.po.ModelEntity; import ai.starwhale.mlops.domain.model.po.ModelVersionEntity; import ai.starwhale.mlops.domain.project.bo.Project; @@ -83,8 +86,8 @@ public void setup() { jobSpecParser = mock(JobSpecParser.class); modelDao = mock(ModelDao.class); datasetDao = mock(DatasetDao.class); - UserJobConverter jobConverter = mock(UserJobConverter.class); - when(jobConverter.convert(any(), any())).thenReturn(UserJobCreateRequest.builder().build()); + UserJobConverter userJobConverter = mock(UserJobConverter.class); + when(userJobConverter.convert(any(), any())).thenReturn(UserJobCreateRequest.builder().build()); fineTuneSpaceMapper = mock(FineTuneSpaceMapper.class); featuresProperties = mock(FeaturesProperties.class); when(featuresProperties.isFineTuneEnabled()).thenReturn(true); @@ -98,7 +101,11 @@ public void setup() { modelDao, "instanceuri", datasetDao, - fineTuneSpaceMapper, jobConverter//todo + fineTuneSpaceMapper, + userJobConverter, + mock(JobConverter.class), + mock(ModelService.class), + mock(DatasetService.class) ); }