Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(controller): add test for JobMapper,JobSwdsVersionMapper,ProjectMapper #1235

Merged
merged 2 commits into from
Sep 19, 2022
Merged

chore(controller): add test for JobMapper,JobSwdsVersionMapper,ProjectMapper #1235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af4ae5e
add test for JobMapper,JobSwdsVersionMapper,ProjectMapper
anda-ren Sep 16, 2022
11cf917
restore evaluation list api
anda-ren Sep 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion server/controller/src/main/java/ai/starwhale/mlops/api/EvaluationController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,5 +86,4 @@ public ResponseEntity<ResponseMessage<PageInfo<SummaryVo>>> listEvaluationSummar
return ResponseEntity.ok(Code.success.asResponse(vos));
}


}
4 changes: 2 additions & 2 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/evaluation/EvaluationService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,11 +117,11 @@ public PageInfo<SummaryVo> listEvaluationSummary(String projectUrl,
SummaryFilter summaryFilter, PageParams pageParams) {
PageHelper.startPage(pageParams.getPageNum(), pageParams.getPageSize());
Long projectId = projectManager.getProjectId(projectUrl);
List<JobEntity> jobEntities = jobMapper.listJobsByStatus(projectId, null,
JobStatus.SUCCESS);
List<JobEntity> jobEntities = jobMapper.listJobs(projectId, null);
return PageUtil.toPageInfo(jobEntities, this::toSummary);
}


private SummaryVo toSummary(JobEntity entity) {
if (summaryCache.containsKey(entity.getId())) {
return summaryCache.get(entity.getId());
Expand Down
5 changes: 0 additions & 5 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/job/mapper/JobMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ public interface JobMapper {

List<JobEntity> listJobs(@Param("projectId") Long projectId, @Param("swmpId") Long swmpId);

List<JobEntity> listJobsByStatus(@Param("projectId") Long projectId, @Param("swmpId") Long swmpId,
@Param("jobStatus") JobStatus jobStatus);

JobEntity findJobById(@Param("jobId") Long jobId);

JobEntity findJobByUuid(@Param("uuid") String uuid);
Expand All @@ -43,8 +40,6 @@ List<JobEntity> listJobsByStatus(@Param("projectId") Long projectId, @Param("swm

void updateJobFinishedTime(@Param("jobIds") List<Long> jobIds, @Param("finishedTime") LocalDateTime finishedTime);

void updateJobResultPath(@Param("jobId") Long jobId, @Param("resultDir") String resultDir);

int updateJobComment(@Param("id") Long id, @Param("comment") String comment);

int updateJobCommentByUuid(@Param("uuid") String uuid, @Param("comment") String comment);
Expand Down
3 changes: 0 additions & 3 deletions server/controller/src/main/java/ai/starwhale/mlops/domain/project/mapper/ProjectMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@ List<ProjectEntity> listProjects(@Param("projectName") String projectName, @Para
List<ProjectEntity> listProjectsByOwner(@Param("userId") Long userId, @Param("order") String order,
@Param("isDeleted") Integer isDeleted);

List<ProjectEntity> listProjectsByOwnerName(@Param("userName") String userName, @Param("order") String order,
@Param("isDeleted") Integer isDeleted);

ProjectEntity findProject(@Param("id") Long id);

ProjectEntity findProjectByName(@NotNull @Param("projectName") String projectName);
Expand Down
15 changes: 0 additions & 15 deletions server/controller/src/main/resources/mapper/JobMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,6 @@
order by job_id desc
</select>

<select id="listJobsByStatus" resultMap="jobResultMap">
<include refid="select_jobs"/>
<where>
and job_is_deleted = 0
and project_id = #{projectId}
</where>
<if test="swmpId != null and swmpId != ''">
and swmp_id = #{swmpId}
</if>
order by job_id desc
</select>

<select id="findJobById" resultMap="jobResultMap">
<include refid="select_jobs"/>
<where>
Expand Down Expand Up @@ -125,9 +113,6 @@
#{item}
</foreach>
</update>
<update id="updateJobResultPath">
update job_info set result_output_path = #{resultDir} WHERE id = #{jobId}
</update>

<update id="updateJobComment">
update job_info set job_comment = #{comment}
Expand Down
3 changes: 3 additions & 0 deletions server/controller/src/main/resources/mapper/JobSwdsVersionMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
version_name as dsv_version_name,
ds.dataset_name as ds_dataset_name,
dsv.storage_auths as dsv_storage_auths,
dsv.index_table as dsv_index_table,
version_tag as dsv_version_tag,
version_meta as dsv_version_meta,
storage_path as dsv_storage_path,
dsv.created_time as dsv_created_time,
dsv.files_uploaded,
dsv.size as dsv_size,
dsv.modified_time as dsv_modified_time
from dataset_version as dsv,
dataset_info as ds,
Expand Down
27 changes: 10 additions & 17 deletions server/controller/src/main/resources/mapper/ProjectMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
<mapper namespace="ai.starwhale.mlops.domain.project.mapper.ProjectMapper">
<insert id="createProject" parameterType="ai.starwhale.mlops.domain.project.po.ProjectEntity"
useGeneratedKeys="true" keyProperty="id">
insert into project_info(project_name, owner_id, privacy, project_description)
values (#{project.projectName}, #{project.ownerId}, #{project.privacy}, #{project.description})
insert into project_info(project_name, owner_id, privacy, project_description, is_default)
values (#{project.projectName}, #{project.ownerId}, #{project.privacy}, #{project.description}, #{project.isDefault})
</insert>

<update id="deleteProject" parameterType="java.lang.Long">
Expand Down Expand Up @@ -40,7 +40,10 @@
owner_id = #{project.ownerId},
</if>
<if test="project.privacy != null">
privacy = #{project.privacy}
privacy = #{project.privacy},
</if>
<if test="project.isDefault != null">
is_default = #{project.isDefault}
</if>
</set>
where id = #{project.id}
Expand Down Expand Up @@ -89,16 +92,6 @@
</if>
</select>

<select id="listProjectsByOwnerName" parameterType="java.lang.String"
resultMap="projectResultMap">
<include refid="select_project"/>
and p.is_deleted = #{isDeleted}
and u.user_name like concat(#{userName}, '%')
<if test="order != null and order != ''">
order by ${order}
</if>
</select>

<select id="findProject" parameterType="java.lang.Long" resultMap="projectResultMap">
<include refid="select_project"/>
and p.id = #{id}
Expand All @@ -119,10 +112,10 @@
<select id="listObjectCounts"
resultType="ai.starwhale.mlops.domain.project.po.ProjectObjectCountEntity">
select p.id as project_id,
c1.count_model,
c2.count_dataset,
c3.count_member,
c4.count_jobs
COALESCE(c1.count_model,0),
COALESCE(c2.count_dataset,0),
COALESCE(c3.count_member,0),
COALESCE(c4.count_jobs,0)
from project_info as p
left join
(select count(*) as count_model, project_id from swmp_info group by project_id) as c1
Expand Down
5 changes: 3 additions & 2 deletions server/controller/src/main/resources/mapper/SwDatasetVersionMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
dataset_id,
dsv.version_order as dsv_version_order,
dsv.owner_id,
dsv.files_uploaded,
dsv.size as dsv_size,
dsv.index_table as dsv_index_table,
dsv.storage_auths as dsv_storage_auths,
Expand Down Expand Up @@ -72,11 +73,11 @@
keyProperty="id" parameterType="ai.starwhale.mlops.domain.swds.po.SwDatasetVersionEntity"
useGeneratedKeys="true">
insert into dataset_version (dataset_id, owner_id, version_name, size, index_table, version_tag,
version_meta, storage_path, files_uploaded)
version_meta, storage_path, files_uploaded, storage_auths)
values (#{version.datasetId}, #{version.ownerId}, #{version.versionName}, #{version.size},
#{version.indexTable}, #{version.versionTag}, #{version.versionMeta},
#{version.storagePath},
#{version.filesUploaded})
#{version.filesUploaded}, #{version.storageAuths})
</insert>

<select id="getVersionById" resultMap="dsVersionResultMap">
Expand Down
2 changes: 1 addition & 1 deletion .../controller/src/test/java/ai/starwhale/mlops/domain/evaluation/EvaluationServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public void testCreateViewConfig() {

@Test
public void testListEvaluationSummary() {
given(jobMapper.listJobsByStatus(same(1L), any(), any()))
given(jobMapper.listJobs(same(1L), any()))
.willReturn(List.of(
JobEntity.builder()
.id(1L)
Expand Down
147 changes: 130 additions & 17 deletions server/controller/src/test/java/ai/starwhale/mlops/domain/job/mapper/JobMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,16 @@
import ai.starwhale.mlops.domain.swmp.po.SwModelPackageVersionEntity;
import ai.starwhale.mlops.domain.user.mapper.UserMapper;
import ai.starwhale.mlops.domain.user.po.UserEntity;
import java.time.LocalDateTime;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mybatis.spring.boot.test.autoconfigure.MybatisTest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase.Replace;
import org.springframework.context.ApplicationContext;

@MybatisTest
@AutoConfigureTestDatabase(replace = Replace.NONE)
Expand All @@ -57,41 +58,146 @@ public class JobMapperTest extends MySqlContainerHolder {
@Autowired
private SwModelPackageVersionMapper swModelPackageVersionMapper;

@Autowired
private ApplicationContext applicationContext;
UserEntity user;
ProjectEntity project;
SwModelPackageEntity swmp;
SwModelPackageVersionEntity swModelPackageVersionEntity;
JobEntity jobPaused;
JobEntity jobCreated;

@Test
public void testListJobs() {
UserEntity user = UserEntity.builder().userEnabled(0).userName("un12").userPwdSalt("x").userPwd("up").build();
@BeforeEach
public void initData() {
user = UserEntity.builder().userEnabled(0).userName("un12").userPwdSalt("x").userPwd("up").build();
userMapper.createUser(user);
ProjectEntity project = ProjectEntity.builder().projectName("pjn").ownerId(user.getId()).privacy(1).isDefault(0)
project = ProjectEntity.builder().projectName("pjn").ownerId(user.getId()).privacy(1).isDefault(0)
.build();
projectMapper.createProject(project);
SwModelPackageEntity swmp = SwModelPackageEntity.builder().swmpName("swmp").projectId(project.getId())
swmp = SwModelPackageEntity.builder().swmpName("swmp").projectId(project.getId())
.ownerId(user.getId()).build();
swModelPackageMapper.addSwModelPackage(swmp);
SwModelPackageVersionEntity swModelPackageVersionEntity = SwModelPackageVersionEntity.builder()
swModelPackageVersionEntity = SwModelPackageVersionEntity.builder()
.swmpId(swmp.getId())
.versionName("vn")
.ownerId(user.getId()).evalJobs("")
.manifest("mf").versionMeta("mt").storagePath("s").build();
swModelPackageVersionMapper.addNewVersion(swModelPackageVersionEntity);
JobEntity job = JobEntity.builder().jobUuid(UUID.randomUUID().toString()).jobStatus(JobStatus.PAUSED)
jobPaused = JobEntity.builder().jobUuid(UUID.randomUUID().toString()).jobStatus(JobStatus.PAUSED)
.resourcePoolId(1L).runtimeVersionId(1L).swmpVersionId(swModelPackageVersionEntity.getId())
.resultOutputPath("").type(JobType.EVALUATION)
.deviceType(0).deviceAmount(1).projectId(project.getId()).ownerId(user.getId()).build();
jobCreated = JobEntity.builder().jobUuid(UUID.randomUUID().toString()).jobStatus(JobStatus.CREATED)
.resourcePoolId(1L).runtimeVersionId(1L).swmpVersionId(swModelPackageVersionEntity.getId())
.resultOutputPath("").type(JobType.EVALUATION)
.deviceType(0).deviceAmount(1).projectId(project.getId()).ownerId(user.getId()).build();
jobMapper.addJob(job);
jobMapper.addJob(jobPaused);
jobMapper.addJob(jobCreated);
}

@Test
public void testListJobs() {
List<JobEntity> jobEntities = jobMapper.listJobs(project.getId(), null);
Assertions.assertEquals(1, jobEntities.size());
validateJob(job, user, project, swModelPackageVersionEntity, jobEntities.get(0));
Assertions.assertEquals(2, jobEntities.size());
jobEntities.forEach(
jobEntity -> validateJob(jobEntity.getJobStatus() == JobStatus.PAUSED ? jobPaused : jobCreated, user,
project, swModelPackageVersionEntity, jobEntity));
jobEntities = jobMapper.listJobs(project.getId(), swModelPackageVersionEntity.getId());
Assertions.assertEquals(1, jobEntities.size());
validateJob(job, user, project, swModelPackageVersionEntity, jobEntities.get(0));
Assertions.assertEquals(2, jobEntities.size());
jobEntities.forEach(
jobEntity -> validateJob(jobEntity.getJobStatus() == JobStatus.PAUSED ? jobPaused : jobCreated, user,
project, swModelPackageVersionEntity, jobEntity));
jobEntities = jobMapper.listJobs(project.getId(), swModelPackageVersionEntity.getId() + 1234L);
Assertions.assertIterableEquals(List.of(), jobEntities);

}

@Test
public void testFindById() {
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobMapper.findJobById(jobCreated.getId()));
}

@Test
public void testFindByUuId() {
validateJob(jobPaused, user, project, swModelPackageVersionEntity,
jobMapper.findJobByUuid(jobPaused.getJobUuid()));
}

@Test
public void testFindByStatusIn() {
List<JobEntity> jobEntities = jobMapper.findJobByStatusIn(List.of(JobStatus.PAUSED));
Assertions.assertEquals(1, jobEntities.size());
validateJob(jobPaused, user, project, swModelPackageVersionEntity, jobEntities.get(0));

jobEntities = jobMapper.findJobByStatusIn(List.of(JobStatus.PAUSED, JobStatus.CREATED));
Assertions.assertEquals(2, jobEntities.size());
jobEntities.forEach(
jobEntity -> validateJob(jobEntity.getJobStatus() == JobStatus.PAUSED ? jobPaused : jobCreated, user,
project, swModelPackageVersionEntity, jobEntity));

jobEntities = jobMapper.findJobByStatusIn(List.of(JobStatus.PAUSED, JobStatus.CREATED, JobStatus.FAIL));
Assertions.assertEquals(2, jobEntities.size());
jobEntities.forEach(
jobEntity -> validateJob(jobEntity.getJobStatus() == JobStatus.PAUSED ? jobPaused : jobCreated, user,
project, swModelPackageVersionEntity, jobEntity));

jobEntities = jobMapper.findJobByStatusIn(List.of(JobStatus.CANCELED));
Assertions.assertEquals(0, jobEntities.size());

}

@Test
public void testUpdateStatus() {
jobMapper.updateJobStatus(List.of(jobCreated.getId()), JobStatus.SUCCESS);
JobEntity jobById = jobMapper.findJobById(jobCreated.getId());
jobCreated.setJobStatus(JobStatus.SUCCESS);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobById);
}

@Test
public void testUpdateFinishedTime() {
LocalDateTime now = LocalDateTime.now();
jobMapper.updateJobFinishedTime(List.of(jobCreated.getId()), now);
JobEntity jobById = jobMapper.findJobById(jobCreated.getId());
jobCreated.setFinishedTime(now);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobById);
}

@Test
public void testeUpdateJobComment() {
String comment = "any comment";
jobMapper.updateJobComment(jobCreated.getId(), comment);
JobEntity jobById = jobMapper.findJobById(jobCreated.getId());
jobCreated.setComment(comment);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobById);
}

@Test
public void testUpdateJobCommentByUuid() {
String comment = "any comment";
jobMapper.updateJobCommentByUuid(jobCreated.getJobUuid(), comment);
JobEntity jobById = jobMapper.findJobById(jobCreated.getId());
jobCreated.setComment(comment);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobById);
}

@Test
public void testRemoveJob() {
jobMapper.removeJob(jobCreated.getId());
jobCreated.setIsDeleted(1);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobMapper.findJobById(jobCreated.getId()));

jobMapper.recoverJob(jobCreated.getId());
jobCreated.setIsDeleted(0);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobMapper.findJobById(jobCreated.getId()));

jobMapper.removeJobByUuid(jobCreated.getJobUuid());
jobCreated.setIsDeleted(1);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobMapper.findJobById(jobCreated.getId()));

jobMapper.recoverJobByUuid(jobCreated.getJobUuid());
jobCreated.setIsDeleted(0);
validateJob(jobCreated, user, project, swModelPackageVersionEntity, jobMapper.findJobById(jobCreated.getId()));
}

private void validateJob(JobEntity expectedJob, UserEntity user, ProjectEntity project,
SwModelPackageVersionEntity swModelPackageVersionEntity, JobEntity jobEntity) {
Assertions.assertEquals(expectedJob.getId(), jobEntity.getId());
Expand All @@ -109,8 +215,15 @@ private void validateJob(JobEntity expectedJob, UserEntity user, ProjectEntity p
Assertions.assertEquals("swmp", jobEntity.getModelName());
Assertions.assertEquals(expectedJob.getComment(), jobEntity.getComment());
Assertions.assertNotNull(jobEntity.getCreatedTime());
Assertions.assertNull(jobEntity.getFinishedTime());
Assertions.assertEquals(0, jobEntity.getIsDeleted());
final int milli500 = 5;
if (null != expectedJob.getFinishedTime()) {
Assertions.assertTrue(
Math.abs(expectedJob.getFinishedTime().compareTo(jobEntity.getFinishedTime())) <= milli500);
} else {
Assertions.assertNull(jobEntity.getFinishedTime());
}
Assertions.assertEquals(null == expectedJob.getIsDeleted() ? 0 : expectedJob.getIsDeleted(),
jobEntity.getIsDeleted());
validUser(user, jobEntity.getOwner());
validateSwModelPackageVersionEntity(swModelPackageVersionEntity, user, jobEntity.getSwmpVersion());
validProject(project, user, jobEntity.getProject());
Expand Down
Loading