Skip to content

Commit

Permalink
enhance: Avoid redundant meta operations of import (milvus-io#33518) (m…
Browse files Browse the repository at this point in the history
…ilvus-io#33519)

issue: milvus-io#33513

pr: milvus-io#33518

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
  • Loading branch information
bigsheeper authored Jun 3, 2024
1 parent b9b76ee commit 30fd4a9
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 3 deletions.
5 changes: 5 additions & 0 deletions internal/datacoord/import_checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,9 @@ func (c *importChecker) checkCollection(collectionID int64, jobs []ImportJob) {
return
}
if !has {
jobs = lo.Filter(jobs, func(job ImportJob, _ int) bool {
return job.GetState() != internalpb.ImportJobState_Failed
})
for _, job := range jobs {
err = c.imeta.UpdateJob(job.GetJobID(), UpdateJobState(internalpb.ImportJobState_Failed),
UpdateJobReason(fmt.Sprintf("collection %d dropped", collectionID)))
Expand Down Expand Up @@ -388,6 +391,8 @@ func (c *importChecker) checkGC(job ImportJob) {
err := c.imeta.RemoveJob(job.GetJobID())
if err != nil {
log.Warn("remove import job failed", zap.Int64("jobID", job.GetJobID()), zap.Error(err))
return
}
log.Info("import job removed", zap.Int64("jobID", job.GetJobID()))
}
}
8 changes: 5 additions & 3 deletions internal/datacoord/import_scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -350,9 +350,11 @@ func (s *importScheduler) processFailed(task ImportTask) {
return
}
}
err := s.imeta.UpdateTask(task.GetTaskID(), UpdateSegmentIDs(nil))
if err != nil {
log.Warn("update import task segments failed", WrapTaskLog(task, zap.Error(err))...)
if len(segments) > 0 {
err := s.imeta.UpdateTask(task.GetTaskID(), UpdateSegmentIDs(nil))
if err != nil {
log.Warn("update import task segments failed", WrapTaskLog(task, zap.Error(err))...)
}
}
}
err := DropImportTask(task, s.cluster, s.imeta)
Expand Down

0 comments on commit 30fd4a9

Please sign in to comment.