Skip to content

Commit

Permalink
add timeCosts for upload
Browse files Browse the repository at this point in the history
Signed-off-by: yangxuan <xuan.yang@zilliz.com>
  • Loading branch information
XuanYang-cn committed Apr 23, 2024
1 parent be1c5aa commit 02e9f9b
Showing 1 changed file with 12 additions and 3 deletions.
15 changes: 12 additions & 3 deletions internal/datanode/compaction/mix_compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ func (t *mixCompactionTask) merge(

downloadTimeCost := time.Duration(0)
serWriteTimeCost := time.Duration(0)
uploadTimeCost := time.Duration(0)

for _, paths := range binlogPaths {
downloadStart := time.Now()
Expand Down Expand Up @@ -340,32 +341,39 @@ func (t *mixCompactionTask) merge(
zap.Strings("path", paths), zap.Error(err))
return nil, err
}
serWriteTimeCost += time.Since(serWriteStart)

uploadStart := time.Now()
if err := t.binlogIO.Upload(ctx, kvs); err != nil {
log.Warn("compact wrong, failed to upload kvs", zap.Error(err))
}
serWriteTimeCost += time.Since(serWriteStart)
uploadTimeCost += time.Since(uploadStart)
mergeFieldBinlogs(allBinlogs, partialBinlogs)
syncBatchCount++
}
}
}

serWriteStart := time.Now()
if !buffer.IsEmpty() {
serWriteStart := time.Now()
kvs, partialBinlogs, err := t.serializeWrite(ctx, buffer)
if err != nil {
log.Warn("compact wrong, failed to serialize write buffer", zap.Error(err))
return nil, err
}
serWriteTimeCost += time.Since(serWriteStart)

uploadStart := time.Now()
if err := t.binlogIO.Upload(ctx, kvs); err != nil {
log.Warn("compact wrong, failed to upload kvs", zap.Error(err))
}
uploadTimeCost += time.Since(uploadStart)

mergeFieldBinlogs(allBinlogs, partialBinlogs)
syncBatchCount++
}

serWriteStart := time.Now()
sPath, err := t.statSerializeWrite(ctx, buffer, remainingRowCount)
if err != nil {
log.Warn("compact wrong, failed to serialize write buffer stats",
Expand All @@ -389,8 +397,9 @@ func (t *mixCompactionTask) merge(
zap.Int64("expired entities", expiredRowCount),
zap.Int("binlog batch count", syncBatchCount),
zap.Duration("download binlogs elapse", downloadTimeCost),
zap.Duration("upload binlogs elapse", uploadTimeCost),
zap.Duration("serWrite elapse", serWriteTimeCost),
zap.Duration("deRead elapse", totalElapse-serWriteTimeCost-downloadTimeCost),
zap.Duration("deRead elapse", totalElapse-serWriteTimeCost-downloadTimeCost-uploadTimeCost),
zap.Duration("total elapse", totalElapse))

return pack, nil
Expand Down

0 comments on commit 02e9f9b

Please sign in to comment.