enhance: fix serialization record span & flushed buffer size metrics (#…

…29482) See also #27675 #29413 --------- Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
milvus-io · Dec 27, 2023 · f6cff25 · f6cff25
1 parent 033456e
commit f6cff25
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 12 deletions.
diff --git a/internal/datanode/syncmgr/storage_serializer.go b/internal/datanode/syncmgr/storage_serializer.go
@@ -18,6 +18,7 @@ package syncmgr
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/samber/lo"
@@ -28,7 +29,10 @@ import (
 	"github.com/milvus-io/milvus/internal/proto/etcdpb"
 	"github.com/milvus-io/milvus/internal/storage"
 	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
+	"github.com/milvus-io/milvus/pkg/util/timerecord"
 )
 
 type storageV1Serializer struct {
@@ -69,6 +73,8 @@ func NewStorageSerializer(metacache metacache.MetaCache, metaWriter MetaWriter)
 
 func (s *storageV1Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack) (Task, error) {
 	task := NewSyncTask()
+	tr := timerecord.NewTimeRecorder("storage_serializer")
+	metricSegLevel := pack.level.String()
 
 	log := log.Ctx(ctx).With(
 		zap.Int64("segmentID", pack.segmentID),
@@ -125,6 +131,8 @@ func (s *storageV1Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack)
 	}
 
 	s.setTaskMeta(task, pack)
+
+	metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(tr.RecordSpan().Milliseconds()))
 	return task, nil
 }
 

diff --git a/internal/datanode/syncmgr/storage_v2_serializer.go b/internal/datanode/syncmgr/storage_v2_serializer.go
@@ -35,7 +35,10 @@ import (
 	iTypeutil "github.com/milvus-io/milvus/internal/util/typeutil"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
+	"github.com/milvus-io/milvus/pkg/util/timerecord"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
 )
 
@@ -68,6 +71,8 @@ func NewStorageV2Serializer(
 
 func (s *storageV2Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack) (Task, error) {
 	task := NewSyncTaskV2()
+	tr := timerecord.NewTimeRecorder("storage_serializer_v2")
+	metricSegLevel := pack.level.String()
 
 	space, err := s.storageV2Cache.GetOrCreateSpace(pack.segmentID, SpaceCreatorFunc(pack.segmentID, s.schema, s.arrowSchema))
 	if err != nil {
@@ -120,6 +125,7 @@ func (s *storageV2Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack)
 	}
 
 	s.setTaskMeta(task, pack)
+	metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(tr.RecordSpan().Milliseconds()))
 	return task, nil
 }
 

diff --git a/internal/datanode/syncmgr/task.go b/internal/datanode/syncmgr/task.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"path"
 
+	"github.com/samber/lo"
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@@ -162,25 +163,23 @@ func (t *SyncTask) Run() (err error) {
 		return err
 	}
 
-	/*
-		var totalSize float64 = 0
-		if t.deleteData != nil {
-			totalSize += float64(t.deleteData.Size())
-		}
-
-		if t.insertData != nil {
-			totalSize += float64(t.insertData.GetMemorySize())
-		}
-		metrics.DataNodeFlushedSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.AllLabel, metricSegLevel).Add(totalSize)
-		metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(t.tr.RecordSpan().Milliseconds()))*/
-
 	err = t.writeLogs()
 	if err != nil {
 		log.Warn("failed to save serialized data into storage", zap.Error(err))
 		t.handleError(err, metricSegLevel)
 		return err
 	}
 
+	var totalSize float64
+	totalSize += lo.SumBy(lo.Values(t.binlogMemsize), func(fieldSize int64) float64 {
+		return float64(fieldSize)
+	})
+	if t.deltaBlob != nil {
+		totalSize += float64(len(t.deltaBlob.Value))
+	}
+
+	metrics.DataNodeFlushedSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.AllLabel, metricSegLevel).Add(totalSize)
+
 	metrics.DataNodeSave2StorageLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(t.tr.RecordSpan().Milliseconds()))
 
 	if t.metaWriter != nil {