From 82743c5c50d26fa433f69a8b977faff1ea49235f Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Mon, 26 Aug 2024 12:22:57 +0800 Subject: [PATCH] fix: Clear channelcp meta and metrics ASAP (#35658) See also: #35588 --------- Signed-off-by: yangxuan --- internal/datacoord/garbage_collector.go | 7 +++++-- internal/datacoord/handler.go | 7 ++++++- internal/datacoord/meta.go | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/datacoord/garbage_collector.go b/internal/datacoord/garbage_collector.go index 2d216c5905e00..4296e69449072 100644 --- a/internal/datacoord/garbage_collector.go +++ b/internal/datacoord/garbage_collector.go @@ -510,9 +510,12 @@ func (gc *garbageCollector) recycleChannelCPMeta(ctx context.Context) { continue } - if err := gc.meta.DropChannelCheckpoint(vChannel); err != nil { + err := gc.meta.DropChannelCheckpoint(vChannel) + if err != nil { // Try to GC in the next gc cycle if drop channel cp meta fail. - log.Warn("failed to drop channel check point during gc", zap.String("vchannel", vChannel), zap.Error(err)) + log.Warn("failed to drop channelcp check point during gc", zap.String("vchannel", vChannel), zap.Error(err)) + } else { + log.Info("GC channel cp", zap.String("vchannel", vChannel)) } } diff --git a/internal/datacoord/handler.go b/internal/datacoord/handler.go index 045e17c268662..f9f82bc4268f3 100644 --- a/internal/datacoord/handler.go +++ b/internal/datacoord/handler.go @@ -458,7 +458,12 @@ func (h *ServerHandler) FinishDropChannel(channel string, collectionID int64) er log.Warn("DropChannel failed", zap.String("vChannel", channel), zap.Error(err)) return err } - log.Info("DropChannel succeeded", zap.String("vChannel", channel)) + err = h.s.meta.DropChannelCheckpoint(channel) + if err != nil { + log.Warn("DropChannel failed to drop channel checkpoint", zap.String("channel", channel), zap.Error(err)) + return err + } + log.Info("DropChannel succeeded", zap.String("channel", channel)) // Channel checkpoints are cleaned up during garbage collection. // clean collection info cache when meet drop collection info diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index e768011e226dc..2a3d39dec00f9 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -1727,6 +1727,7 @@ func (m *meta) DropChannelCheckpoint(vChannel string) error { return err } delete(m.channelCPs.checkpoints, vChannel) + metrics.DataCoordCheckpointUnixSeconds.DeleteLabelValues(fmt.Sprint(paramtable.GetNodeID()), vChannel) log.Info("DropChannelCheckpoint done", zap.String("vChannel", vChannel)) return nil }