Skip to content

Commit

Permalink
Add config for disable BF load
Browse files Browse the repository at this point in the history
Signed-off-by: xiaofan-luan <xiaofan.luan@zilliz.com>
  • Loading branch information
xiaofan-luan committed Mar 26, 2023
1 parent cb11abe commit 70d602b
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 0 deletions.
6 changes: 6 additions & 0 deletions internal/querynode/segment.go
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,12 @@ func (s *Segment) InitCurrentStat() {
func (s *Segment) isPKExist(pk primaryKey) bool {
s.statLock.Lock()
defer s.statLock.Unlock()

if Params.DataNodeCfg.SkipBFStatsLoad {
log.Warn("processing delete while skip load BF, may affect performance", zap.Any("pk", pk), zap.Int64("segmentID", s.segmentID))
return true
}

if s.currentStat != nil && s.currentStat.PkExist(pk) {
return true
}
Expand Down
5 changes: 5 additions & 0 deletions internal/querynode/segment_loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,11 @@ func (loader *segmentLoader) loadSegmentBloomFilter(ctx context.Context, segment
return nil
}

if Params.DataNodeCfg.SkipBFStatsLoad {
log.Info("skip load BF with config set ", zap.Int64("segmentID", segment.segmentID))
return nil
}

startTs := time.Now()
values, err := loader.cm.MultiRead(ctx, binlogPaths)
if err != nil {
Expand Down
8 changes: 8 additions & 0 deletions internal/util/paramtable/component_param.go
Original file line number Diff line number Diff line change
Expand Up @@ -1567,6 +1567,8 @@ type dataNodeConfig struct {
// datanote send timetick interval per channel
DataNodeTimeTickInterval int

SkipBFStatsLoad bool

CreatedTime time.Time
UpdatedTime time.Time

Expand All @@ -1589,6 +1591,8 @@ func (p *dataNodeConfig) init(base *BaseTable) {
p.initIOConcurrency()
p.initDataNodeTimeTickInterval()

p.initSkipBFStatsLoad()

p.initChannelWatchPath()
p.initMemoryForceSyncEnable()
p.initMemoryWatermark()
Expand Down Expand Up @@ -1653,6 +1657,10 @@ func (p *dataNodeConfig) initDataNodeTimeTickInterval() {
p.DataNodeTimeTickInterval = p.Base.ParseIntWithDefault("datanode.timetick.interval", 500)
}

func (p *dataNodeConfig) initSkipBFStatsLoad() {
p.SkipBFStatsLoad = p.Base.ParseBool("dataNode.skip.BFStats.Load", false)
}

func (p *dataNodeConfig) SetNodeID(id UniqueID) {
p.NodeID.Store(id)
}
Expand Down

0 comments on commit 70d602b

Please sign in to comment.