Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

statistics: optimize stats delta dumping with batch processing #58791

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open
Prev Previous commit
Next Next commit
statistics: add logging for slow historical stats recording and dumpi…
…ng operations

Signed-off-by: Rustin170506 <techregister@pm.me>
  • Loading branch information
Rustin170506 committed Jan 9, 2025
commit a035e98e65a28aa02eeb5f30e7f270b579daac69
15 changes: 15 additions & 0 deletions pkg/statistics/handle/usage/session_stats_collect.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ func (s *statsUsageImpl) DumpStatsDeltaToKV(dumpAll bool) error {
return errors.Trace(err)
}

startRecordHistoricalStatsMeta := time.Now()
// Record historical stats meta for all tables one by one.
// FIXME: Although this feature is currently disabled, it would be beneficial to implement it in batches for efficiency.
for _, update := range batchUpdates {
Expand All @@ -189,6 +190,11 @@ func (s *statsUsageImpl) DumpStatsDeltaToKV(dumpAll bool) error {
s.statsHandle.RecordHistoricalStatsMeta(update.TableID, statsVersion, "flush stats", false)
}
}
if time.Since(startRecordHistoricalStatsMeta) > tooSlowThreshold {
statslogutil.SingletonStatsSamplerLogger().Warn("Recording historical stats meta is too slow",
zap.Int("tableCount", len(batchUpdates)),
zap.Duration("duration", time.Since(startRecordHistoricalStatsMeta)))
}
}

return nil
Expand All @@ -211,6 +217,15 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV(
if len(updates) == 0 {
return 0, nil
}
const queryTooSlowThreshold = 5 * time.Second
start := time.Now()
defer func() {
if time.Since(start) > queryTooSlowThreshold {
statslogutil.SingletonStatsSamplerLogger().Warn("Dumping stats delta to KV query is too slow",
zap.Int("tableCount", len(updates)),
zap.Duration("duration", time.Since(start)))
}
}()
statsVersion, err = utilstats.GetStartTS(sctx)
if err != nil {
return 0, errors.Trace(err)
Expand Down