Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tablet throttler: only start watching SrvKeyspace once it's confirmed to exist #13384

Merged
merged 1 commit into from
Jun 28, 2023
Merged

Tablet throttler: only start watching SrvKeyspace once it's confirmed to exist #13384

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Tablet throttler: only start watching SrvKeyspace once it's confirmed… 
… to exist

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
  • Loading branch information
@shlomi-noach
shlomi-noach committed Jun 27, 2023
commit 9911cf3ca1abbd59ff7f204b1e618aee85764479
19 changes: 11 additions & 8 deletions go/vt/vttablet/tabletserver/throttle/throttler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,11 @@ type Throttler struct {

lastCheckTimeNano int64

initMutex sync.Mutex
enableMutex sync.Mutex
cancelEnableContext context.CancelFunc
throttledAppsMutex sync.Mutex
initMutex sync.Mutex
enableMutex sync.Mutex
cancelEnableContext context.CancelFunc
throttledAppsMutex sync.Mutex
watchSrvKeyspaceOnce sync.Once

nonLowPriorityAppRequestsThrottled *cache.Cache
httpClient *http.Client
Expand Down Expand Up @@ -246,10 +247,6 @@ func (throttler *Throttler) initThrottleTabletTypes() {
func (throttler *Throttler) InitDBConfig(keyspace, shard string) {
throttler.keyspace = keyspace
throttler.shard = shard

if throttlerConfigViaTopo {
throttler.srvTopoServer.WatchSrvKeyspace(context.Background(), throttler.cell, throttler.keyspace, throttler.WatchSrvKeyspaceCallback)
}
}

func (throttler *Throttler) GetMetricsQuery() string {
Expand Down Expand Up @@ -479,8 +476,14 @@ func (throttler *Throttler) Open() error {
throttler.initMutex.Lock()
defer throttler.initMutex.Unlock()
throttler.applyThrottlerConfig(ctx, throttlerConfig) // may issue an Enable
go throttler.watchSrvKeyspaceOnce.Do(func() {
// We start watching SrvKeyspace only after we know it's been created. Now is that time!
throttler.srvTopoServer.WatchSrvKeyspace(context.Background(), throttler.cell, throttler.keyspace, throttler.WatchSrvKeyspaceCallback)
})
return
}
// It's possible, especially in CI, that this throttler opened before the SrvKeyspace entry is created in topo.
// We thus retry until the entry is found.
log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): error reading throttler config. Will retry in %v. Err=%+v", retryInterval, err)
<-retryTicker.C
}
Expand Down