cdc: fixes minor bugs #10168 and #10169 (#10170) (#10191)

close #10168
pingcap · Dec 4, 2023 · 7b2f258 · 7b2f258
1 parent 8c006b3
commit 7b2f258
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 5 deletions.
diff --git a/cdc/kv/client.go b/cdc/kv/client.go
@@ -73,6 +73,8 @@ const (
 	// failed region will be reloaded via `BatchLoadRegionsWithKeyRange` API. So we
 	// don't need to force reload region anymore.
 	regionScheduleReload = false
+
+	scanRegionsConcurrency = 1024
 )
 
 // time interval to force kv client to terminate gRPC stream and reconnect
@@ -431,6 +433,8 @@ func (s *eventFeedSession) eventFeed(ctx context.Context) error {
 	g.Go(func() error { return s.logSlowRegions(ctx) })
 
 	g.Go(func() error {
+		g, ctx := errgroup.WithContext(ctx)
+		g.SetLimit(scanRegionsConcurrency)
 		for {
 			select {
 			case <-ctx.Done():

diff --git a/cdc/kv/shared_client.go b/cdc/kv/shared_client.go
@@ -244,7 +244,7 @@ func (s *SharedClient) Run(ctx context.Context) error {
 		s.workers = append(s.workers, worker)
 	}
 
-	g.Go(func() error { return s.handleRequestRanges(ctx, g) })
+	g.Go(func() error { return s.handleRequestRanges(ctx) })
 	g.Go(func() error { return s.dispatchRequest(ctx) })
 	g.Go(func() error { return s.requestRegionToStore(ctx, g) })
 	g.Go(func() error { return s.handleErrors(ctx) })
@@ -427,7 +427,9 @@ func (s *SharedClient) broadcastRequest(r *requestedStore, sri singleRegionInfo)
 	}
 }
 
-func (s *SharedClient) handleRequestRanges(ctx context.Context, g *errgroup.Group) error {
+func (s *SharedClient) handleRequestRanges(ctx context.Context) error {
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(scanRegionsConcurrency)
 	for {
 		select {
 		case <-ctx.Done():

diff --git a/cdc/processor/sinkmanager/tasks.go b/cdc/processor/sinkmanager/tasks.go
@@ -39,8 +39,8 @@ var (
 	maxUpdateIntervalSize = defaultMaxUpdateIntervalSize
 
 	// Sink manager schedules table tasks based on lag. Limit the max task range
-	// can be helpful to reduce changefeed latency.
-	maxTaskTimeRange = 5 * time.Second
+	// can be helpful to reduce changefeed latency for large initial data.
+	maxTaskTimeRange = 30 * time.Minute
 )
 
 // Used to record the progress of the table.

diff --git a/cdc/processor/sinkmanager/tasks_test.go b/cdc/processor/sinkmanager/tasks_test.go
@@ -37,7 +37,7 @@ func TestValidateAndAdjustBound(t *testing.T) {
 				StartTs:  439333515018895365,
 				CommitTs: 439333515018895366,
 			},
-			taskTimeRange: 10 * time.Second,
+			taskTimeRange: 60 * time.Minute,
 			expectAdjust:  true,
 		},
 		{