pdutil/backend: enlarge max retry time and fix nested retriable error (…

…#48210) (#48227) close #46950
pingcap · Nov 3, 2023 · a633433 · a633433
1 parent 090f77e
commit a633433
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/br/pkg/lightning/common/retry.go b/br/pkg/lightning/common/retry.go
@@ -103,7 +103,9 @@ func isSingleRetryableError(err error) bool {
 		if nerr.Timeout() {
 			return true
 		}
-		if syscallErr, ok := goerrors.Unwrap(err).(*os.SyscallError); ok {
+		// the error might be nested, such as *url.Error -> *net.OpError -> *os.SyscallError
+		var syscallErr *os.SyscallError
+		if goerrors.As(nerr, &syscallErr) {
 			return syscallErr.Err == syscall.ECONNREFUSED || syscallErr.Err == syscall.ECONNRESET
 		}
 		return false

diff --git a/br/pkg/lightning/common/retry_test.go b/br/pkg/lightning/common/retry_test.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/url"
 	"testing"
 
 	"github.com/go-sql-driver/mysql"
@@ -66,6 +67,9 @@ func TestIsRetryableError(t *testing.T) {
 	_, err := net.Dial("tcp", "localhost:65533")
 	require.Error(t, err)
 	require.True(t, IsRetryableError(err))
+	// wrap net.OpErr inside url.Error
+	urlErr := &url.Error{Op: "post", Err: err}
+	require.True(t, IsRetryableError(urlErr))
 
 	// MySQL Errors
 	require.False(t, IsRetryableError(&mysql.MySQLError{}))

diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go
@@ -52,8 +52,7 @@ const (
 	pauseTimeout         = 5 * time.Minute
 
 	// pd request retry time when connection fail
-	pdRequestRetryTime = 10
-
+	pdRequestRetryTime = 120
 	// set max-pending-peer-count to a large value to avoid scatter region failed.
 	maxPendingPeerUnlimited uint64 = math.MaxInt32
 )
@@ -170,6 +169,7 @@ func pdRequestWithCode(
 	}
 	var resp *http.Response
 	count := 0
+	// the total retry duration: 120*1 = 2min
 	for {
 		resp, err = cli.Do(req) //nolint:bodyclose
 		count++