lightning: make OpLevelOptional suppress the error of DoChecksum (pin…

…gcap#45486) (pingcap#45865) close pingcap#45382
ti-chi-bot · Aug 15, 2023 · 95956e2 · 95956e2
1 parent c4b5de9
commit 95956e2
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 8 deletions.
diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go
@@ -43,6 +43,8 @@ import (
 	"github.com/pingcap/tidb/util/mathutil"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 type TableRestore struct {
@@ -800,11 +802,27 @@ func (tr *TableRestore) postProcess(
 				return false, err
 			}
 			err = tr.compareChecksum(remoteChecksum, localChecksum)
+			failpoint.Inject("checksum-error", func() {
+				tr.logger.Info("failpoint checksum-error injected.")
+				remoteChecksum = nil
+				err = status.Error(codes.Unknown, "Compare checksum meets error.")
+			})
 			// with post restore level 'optional', we will skip checksum error
-			if rc.cfg.PostRestore.Checksum == config.OpLevelOptional {
-				if err != nil {
-					tr.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err))
-					err = nil
+			if err != nil {
+				if rc.cfg.PostRestore.Checksum != config.OpLevelOptional {
+					return false, err
+				}
+				tr.logger.Warn("do checksum failed, will skip this error and go on", log.ShortError(err))
+				err = nil
+			}
+			if remoteChecksum != nil {
+				err = tr.compareChecksum(remoteChecksum, localChecksum)
+				// with post restore level 'optional', we will skip checksum error
+				if rc.cfg.PostRestore.Checksum == config.OpLevelOptional {
+					if err != nil {
+						tr.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err))
+						err = nil
+					}
 				}
 			}
 		} else {
@@ -846,11 +864,12 @@ func (tr *TableRestore) postProcess(
 		case forcePostProcess || !rc.cfg.PostRestore.PostProcessAtLast:
 			err := tr.analyzeTable(ctx, rc.tidbGlue.GetSQLExecutor())
 			// witch post restore level 'optional', we will skip analyze error
-			if rc.cfg.PostRestore.Analyze == config.OpLevelOptional {
-				if err != nil {
-					tr.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err))
-					err = nil
+			if err != nil {
+				if rc.cfg.PostRestore.Analyze != config.OpLevelOptional {
+					return false, err
 				}
+				tr.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err))
+				err = nil
 			}
 			saveCpErr := rc.saveStatusCheckpoint(ctx, tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusAnalyzed)
 			if err = firstErr(err, saveCpErr); err != nil {

diff --git a/br/tests/lightning_routes/config.toml b/br/tests/lightning_routes/config.toml
@@ -8,3 +8,6 @@ schema-pattern = "routes_a*"
 table-pattern = "t*"
 target-schema = "routes_b"
 target-table = "u"
+
+[post-restore]
+checksum = "optional"
diff --git a/br/tests/lightning_routes/run.sh b/br/tests/lightning_routes/run.sh
@@ -4,12 +4,17 @@
 
 set -eux
 
+echo "testing checksum-error..."
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/lightning/restore/checksum-error=1*return()"
+
 run_sql 'DROP DATABASE IF EXISTS routes_a0;'
 run_sql 'DROP DATABASE IF EXISTS routes_a1;'
 run_sql 'DROP DATABASE IF EXISTS routes_b;'
 
 run_lightning
 
+echo "test checksum-error success!"
+
 run_sql 'SELECT count(1), sum(x) FROM routes_b.u;'
 check_contains 'count(1): 4'
 check_contains 'sum(x): 259'