Skip to content

Commit

Permalink
disttask/ddl: fail task for panic (pingcap#48135)
Browse files Browse the repository at this point in the history
  • Loading branch information
ywqzzy authored and ti-chi-bot committed Nov 1, 2023
1 parent 8524952 commit 5c2c01a
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 0 deletions.
17 changes: 17 additions & 0 deletions pkg/ddl/backfilling_operators.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
util2 "github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/disttask/operator"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
Expand All @@ -45,6 +46,7 @@ import (
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/memory"
Expand Down Expand Up @@ -402,6 +404,13 @@ type tableScanWorker struct {
}

func (w *tableScanWorker) HandleTask(task TableScanTask, sender func(IndexRecordChunk)) {
defer tidbutil.Recover(metrics.LblAddIndex, "handleTableScanTaskWithRecover", func() {
w.ctx.onError(errors.New("met panic in tableScanWorker"))
}, false)

failpoint.Inject("injectPanicForTableScan", func() {
panic("mock panic")
})
if w.se == nil {
sessCtx, err := w.sessPool.Get()
if err != nil {
Expand Down Expand Up @@ -604,6 +613,14 @@ func (w *indexIngestWorker) HandleTask(rs IndexRecordChunk, send func(IndexWrite
w.srcChunkPool <- rs.Chunk
}
}()
defer tidbutil.Recover(metrics.LblAddIndex, "handleIndexIngtestTaskWithRecover", func() {
w.ctx.onError(errors.New("met panic in indexIngestWorker"))
}, false)

failpoint.Inject("injectPanicForIndexIngest", func() {
panic("mock panic")
})

result := IndexWriteResult{
ID: rs.ID,
}
Expand Down
9 changes: 9 additions & 0 deletions tests/realtikvtest/addindextest1/disttask_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ func TestAddIndexDistBasic(t *testing.T) {
tk.MustExec("alter table t1 add index idx1(a);")
tk.MustExec("admin check index t1 idx1;")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/MockRunSubtaskContextCanceled"))

require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan", "return()"))
tk.MustExecToErr("alter table t1 add index idx2(a);")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan"))

require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest", "return()"))
tk.MustExecToErr("alter table t1 add index idx2(a);")
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest"))

tk.MustExec(`set global tidb_enable_dist_task=0;`)
}

Expand Down

0 comments on commit 5c2c01a

Please sign in to comment.