check_dev panic sometimes, might related to badger

[D3Hunter]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

it appears many times, such as this from 5 days ago
https://do.pingcap.net/jenkins/blue/organizations/jenkins/pingcap%2Ftidb%2Fghpr_check/detail/ghpr_check/19142/pipeline/

fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x80 addr=0x0 pc=0x22e5c85]

goroutine 388904 gp=0xc0072ca380 m=17 mp=0xc001f80008 [running]:
runtime.throw({0x700c860?, 0x2330cd2?})
	/usr/local/go/src/runtime/panic.go:1067 +0x48 fp=0xc047705fa8 sp=0xc047705f78 pc=0x232bbe8
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:884 +0x3c9 fp=0xc047706008 sp=0xc047705fa8 pc=0x232eb29
runtime.fpTracebackPartialExpand(0x0, 0x22c35e5?, {0xc000991200, 0x86, 0x1?})
	/usr/local/go/src/runtime/mprof.go:592 +0x45 fp=0xc047706068 sp=0xc047706008 pc=0x22e5c85
runtime.saveblockevent(0x11026, 0xa, 0x5, 0x3)
	/usr/local/go/src/runtime/mprof.go:563 +0x1a5 fp=0xc0477060b8 sp=0xc047706068 pc=0x22e5b25
sync.event(0xb4c30e0?, 0x28?)
	/usr/local/go/src/runtime/mprof.go:902 +0xc5 fp=0xc0477060e8 sp=0xc0477060b8 pc=0x232a925
runtime.semrelease1(0xc01c6e6c00?, 0x0, 0x1)
	/usr/local/go/src/runtime/sema.go:241 +0x145 fp=0xc047706140 sp=0xc0477060e8 pc=0x2307ba5
sync.runtime_Semrelease(0xc01c6d16d0?, 0xa0?, 0x3ef3426?)
	/usr/local/go/src/runtime/sema.go:90 +0x13 fp=0xc047706168 sp=0xc047706140 pc=0x232e093
sync.(*Mutex).unlockSlow(0xc01a673300?, 0x47706228?)
	/usr/local/go/src/sync/mutex.go:248 +0x9b fp=0xc047706190 sp=0xc047706168 pc=0x2355a7b
sync.(*Mutex).Unlock(...)
	/usr/local/go/src/sync/mutex.go:225
sync.(*Once).doSlow.deferwrap1()
	/usr/local/go/src/sync/once.go:73 +0x2f fp=0xc0477061b0 sp=0xc047706190 pc=0x2355c8f
sync.(*Once).doSlow(0x3239e82ee39b991a?, 0x0?)
	/usr/local/go/src/sync/once.go:78 +0xec fp=0xc047706210 sp=0xc0477061b0 pc=0x2355bec
sync.(*Once).Do(...)
	/usr/local/go/src/sync/once.go:67
github.com/pingcap/badger/table/sstable.(*Table).getIndex(0xc01a673300)
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/table/sstable/table.go:360 +0x65 fp=0xc047706270 sp=0xc047706210 pc=0x3ef32a5
github.com/pingcap/badger/table/sstable.(*Table).pointGet(0xc01a673300, {{0xc01c6ec08c?, 0x25?, 0x25?}, 0xffffffffffffffff?}, 0xd115fd662f6d84b)
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/table/sstable/table.go:243 +0xa5 fp=0xc0477063f0 sp=0xc047706270 pc=0x3ef1925
github.com/pingcap/badger/table/sstable.(*Table).Get(0xc01a673300, {{0xc01c6ec08c?, 0x10?, 0xc05e58e638?}, 0x22c88cb?}, 0x7f1ff81e6958?)
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/table/sstable/table.go:215 +0xa7 fp=0xc0477065b8 sp=0xc0477063f0 pc=0x3ef1147
github.com/pingcap/badger.(*levelHandler).getInTable(0xc0019341c0, {{0xc01c6ec08c?, 0x3ef9357?, 0xc04a7f5b00?}, 0xc0430a2790?}, 0xd115fd662f6d84b, {0x78e3270, 0xc01a673300})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/level_handler.go:330 +0xce fp=0xc047706730 sp=0xc0477065b8 pc=0x3f164ee
github.com/pingcap/badger.(*levelHandler).multiGetLevel0(0xc0019341c0, {0xc01bbdb260, 0x2, 0x0?}, {0xc03c55d4f0?, 0xc05e58e878?, 0x23afbcf?})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/level_handler.go:360 +0x1ea fp=0xc047706840 sp=0xc047706730 pc=0x3f16a4a
github.com/pingcap/badger.(*levelHandler).multiGet(0xc0019341c0, {0xc01bbdb260, 0x2, 0x2})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/level_handler.go:343 +0x53 fp=0xc047706888 sp=0xc047706840 pc=0x3f167f3
github.com/pingcap/badger.(*levelsController).multiGet(0xc001d02c40, {0xc01bbdb260, 0x2, 0x2})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/levels.go:749 +0x73 fp=0xc0477068e8 sp=0xc047706888 pc=0x3f1db53
github.com/pingcap/badger.(*DB).multiGet(0xc001871208, {0xc01bbdb260, 0x2, 0x2})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/db.go:746 +0x188 fp=0xc047706a60 sp=0xc0477068e8 pc=0x3f0df68
github.com/pingcap/badger.(*Txn).MultiGet(0xc047435170, {0xc0537e51d0, 0x2, 0xc05e58ec38?})
	/go/pkg/mod/github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/transaction.go:416 +0x23d fp=0xc047706b40 sp=0xc047706a60 pc=0x3f22c9d
github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv.(*MVCCStore).getDBItems(0xb4b37a0?, 0xc01c6c6c80?, {0xc03c55d4c0, 0x2, 0x0?})
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/unistore/tikv/mvcc.go:174 +0x106 fp=0xc047706b78 sp=0xc047706b40 pc=0x3fbd566
github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv.(*MVCCStore).prewriteOptimistic(0xc001939a00, 0xc01c6c6c80, {0xc03c55d4c0, 0x2, 0x2}, 0xc01c6e9200)
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/unistore/tikv/mvcc.go:791 +0x259 fp=0xc047706c48 sp=0xc047706b78 pc=0x3fc38b9
github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv.(*MVCCStore).Prewrite(0xc001939a00, 0xc01c6c6c80, 0xc01c6e9200)
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/unistore/tikv/mvcc.go:751 +0x1dd fp=0xc047706d68 sp=0xc047706c48 pc=0x3fc337d
github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv.(*Server).KvPrewrite(0xc0018960c0, {0x649c22260c00003?, 0x2a?}, 0xc01c6e9200)
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/unistore/tikv/server.go:355 +0x157 fp=0xc047706e00 sp=0xc047706d68 pc=0x3fd6db7
github.com/pingcap/tidb/pkg/store/mockstore/unistore.(*RPCClient).SendRequest(0xc001d43aa0, {0x78bad40, 0xc01c6d1630}, {0xc001898048, 0x6}, 0xc0221bce00, 0x6fc23ac00)
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/unistore/rpc.go:169 +0x36d fp=0xc047706eb8 sp=0xc047706e00 pc=0x4077dcd
github.com/pingcap/tidb/pkg/store/mockstore.(*clientRedirector).SendRequest(0x1?, {0x78bad40?, 0xc01c6d1630?}, {0xc001898048?, 0x7?}, 0xc01c6e6ae0?, 0x2fb1480?)
	/home/jenkins/agent/workspace/pingcap/tidb/ghpr_check/tidb/pkg/store/mockstore/redirector.go:72 +0x112 fp=0xc047706f10 sp=0xc047706eb8 pc=0x4120352
github.com/tikv/client-go/v2/tikv.(*CodecClient).SendRequest(0xc001d34b00, {0x78bad40, 0xc01c6d1630}, {0xc001898048, 0x6}, 0xc03bf993a0?, 0x6fc23ac00)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/tikv/test_util.go:62 +0x86 fp=0xc047706f60 sp=0xc047706f10 pc=0x3178d26
github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest.func1({0xc001898048?, 0x78bad40?}, 0xc01c6d1630?)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/client/client_interceptor.go:57 +0x43 fp=0xc047706fa8 sp=0xc047706f60 pc=0x2fcb1e3
github.com/tikv/client-go/v2/internal/client.buildResourceControlInterceptor.func1.1({0xc001898048, 0x6}, 0xc0221bcc40)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/client/client_interceptor.go:124 +0x1cd fp=0xc047707058 sp=0xc047706fa8 pc=0x2fcb7ad
github.com/tikv/client-go/v2/internal/client.interceptedClient.SendRequest({{0x78bbab0?, 0xc001d34b00?}}, {0x78bad40, 0xc01c6d1630}, {0xc001898048, 0x6}, 0xc0221bcc40, 0x6fc23ac00)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/client/client_interceptor.go:58 +0x1c2 fp=0xc0477070d8 sp=0xc047707058 pc=0x2fcb082
github.com/tikv/client-go/v2/internal/client.(*interceptedClient).SendRequest(0x78bbb58?, {0x78bad40?, 0xc01c6d1630?}, {0xc001898048?, 0xc001898048?}, 0xc05e58f198?, 0x245be11?)
	<autogenerated>:1 +0x53 fp=0xc047707128 sp=0xc0477070d8 pc=0x2fd0dd3
github.com/tikv/client-go/v2/internal/client.reqCollapse.SendRequest({{0x78bbb58?, 0xc00193c170?}}, {0x78bad40, 0xc01c6d1630}, {0xc001898048, 0x6}, 0xc0221bcc40, 0x6fc23ac00)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/client/client_collapse.go:74 +0xbd fp=0xc047707188 sp=0xc047707128 pc=0x2fca4bd
github.com/tikv/client-go/v2/internal/client.(*reqCollapse).SendRequest(0x30?, {0x78bad40?, 0xc01c6d1630?}, {0xc001898048?, 0xc05e58f250?}, 0x22c88cb?, 0x7f1fe2817688?)
	<autogenerated>:1 +0x53 fp=0xc0477071d8 sp=0xc047707188 pc=0x2fd0833
github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).sendReqToRegion(0xc0365af5e0, 0xc047435050, 0xc04240cb00, 0xc0221bcc40, 0x6fc23ac00)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/locate/region_request.go:1114 +0x8d2 fp=0xc0477075b0 sp=0xc0477071d8 pc=0x2ff9572
github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).SendReqCtx(0xc0365af5e0, 0xc047435050, 0xc0221bcc40, {0x2a, 0x1, 0x2}, 0x6fc23ac00, 0x0, {0x0, 0x0, ...})
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/locate/region_request.go:870 +0x14bb fp=0xc0477079e8 sp=0xc0477075b0 pc=0x2ff5c7b
github.com/tikv/client-go/v2/internal/locate.(*RegionRequestSender).SendReq(...)
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/internal/locate/region_request.go:432
github.com/tikv/client-go/v2/txnkv/transaction.actionPrewrite.handleSingleBatch({0x0?, 0x1?, 0x0?}, 0xc01bfb3000, 0xc047435050, {{0x2a, 0x1, 0x2}, {0x78e0468, 0xc044bbd880}, ...})
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/txnkv/transaction/prewrite.go:300 +0xc27 fp=0xc047707e98 sp=0xc0477079e8 pc=0x3089de7
github.com/tikv/client-go/v2/txnkv/transaction.(*actionPrewrite).handleSingleBatch(0xc047434f30?, 0xc001e19f08?, 0xc001e19ee8?, {{0x2a, 0x1, 0x2}, {0x78e0468, 0xc044bbd880}, 0x0})
	<autogenerated>:1 +0x91 fp=0xc047707ef0 sp=0xc047707e98 pc=0x30994b1
github.com/tikv/client-go/v2/txnkv/transaction.(*batchExecutor).startWorker.func1()
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/txnkv/transaction/2pc.go:2216 +0x106 fp=0xc047707fe0 sp=0xc047707ef0 pc=0x3078a66
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc047707fe8 sp=0xc047707fe0 pc=0x23355e1
created by github.com/tikv/client-go/v2/txnkv/transaction.(*batchExecutor).startWorker in goroutine 388902
	/go/pkg/mod/github.com/tikv/client-go/v2@v2.0.8-0.20241008085809-c3e10ae7c8fc/txnkv/transaction/2pc.go:2199 +0x72

2. What did you expect to see? (Required)

3. What did you see instead (Required)

4. What is your TiDB version? (Required)

master

[lance6716]

maybe similar to golang/go#69629 because the fatal message is also fatal error: unexpected signal during runtime execution

[lance6716]

I'm not sure if we should upgrade github.com/dgryski/go-farm of pingcap/badger every repo to include dgryski/go-farm#21 . Just a guess.

[lance6716]

I have added "release-blocker" because I think release-8.4 is also affected. Let's wait some days to check if master branch has solved this problem