5m is not enough for DM unit tests

[lance6716]

Which jobs are flaking?

github.com/pingcap/tiflow/dm/dm/master TestMaster.TestWaitWorkersReadyV1Import

Which test(s) are flaking?

DM-UT
release-6.1

Jenkins logs or GitHub Actions link

[2022-06-03T03:38:13.699Z] goroutine 22163 [sleep]:

[2022-06-03T03:38:13.699Z] time.Sleep(0x59682f00)

[2022-06-03T03:38:13.699Z] 	/usr/local/go/src/runtime/time.go:194 +0x12e

[2022-06-03T03:38:13.699Z] github.com/pingcap/tiflow/dm/dm/master.(*testMaster).TestWaitWorkersReadyV1Import.func2()

[2022-06-03T03:38:13.699Z] 	/home/jenkins/agent/workspace/cdc_ghpr_test/go/src/github.com/pingcap/tiflow/dm/dm/master/bootstrap_test.go:150 +0x65

[2022-06-03T03:38:13.699Z] created by github.com/pingcap/tiflow/dm/dm/master.(*testMaster).TestWaitWorkersReadyV1Import

[2022-06-03T03:38:13.699Z] 	/home/jenkins/agent/workspace/cdc_ghpr_test/go/src/github.com/pingcap/tiflow/dm/dm/master/bootstrap_test.go:149 +0xa5b

https://ci2.pingcap.net/blue/organizations/jenkins/cdc_ghpr_test/detail/cdc_ghpr_test/16595/pipeline/

Anything else we need to know

• Does this test exist for other branches as well?

• Has there been a high frequency of failure lately?

[lance6716]

another case

https://ci.pingcap.net/blue/organizations/jenkins/ut-check/detail/ut-check/17240/pipeline/

seems an etcd action is still running when timeout

[2022-06-14T07:35:36.042Z] goroutine 22053 [select]:

[2022-06-14T07:35:36.042Z] google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00259ed80)

[2022-06-14T07:35:36.042Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/internal/transport/transport.go:324 +0x10e

[2022-06-14T07:35:36.042Z] google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)

[2022-06-14T07:35:36.042Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/internal/transport/transport.go:339

[2022-06-14T07:35:36.042Z] google.golang.org/grpc.(*csAttempt).recvMsg(0xc0006c1d90, {0x6259c60?, 0xc003321a90}, 0x0)

[2022-06-14T07:35:36.042Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/stream.go:980 +0x13e

[2022-06-14T07:35:36.043Z] google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00259ebec?)

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/stream.go:845 +0x57

[2022-06-14T07:35:36.043Z] google.golang.org/grpc.(*clientStream).withRetry(0xc00259eb40, 0xc001a15a80, 0xc001a15a50)

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/stream.go:709 +0xd7

[2022-06-14T07:35:36.043Z] google.golang.org/grpc.(*clientStream).RecvMsg(0xc00259eb40, {0x6259c60, 0xc003321a90})

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/stream.go:844 +0x186

[2022-06-14T07:35:36.043Z] google.golang.org/grpc.invoke({0x794c750, 0xc001b01980}, {0x63ec653, 0x14}, {0x6259b00, 0xc0010f6150}, {0x6259c60, 0xc003321a90}, 0x0?, {0xc002e7ac80, ...})

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/call.go:73 +0x14e

[2022-06-14T07:35:36.043Z] go.etcd.io/etcd/client/v3.(*Client).unaryClientInterceptor.func1({0x794c718, 0xc003521a40}, {0x63ec653, 0x14}, {0x6259b00, 0xc0010f6150}, {0x6259c60, 0xc003321a90}, 0xc001b61c00, 0x6535be8, ...)

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.2/retry_interceptor.go:58 +0x51c

[2022-06-14T07:35:36.043Z] google.golang.org/grpc.(*ClientConn).Invoke(0xc001b61c00, {0x794c718, 0xc003521a40}, {0x63ec653, 0x14}, {0x6259b00, 0xc0010f6150}, {0x6259c60, 0xc003321a90}, {0xa090240, ...})

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/google.golang.org/grpc@v1.46.2/call.go:35 +0x26c

[2022-06-14T07:35:36.043Z] go.etcd.io/etcd/api/v3/etcdserverpb.(*kVClient).Txn(0xc0012c0238, {0x794c718, 0xc003521a40}, 0x0?, {0xa090240, 0x3, 0x3})

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.2/etcdserverpb/rpc.pb.go:6487 +0xf5

[2022-06-14T07:35:36.043Z] go.etcd.io/etcd/client/v3.(*retryKVClient).Txn(0xc00261fa60, {0x794c718, 0xc003521a40}, 0xc0015ac6e3?, {0xa090240, 0x3, 0x3})

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.2/retry.go:117 +0x96

[2022-06-14T07:35:36.043Z] go.etcd.io/etcd/client/v3.(*txn).Commit(0xc0015ac6c0)

[2022-06-14T07:35:36.043Z] 	/go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.2/txn.go:146 +0x3a4

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/pkg/etcdutil.ThenOpFunc.func1(0xc0030e8fc0, 0xc000ef6c40)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/pkg/etcdutil/etcdutil.go:117 +0xe2

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/pkg/etcdutil.DoTxnWithRepeatable.func1(0xc001a16338?)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/pkg/etcdutil/etcdutil.go:104 +0x49

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/pkg/retry.(*FiniteRetryStrategy).Apply(0x794c6a8, 0xc0030e8fc0, {0xc001a165a8?, 0x16bae5e?, 0x30?, 0x6534820?}, 0xc001a165e8)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/pkg/retry/strategy.go:84 +0xbe

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/pkg/etcdutil.DoTxnWithRepeatable(0xc000ef6c40, 0xc000778d40)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/pkg/etcdutil/etcdutil.go:103 +0x20a

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/pkg/ha.PutSourceCfg(0x5e92a00?, 0xc0033edb80)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/pkg/ha/source.go:37 +0x217

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/dm/master/scheduler.(*Scheduler).addSource(0xc00325f6c0, 0xc0033edb80)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/dm/master/scheduler/scheduler.go:384 +0x1de

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/dm/master/scheduler.(*Scheduler).AddSourceCfg(0xc00325f6c0, 0xc0033edb80)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/dm/master/scheduler/scheduler.go:340 +0xde

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/dm/master.(*testMaster).testMockScheduler(0xc000eab440, {0x794c6a8, 0xc0011332c0}, 0xc000b6c260, 0xc00112a4b0, {0xc002d9e140?, 0x2, 0x2}, {0xc001a17048, 0x2, ...}, ...)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/dm/master/server_test.go:376 +0x6a5

[2022-06-14T07:35:36.043Z] github.com/pingcap/tiflow/dm/dm/master.(*testMaster).TestStartTask(0xc000eab440, 0x16?)

[2022-06-14T07:35:36.043Z] 	/home/jenkins/agent/workspace/ut-check/tiflow/dm/dm/master/server_test.go:882 +0x63e