test(dm): fix unstable tests (#5865) (#5872)

ref #5089, close #5746, close #5793
pingcap · Jun 27, 2022 · 8f2b127 · 8f2b127
1 parent 4babf88
commit 8f2b127
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 8 deletions.
diff --git a/Makefile b/Makefile
@@ -304,7 +304,7 @@ define run_dm_unit_test
 	mkdir -p $(DM_TEST_DIR)
 	$(FAILPOINT_ENABLE)
 	@export log_level=error; \
-	$(GOTEST) -timeout 5m -covermode=atomic -coverprofile="$(DM_TEST_DIR)/cov.unit_test.out" $(1) \
+	$(GOTEST) -timeout 10m -covermode=atomic -coverprofile="$(DM_TEST_DIR)/cov.unit_test.out" $(1) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	$(FAILPOINT_DISABLE)
 endef
@@ -320,7 +320,7 @@ dm_unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/goc
 	mkdir -p $(DM_TEST_DIR)
 	$(FAILPOINT_ENABLE)
 	@export log_level=error; \
-	CGO_ENABLED=1 tools/bin/gotestsum --junitfile dm-junit-report.xml -- -v -timeout 5m -p $(P) --race \
+	CGO_ENABLED=1 tools/bin/gotestsum --junitfile dm-junit-report.xml -- -v -timeout 10m -p $(P) --race \
 	-covermode=atomic -coverprofile="$(DM_TEST_DIR)/cov.unit_test.out" $(DM_PACKAGES) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	tools/bin/gocov convert "$(DM_TEST_DIR)/cov.unit_test.out" | tools/bin/gocov-xml > dm-coverage.xml

diff --git a/dm/dm/master/server.go b/dm/dm/master/server.go
@@ -63,9 +63,6 @@ import (
 )
 
 const (
-	// the session's TTL in seconds for leader election.
-	// NOTE: select this value carefully when adding a mechanism relying on leader election.
-	electionTTL = 60
 	// the DM-master leader election key prefix
 	// DM-master cluster : etcd cluster = 1 : 1 now.
 	electionKey = "/dm-master/leader"
@@ -82,6 +79,10 @@ var (
 
 	useTLS atomic.Bool
 
+	// the session's TTL in seconds for leader election.
+	// NOTE: select this value carefully when adding a mechanism relying on leader election.
+	electionTTL = 60
+
 	// typically there's only one server running in one process, but testMaster.TestOfflineMember starts 3 servers,
 	// so we need sync.Once to prevent data race.
 	registerOnce      sync.Once

diff --git a/dm/dm/master/server_test.go b/dm/dm/master/server_test.go
@@ -161,9 +161,10 @@ var (
 )
 
 type testMaster struct {
-	workerClients   map[string]workerrpc.Client
-	saveMaxRetryNum int
-	testT           *testing.T
+	workerClients     map[string]workerrpc.Client
+	saveMaxRetryNum   int
+	electionTTLBackup int
+	testT             *testing.T
 
 	testEtcdCluster *integration.ClusterV3
 	etcdTestCli     *clientv3.Client
@@ -195,12 +196,15 @@ func (t *testMaster) SetUpSuite(c *check.C) {
 	c.Assert(log.InitLogger(&log.Config{}), check.IsNil)
 	t.workerClients = make(map[string]workerrpc.Client)
 	t.saveMaxRetryNum = maxRetryNum
+	t.electionTTLBackup = electionTTL
+	electionTTL = 3
 	maxRetryNum = 2
 	checkAndAdjustSourceConfigForDMCtlFunc = checkAndNoAdjustSourceConfigMock
 }
 
 func (t *testMaster) TearDownSuite(c *check.C) {
 	maxRetryNum = t.saveMaxRetryNum
+	electionTTL = t.electionTTLBackup
 	checkAndAdjustSourceConfigForDMCtlFunc = checkAndAdjustSourceConfig
 }
 

diff --git a/dm/tests/_utils/wait_process_exit b/dm/tests/_utils/wait_process_exit
@@ -16,5 +16,9 @@ while [ $WAIT_COUNT -lt 120 ]; do
 	((WAIT_COUNT++))
 done
 
+<<<<<<< HEAD
 echo "process $process didn't exit after 120 seconds, current processlist: $(pgrep $process)"
+=======
+echo "process $process didn't exit after 120 seconds, current processlist: $(ps aux | grep $process | grep -v 'grep')"
+>>>>>>> 65e67fc4b (test(dm): fix unstable tests (#5865))
 exit 1