Fix partition recovery tests (#2820)

The expected round number is captured before the node stops. However,
it is likely that the node advances to the next round before it is
stopped. When this happens, the test will fail.

This change gets the most up-to-date round number after the node is
stopped, but before inducePartitionTime timeout is waited.

inducePartitionTime is the wait to make sure the expected behavior is
obtained. The round number is captured before this wait.

However, I could not identify in this PR why TestBasicPartitionRecovery has failed. Could not find anything in the test, and the failure logs have nothing.
I suspect that the failure in the other tests triggered the failure, and fixed the other tests, but cannot be sure.

As for the data race, it is fixed in #2844.

Fixes #2384 and #2545

Author: algonautshant

test/e2e-go/features/partitionRecovery/partitionRecovery_test.go

@@ -135,6 +135,10 @@ func runTestWithStaggeredStopStart(t *testing.T, fixture *fixtures.RestClientFix
 	// Stop Node1
 	nc1.FullStop()
 
+	status, err := fixture.LibGoalClient.Status()
+	a.NoError(err)
+	roundAfterStop := status.LastRound
+
 	time.Sleep(inducePartitionTime)
 
 	// Use the fixture to start the node again so it supplies the correct peer addresses
@@ -152,10 +156,10 @@ func runTestWithStaggeredStopStart(t *testing.T, fixture *fixtures.RestClientFix
 	a.NoError(err)
 
 	// Now wait for us to make progress again.
-	status, err := fixture.LibGoalClient.Status()
+	status, err = fixture.LibGoalClient.Status()
 	a.NoError(err)
 
-	a.Equal(waitForRound, status.LastRound, "We should not have made progress since stopping the first node")
+	a.Equal(roundAfterStop, status.LastRound, "We should not have made progress since stopping the first node")
 
 	err = fixture.WaitForRound(status.LastRound+1, partitionRecoveryTime)
 	a.NoError(err)
@@ -193,6 +197,10 @@ func TestBasicPartitionRecoveryPartOffline(t *testing.T) {
 	// Stop Node1
 	nc1.FullStop()
 
+	status, err := fixture.LibGoalClient.Status()
+	a.NoError(err)
+	roundAfterStop := status.LastRound
+
 	// Stop the 2nd node and give network a chance to stall
 	nc2, err := fixture.GetNodeController("Node2")
 	a.NoError(err)
@@ -205,10 +213,10 @@ func TestBasicPartitionRecoveryPartOffline(t *testing.T) {
 	a.NoError(err)
 
 	// Now wait for us to make progress again.
-	status, err := fixture.LibGoalClient.Status()
+	status, err = fixture.LibGoalClient.Status()
 	a.NoError(err)
 
-	a.Equal(waitForRound, status.LastRound, "We should not have made progress since stopping the first node")
+	a.Equal(roundAfterStop, status.LastRound, "We should not have made progress since stopping the first node")
 
 	err = fixture.WaitForRound(status.LastRound+1, partitionRecoveryTime)
 	a.NoError(err)
@@ -253,12 +261,16 @@ func TestPartitionHalfOffline(t *testing.T) {
 	a.NoError(err)
 	nc3.FullStop()
 
+	status, err := client.Status()
+	a.NoError(err)
+	roundAfterStop := status.LastRound
+
 	time.Sleep(inducePartitionTime)
 
 	// Get main client to monitor
-	status, err := client.Status()
+	status, err = client.Status()
 	a.NoError(err)
-	a.Equal(waitForRound, status.LastRound, "We should not have made progress since stopping the nodes")
+	a.Equal(roundAfterStop, status.LastRound, "We should not have made progress since stopping the nodes")
 
 	// Start 40 of 50% of the stake
 	_, err = fixture.StartNode(nc1.GetDataDir())