testing: Switch upgrade test to the testnet fixture

Author: maru-ava

.github/workflows/test.upgrade.yml

@@ -28,4 +28,7 @@ jobs:
         run: ./scripts/build.sh
       - name: Run upgrade tests
         shell: bash
-        run: scripts/tests.upgrade.sh 1.10.1 ./build/avalanchego
+        # 1.10.7 is the first version compatible with the testnet
+        # fixture by virtue of writing a process context file on node
+        # start.
+        run: scripts/tests.upgrade.sh 1.10.7 ./build/avalanchego

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/DataDog/zstd v1.5.2
 	github.com/Microsoft/go-winio v0.5.2
 	github.com/NYTimes/gziphandler v1.1.1
-	github.com/ava-labs/avalanche-network-runner-sdk v0.3.0
 	github.com/ava-labs/coreth v0.12.5-rc.1.0.20230814185807-00214b143a8a
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20230105152938-00a24d05a8c7
 	github.com/btcsuite/btcd/btcutil v1.1.3

go.sum

@@ -61,8 +61,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/ava-labs/avalanche-network-runner-sdk v0.3.0 h1:TVi9JEdKNU/RevYZ9PyW4pULbEdS+KQDA9Ki2DUvuAs=
-github.com/ava-labs/avalanche-network-runner-sdk v0.3.0/go.mod h1:SgKJvtqvgo/Bl/c8fxEHCLaSxEbzimYfBopcfrajxQk=
 github.com/ava-labs/coreth v0.12.5-rc.1.0.20230814185807-00214b143a8a h1:WNZDFqolp4rumlJVzJ92hl9VDkL904yIQFd7CmBr3rk=
 github.com/ava-labs/coreth v0.12.5-rc.1.0.20230814185807-00214b143a8a/go.mod h1:K7Xm2jqx90wxKZXfLvkLEL+zlM5843gGq9XkqVDwKds=
 github.com/ava-labs/ledger-avalanche/go v0.0.0-20230105152938-00a24d05a8c7 h1:EdxD90j5sClfL5Ngpz2TlnbnkNYdFPDXa0jDOjam65c=

scripts/install_anr.sh

@@ -52,11 +52,6 @@ elif [[ ${GOOS} == "darwin" ]]; then
 fi
 find /tmp/avalanchego-v${VERSION}
 
-#################################
-echo "installing avalanche-network-runner"
-ANR_WORKDIR="/tmp"
-./scripts/install_anr.sh
-
 # Sourcing constants.sh ensures that the necessary CGO flags are set to
 # build the portable version of BLST. Without this, ginkgo may fail to
 # build the test binary if run on a host (e.g. github worker) that lacks
@@ -70,31 +65,14 @@ go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
 ACK_GINKGO_RC=true ginkgo build ./tests/upgrade
 ./tests/upgrade/upgrade.test --help
 
-#################################
-# run "avalanche-network-runner" server
-echo "launch avalanche-network-runner in the background"
-$ANR_WORKDIR/avalanche-network-runner \
-  server \
-  --log-level debug \
-  --port=":12340" \
-  --disable-grpc-gateway &
-PID=${!}
-
 #################################
 # By default, it runs all upgrade test cases!
 echo "running upgrade tests against the local cluster with ${NEW_BINARY}"
 ./tests/upgrade/upgrade.test \
   --ginkgo.v \
-  --log-level debug \
-  --network-runner-grpc-endpoint="0.0.0.0:12340" \
-  --network-runner-avalanchego-path=/tmp/avalanchego-v${VERSION}/avalanchego \
-  --network-runner-avalanchego-path-to-upgrade=${NEW_BINARY} \
-  --network-runner-avalanchego-log-level="WARN" || EXIT_CODE=$?
-
-# "e2e.test" already terminates the cluster
-# just in case tests are aborted, manually terminate them again
-pkill -P ${PID} || true
-kill -2 ${PID}
+  --avalanchego-path=/tmp/avalanchego-v${VERSION}/avalanchego \
+  --avalanchego-path-to-upgrade-to=${NEW_BINARY} \
+  || EXIT_CODE=$?
 
 if [[ "${EXIT_CODE:-}" -gt 0 ]]; then
   echo "FAILURE with exit code ${EXIT_CODE}"

scripts/tests.upgrade.sh

@@ -8,6 +8,9 @@ import (
 	"context"
 	"encoding/json"
 	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
@@ -111,3 +114,54 @@ func (te *TestEnvironment) NewWallet(keychain *secp256k1fx.Keychain) primary.Wal
 	te.require.NoError(err)
 	return wallet
 }
+
+// Start a local test-managed network with the provided avalanchego binary.
+func StartLocalNetwork(avalancheGoExecPath string, archiveNetworkDir bool) *local.LocalNetwork {
+	require := require.New(ginkgo.GinkgoT())
+
+	tests.Outf("{{magenta}}Starting network with %q{{/}}\n", avalancheGoExecPath)
+
+	var networkRootDir string
+	if archiveNetworkDir {
+		// Since the network dir will be archived on teardown, cleaning up the
+		// directory won't prevent troubleshooting.
+		networkRootDir = ginkgo.GinkgoT().TempDir()
+	} else {
+		// Use a temp directory that will not be removed on teardown to aid in
+		// troubleshooting.
+		var err error
+		networkRootDir, err = os.MkdirTemp("", "networks")
+		require.NoError(err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), local.DefaultNetworkStartTimeout)
+	defer cancel()
+	network, err := local.StartNetwork(
+		ctx,
+		ginkgo.GinkgoWriter,
+		networkRootDir,
+		&local.LocalNetwork{
+			LocalConfig: local.LocalConfig{
+				ExecPath: avalancheGoExecPath,
+			},
+		},
+		testnet.DefaultNodeCount,
+		testnet.DefaultFundedKeyCount,
+	)
+	require.NoError(err)
+	ginkgo.DeferCleanup(func() {
+		tests.Outf("Shutting down network\n")
+		require.NoError(network.Stop())
+		if archiveNetworkDir {
+			scriptPath, err := filepath.Abs("../../scripts/archive_network_dir.sh")
+			require.NoError(err)
+			output, err := exec.Command(scriptPath, network.Dir).Output()
+			require.NoError(err)
+			tests.Outf("%s", output)
+		}
+	})
+
+	tests.Outf("{{green}}Successfully started network{{/}}\n")
+
+	return network
+}

tests/e2e/e2e.go

@@ -4,13 +4,10 @@
 package e2e_test
 
 import (
-	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
-	"os/exec"
-	"path/filepath"
 	"testing"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
@@ -22,7 +19,6 @@ import (
 	"github.com/ava-labs/avalanchego/tests"
 	"github.com/ava-labs/avalanchego/tests/e2e"
 	"github.com/ava-labs/avalanchego/tests/fixture"
-	"github.com/ava-labs/avalanchego/tests/fixture/testnet"
 	"github.com/ava-labs/avalanchego/tests/fixture/testnet/local"
 
 	// ensure test packages are scanned by ginkgo
@@ -97,37 +93,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		network, err = local.ReadNetwork(persistentNetworkDir)
 		require.NoError(err)
 	} else {
-		tests.Outf("{{magenta}}Starting network with %q{{/}}\n", avalancheGoExecPath)
-
-		ctx, cancel := context.WithTimeout(context.Background(), local.DefaultNetworkStartTimeout)
-		defer cancel()
-		var err error
-		network, err = local.StartNetwork(
-			ctx,
-			ginkgo.GinkgoWriter,
-			ginkgo.GinkgoT().TempDir(),
-			&local.LocalNetwork{
-				LocalConfig: local.LocalConfig{
-					ExecPath: avalancheGoExecPath,
-				},
-			},
-			testnet.DefaultNodeCount,
-			testnet.DefaultFundedKeyCount,
-		)
-		require.NoError(err)
-		ginkgo.DeferCleanup(func() {
-			tests.Outf("Shutting down network\n")
-			require.NoError(network.Stop())
-			if archiveNetworkDirOnTeardown {
-				scriptPath, err := filepath.Abs("../../scripts/archive_network_dir.sh")
-				require.NoError(err)
-				output, err := exec.Command(scriptPath, network.Dir).Output()
-				require.NoError(err)
-				tests.Outf("%s", output)
-			}
-		})
-
-		tests.Outf("{{green}}Successfully started network{{/}}\n")
+		network = e2e.StartLocalNetwork(avalancheGoExecPath, archiveNetworkDirOnTeardown)
 	}
 
 	uris := network.GetURIs()

tests/e2e/e2e_test.go

@@ -632,3 +632,22 @@ func (ln *LocalNetwork) ReadAll() error {
 	}
 	return ln.ReadNodes()
 }
+
+func (ln *LocalNetwork) GetBootstrapIPsAndIds() ([]string, []string, error) {
+	// Collect staking addresses of running nodes for use in bootstrapping a node
+	if err := ln.ReadNodes(); err != nil {
+		return nil, nil, fmt.Errorf("failed to read local network nodes: %w", err)
+	}
+	bootstrapIPs := make([]string, 0, len(ln.Nodes))
+	bootstrapIDs := make([]string, 0, len(ln.Nodes))
+	for _, node := range ln.Nodes {
+		if len(node.StakingAddress) == 0 {
+			// Node is not running
+			continue
+		}
+
+		bootstrapIPs = append(bootstrapIPs, node.StakingAddress)
+		bootstrapIDs = append(bootstrapIDs, node.NodeID.String())
+	}
+	return bootstrapIPs, bootstrapIDs, nil
+}

tests/fixture/testnet/local/network.go

@@ -311,6 +311,31 @@ func (n *LocalNode) IsHealthy(ctx context.Context) (bool, error) {
 	return false, err
 }
 
+// WaitForHealthy blocks until IsHealthy returns true or an error (including context timeout) is observed.
+func (n *LocalNode) WaitForHealthy(ctx context.Context) error {
+	if _, ok := ctx.Deadline(); !ok {
+		return fmt.Errorf("unable to wait for health for node %q with a context without a deadline", n.NodeID)
+	}
+	ticker := time.NewTicker(DefaultNodeTickerInterval)
+	defer ticker.Stop()
+
+	for {
+		healthy, err := n.IsHealthy(ctx)
+		if err != nil && !errors.Is(err, errProcessNotRunning) {
+			return fmt.Errorf("failed to wait for health of node %q: %w", n.NodeID, err)
+		}
+		if healthy {
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("failed to wait for health of node %q before timeout: %w", n.NodeID, ctx.Err())
+		case <-ticker.C:
+		}
+	}
+}
+
 func (n *LocalNode) WaitForProcessContext(ctx context.Context) error {
 	ticker := time.NewTicker(DefaultNodeTickerInterval)
 	defer ticker.Stop()