e2e: Add local network fixture

To better enable e2e test development, this changeset adds a new
orchestration mechanism for local test networks. Nodes are launched as
independent processes and all configuration is serialized to disk to
avoid requiring a persistent rpc daemon.

Please review the README changes for more details.

Author: maru-ava

go.mod

@@ -39,6 +39,8 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spaolacci/murmur3 v1.1.0
+	github.com/spf13/cast v1.5.0
+	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.8.1
@@ -101,6 +103,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -122,7 +125,6 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sanity-io/litter v1.5.1 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/subosito/gotenv v1.3.0 // indirect

go.sum

@@ -27,26 +27,6 @@ export CGO_CFLAGS="-O -D__BLST_PORTABLE__"
 # clear error due to the default value change in go1.20.
 export CGO_ENABLED=1
 
-#################################
-# download avalanche-network-runner
-# https://github.com/ava-labs/avalanche-network-runner
-# TODO: migrate to upstream avalanche-network-runner
-GOARCH=$(go env GOARCH)
-GOOS=$(go env GOOS)
-NETWORK_RUNNER_VERSION=1.3.5-rc.0
-DOWNLOAD_PATH=/tmp/avalanche-network-runner.tar.gz
-DOWNLOAD_URL="https://github.com/ava-labs/avalanche-network-runner/releases/download/v${NETWORK_RUNNER_VERSION}/avalanche-network-runner_${NETWORK_RUNNER_VERSION}_${GOOS}_${GOARCH}.tar.gz"
-
-rm -f ${DOWNLOAD_PATH}
-rm -f /tmp/avalanche-network-runner
-
-echo "downloading avalanche-network-runner ${NETWORK_RUNNER_VERSION} at ${DOWNLOAD_URL} to ${DOWNLOAD_PATH}"
-curl --fail -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH}
-
-echo "extracting downloaded avalanche-network-runner"
-tar xzvf ${DOWNLOAD_PATH} -C /tmp
-/tmp/avalanche-network-runner -h
-
 GOPATH="$(go env GOPATH)"
 PATH="${GOPATH}/bin:${PATH}"
 
@@ -57,29 +37,14 @@ go install -v github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
 ACK_GINKGO_RC=true ginkgo build ./tests/e2e
 ./tests/e2e/e2e.test --help
 
-#################################
-# run "avalanche-network-runner" server
-echo "launch avalanche-network-runner in the background"
-/tmp/avalanche-network-runner \
-server \
---log-level debug \
---port=":12342" \
---disable-grpc-gateway &
-PID=${!}
-
 #################################
 echo "running e2e tests against the local cluster with ${AVALANCHEGO_PATH}"
 ./tests/e2e/e2e.test \
 --ginkgo.v \
---log-level debug \
---network-runner-grpc-endpoint="0.0.0.0:12342" \
---network-runner-avalanchego-path=${AVALANCHEGO_PATH} \
---network-runner-avalanchego-log-level="WARN" \
+--avalanchego-path=${AVALANCHEGO_PATH} \
 --test-keys-file=tests/test.insecure.secp256k1.keys \
 && EXIT_CODE=$? || EXIT_CODE=$?
 
-kill ${PID}
-
 if [[ ${EXIT_CODE} -gt 0 ]]; then
   echo "FAILURE with exit code ${EXIT_CODE}"
   exit ${EXIT_CODE}

scripts/tests.e2e.sh

@@ -1,6 +1,6 @@
 # Avalanche e2e test suites
 
-- Works for any environments (e.g., local, test network).
+- Works with fixture-managed networks (currently only local, with kubernetes a future target)
 - Compiles to a single binary with customizable configurations.
 
 ## Running tests
@@ -11,7 +11,6 @@ ACK_GINKGO_RC=true ginkgo build ./tests/e2e
 ./tests/e2e/e2e.test --help
 
 ./tests/e2e/e2e.test \
---network-runner-grpc-endpoint="0.0.0.0:12340" \
 --avalanchego-path=./build/avalanchego
 ```
 
@@ -29,8 +28,47 @@ Create a new package to implement feature-specific tests, or add tests to an exi
     ├── README.md
     ├── e2e.go
     ├── e2e_test.go
-    └── ping
-        └── suites.go
+    └── x
+        └── transfer.go
+            └── virtuous.go
 ```
 
-`e2e.go` defines common configurations (e.g., network-runner client) for other test packages. `ping/suites.go` defines ping tests, annotated by `[Ping]`, which can be selected by `./tests/e2e/e2e.test --ginkgo.focus "\[Local\] \[Ping\]"`.
+`e2e.go` defines common configuration for other test packages. `x/transfer/virtuous.go` defines X-Chain transfer tests, labeled with `x`, which can be selected by `./tests/e2e/e2e.test --ginkgo.label-filter "x"`.
+
+## Testing against a persistent network
+
+By default, a new ephemeral test network will be started before each
+test run. When developing e2e tests, it may be helpful to create a
+persistent test network to test against. This can increase the speed
+of iteration by removing the requirement to start a new network for
+every invocation of the test under development.
+
+To use a persistent network:
+
+```bash
+# From the root of the avalanchego repo
+
+# Build the avctl binary
+$ go build -o build/avctl ./tests/e2e/fixture/cmd/*.go  && avctl start
+
+# Start a new network
+$ ./build/avctl start-network --avalanchego-path=/path/to/avalanchego
+...
+Started network 1337 @ /home/me/.avctl/1337-20230705-213938-1707053131
+
+Configure avctl to target this network by default with one of the following statements:
+ - source /home/me/.avctl/1337-20230705-213938-1707053131/network.env
+ - export AVCTL_NETWORK_DIR=/home/me/.avctl/1337-20230705-213938-1707053131
+
+# Start a new test run to use the persistent network
+ginkgo build ./tests/e2e/ && \
+  ./tests/e2e/e2e.test \
+    --avalanchego-path=/path/to/avalanchego \
+    --test-keys-file=$PWD/tests/test.insecure.secp256k1.keys \
+    --network-dir=/path/to/network \
+    --ginkgo-focus-file=[name of file containing test] \
+    --use-persistent-network
+
+# Also possible to set the AVALANCHEGO_PATH env var instead of supplying --avalanchego-path
+# Also possible to set the AVCTL_NETWORK_DIR env var instead of supplying --network-dir
+```

tests/e2e/README.md

@@ -24,21 +24,19 @@ import (
 )
 
 var _ = ginkgo.Describe("[Banff]", func() {
+	ginkgo.BeforeEach(func() {
+		e2e.Env.EnsurePristineNetwork()
+	})
+
 	ginkgo.It("can send custom assets X->P and P->X",
 		// use this for filtering tests by labels
 		// ref. https://onsi.github.io/ginkgo/#spec-labels
 		ginkgo.Label(
-			"require-network-runner",
 			"xp",
 			"banff",
 		),
 		func() {
-			ginkgo.By("reload initial snapshot for test independence", func() {
-				err := e2e.Env.RestoreInitialState(true /*switchOffNetworkFirst*/)
-				gomega.Expect(err).Should(gomega.BeNil())
-			})
-
-			uris := e2e.Env.GetURIs()
+			uris := e2e.Env.GetURIsRW()
 			gomega.Expect(uris).ShouldNot(gomega.BeEmpty())
 
 			kc := secp256k1fx.NewKeychain(genesis.EWOQKey)